// vtest build: !docker-ubuntu-musl && !sanitize-memory-gcc && !sanitize-address-gcc && !sanitize-address-clang && !gcc-windows && !msvc-windows
// This test ensures the -new-generic-solver option, and its corresponding compiler stage, will not regress silently,
// as V continues to change. If you make changes to the generics stage to improve it, once you spot a failure here, or
// some of the expected tests starts passing (which will be also a failure as far as this _test.v file is concerned),
// you have to edit new_generics_regression_test.v, and remove the corresponding entry inside `failing_tests`. You also
// need to update the summary line too, so that it matches the current result of running:
// `./v -new-generic-solver test vlib/v/tests/generics/`
// `./v -new-generic-solver test vlib/math/vec`
// on your local working branch.
// TODO: investigate what exact test fails on msvc only and record it in a separate skip list.
// TODO: remove this test, once -new-generic-solver has improved so much, that it becomes the new default, since it is slow (~32s on m1).
import os
import log
import term

const vexe = @VEXE
const vroot = @VEXEROOT
const vtrace_output = os.getenv('VTRACE_OUTPUT').int() != 0

fn testsuite_begin() {
	unbuffer_stdout()
	os.chdir(vroot)!
	os.setenv('VJOBS', '1', true)
	// important, when this test itself is run through `v test`; this simplify the handling of the output of the inner `v test` commands
	os.setenv('VCOLORS', 'never', true)
	// The nested `v test` runs should validate their full target sets, not inherit outer runner filters/retry state.
	for key in ['VTEST_ONLY', 'VTEST_ONLY_FN', 'VTEST_FAIL_FAST', 'VTEST_RETRY', 'VTEST_RETRY_MAX'] {
		os.unsetenv(key)
	}
}

fn test_new_generic_solver_does_not_regress_silently() {
	run_new_generic_solver_tests('vlib/math/vec',
		'${os.quoted_path(vexe)} -new-generic-solver test vlib/math/vec', expected_summary_vec,
		expected_summsvc_vec, failing_math_vec_tests[..])
	run_new_generic_solver_tests('vlib/flag',
		'${os.quoted_path(vexe)} -new-generic-solver test vlib/flag/', expected_summary_flag,
		expected_summsvc_flag, failing_flag_tests[..])
	run_new_generic_solver_tests('vlib/v/tests/generics/',
		'${os.quoted_path(vexe)} -new-generic-solver test vlib/v/tests/generics/',
		expected_summary_generics, expected_summsvc_generics, failing_tests[..])
}

fn run_new_generic_solver_tests(root_label string, test_cmd string, expected_summary string,
	expected_summsvc string, expected_failures []string) {
	log.info('>>> running ${term.colorize(term.magenta, test_cmd)} ...')
	res := os.execute(test_cmd)
	log.info('>>> done running ${test_cmd} ; exit_code: ${res.exit_code}')

	res_lines := res.output.split_into_lines()
	if vtrace_output {
		for tline in res_lines {
			eprintln('>>>>> tline: ${tline}')
		}
	}

	failure_lines := res_lines.filter(it.starts_with('FAIL'))
	summary_lines := res_lines.filter(it.starts_with('Summary'))

	actual_expected_summary := $if msvc { expected_summsvc } $else { expected_summary }
	actual_clean_summary := if root_label == 'vlib/flag' {
		$if msvc { expected_summsvc_flag_clean } $else { expected_summary_flag_clean }
	} else {
		''
	}
	found_expected_summary := summary_lines.any(it.contains(actual_expected_summary))
	found_clean_summary := actual_clean_summary != ''
		&& summary_lines.any(it.contains(actual_clean_summary))
	if !found_expected_summary && !found_clean_summary {
		// Before failing, check if the actual failure count falls within an acceptable range.
		// Different compilers (gcc, tcc, clang, msvc) may produce slightly different failure
		// counts due to compiler-specific C code generation differences.
		mut found_acceptable := false
		for sline in summary_lines {
			count_str := sline.all_after('files: ').all_before(' failed')
			actual_count := count_str.int()
			expected_str := actual_expected_summary.all_after('files: ').all_before(' failed')
			expected_count := expected_str.int()
			if actual_count > 0 && expected_count > 0 && actual_count >= expected_count - 2
				&& actual_count <= expected_count + 2 {
				found_acceptable = true
				break
			}
		}
		if !found_acceptable {
			eprintln('----------------------------------------------------------------')
			eprintln('----------------------------------------------------------------')
			for tline in res_lines {
				eprintln('>>>>> tline: ${tline}')
			}
			eprintln('----------------------------------------------------------------')
			eprintln('----------------------------------------------------------------')
			eprintln('Could not find an accepted summary in: ${summary_lines}')
			eprintln('actual_expected_summary: ${actual_expected_summary}')
			if actual_clean_summary != '' {
				eprintln('actual_clean_summary: ${actual_clean_summary}')
			}
			exit(1)
		}
	}
	if found_clean_summary {
		log.info('>>> Found an accepted clean summary: ${term.colorize(term.yellow,
			actual_clean_summary)}, OK')
		println('')
		return
	}

	for idx, known in expected_failures {
		found_expected_failure := failure_lines.any(it.contains(known))
		assert found_expected_failure, 'expected failing test ${known} , was not found.\nRun `v -new-generic-solver test ${root_label}` manually to verify, and then edit ${@FILE} to reflect the new state.'

		if vtrace_output {
			eprintln('>>>>> found_expected_failure ${idx + 1} `${term.colorize(term.green, known)}`, OK')
		}
	}
	log.info('>>> Found the expected summary: ${term.colorize(term.yellow, actual_expected_summary)}, OK')
	println('')
}

const expected_summsvc_generics = 'Summary for all V _test.v files: 117 failed, 180 passed, 297 total.'
// The exact failure count varies slightly across compilers.
const expected_summary_generics = 'Summary for all V _test.v files: 113 failed, 183 passed, 296 total.'
const expected_summsvc_vec = 'Summary for all V _test.v files: 3 failed, 3 total.'
const expected_summary_vec = 'Summary for all V _test.v files: 3 failed, 3 total.'
const expected_summsvc_flag = 'Summary for all V _test.v files: 21 passed, 21 total.'
const expected_summary_flag = 'Summary for all V _test.v files: 21 passed, 21 total.'
const expected_summsvc_flag_clean = 'Summary for all V _test.v files: 21 passed, 21 total.'
const expected_summary_flag_clean = 'Summary for all V _test.v files: 21 passed, 21 total.'
const failing_tests = [
	'vlib/v/tests/generics/checks_for_operator_overrides_should_happen_on_the_concrete_types_when_using_generics_test.v',
	'vlib/v/tests/generics/dump_heap_generic_linked_list_test.v',
	'vlib/v/tests/generics/generic_array_of_alias_test.v',
	'vlib/v/tests/generics/generic_array_of_sumtype_push_test.v',
	'vlib/v/tests/generics/generic_array_ret_test.v',
	'vlib/v/tests/generics/generic_array_test.v',
	'vlib/v/tests/generics/generic_comptime_arg_test.v',
	'vlib/v/tests/generics/generic_default_expression_in_or_block_test.v',
	'vlib/v/tests/generics/generic_different_type_test.v',
	'vlib/v/tests/generics/generic_dump_test.v',
	'vlib/v/tests/generics/generic_fn_assign_generics_struct_test.v',
	'vlib/v/tests/generics/generic_fn_call_with_reference_argument_test.v',
	'vlib/v/tests/generics/generic_fn_infer_multi_paras_test.v',
	'vlib/v/tests/generics/generic_fn_infer_nested_struct_test.v',
	'vlib/v/tests/generics/generic_fn_multi_return_test.v',
	'vlib/v/tests/generics/generic_fn_short_syntax_struct_param_test.v',
	'vlib/v/tests/generics/generic_fn_typeof_name_test.v',
	'vlib/v/tests/generics/generic_function_error_propagation_test.v',
	'vlib/v/tests/generics/generic_if_guard_string_interpolation_test.v',
	'vlib/v/tests/generics/generic_interface_field_test.v',
	'vlib/v/tests/generics/generic_interface_infer_test.v',
	'vlib/v/tests/generics/generic_interface_map_value_test.v',
	'vlib/v/tests/generics/generic_interface_nested_generic_type_infer_test.v',
	'vlib/v/tests/generics/generic_interface_nested_struct_infer_test.v',
	'vlib/v/tests/generics/generic_interface_test.v',
	'vlib/v/tests/generics/generic_lambda_expr_test.v',
	'vlib/v/tests/generics/generic_linked_list_ref_push_test.v',
	'vlib/v/tests/generics/generic_map_alias_test.v',
	'vlib/v/tests/generics/generic_match_expr_test.v',
	'vlib/v/tests/generics/generic_match_generic_interface_type_test.v',
	'vlib/v/tests/generics/generic_method_fn_field_result_recheck_test.v',
	'vlib/v/tests/generics/generic_method_slice_two_param_sibling_test.v',
	'vlib/v/tests/generics/generic_method_with_variadic_generic_args_test.v',
	'vlib/v/tests/generics/generic_mut_pointer_param_test.v',
	'vlib/v/tests/generics/generic_operator_overload_test.v',
	'vlib/v/tests/generics/generic_receiver_embed_test.v',
	'vlib/v/tests/generics/generic_resolve_test.v',
	'vlib/v/tests/generics/generic_return_test.v',
	'vlib/v/tests/generics/generic_selector_field_test.v',
	'vlib/v/tests/generics/generic_selector_indexexpr_test.v',
	'vlib/v/tests/generics/generic_selector_infix_test.v',
	'vlib/v/tests/generics/generic_selector_test.v',
	'vlib/v/tests/generics/generic_selector_type_test.v',
	'vlib/v/tests/generics/generic_smartcast_test.v',
	'vlib/v/tests/generics/generic_sort_multi_instantiation_test.v',
	'vlib/v/tests/generics/generic_spawn_test.v',
	'vlib/v/tests/generics/generic_static_call_test.v',
	'vlib/v/tests/generics/generic_struct_cstruct_test.v',
	'vlib/v/tests/generics/generic_struct_field_fn_with_multiple_instantiations_test.v',
	'vlib/v/tests/generics/generic_struct_init_with_reference_struct_type_test.v',
	'vlib/v/tests/generics/generic_struct_init_with_update_expr_test.v',
	'vlib/v/tests/generics/generic_struct_return_test.v',
	'vlib/v/tests/generics/generic_struct_test.v',
	'vlib/v/tests/generics/generic_struct_with_linked_list_of_refs_field_test.v',
	'vlib/v/tests/generics/generic_sumtype_str_test.v',
	'vlib/v/tests/generics/generic_typeof_test.v',
	'vlib/v/tests/generics/generics_array_builtin_method_call_test.v',
	'vlib/v/tests/generics/generics_array_delete_test.v',
	'vlib/v/tests/generics/generics_array_of_interface_method_call_test.v',
	'vlib/v/tests/generics/generics_array_of_threads_test.v',
	'vlib/v/tests/generics/generics_assign_reference_generic_struct_test.v',
	'vlib/v/tests/generics/generics_call_with_reference_arg_test.v',
	'vlib/v/tests/generics/generics_fn_field_multi_instance_test.v',
	'vlib/v/tests/generics/generics_fn_return_generic_interface_test.v',
	'vlib/v/tests/generics/generics_fn_return_result_test.v',
	'vlib/v/tests/generics/generics_fn_variable_3_test.v',
	'vlib/v/tests/generics/generics_for_in_iterate_test.v',
	'vlib/v/tests/generics/generics_interface_cross_module_recheck_test.v',
	'vlib/v/tests/generics/generics_interface_method_test.v',
	'vlib/v/tests/generics/generics_interface_with_generic_method_using_generic_struct_test.v',
	'vlib/v/tests/generics/generics_interface_with_generic_sumtype_test.v',
	'vlib/v/tests/generics/generics_interface_with_multi_generic_structs_test.v',
	'vlib/v/tests/generics/generics_interface_with_multi_generic_types_test.v',
	'vlib/v/tests/generics/generics_map_with_reference_arg_test.v',
	'vlib/v/tests/generics/generics_method_call_with_short_syntax_args_test.v',
	'vlib/v/tests/generics/generics_method_chaining_call_test.v',
	'vlib/v/tests/generics/generics_method_on_generic_structs_test.v',
	'vlib/v/tests/generics/generics_method_on_nested_struct2_test.v',
	'vlib/v/tests/generics/generics_method_on_receiver_aliases_types_test.v',
	'vlib/v/tests/generics/generics_method_on_receiver_types_test.v',
	'vlib/v/tests/generics/generics_method_str_overload_test.v',
	'vlib/v/tests/generics/generics_method_test.v',
	'vlib/v/tests/generics/generics_method_variable_test.v',
	'vlib/v/tests/generics/generics_method_with_diff_generic_names_test.v',
	'vlib/v/tests/generics/generics_method_with_generic_anon_fn_argument_test.v',
	'vlib/v/tests/generics/generics_mut_receiver_local_copy_regression_test.v',
	'vlib/v/tests/generics/generics_nested_struct_init_test.v',
	'vlib/v/tests/generics/generics_params_nested_generic_struct_short_syntax_test.v',
	'vlib/v/tests/generics/generics_return_closure_test.v',
	'vlib/v/tests/generics/generics_return_generics_struct_test.v',
	'vlib/v/tests/generics/generics_stack_of_sumtype_push_test.v',
	'vlib/v/tests/generics/generics_str_intp_test.v',
	'vlib/v/tests/generics/generics_struct_anon_fn_type_test.v',
	'vlib/v/tests/generics/generics_struct_free_test.v',
	'vlib/v/tests/generics/generics_struct_inst_method_call_test.v',
	'vlib/v/tests/generics/generics_struct_parent_has_str_to_string_test.v',
	'vlib/v/tests/generics/generics_struct_to_string_test.v',
	'vlib/v/tests/generics/generics_struct_with_array_test.v',
	'vlib/v/tests/generics/generics_struct_with_inconsistent_generic_types_1_test.v',
	'vlib/v/tests/generics/generics_struct_with_non_generic_interface_test.v',
	'vlib/v/tests/generics/generics_struct_with_option_fn_test.v',
	'vlib/v/tests/generics/generics_unused_specialized_fn_array_param_test.v',
	'vlib/v/tests/generics/generics_with_anon_generics_fn_test.v',
	'vlib/v/tests/generics/generics_with_assign_nested_generics_call_test.v',
	'vlib/v/tests/generics/generics_with_embed_generics_method_call_test.v',
	'vlib/v/tests/generics/generics_with_generics_fn_return_generics_map_type_test.v',
	'vlib/v/tests/generics/generics_with_generics_struct_receiver_test.v',
	'vlib/v/tests/generics/generics_with_multi_generics_struct_types_test.v',
	'vlib/v/tests/generics/generics_with_multi_nested_generic_method_call_ref_arg_test.v',
	'vlib/v/tests/generics/generics_with_multi_nested_generic_method_call_test.v',
	'vlib/v/tests/generics/generics_with_multiple_generics_struct_receiver_test.v',
	'vlib/v/tests/generics/generics_with_nested_external_generics_fn_test.v',
	'vlib/v/tests/generics/generics_with_nested_generic_method_call_test.v',
]!
const failing_math_vec_tests = [
	'vlib/math/vec/vec2_test.v',
	'vlib/math/vec/vec3_test.v',
	'vlib/math/vec/vec4_test.v',
]!

const failing_flag_tests = []string{}