From 2a487583a48cd021f6109712d938b168de2b9800 Mon Sep 17 00:00:00 2001 From: Mark VanderVoord Date: Tue, 7 May 2024 13:25:54 -0400 Subject: [PATCH] :art: Update project files to include correct reporting options. :racehorse: Improve some test performance. --- assets/project_as_gem.yml | 20 +++++-- assets/project_with_guts.yml | 20 +++++-- assets/project_with_guts_gcov.yml | 20 +++++-- examples/temp_sensor/project.yml | 20 +++++-- spec/gcov/gcov_test_cases_spec.rb | 10 ++-- spec/spec_system_helper.rb | 94 ++++++++++++------------------- 6 files changed, 96 insertions(+), 88 deletions(-) diff --git a/assets/project_as_gem.yml b/assets/project_as_gem.yml index d7327a025..93de003fe 100644 --- a/assets/project_as_gem.yml +++ b/assets/project_as_gem.yml @@ -64,13 +64,21 @@ #- fake_function_framework # use FFF instead of CMock # Report options (You'll want to choose one stdout option, but may choose multiple stored options if desired) - #- test_suite_reporter - #- report_tests_raw_output_log - - report_tests_pretty_stdout - #- report_tests_ide_stdout + #- report_build_warnings_log #- report_tests_gtestlike_stdout - #- teamcity_tests_report - #- warnings_report + #- report_tests_ide_stdout + #- report_tests_log_factory + - report_tests_pretty_stdout + #- report_tests_raw_output_log + #- report_tests_teamcity_stdout + +# Specify which reports you'd like from the log factory +:report_tests_log_factory: + :reports: + - json + - junit + - cppunit + - html # override the default extensions for your system and toolchain :extension: diff --git a/assets/project_with_guts.yml b/assets/project_with_guts.yml index 2dc73cb6e..ed3bd080f 100644 --- a/assets/project_with_guts.yml +++ b/assets/project_with_guts.yml @@ -64,13 +64,21 @@ #- fake_function_framework # use FFF instead of CMock # Report options (You'll want to choose one stdout option, but may choose multiple stored options if desired) - #- test_suite_reporter - - report_tests_raw_output_log - - report_tests_pretty_stdout - #- report_tests_ide_stdout + #- report_build_warnings_log #- report_tests_gtestlike_stdout - #- teamcity_tests_report - #- warnings_report + #- report_tests_ide_stdout + #- report_tests_log_factory + - report_tests_pretty_stdout + #- report_tests_raw_output_log + #- report_tests_teamcity_stdout + +# Specify which reports you'd like from the log factory +:report_tests_log_factory: + :reports: + - json + - junit + - cppunit + - html # override the default extensions for your system and toolchain :extension: diff --git a/assets/project_with_guts_gcov.yml b/assets/project_with_guts_gcov.yml index 7d0eadd25..f3fdb06da 100644 --- a/assets/project_with_guts_gcov.yml +++ b/assets/project_with_guts_gcov.yml @@ -64,13 +64,21 @@ #- fake_function_framework # use FFF instead of CMock # Report options (You'll want to choose one stdout option, but may choose multiple stored options if desired) - #- test_suite_reporter - #- report_tests_raw_output_log - - report_tests_pretty_stdout - #- report_tests_ide_stdout + #- report_build_warnings_log #- report_tests_gtestlike_stdout - #- teamcity_tests_report - #- warnings_report + #- report_tests_ide_stdout + #- report_tests_log_factory + - report_tests_pretty_stdout + #- report_tests_raw_output_log + #- report_tests_teamcity_stdout + +# Specify which reports you'd like from the log factory +:report_tests_log_factory: + :reports: + - json + - junit + - cppunit + - html # override the default extensions for your system and toolchain :extension: diff --git a/examples/temp_sensor/project.yml b/examples/temp_sensor/project.yml index 2ff20b2b8..d98e314fd 100644 --- a/examples/temp_sensor/project.yml +++ b/examples/temp_sensor/project.yml @@ -64,13 +64,21 @@ #- fake_function_framework # use FFF instead of CMock # Report options (You'll want to choose one stdout option, but may choose multiple stored options if desired) - #- test_suite_reporter - #- report_tests_raw_output_log - - report_tests_pretty_stdout - #- report_tests_ide_stdout + #- report_build_warnings_log #- report_tests_gtestlike_stdout - #- teamcity_tests_report - #- warnings_report + #- report_tests_ide_stdout + #- report_tests_log_factory + - report_tests_pretty_stdout + #- report_tests_raw_output_log + #- report_tests_teamcity_stdout + +# Specify which reports you'd like from the log factory +:report_tests_log_factory: + :reports: + - json + - junit + - cppunit + - html # override the default extensions for your system and toolchain :extension: diff --git a/spec/gcov/gcov_test_cases_spec.rb b/spec/gcov/gcov_test_cases_spec.rb index 18be4dea5..d6aa37c36 100644 --- a/spec/gcov/gcov_test_cases_spec.rb +++ b/spec/gcov/gcov_test_cases_spec.rb @@ -207,8 +207,8 @@ def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, + :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling gcov:all 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called @@ -238,8 +238,8 @@ def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, + :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling gcov:all --exclude_test_case=test_add_numbers_adds_numbers 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called @@ -269,7 +269,7 @@ def can_create_gcov_html_report_from_test_runner_with_enabled_debug_and_cmd_args FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:test_runner => { :cmdline_args => true }}) add_test_case = "\nvoid test_difference_between_two_numbers(void)\n"\ "{\n" \ diff --git a/spec/spec_system_helper.rb b/spec/spec_system_helper.rb index 08ccc48c4..c4acf6ed5 100644 --- a/spec/spec_system_helper.rb +++ b/spec/spec_system_helper.rb @@ -22,14 +22,6 @@ def convert_slashes(path) end end -def add_project_settings(project_file_path, settings, show_final=false) - yaml_wrapper = YamlWrapper.new - project_hash = yaml_wrapper.load(project_file_path) - project_hash.deep_merge!(settings) - puts "\n\n#{project_hash.to_yaml}\n\n" if show_final - yaml_wrapper.dump(project_file_path, project_hash) -end - class GemDirLayout attr_reader :gem_dir_base_name @@ -142,34 +134,26 @@ def with_constrained_env end end - def modify_project_yml_for_test(prefix, key, new_value) - add_line = nil - updated = false - updated_yml = [] - File.read('project.yml').split("\n").each_with_index do |line, i| - m = line.match /\:#{key.to_s}\:\s*(.*)/ - unless m.nil? - line = line.gsub(m[1], new_value) - updated = true - end - - m = line.match /(\s*)\:#{prefix.to_s}\:/ - unless m.nil? - add_line = [i+1, m[1]+' '] - end + ############################################################ + # Functions for manipulating project.yml files during tests: + def merge_project_yml_for_test(settings, show_final=false) + yaml_wrapper = YamlWrapper.new + project_hash = yaml_wrapper.load('project.yml') + project_hash.deep_merge!(settings) + puts "\n\n#{project_hash.to_yaml}\n\n" if show_final + yaml_wrapper.dump('project.yml', project_hash) + end - updated_yml.append(line) - end - unless updated - if add_line.nil? - updated_yml.insert(updated_yml.length - 1, ":#{prefix.to_s}:\n :#{key.to_s}: #{new_value}") - else - updated_yml.insert(add_line[0], "#{add_line[1]}:#{key}: #{new_value}") - end - end + def append_project_yml_for_test(new_args) + fake_prj_yml= "#{File.read('project.yml')}\n#{new_args}" + File.write('project.yml', fake_prj_yml, mode: 'w') + end - File.write('project.yml', updated_yml.join("\n"), mode: 'w') + def uncomment_project_yml_option_for_test(option) + fake_prj_yml= File.read('project.yml').gsub(/\##{option}/,option) + File.write('project.yml', fake_prj_yml, mode: 'w') end + ############################################################ end module CeedlingTestCases @@ -329,7 +313,7 @@ def can_test_projects_with_unity_exec_time FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' settings = { :unity => { :defines => [ "UNITY_INCLUDE_EXEC_TIME" ] } } - add_project_settings("project.yml", settings) + @c.merge_project_yml_for_test(settings) output = `bundle exec ruby -S ceedling 2>&1` expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here @@ -350,7 +334,7 @@ def can_test_projects_with_test_and_vendor_defines_with_success settings = { :unity => { :defines => [ "UNITY_INCLUDE_PRINT_FORMATTED" ] }, :defines => { :test => { :example_file_unity_printf => [ "TEST" ] } } } - add_project_settings("project.yml", settings) + @c.merge_project_yml_for_test(settings) output = `bundle exec ruby -S ceedling 2>&1` expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here @@ -368,7 +352,7 @@ def can_test_projects_with_enabled_auto_link_deep_deependency_with_success FileUtils.copy_entry test_asset_path("auto_link_deep_dependencies/src/"), 'src/' FileUtils.cp_r test_asset_path("auto_link_deep_dependencies/test/."), 'test/' settings = { :project => { :auto_link_deep_dependencies => true } } - add_project_settings("project.yml", settings) + @c.merge_project_yml_for_test(settings) output = `bundle exec ruby -S ceedling 2>&1` expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here @@ -389,7 +373,7 @@ def can_test_projects_with_test_name_replaced_defines_with_success 'test_adc_hardware_special.c' => [ "TEST", "SPECIFIC_CONFIG" ], } } } - add_project_settings("project.yml", settings) + @c.merge_project_yml_for_test(settings) output = `bundle exec ruby -S ceedling 2>&1` expect($?.exitstatus).to match(0) # Since a test either passes or is ignored, we return success here @@ -492,6 +476,8 @@ def uses_report_tests_raw_output_log_plugin FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_verbose.c"), 'test/' + @c.uncomment_project_yml_option_for_test('- report_tests_raw_output_log') + output = `bundle exec ruby -S ceedling test:all 2>&1` expect($?.exitstatus).to match(0) # Since a test either pass or are ignored, we return success here expect(output).to match(/TESTED:\s+\d/) @@ -542,9 +528,7 @@ def can_run_single_test_with_full_test_case_name_from_test_file_with_success_cmd FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' enable_unity_extra_args = "\n:test_runner:\n"\ " :cmdline_args: true\n" - fake_prj_yml= File.read('project.yml').split("\n") - fake_prj_yml.insert(fake_prj_yml.length() -1, enable_unity_extra_args) - File.write('project.yml', fake_prj_yml.join("\n"), mode: 'w') + @c.append_project_yml_for_test(enable_unity_extra_args) output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=test_add_numbers_adds_numbers 2>&1` @@ -565,9 +549,7 @@ def can_run_single_test_with_partiall_test_case_name_from_test_file_with_enabled FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' enable_unity_extra_args = "\n:test_runner:\n"\ " :cmdline_args: true\n" - fake_prj_yml= File.read('project.yml').split("\n") - fake_prj_yml.insert(fake_prj_yml.length() -1, enable_unity_extra_args) - File.write('project.yml', fake_prj_yml.join("\n"), mode: 'w') + @c.append_project_yml_for_test(enable_unity_extra_args) output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=_adds_numbers 2>&1` @@ -588,9 +570,7 @@ def none_of_test_is_executed_if_test_case_name_passed_does_not_fit_defined_in_te FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' enable_unity_extra_args = "\n:test_runner:\n"\ " :cmdline_args: true\n" - fake_prj_yml= File.read('project.yml').split("\n") - fake_prj_yml.insert(fake_prj_yml.length() -1, enable_unity_extra_args) - File.write('project.yml', fake_prj_yml.join("\n"), mode: 'w') + @c.append_project_yml_for_test(enable_unity_extra_args) output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=zumzum 2>&1` @@ -608,9 +588,7 @@ def none_of_test_is_executed_if_test_case_name_and_exclude_test_case_name_is_the FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' enable_unity_extra_args = "\n:test_runner:\n"\ " :cmdline_args: true\n" - fake_prj_yml= File.read('project.yml').split("\n") - fake_prj_yml.insert(fake_prj_yml.length() -1, enable_unity_extra_args) - File.write('project.yml', fake_prj_yml.join("\n"), mode: 'w') + @c.append_project_yml_for_test(enable_unity_extra_args) output = `bundle exec ruby -S ceedling test:test_example_file_success --test_case=_adds_numbers --exclude_test_case=_adds_numbers 2>&1` @@ -647,9 +625,7 @@ def exclude_test_case_name_filter_works_and_only_one_test_case_is_executed FileUtils.cp test_asset_path("test_example_file_success.c"), 'test/' enable_unity_extra_args = "\n:test_runner:\n"\ " :cmdline_args: true\n" - fake_prj_yml= File.read('project.yml').split("\n") - fake_prj_yml.insert(fake_prj_yml.length() -1, enable_unity_extra_args) - File.write('project.yml', fake_prj_yml.join("\n"), mode: 'w') + @c.append_project_yml_for_test(enable_unity_extra_args) output = `bundle exec ruby -S ceedling test:all --exclude_test_case=test_add_numbers_adds_numbers 2>&1` @@ -704,7 +680,7 @@ def test_run_of_projects_fail_because_of_sigsegv_with_report FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }}) output = `bundle exec ruby -S ceedling test:all 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called @@ -724,8 +700,8 @@ def execute_all_test_cases_from_crashing_test_runner_and_return_test_report_with FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, + :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called @@ -749,8 +725,8 @@ def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_test_case_ FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, + :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all --test_case=test_add_numbers_will_fail 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called @@ -774,8 +750,8 @@ def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_exclude_te FileUtils.cp test_asset_path("example_file.c"), 'src/' FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' - @c.modify_project_yml_for_test(:project, :use_backtrace, 'TRUE') - @c.modify_project_yml_for_test(:test_runner, :cmdline_args, 'TRUE') + @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, + :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all --exclude_test_case=add_numbers_adds_numbers 2>&1` expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called