From 4e09fe2cb7240982ddb8e188d67c07cd965807f3 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Mon, 15 Jul 2024 21:25:51 +0100 Subject: [PATCH] Support tests referencing external test data using `dataPath` (#127) See: https://github.com/Intelligence-AI/jsonschema/discussions/110 Signed-off-by: Juan Cruz Viotti --- docs/test.markdown | 7 ++- src/command_test.cc | 63 +++++++++++++++++-- test/CMakeLists.txt | 4 ++ .../test/fail_test_case_data_and_data_path.sh | 35 +++++++++++ test/test/fail_test_case_no_data.sh | 2 +- .../fail_test_case_non_string_data_path.sh | 34 ++++++++++ test/test/pass_single_data_path.sh | 50 +++++++++++++++ test/test/pass_single_data_path_verbose.sh | 55 ++++++++++++++++ 8 files changed, 242 insertions(+), 8 deletions(-) create mode 100755 test/test/fail_test_case_data_and_data_path.sh create mode 100755 test/test/fail_test_case_non_string_data_path.sh create mode 100755 test/test/pass_single_data_path.sh create mode 100755 test/test/pass_single_data_path_verbose.sh diff --git a/docs/test.markdown b/docs/test.markdown index b77bed8..a232322 100644 --- a/docs/test.markdown +++ b/docs/test.markdown @@ -43,7 +43,12 @@ To create a test definition, you must write JSON documents that look like this: "data": { "type": 1 } - } + }, + { + "description": "Load from an external file, relative to the test", + "valid": true, + "dataPath": "../my-data.json" + }, ] } ``` diff --git a/src/command_test.cc b/src/command_test.cc index ec28a8c..aa009b7 100644 --- a/src/command_test.cc +++ b/src/command_test.cc @@ -2,8 +2,9 @@ #include #include -#include // EXIT_SUCCESS, EXIT_FAILURE -#include // std::cerr, std::cout +#include // EXIT_SUCCESS, EXIT_FAILURE +#include // std::filesystem +#include // std::cerr, std::cout #include "command.h" #include "utils.h" @@ -33,6 +34,33 @@ get_schema_object(const sourcemeta::jsontoolkit::URI &identifier, return std::nullopt; } +static auto get_data(const sourcemeta::jsontoolkit::JSON &test_case, + const std::filesystem::path &base, + const bool verbose) -> sourcemeta::jsontoolkit::JSON { + assert(base.is_absolute()); + assert(test_case.is_object()); + assert(test_case.defines("data") || test_case.defines("dataPath")); + if (test_case.defines("data")) { + return test_case.at("data"); + } + + assert(test_case.defines("dataPath")); + assert(test_case.at("dataPath").is_string()); + + const std::filesystem::path data_path{std::filesystem::weakly_canonical( + base / test_case.at("dataPath").to_string())}; + if (verbose) { + std::cerr << "Reading test instance file: " << data_path.string() << "\n"; + } + + try { + return sourcemeta::jsontoolkit::from_file(data_path); + } catch (...) { + std::cout << "\n"; + throw; + } +} + auto intelligence::jsonschema::cli::test( const std::span &arguments) -> int { const auto options{parse_options(arguments, {"h", "http"})}; @@ -146,9 +174,31 @@ auto intelligence::jsonschema::cli::test( return EXIT_FAILURE; } - if (!test_case.defines("data")) { - std::cout << "\nerror: Test case documents must contain a `data` " - "property\n at test case #" + if (!test_case.defines("data") && !test_case.defines("dataPath")) { + std::cout << "\nerror: Test case documents must contain a `data` or " + "`dataPath` property\n at test case #" + << index << "\n\n"; + std::cout << "Learn more here: " + "https://github.com/Intelligence-AI/jsonschema/blob/main/" + "docs/test.markdown\n"; + return EXIT_FAILURE; + } + + if (test_case.defines("data") && test_case.defines("dataPath")) { + std::cout + << "\nerror: Test case documents must contain either a `data` or " + "`dataPath` property, but not both\n at test case #" + << index << "\n\n"; + std::cout << "Learn more here: " + "https://github.com/Intelligence-AI/jsonschema/blob/main/" + "docs/test.markdown\n"; + return EXIT_FAILURE; + } + + if (test_case.defines("dataPath") && + !test_case.at("dataPath").is_string()) { + std::cout << "\nerror: Test case documents must set the `dataPath` " + "property to a string\n at test case #" << index << "\n\n"; std::cout << "Learn more here: " "https://github.com/Intelligence-AI/jsonschema/blob/main/" @@ -189,7 +239,8 @@ auto intelligence::jsonschema::cli::test( std::ostringstream error; const auto case_result{sourcemeta::jsontoolkit::evaluate( - schema_template, test_case.at("data"), + schema_template, + get_data(test_case, entry.first.parent_path(), verbose), sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast, pretty_evaluate_callback(error, {"$ref"}))}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bc3ec1f..5362e91 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -82,7 +82,9 @@ add_jsonschema_test_unix(test/fail_no_tests) add_jsonschema_test_unix(test/fail_tests_non_array) add_jsonschema_test_unix(test/fail_test_case_non_object) add_jsonschema_test_unix(test/fail_test_case_no_data) +add_jsonschema_test_unix(test/fail_test_case_data_and_data_path) add_jsonschema_test_unix(test/fail_test_case_non_string_description) +add_jsonschema_test_unix(test/fail_test_case_non_string_data_path) add_jsonschema_test_unix(test/fail_test_case_no_valid) add_jsonschema_test_unix(test/fail_test_case_non_boolean_valid) add_jsonschema_test_unix(test/fail_true_resolve_fragment) @@ -95,6 +97,8 @@ add_jsonschema_test_unix(test/pass_single_resolve_fragment_verbose) add_jsonschema_test_unix(test/pass_single_comment_verbose) add_jsonschema_test_unix(test/pass_single_no_description_verbose) add_jsonschema_test_unix(test/pass_single_no_test_description_verbose) +add_jsonschema_test_unix(test/pass_single_data_path) +add_jsonschema_test_unix(test/pass_single_data_path_verbose) add_jsonschema_test_unix(test/pass_multi_directory_resolve) add_jsonschema_test_unix(test/pass_multi_directory_resolve_verbose) diff --git a/test/test/fail_test_case_data_and_data_path.sh b/test/test/fail_test_case_data_and_data_path.sh new file mode 100755 index 0000000..7e7ad20 --- /dev/null +++ b/test/test/fail_test_case_data_and_data_path.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "http://json-schema.org/draft-04/schema#", + "tests": [ + { + "valid": true, + "data": {}, + "dataPath": "./foo.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: +error: Test case documents must contain either a \`data\` or \`dataPath\` property, but not both + at test case #1 + +Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/fail_test_case_no_data.sh b/test/test/fail_test_case_no_data.sh index 0d6e1aa..a8e5420 100755 --- a/test/test/fail_test_case_no_data.sh +++ b/test/test/fail_test_case_no_data.sh @@ -32,7 +32,7 @@ test "$CODE" = "1" || exit 1 cat << EOF > "$TMP/expected.txt" $(realpath "$TMP")/test.json: -error: Test case documents must contain a \`data\` property +error: Test case documents must contain a \`data\` or \`dataPath\` property at test case #3 Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown diff --git a/test/test/fail_test_case_non_string_data_path.sh b/test/test/fail_test_case_non_string_data_path.sh new file mode 100755 index 0000000..a30edee --- /dev/null +++ b/test/test/fail_test_case_non_string_data_path.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "http://json-schema.org/draft-04/schema#", + "tests": [ + { + "valid": true, + "dataPath": 1 + } + ] +} +EOF + +"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: +error: Test case documents must set the \`dataPath\` property to a string + at test case #1 + +Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/pass_single_data_path.sh b/test/test/pass_single_data_path.sh new file mode 100755 index 0000000..d25cbb3 --- /dev/null +++ b/test/test/pass_single_data_path.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/data-valid.json" +"Hello World" +EOF + +cat << 'EOF' > "$TMP/data-invalid.json" +{ "type": "Hello World" } +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "First test", + "valid": true, + "dataPath": "./data-valid.json" + }, + { + "description": "Second test", + "valid": false, + "dataPath": "./data-invalid.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" 1> "$TMP/output.txt" 2>&1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: PASS 2/2 +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/pass_single_data_path_verbose.sh b/test/test/pass_single_data_path_verbose.sh new file mode 100755 index 0000000..6e1fe5f --- /dev/null +++ b/test/test/pass_single_data_path_verbose.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/data-valid.json" +"Hello World" +EOF + +cat << 'EOF' > "$TMP/data-invalid.json" +{ "type": "Hello World" } +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "First test", + "valid": true, + "dataPath": "./data-valid.json" + }, + { + "description": "Second test", + "valid": false, + "dataPath": "./data-invalid.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --verbose 1> "$TMP/output.txt" 2>&1 + +cat << EOF > "$TMP/expected.txt" +Importing schema into the resolution context: $(realpath "$TMP")/schema.json +$(realpath "$TMP")/test.json: +Reading test instance file: $(realpath "$TMP")/data-valid.json + 1/2 PASS First test +Reading test instance file: $(realpath "$TMP")/data-invalid.json + 2/2 PASS Second test +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt"