Support tests referencing external test data using dataPath (#127)

See: #110
Signed-off-by: Juan Cruz Viotti <[email protected]>

docs/test.markdown

@@ -43,7 +43,12 @@ To create a test definition, you must write JSON documents that look like this:
       "data": {
         "type": 1
       }
-    }
+    },
+    {
+      "description": "Load from an external file, relative to the test",
+      "valid": true,
+      "dataPath": "../my-data.json"
+    },
   ]
 }
 ```

src/command_test.cc

@@ -2,8 +2,9 @@
 #include <sourcemeta/jsontoolkit/jsonschema.h>
 #include <sourcemeta/jsontoolkit/uri.h>
 
-#include <cstdlib>  // EXIT_SUCCESS, EXIT_FAILURE
-#include <iostream> // std::cerr, std::cout
+#include <cstdlib>    // EXIT_SUCCESS, EXIT_FAILURE
+#include <filesystem> // std::filesystem
+#include <iostream>   // std::cerr, std::cout
 
 #include "command.h"
 #include "utils.h"
@@ -33,6 +34,33 @@ get_schema_object(const sourcemeta::jsontoolkit::URI &identifier,
   return std::nullopt;
 }
 
+static auto get_data(const sourcemeta::jsontoolkit::JSON &test_case,
+                     const std::filesystem::path &base,
+                     const bool verbose) -> sourcemeta::jsontoolkit::JSON {
+  assert(base.is_absolute());
+  assert(test_case.is_object());
+  assert(test_case.defines("data") || test_case.defines("dataPath"));
+  if (test_case.defines("data")) {
+    return test_case.at("data");
+  }
+
+  assert(test_case.defines("dataPath"));
+  assert(test_case.at("dataPath").is_string());
+
+  const std::filesystem::path data_path{std::filesystem::weakly_canonical(
+      base / test_case.at("dataPath").to_string())};
+  if (verbose) {
+    std::cerr << "Reading test instance file: " << data_path.string() << "\n";
+  }
+
+  try {
+    return sourcemeta::jsontoolkit::from_file(data_path);
+  } catch (...) {
+    std::cout << "\n";
+    throw;
+  }
+}
+
 auto intelligence::jsonschema::cli::test(
     const std::span<const std::string> &arguments) -> int {
   const auto options{parse_options(arguments, {"h", "http"})};
@@ -146,9 +174,31 @@ auto intelligence::jsonschema::cli::test(
         return EXIT_FAILURE;
       }
 
-      if (!test_case.defines("data")) {
-        std::cout << "\nerror: Test case documents must contain a `data` "
-                     "property\n  at test case #"
+      if (!test_case.defines("data") && !test_case.defines("dataPath")) {
+        std::cout << "\nerror: Test case documents must contain a `data` or "
+                     "`dataPath` property\n  at test case #"
+                  << index << "\n\n";
+        std::cout << "Learn more here: "
+                     "https://github.com/Intelligence-AI/jsonschema/blob/main/"
+                     "docs/test.markdown\n";
+        return EXIT_FAILURE;
+      }
+
+      if (test_case.defines("data") && test_case.defines("dataPath")) {
+        std::cout
+            << "\nerror: Test case documents must contain either a `data` or "
+               "`dataPath` property, but not both\n  at test case #"
+            << index << "\n\n";
+        std::cout << "Learn more here: "
+                     "https://github.com/Intelligence-AI/jsonschema/blob/main/"
+                     "docs/test.markdown\n";
+        return EXIT_FAILURE;
+      }
+
+      if (test_case.defines("dataPath") &&
+          !test_case.at("dataPath").is_string()) {
+        std::cout << "\nerror: Test case documents must set the `dataPath` "
+                     "property to a string\n  at test case #"
                   << index << "\n\n";
         std::cout << "Learn more here: "
                      "https://github.com/Intelligence-AI/jsonschema/blob/main/"
@@ -189,7 +239,8 @@ auto intelligence::jsonschema::cli::test(
 
       std::ostringstream error;
       const auto case_result{sourcemeta::jsontoolkit::evaluate(
-          schema_template, test_case.at("data"),
+          schema_template,
+          get_data(test_case, entry.first.parent_path(), verbose),
           sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
           pretty_evaluate_callback(error, {"$ref"}))};
 

test/CMakeLists.txt

@@ -82,7 +82,9 @@ add_jsonschema_test_unix(test/fail_no_tests)
 add_jsonschema_test_unix(test/fail_tests_non_array)
 add_jsonschema_test_unix(test/fail_test_case_non_object)
 add_jsonschema_test_unix(test/fail_test_case_no_data)
+add_jsonschema_test_unix(test/fail_test_case_data_and_data_path)
 add_jsonschema_test_unix(test/fail_test_case_non_string_description)
+add_jsonschema_test_unix(test/fail_test_case_non_string_data_path)
 add_jsonschema_test_unix(test/fail_test_case_no_valid)
 add_jsonschema_test_unix(test/fail_test_case_non_boolean_valid)
 add_jsonschema_test_unix(test/fail_true_resolve_fragment)
@@ -95,6 +97,8 @@ add_jsonschema_test_unix(test/pass_single_resolve_fragment_verbose)
 add_jsonschema_test_unix(test/pass_single_comment_verbose)
 add_jsonschema_test_unix(test/pass_single_no_description_verbose)
 add_jsonschema_test_unix(test/pass_single_no_test_description_verbose)
+add_jsonschema_test_unix(test/pass_single_data_path)
+add_jsonschema_test_unix(test/pass_single_data_path_verbose)
 add_jsonschema_test_unix(test/pass_multi_directory_resolve)
 add_jsonschema_test_unix(test/pass_multi_directory_resolve_verbose)
 

test/test/fail_test_case_data_and_data_path.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "target": "http://json-schema.org/draft-04/schema#",
+  "tests": [
+    {
+      "valid": true,
+      "data": {},
+      "dataPath": "./foo.json"
+    }
+  ]
+}
+EOF
+
+"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+$(realpath "$TMP")/test.json:
+error: Test case documents must contain either a \`data\` or \`dataPath\` property, but not both
+  at test case #1
+
+Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/test/fail_test_case_no_data.sh

@@ -32,7 +32,7 @@ test "$CODE" = "1" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
 $(realpath "$TMP")/test.json:
-error: Test case documents must contain a \`data\` property
+error: Test case documents must contain a \`data\` or \`dataPath\` property
   at test case #3
 
 Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown

test/test/fail_test_case_non_string_data_path.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "target": "http://json-schema.org/draft-04/schema#",
+  "tests": [
+    {
+      "valid": true,
+      "dataPath": 1
+    }
+  ]
+}
+EOF
+
+"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+$(realpath "$TMP")/test.json:
+error: Test case documents must set the \`dataPath\` property to a string
+  at test case #1
+
+Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/test/pass_single_data_path.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "id": "https://example.com",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "string"
+}
+EOF
+
+cat << 'EOF' > "$TMP/data-valid.json"
+"Hello World"
+EOF
+
+cat << 'EOF' > "$TMP/data-invalid.json"
+{ "type": "Hello World" }
+EOF
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "target": "https://example.com",
+  "tests": [
+    {
+      "description": "First test",
+      "valid": true,
+      "dataPath": "./data-valid.json"
+    },
+    {
+      "description": "Second test",
+      "valid": false,
+      "dataPath": "./data-invalid.json"
+    }
+  ]
+}
+EOF
+
+"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" 1> "$TMP/output.txt" 2>&1
+
+cat << EOF > "$TMP/expected.txt"
+$(realpath "$TMP")/test.json: PASS 2/2
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/test/pass_single_data_path_verbose.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "id": "https://example.com",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "string"
+}
+EOF
+
+cat << 'EOF' > "$TMP/data-valid.json"
+"Hello World"
+EOF
+
+cat << 'EOF' > "$TMP/data-invalid.json"
+{ "type": "Hello World" }
+EOF
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "target": "https://example.com",
+  "tests": [
+    {
+      "description": "First test",
+      "valid": true,
+      "dataPath": "./data-valid.json"
+    },
+    {
+      "description": "Second test",
+      "valid": false,
+      "dataPath": "./data-invalid.json"
+    }
+  ]
+}
+EOF
+
+"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --verbose 1> "$TMP/output.txt" 2>&1
+
+cat << EOF > "$TMP/expected.txt"
+Importing schema into the resolution context: $(realpath "$TMP")/schema.json
+$(realpath "$TMP")/test.json:
+Reading test instance file: $(realpath "$TMP")/data-valid.json
+  1/2 PASS First test
+Reading test instance file: $(realpath "$TMP")/data-invalid.json
+  2/2 PASS Second test
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"