1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package dev.metaschema.cli;
7   
8   import static org.assertj.core.api.Assertions.assertThat;
9   import static org.junit.jupiter.api.Assertions.assertAll;
10  import static org.junit.jupiter.api.Assertions.assertEquals;
11  import static org.junit.jupiter.api.Assertions.assertFalse;
12  import static org.junit.jupiter.api.Assertions.assertNull;
13  import static org.junit.jupiter.api.Assertions.assertTrue;
14  
15  import org.json.JSONArray;
16  import org.json.JSONObject;
17  import org.json.JSONTokener;
18  import org.junit.jupiter.api.Test;
19  import org.junit.jupiter.api.parallel.Execution;
20  import org.junit.jupiter.api.parallel.ExecutionMode;
21  import org.junit.jupiter.params.ParameterizedTest;
22  import org.junit.jupiter.params.provider.Arguments;
23  import org.junit.jupiter.params.provider.MethodSource;
24  
25  import java.io.IOException;
26  import java.io.OutputStream;
27  import java.io.PrintStream;
28  import java.io.Reader;
29  import java.nio.charset.StandardCharsets;
30  import java.nio.file.Files;
31  import java.nio.file.Path;
32  import java.nio.file.Paths;
33  import java.util.LinkedList;
34  import java.util.List;
35  import java.util.StringJoiner;
36  import java.util.stream.Stream;
37  
38  import dev.harrel.jsonschema.Dialects;
39  import dev.harrel.jsonschema.JsonNode;
40  import dev.harrel.jsonschema.Validator;
41  import dev.harrel.jsonschema.ValidatorFactory;
42  import dev.harrel.jsonschema.providers.OrgJsonNode;
43  import dev.metaschema.cli.processor.ExitCode;
44  import dev.metaschema.cli.processor.ExitStatus;
45  import edu.umd.cs.findbugs.annotations.NonNull;
46  import nl.altindag.log.LogCaptor;
47  
48  /**
49   * Unit test for simple CLI.
50   */
51  @Execution(value = ExecutionMode.SAME_THREAD, reason = "Log capturing needs to be single threaded")
52  public class CLITest {
53    private static final ExitCode NO_EXCEPTION_CLASS = null;
54  
55    /**
56     * A PrintStream that discards all output, used to suppress CLI console output
57     * during tests.
58     */
59    @SuppressWarnings("resource")
60    private static final PrintStream NULL_STREAM = new PrintStream(new OutputStream() {
61      @Override
62      public void write(int b) {
63        // discard
64      }
65  
66      @Override
67      public void write(byte[] b, int off, int len) {
68        // discard
69      }
70    });
71  
72    void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode, @NonNull String[] args) {
73      status.generateMessage(true);
74      Throwable thrown = status.getThrowable();
75      assertAll(
76          () -> assertEquals(expectedCode, status.getExitCode(),
77              () -> buildExitCodeMismatchMessage(status, expectedCode, thrown, args)),
78          () -> assertNull(thrown,
79              () -> buildUnexpectedThrowableMessage(thrown, args)));
80    }
81  
82    void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode,
83        @NonNull Class<? extends Throwable> thrownClass, @NonNull String[] args) {
84      Throwable thrown = status.getThrowable();
85      assertAll(
86          () -> assertEquals(expectedCode, status.getExitCode(),
87              () -> buildExitCodeMismatchMessage(status, expectedCode, thrown, args)),
88          () -> assertEquals(thrownClass, thrown == null ? null : thrown.getClass(),
89              () -> buildThrowableMismatchMessage(thrownClass, thrown, args)));
90    }
91  
92    private static String buildExitCodeMismatchMessage(@NonNull ExitStatus status, @NonNull ExitCode expectedCode,
93        Throwable thrown, @NonNull String[] args) {
94      StringBuilder sb = new StringBuilder();
95      sb.append("exit code mismatch: expected <").append(expectedCode).append("> but was <")
96          .append(status.getExitCode()).append(">");
97      sb.append("\nCommand args: ").append(String.join(" ", args));
98      if (status.getMessage() != null) {
99        sb.append("\nStatus message: ").append(status.getMessage());
100     }
101     if (thrown != null) {
102       sb.append("\nThrowable: ").append(thrown.getClass().getName()).append(": ").append(thrown.getMessage());
103       sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
104     }
105     return sb.toString();
106   }
107 
108   private static String buildUnexpectedThrowableMessage(Throwable thrown, @NonNull String[] args) {
109     if (thrown == null) {
110       return "expected null Throwable";
111     }
112     StringBuilder sb = new StringBuilder();
113     sb.append("expected null Throwable but got: ").append(thrown.getClass().getName())
114         .append(": ").append(thrown.getMessage());
115     sb.append("\nCommand args: ").append(String.join(" ", args));
116     sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
117     return sb.toString();
118   }
119 
120   private static String buildThrowableMismatchMessage(Class<? extends Throwable> expectedClass, Throwable thrown,
121       @NonNull String[] args) {
122     StringBuilder sb = new StringBuilder();
123     sb.append("expected Throwable mismatch: expected <")
124         .append(expectedClass == null ? "null" : expectedClass.getName())
125         .append("> but was <")
126         .append(thrown == null ? "null" : thrown.getClass().getName())
127         .append(">");
128     sb.append("\nCommand args: ").append(String.join(" ", args));
129     if (thrown != null) {
130       sb.append("\nMessage: ").append(thrown.getMessage());
131       sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
132     }
133     return sb.toString();
134   }
135 
136   private static String getStackTraceAsString(Throwable throwable) {
137     java.io.StringWriter sw = new java.io.StringWriter();
138     throwable.printStackTrace(new java.io.PrintWriter(sw));
139     return sw.toString();
140   }
141 
142   private static Stream<Arguments> providesValues() {
143     List<Arguments> values = new LinkedList<>() {
144       {
145         add(Arguments.of(new String[] {}, ExitCode.INVALID_COMMAND,
146             NO_EXCEPTION_CLASS));
147         add(Arguments.of(new String[] { "-h" }, ExitCode.OK, NO_EXCEPTION_CLASS));
148         add(Arguments.of(new String[] { "generate-schema", "--help" }, ExitCode.OK,
149             NO_EXCEPTION_CLASS));
150         add(Arguments.of(new String[] { "generate-diagram", "--help" }, ExitCode.OK,
151             NO_EXCEPTION_CLASS));
152         add(Arguments.of(new String[] { "validate", "--help" }, ExitCode.OK,
153             NO_EXCEPTION_CLASS));
154         add(Arguments.of(new String[] { "validate-content", "--help" }, ExitCode.OK,
155             NO_EXCEPTION_CLASS));
156         add(Arguments.of(new String[] { "convert", "--help" }, ExitCode.OK,
157             NO_EXCEPTION_CLASS));
158         add(Arguments.of(new String[] { "metapath", "list-functions", "--help" }, ExitCode.OK,
159             NO_EXCEPTION_CLASS));
160         add(Arguments.of(new String[] { "metapath", "eval", "--help" }, ExitCode.OK,
161             NO_EXCEPTION_CLASS));
162         add(Arguments.of(new String[] { "list-allowed-values", "--help" }, ExitCode.OK,
163             NO_EXCEPTION_CLASS));
164         add(Arguments.of(
165             new String[] { "list-allowed-values",
166                 "src/test/resources/content/schema-validation-module.xml"
167             },
168             ExitCode.OK, NO_EXCEPTION_CLASS));
169         // list-allowed-values with no module argument should fail fast with
170         // INVALID_ARGUMENTS (missing required positional arg)
171         add(Arguments.of(
172             new String[] { "list-allowed-values" },
173             ExitCode.INVALID_ARGUMENTS, NO_EXCEPTION_CLASS));
174         add(Arguments.of(
175             new String[] { "validate",
176                 "../databind/src/test/resources/metaschema/fields_with_flags/metaschema.xml"
177             },
178             ExitCode.OK, NO_EXCEPTION_CLASS));
179         add(Arguments.of(
180             new String[] { "generate-schema", "--overwrite", "--as",
181                 "JSON",
182                 "../databind/src/test/resources/metaschema/fields_with_flags/metaschema.xml",
183                 "target/schema-test.json" },
184             ExitCode.OK, NO_EXCEPTION_CLASS));
185         add(Arguments.of(
186             new String[] { "validate-content", "--as=xml",
187                 "-m=../databind/src/test/resources/metaschema/bad_index-has-key/metaschema.xml",
188                 "../databind/src/test/resources/metaschema/bad_index-has-key/example.xml",
189                 "--show-stack-trace" },
190             ExitCode.FAIL, NO_EXCEPTION_CLASS));
191         add(Arguments.of(
192             new String[] { "validate-content", "--as=json",
193                 "-m=../databind/src/test/resources/metaschema/bad_index-has-key/metaschema.xml",
194                 "../databind/src/test/resources/metaschema/bad_index-has-key/example.json", "--show-stack-trace" },
195             ExitCode.FAIL, NO_EXCEPTION_CLASS));
196         add(Arguments.of(
197             new String[] { "validate",
198                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
199                 "--show-stack-trace" },
200             ExitCode.OK, NO_EXCEPTION_CLASS));
201         add(Arguments.of(
202             new String[] { "generate-schema",
203                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
204                 "--as", "xml",
205             },
206             ExitCode.OK, NO_EXCEPTION_CLASS));
207         add(Arguments.of(
208             new String[] { "generate-schema",
209                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
210                 "--as", "json",
211             },
212             ExitCode.OK, NO_EXCEPTION_CLASS));
213         add(Arguments.of(
214             new String[] { "generate-diagram",
215                 "../databind/src/test/resources/metaschema/simple/metaschema.xml"
216             },
217             ExitCode.OK, NO_EXCEPTION_CLASS));
218         add(Arguments.of(
219             new String[] { "validate-content",
220                 "-m",
221                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
222                 "../databind/src/test/resources/metaschema/simple/example.json",
223                 "--as=json"
224             },
225             ExitCode.OK, NO_EXCEPTION_CLASS));
226         add(Arguments.of(
227             new String[] { "validate-content",
228                 "-m",
229                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
230                 "../databind/src/test/resources/metaschema/simple/example.xml",
231                 "--as=xml"
232             },
233             ExitCode.OK, NO_EXCEPTION_CLASS));
234         add(Arguments.of(
235             new String[] { "validate-content",
236                 "-m",
237                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
238                 "https://bad.domain.example.net/example.xml",
239                 "--as=xml"
240             },
241             ExitCode.IO_ERROR, java.net.UnknownHostException.class));
242         add(Arguments.of(
243             new String[] { "validate-content",
244                 "-m",
245                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
246                 "https://github.com/no-example.xml",
247                 "--as=xml"
248             },
249             ExitCode.IO_ERROR, java.io.FileNotFoundException.class));
250         add(Arguments.of(
251             new String[] { "validate-content",
252                 "-m",
253                 "src/test/resources/content/schema-validation-module.xml",
254                 "src/test/resources/content/schema-validation-module-missing-required.xml",
255                 "--as=xml"
256             },
257             // fail due to schema validation issue
258             ExitCode.FAIL, NO_EXCEPTION_CLASS));
259         add(Arguments.of(
260             new String[] { "validate-content",
261                 "-m",
262                 "src/test/resources/content/schema-validation-module.xml",
263                 "src/test/resources/content/schema-validation-module-missing-required.xml",
264                 "--as=xml",
265                 "--disable-schema-validation"
266             },
267             // fail due to missing element during parsing
268             ExitCode.FAIL, NO_EXCEPTION_CLASS));
269         add(Arguments.of(
270             new String[] { "validate-content",
271                 "-m",
272                 "src/test/resources/content/schema-validation-module.xml",
273                 "src/test/resources/content/schema-validation-module-missing-required.xml",
274                 "--as=xml",
275                 "--disable-schema-validation",
276                 "--disable-constraint-validation"
277             },
278             ExitCode.OK, NO_EXCEPTION_CLASS));
279         add(Arguments.of(
280             new String[] { "metapath", "list-functions" },
281             ExitCode.OK, NO_EXCEPTION_CLASS));
282         add(Arguments.of(
283             new String[] { "convert",
284                 "-m",
285                 "../core/metaschema/schema/metaschema/metaschema-module-metaschema.xml",
286                 "--to=yaml",
287                 "../core/metaschema/schema/metaschema/metaschema-module-metaschema.xml",
288             },
289             ExitCode.OK, NO_EXCEPTION_CLASS));
290         // Test markup-line datatype validation with YAML module
291         add(Arguments.of(
292             new String[] { "validate-content",
293                 "-m",
294                 "src/test/resources/content/test-markup-line-module.yaml",
295                 "src/test/resources/content/test-markup-line-content.json",
296                 "--as=json"
297             },
298             ExitCode.OK, NO_EXCEPTION_CLASS));
299         // Test --sarif-timing without -o produces an error
300         add(Arguments.of(
301             new String[] { "validate-content",
302                 "-m",
303                 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
304                 "../databind/src/test/resources/metaschema/simple/example.json",
305                 "--as=json",
306                 "--sarif-timing"
307             },
308             ExitCode.INVALID_ARGUMENTS, NO_EXCEPTION_CLASS));
309       }
310     };
311     return values.stream();
312   }
313 
314   @ParameterizedTest
315   @MethodSource("providesValues")
316   void testAllCommands(@NonNull String[] args, @NonNull ExitCode expectedExitCode,
317       Class<? extends Throwable> expectedThrownClass) {
318     String[] defaultArgs = { "--show-stack-trace" };
319     String[] fullArgs = Stream.of(args, defaultArgs).flatMap(Stream::of)
320         .toArray(String[]::new);
321     if (expectedThrownClass == null) {
322       evaluateResult(CLI.runCli(NULL_STREAM, fullArgs), expectedExitCode, fullArgs);
323     } else {
324       evaluateResult(CLI.runCli(NULL_STREAM, fullArgs), expectedExitCode, expectedThrownClass, fullArgs);
325     }
326   }
327 
328   @Test
329   void testValidateContent() {
330     try (LogCaptor captor = LogCaptor.forRoot()) {
331       String[] cliArgs = { "validate-content",
332           "-m",
333           "src/test/resources/content/215-module.xml",
334           "src/test/resources/content/215.xml",
335           "--disable-schema-validation"
336       };
337       CLI.runCli(NULL_STREAM, cliArgs);
338       assertThat(captor.getErrorLogs().toString())
339           .contains("expect-default-non-zero: Expect constraint '. > 0' did not match the data",
340               "expect-custom-non-zero: No default message, custom error message for expect-custom-non-zero constraint.",
341               "matches-default-regex-letters-only: Value '1' did not match the pattern",
342               "matches-custom-regex-letters-only: No default message, custom error message for" +
343                   " matches-custom-regex-letters-only constraint.",
344               "cardinality-default-two-minimum: The cardinality '1' is below the required minimum '2' for items" +
345                   " matching",
346               "index-items-default: Index 'index-items-default' has duplicate key for items",
347               "index-items-custom: No default message, custom error message for index-item-custom.",
348               "is-unique-default: Unique constraint violation at paths",
349               "is-unique-custom: No default message, custom error message for is-unique-custom.",
350               "index-has-key-default: Key reference [2] not found in index 'index-items-default' for item",
351               "index-has-key-custom: No default message, custom error message for index-has-key-custom.");
352     }
353   }
354 
355   @Test
356   void testListAllowedValuesOutput() throws Exception {
357     java.nio.file.Path outputFile = java.nio.file.Path.of("target/test-list-allowed-values.yaml");
358     String[] cliArgs = { "list-allowed-values",
359         "src/test/resources/content/schema-validation-module.xml",
360         outputFile.toString(),
361         "--overwrite",
362         "--show-stack-trace"
363     };
364     ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
365     evaluateResult(status, ExitCode.OK, cliArgs);
366 
367     String content = java.nio.file.Files.readString(outputFile);
368     assertThat(content)
369         .contains("locations:")
370         .contains("/root/optional:")
371         .contains("type: allowed-values")
372         .contains("target: optional")
373         .contains("yes")
374         .contains("allow-other: false");
375   }
376 
377   @Test
378   void testListAllowedValuesIncludesIdentifierWhenPresent() throws Exception {
379     Path outputFile = Paths.get("target/test-list-allowed-values-identifier.yaml");
380     Files.deleteIfExists(outputFile);
381     String[] cliArgs = { "list-allowed-values",
382         "src/test/resources/content/list-allowed-values-module.xml",
383         outputFile.toString(),
384         "--show-stack-trace"
385     };
386     evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
387 
388     String content = Files.readString(outputFile);
389     // Field-level allowed-values constraint (with id) on 'color'
390     assertThat(content)
391         .as("Field-level constraint with identifier should be emitted")
392         .contains("/root/color:")
393         .contains("identifier: color-values")
394         .contains("red")
395         .contains("green")
396         .contains("blue");
397     // Flag-level constraint carries its identifier, allow-other preserved from XML
398     assertThat(content)
399         .as("Flag-targeted constraint identifier and allow-other should be preserved")
400         .contains("identifier: status-phase")
401         .contains("allow-other: true");
402   }
403 
404   @Test
405   void testListAllowedValuesGroupsMultipleConstraintsUnderSameTarget() throws Exception {
406     Path outputFile = Paths.get("target/test-list-allowed-values-grouping.yaml");
407     Files.deleteIfExists(outputFile);
408     String[] cliArgs = { "list-allowed-values",
409         "src/test/resources/content/list-allowed-values-module.xml",
410         outputFile.toString(),
411         "--show-stack-trace"
412     };
413     evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
414 
415     String content = Files.readString(outputFile);
416     // Two allowed-values constraints target @status; both should appear under the
417     // same
418     // location key rather than producing two separate /root/@status entries.
419     long statusKeyCount = content.lines()
420         .filter(line -> line.trim().equals("/root/@status:"))
421         .count();
422     assertEquals(1, statusKeyCount,
423         "Constraints sharing a target must group under a single location key");
424     // Values from both constraints ('init', 'run' from first; 'done' from second)
425     // should
426     // all surface in the combined output for that target.
427     assertThat(content)
428         .contains("init")
429         .contains("run")
430         .contains("done");
431   }
432 
433   @Test
434   void testListAllowedValuesOverwriteRequiredForExistingFile() throws Exception {
435     Path outputFile = Paths.get("target/test-list-allowed-values-overwrite.yaml");
436     // Pre-create the destination so the first run lands on an existing file
437     Files.writeString(outputFile, "pre-existing content\n");
438 
439     String[] argsWithoutOverwrite = { "list-allowed-values",
440         "src/test/resources/content/schema-validation-module.xml",
441         outputFile.toString(),
442         "--show-stack-trace"
443     };
444     ExitStatus withoutOverwrite = CLI.runCli(NULL_STREAM, argsWithoutOverwrite);
445     assertEquals(ExitCode.INVALID_ARGUMENTS, withoutOverwrite.getExitCode(),
446         "Writing to an existing file without --overwrite must fail with INVALID_ARGUMENTS");
447     // Pre-existing content must not have been mutated by the failed run
448     assertEquals("pre-existing content\n", Files.readString(outputFile),
449         "Failed run must not have modified the destination file");
450 
451     String[] argsWithOverwrite = { "list-allowed-values",
452         "src/test/resources/content/schema-validation-module.xml",
453         outputFile.toString(),
454         "--overwrite",
455         "--show-stack-trace"
456     };
457     evaluateResult(CLI.runCli(NULL_STREAM, argsWithOverwrite), ExitCode.OK, argsWithOverwrite);
458     // With --overwrite the file should now contain YAML rather than the
459     // pre-existing text
460     assertThat(Files.readString(outputFile))
461         .doesNotContain("pre-existing content")
462         .contains("locations:");
463   }
464 
465   @Test
466   void testListAllowedValuesConsoleOutputYaml() {
467     try (LogCaptor captor = LogCaptor.forClass(
468         dev.metaschema.cli.commands.ListAllowedValuesCommand.class)) {
469       String[] cliArgs = { "list-allowed-values",
470           "src/test/resources/content/schema-validation-module.xml",
471           "--show-stack-trace"
472       };
473       evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
474 
475       // When no destination file is given, YAML output goes to the logger at INFO
476       // level
477       assertThat(captor.getInfoLogs().toString())
478           .contains("locations:")
479           .contains("/root/optional:")
480           .contains("type: allowed-values");
481     }
482   }
483 
484   @Test
485   void testValidateConstraints() {
486     try (LogCaptor captor = LogCaptor.forRoot()) {
487       String[] cliArgs = { "validate",
488           "src/test/resources/content/constraint-example.xml",
489           "-c",
490           "src/test/resources/content/constraint-constraints.xml",
491           "--disable-schema-validation",
492       };
493       CLI.runCli(NULL_STREAM, cliArgs);
494       assertThat(captor.getErrorLogs().toString())
495           .contains("This constraint SHOULD be violated if test passes.");
496     }
497   }
498 
499   @Test
500   void testSarifTimingOutput() throws IOException {
501     Path sarifOutput = Paths.get("target/test-sarif-timing.sarif");
502 
503     // Use a module with inline let statements and constraints, plus external
504     // constraints with additional let statements to exercise all timing features
505     String[] cliArgs = {
506         "validate-content",
507         "-m",
508         "src/test/resources/content/timing-test-module.xml",
509         "src/test/resources/content/timing-test-content.json",
510         "--as=json",
511         "-c", "src/test/resources/content/timing-test-constraints.xml",
512         "-o", sarifOutput.toString(),
513         "--sarif-timing",
514         "--sarif-include-pass",
515         "--show-stack-trace"
516     };
517 
518     ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
519     evaluateResult(status, ExitCode.OK, cliArgs);
520 
521     // Verify SARIF output file was created and contains timing data
522     assertTrue(Files.exists(sarifOutput), "SARIF output file should exist");
523 
524     String sarifContent = new String(Files.readAllBytes(sarifOutput), StandardCharsets.UTF_8);
525     JSONObject sarif = new JSONObject(sarifContent);
526 
527     JSONArray runs = sarif.getJSONArray("runs");
528     JSONObject run = runs.getJSONObject(0);
529 
530     // Verify invocations with timing are present
531     assertTrue(run.has("invocations"), "Run should have invocations when --sarif-timing is enabled");
532     JSONArray invocations = run.getJSONArray("invocations");
533     assertEquals(1, invocations.length(), "Should have exactly one invocation");
534 
535     JSONObject invocation = invocations.getJSONObject(0);
536     assertTrue(invocation.has("startTimeUtc"), "Invocation should have startTimeUtc");
537     assertTrue(invocation.has("endTimeUtc"), "Invocation should have endTimeUtc");
538     assertTrue(invocation.getBoolean("executionSuccessful"), "executionSuccessful should be true");
539     assertTrue(invocation.has("toolExecutionNotifications"),
540         "Invocation should have toolExecutionNotifications for phase timing");
541 
542     // Verify phase timing notifications exist
543     JSONArray notifications = invocation.getJSONArray("toolExecutionNotifications");
544     assertTrue(notifications.length() > 0, "Should have at least one phase timing notification");
545 
546     // Verify let-statement timing is captured (module has 2 inline lets +
547     // constraints has 2 external lets)
548     boolean foundLetTiming = false;
549     for (int idx = 0; idx < notifications.length(); idx++) {
550       JSONObject notification = notifications.getJSONObject(idx);
551       String text = notification.getJSONObject("message").getString("text");
552       if (text.startsWith("$") && text.contains(" := ")) {
553         foundLetTiming = true;
554         break;
555       }
556     }
557     assertTrue(foundLetTiming, "Should have let-statement timing notifications");
558 
559     // Verify per-result timing on at least one result (using --sarif-include-pass
560     // since test content passes all constraints)
561     JSONArray results = run.getJSONArray("results");
562     boolean foundPerResultTiming = false;
563     for (int idx = 0; idx < results.length(); idx++) {
564       JSONObject result = results.getJSONObject(idx);
565       if (result.has("properties")) {
566         JSONObject props = result.getJSONObject("properties");
567         if (props.has("timing")) {
568           foundPerResultTiming = true;
569           JSONObject timing = props.getJSONObject("timing");
570           assertTrue(timing.has("totalMs"), "Per-result timing should have totalMs");
571           break;
572         }
573       }
574     }
575     assertTrue(foundPerResultTiming,
576         "At least one result should have per-result timing when --sarif-timing is used");
577 
578     // Validate against official SARIF 2.1.0 schema
579     Path sarifSchema = Paths.get("../databind-modules/modules/sarif/sarif-schema-2.1.0.json");
580 
581     try (Reader schemaReader = Files.newBufferedReader(sarifSchema, StandardCharsets.UTF_8)) {
582       JsonNode schemaNode = new OrgJsonNode(new JSONObject(new JSONTokener(schemaReader)));
583       JsonNode instanceNode = new OrgJsonNode(new JSONObject(sarifContent));
584 
585       Validator.Result result = new ValidatorFactory()
586           .withJsonNodeFactory(new OrgJsonNode.Factory())
587           .withDialect(new Dialects.Draft2020Dialect())
588           .validate(schemaNode, instanceNode);
589       StringJoiner sj = new StringJoiner("\n");
590       for (dev.harrel.jsonschema.Error finding : result.getErrors()) {
591         sj.add(String.format("[%s]%s %s for schema '%s'",
592             finding.getInstanceLocation(),
593             finding.getKeyword() == null ? "" : " " + finding.getKeyword() + ":",
594             finding.getError(),
595             finding.getSchemaLocation()));
596       }
597       assertTrue(result.isValid(),
598           () -> "SARIF timing output failed schema validation. Errors:\n" + sj.toString());
599     }
600   }
601 
602   @Test
603   void testSarifAlwaysOnInvocations() throws IOException {
604     Path sarifOutput = Paths.get("target/test-sarif-always-on.sarif");
605 
606     // Run CLI with SARIF output but WITHOUT --sarif-timing
607     String[] cliArgs = {
608         "validate-content",
609         "-m",
610         "src/test/resources/content/timing-test-module.xml",
611         "src/test/resources/content/timing-test-content.json",
612         "--as=json",
613         "-o", sarifOutput.toString(),
614         "--show-stack-trace"
615     };
616 
617     ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
618     evaluateResult(status, ExitCode.OK, cliArgs);
619 
620     assertTrue(Files.exists(sarifOutput), "SARIF output file should exist");
621 
622     String sarifContent = new String(Files.readAllBytes(sarifOutput), StandardCharsets.UTF_8);
623     JSONObject sarif = new JSONObject(sarifContent);
624 
625     JSONObject run = sarif.getJSONArray("runs").getJSONObject(0);
626 
627     // Always-on: invocations should always be present
628     assertTrue(run.has("invocations"), "Run should always have invocations (always-on timing)");
629     JSONArray invocations = run.getJSONArray("invocations");
630     assertEquals(1, invocations.length());
631 
632     JSONObject invocation = invocations.getJSONObject(0);
633     assertTrue(invocation.has("startTimeUtc"), "Invocation should always have startTimeUtc");
634     assertTrue(invocation.has("endTimeUtc"), "Invocation should always have endTimeUtc");
635     assertTrue(invocation.getBoolean("executionSuccessful"));
636 
637     // Without --sarif-timing, should NOT have timing notifications
638     assertFalse(invocation.has("toolExecutionNotifications"),
639         "Invocation should not have timing notifications without --sarif-timing");
640   }
641 }