1
2
3
4
5
6 package dev.metaschema.cli;
7
8 import static org.assertj.core.api.Assertions.assertThat;
9 import static org.junit.jupiter.api.Assertions.assertAll;
10 import static org.junit.jupiter.api.Assertions.assertEquals;
11 import static org.junit.jupiter.api.Assertions.assertFalse;
12 import static org.junit.jupiter.api.Assertions.assertNull;
13 import static org.junit.jupiter.api.Assertions.assertTrue;
14
15 import org.json.JSONArray;
16 import org.json.JSONObject;
17 import org.json.JSONTokener;
18 import org.junit.jupiter.api.Test;
19 import org.junit.jupiter.api.parallel.Execution;
20 import org.junit.jupiter.api.parallel.ExecutionMode;
21 import org.junit.jupiter.params.ParameterizedTest;
22 import org.junit.jupiter.params.provider.Arguments;
23 import org.junit.jupiter.params.provider.MethodSource;
24
25 import java.io.IOException;
26 import java.io.OutputStream;
27 import java.io.PrintStream;
28 import java.io.Reader;
29 import java.nio.charset.StandardCharsets;
30 import java.nio.file.Files;
31 import java.nio.file.Path;
32 import java.nio.file.Paths;
33 import java.util.LinkedList;
34 import java.util.List;
35 import java.util.StringJoiner;
36 import java.util.stream.Stream;
37
38 import dev.harrel.jsonschema.Dialects;
39 import dev.harrel.jsonschema.JsonNode;
40 import dev.harrel.jsonschema.Validator;
41 import dev.harrel.jsonschema.ValidatorFactory;
42 import dev.harrel.jsonschema.providers.OrgJsonNode;
43 import dev.metaschema.cli.processor.ExitCode;
44 import dev.metaschema.cli.processor.ExitStatus;
45 import edu.umd.cs.findbugs.annotations.NonNull;
46 import nl.altindag.log.LogCaptor;
47
48
49
50
51 @Execution(value = ExecutionMode.SAME_THREAD, reason = "Log capturing needs to be single threaded")
52 public class CLITest {
53 private static final ExitCode NO_EXCEPTION_CLASS = null;
54
55
56
57
58
59 @SuppressWarnings("resource")
60 private static final PrintStream NULL_STREAM = new PrintStream(new OutputStream() {
61 @Override
62 public void write(int b) {
63
64 }
65
66 @Override
67 public void write(byte[] b, int off, int len) {
68
69 }
70 });
71
72 void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode, @NonNull String[] args) {
73 status.generateMessage(true);
74 Throwable thrown = status.getThrowable();
75 assertAll(
76 () -> assertEquals(expectedCode, status.getExitCode(),
77 () -> buildExitCodeMismatchMessage(status, expectedCode, thrown, args)),
78 () -> assertNull(thrown,
79 () -> buildUnexpectedThrowableMessage(thrown, args)));
80 }
81
82 void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode,
83 @NonNull Class<? extends Throwable> thrownClass, @NonNull String[] args) {
84 Throwable thrown = status.getThrowable();
85 assertAll(
86 () -> assertEquals(expectedCode, status.getExitCode(),
87 () -> buildExitCodeMismatchMessage(status, expectedCode, thrown, args)),
88 () -> assertEquals(thrownClass, thrown == null ? null : thrown.getClass(),
89 () -> buildThrowableMismatchMessage(thrownClass, thrown, args)));
90 }
91
92 private static String buildExitCodeMismatchMessage(@NonNull ExitStatus status, @NonNull ExitCode expectedCode,
93 Throwable thrown, @NonNull String[] args) {
94 StringBuilder sb = new StringBuilder();
95 sb.append("exit code mismatch: expected <").append(expectedCode).append("> but was <")
96 .append(status.getExitCode()).append(">");
97 sb.append("\nCommand args: ").append(String.join(" ", args));
98 if (status.getMessage() != null) {
99 sb.append("\nStatus message: ").append(status.getMessage());
100 }
101 if (thrown != null) {
102 sb.append("\nThrowable: ").append(thrown.getClass().getName()).append(": ").append(thrown.getMessage());
103 sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
104 }
105 return sb.toString();
106 }
107
108 private static String buildUnexpectedThrowableMessage(Throwable thrown, @NonNull String[] args) {
109 if (thrown == null) {
110 return "expected null Throwable";
111 }
112 StringBuilder sb = new StringBuilder();
113 sb.append("expected null Throwable but got: ").append(thrown.getClass().getName())
114 .append(": ").append(thrown.getMessage());
115 sb.append("\nCommand args: ").append(String.join(" ", args));
116 sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
117 return sb.toString();
118 }
119
120 private static String buildThrowableMismatchMessage(Class<? extends Throwable> expectedClass, Throwable thrown,
121 @NonNull String[] args) {
122 StringBuilder sb = new StringBuilder();
123 sb.append("expected Throwable mismatch: expected <")
124 .append(expectedClass == null ? "null" : expectedClass.getName())
125 .append("> but was <")
126 .append(thrown == null ? "null" : thrown.getClass().getName())
127 .append(">");
128 sb.append("\nCommand args: ").append(String.join(" ", args));
129 if (thrown != null) {
130 sb.append("\nMessage: ").append(thrown.getMessage());
131 sb.append("\nStack trace:\n").append(getStackTraceAsString(thrown));
132 }
133 return sb.toString();
134 }
135
136 private static String getStackTraceAsString(Throwable throwable) {
137 java.io.StringWriter sw = new java.io.StringWriter();
138 throwable.printStackTrace(new java.io.PrintWriter(sw));
139 return sw.toString();
140 }
141
142 private static Stream<Arguments> providesValues() {
143 List<Arguments> values = new LinkedList<>() {
144 {
145 add(Arguments.of(new String[] {}, ExitCode.INVALID_COMMAND,
146 NO_EXCEPTION_CLASS));
147 add(Arguments.of(new String[] { "-h" }, ExitCode.OK, NO_EXCEPTION_CLASS));
148 add(Arguments.of(new String[] { "generate-schema", "--help" }, ExitCode.OK,
149 NO_EXCEPTION_CLASS));
150 add(Arguments.of(new String[] { "generate-diagram", "--help" }, ExitCode.OK,
151 NO_EXCEPTION_CLASS));
152 add(Arguments.of(new String[] { "validate", "--help" }, ExitCode.OK,
153 NO_EXCEPTION_CLASS));
154 add(Arguments.of(new String[] { "validate-content", "--help" }, ExitCode.OK,
155 NO_EXCEPTION_CLASS));
156 add(Arguments.of(new String[] { "convert", "--help" }, ExitCode.OK,
157 NO_EXCEPTION_CLASS));
158 add(Arguments.of(new String[] { "metapath", "list-functions", "--help" }, ExitCode.OK,
159 NO_EXCEPTION_CLASS));
160 add(Arguments.of(new String[] { "metapath", "eval", "--help" }, ExitCode.OK,
161 NO_EXCEPTION_CLASS));
162 add(Arguments.of(new String[] { "list-allowed-values", "--help" }, ExitCode.OK,
163 NO_EXCEPTION_CLASS));
164 add(Arguments.of(
165 new String[] { "list-allowed-values",
166 "src/test/resources/content/schema-validation-module.xml"
167 },
168 ExitCode.OK, NO_EXCEPTION_CLASS));
169
170
171 add(Arguments.of(
172 new String[] { "list-allowed-values" },
173 ExitCode.INVALID_ARGUMENTS, NO_EXCEPTION_CLASS));
174 add(Arguments.of(
175 new String[] { "validate",
176 "../databind/src/test/resources/metaschema/fields_with_flags/metaschema.xml"
177 },
178 ExitCode.OK, NO_EXCEPTION_CLASS));
179 add(Arguments.of(
180 new String[] { "generate-schema", "--overwrite", "--as",
181 "JSON",
182 "../databind/src/test/resources/metaschema/fields_with_flags/metaschema.xml",
183 "target/schema-test.json" },
184 ExitCode.OK, NO_EXCEPTION_CLASS));
185 add(Arguments.of(
186 new String[] { "validate-content", "--as=xml",
187 "-m=../databind/src/test/resources/metaschema/bad_index-has-key/metaschema.xml",
188 "../databind/src/test/resources/metaschema/bad_index-has-key/example.xml",
189 "--show-stack-trace" },
190 ExitCode.FAIL, NO_EXCEPTION_CLASS));
191 add(Arguments.of(
192 new String[] { "validate-content", "--as=json",
193 "-m=../databind/src/test/resources/metaschema/bad_index-has-key/metaschema.xml",
194 "../databind/src/test/resources/metaschema/bad_index-has-key/example.json", "--show-stack-trace" },
195 ExitCode.FAIL, NO_EXCEPTION_CLASS));
196 add(Arguments.of(
197 new String[] { "validate",
198 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
199 "--show-stack-trace" },
200 ExitCode.OK, NO_EXCEPTION_CLASS));
201 add(Arguments.of(
202 new String[] { "generate-schema",
203 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
204 "--as", "xml",
205 },
206 ExitCode.OK, NO_EXCEPTION_CLASS));
207 add(Arguments.of(
208 new String[] { "generate-schema",
209 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
210 "--as", "json",
211 },
212 ExitCode.OK, NO_EXCEPTION_CLASS));
213 add(Arguments.of(
214 new String[] { "generate-diagram",
215 "../databind/src/test/resources/metaschema/simple/metaschema.xml"
216 },
217 ExitCode.OK, NO_EXCEPTION_CLASS));
218 add(Arguments.of(
219 new String[] { "validate-content",
220 "-m",
221 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
222 "../databind/src/test/resources/metaschema/simple/example.json",
223 "--as=json"
224 },
225 ExitCode.OK, NO_EXCEPTION_CLASS));
226 add(Arguments.of(
227 new String[] { "validate-content",
228 "-m",
229 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
230 "../databind/src/test/resources/metaschema/simple/example.xml",
231 "--as=xml"
232 },
233 ExitCode.OK, NO_EXCEPTION_CLASS));
234 add(Arguments.of(
235 new String[] { "validate-content",
236 "-m",
237 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
238 "https://bad.domain.example.net/example.xml",
239 "--as=xml"
240 },
241 ExitCode.IO_ERROR, java.net.UnknownHostException.class));
242 add(Arguments.of(
243 new String[] { "validate-content",
244 "-m",
245 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
246 "https://github.com/no-example.xml",
247 "--as=xml"
248 },
249 ExitCode.IO_ERROR, java.io.FileNotFoundException.class));
250 add(Arguments.of(
251 new String[] { "validate-content",
252 "-m",
253 "src/test/resources/content/schema-validation-module.xml",
254 "src/test/resources/content/schema-validation-module-missing-required.xml",
255 "--as=xml"
256 },
257
258 ExitCode.FAIL, NO_EXCEPTION_CLASS));
259 add(Arguments.of(
260 new String[] { "validate-content",
261 "-m",
262 "src/test/resources/content/schema-validation-module.xml",
263 "src/test/resources/content/schema-validation-module-missing-required.xml",
264 "--as=xml",
265 "--disable-schema-validation"
266 },
267
268 ExitCode.FAIL, NO_EXCEPTION_CLASS));
269 add(Arguments.of(
270 new String[] { "validate-content",
271 "-m",
272 "src/test/resources/content/schema-validation-module.xml",
273 "src/test/resources/content/schema-validation-module-missing-required.xml",
274 "--as=xml",
275 "--disable-schema-validation",
276 "--disable-constraint-validation"
277 },
278 ExitCode.OK, NO_EXCEPTION_CLASS));
279 add(Arguments.of(
280 new String[] { "metapath", "list-functions" },
281 ExitCode.OK, NO_EXCEPTION_CLASS));
282 add(Arguments.of(
283 new String[] { "convert",
284 "-m",
285 "../core/metaschema/schema/metaschema/metaschema-module-metaschema.xml",
286 "--to=yaml",
287 "../core/metaschema/schema/metaschema/metaschema-module-metaschema.xml",
288 },
289 ExitCode.OK, NO_EXCEPTION_CLASS));
290
291 add(Arguments.of(
292 new String[] { "validate-content",
293 "-m",
294 "src/test/resources/content/test-markup-line-module.yaml",
295 "src/test/resources/content/test-markup-line-content.json",
296 "--as=json"
297 },
298 ExitCode.OK, NO_EXCEPTION_CLASS));
299
300 add(Arguments.of(
301 new String[] { "validate-content",
302 "-m",
303 "../databind/src/test/resources/metaschema/simple/metaschema.xml",
304 "../databind/src/test/resources/metaschema/simple/example.json",
305 "--as=json",
306 "--sarif-timing"
307 },
308 ExitCode.INVALID_ARGUMENTS, NO_EXCEPTION_CLASS));
309 }
310 };
311 return values.stream();
312 }
313
314 @ParameterizedTest
315 @MethodSource("providesValues")
316 void testAllCommands(@NonNull String[] args, @NonNull ExitCode expectedExitCode,
317 Class<? extends Throwable> expectedThrownClass) {
318 String[] defaultArgs = { "--show-stack-trace" };
319 String[] fullArgs = Stream.of(args, defaultArgs).flatMap(Stream::of)
320 .toArray(String[]::new);
321 if (expectedThrownClass == null) {
322 evaluateResult(CLI.runCli(NULL_STREAM, fullArgs), expectedExitCode, fullArgs);
323 } else {
324 evaluateResult(CLI.runCli(NULL_STREAM, fullArgs), expectedExitCode, expectedThrownClass, fullArgs);
325 }
326 }
327
328 @Test
329 void testValidateContent() {
330 try (LogCaptor captor = LogCaptor.forRoot()) {
331 String[] cliArgs = { "validate-content",
332 "-m",
333 "src/test/resources/content/215-module.xml",
334 "src/test/resources/content/215.xml",
335 "--disable-schema-validation"
336 };
337 CLI.runCli(NULL_STREAM, cliArgs);
338 assertThat(captor.getErrorLogs().toString())
339 .contains("expect-default-non-zero: Expect constraint '. > 0' did not match the data",
340 "expect-custom-non-zero: No default message, custom error message for expect-custom-non-zero constraint.",
341 "matches-default-regex-letters-only: Value '1' did not match the pattern",
342 "matches-custom-regex-letters-only: No default message, custom error message for" +
343 " matches-custom-regex-letters-only constraint.",
344 "cardinality-default-two-minimum: The cardinality '1' is below the required minimum '2' for items" +
345 " matching",
346 "index-items-default: Index 'index-items-default' has duplicate key for items",
347 "index-items-custom: No default message, custom error message for index-item-custom.",
348 "is-unique-default: Unique constraint violation at paths",
349 "is-unique-custom: No default message, custom error message for is-unique-custom.",
350 "index-has-key-default: Key reference [2] not found in index 'index-items-default' for item",
351 "index-has-key-custom: No default message, custom error message for index-has-key-custom.");
352 }
353 }
354
355 @Test
356 void testListAllowedValuesOutput() throws Exception {
357 java.nio.file.Path outputFile = java.nio.file.Path.of("target/test-list-allowed-values.yaml");
358 String[] cliArgs = { "list-allowed-values",
359 "src/test/resources/content/schema-validation-module.xml",
360 outputFile.toString(),
361 "--overwrite",
362 "--show-stack-trace"
363 };
364 ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
365 evaluateResult(status, ExitCode.OK, cliArgs);
366
367 String content = java.nio.file.Files.readString(outputFile);
368 assertThat(content)
369 .contains("locations:")
370 .contains("/root/optional:")
371 .contains("type: allowed-values")
372 .contains("target: optional")
373 .contains("yes")
374 .contains("allow-other: false");
375 }
376
377 @Test
378 void testListAllowedValuesIncludesIdentifierWhenPresent() throws Exception {
379 Path outputFile = Paths.get("target/test-list-allowed-values-identifier.yaml");
380 Files.deleteIfExists(outputFile);
381 String[] cliArgs = { "list-allowed-values",
382 "src/test/resources/content/list-allowed-values-module.xml",
383 outputFile.toString(),
384 "--show-stack-trace"
385 };
386 evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
387
388 String content = Files.readString(outputFile);
389
390 assertThat(content)
391 .as("Field-level constraint with identifier should be emitted")
392 .contains("/root/color:")
393 .contains("identifier: color-values")
394 .contains("red")
395 .contains("green")
396 .contains("blue");
397
398 assertThat(content)
399 .as("Flag-targeted constraint identifier and allow-other should be preserved")
400 .contains("identifier: status-phase")
401 .contains("allow-other: true");
402 }
403
404 @Test
405 void testListAllowedValuesGroupsMultipleConstraintsUnderSameTarget() throws Exception {
406 Path outputFile = Paths.get("target/test-list-allowed-values-grouping.yaml");
407 Files.deleteIfExists(outputFile);
408 String[] cliArgs = { "list-allowed-values",
409 "src/test/resources/content/list-allowed-values-module.xml",
410 outputFile.toString(),
411 "--show-stack-trace"
412 };
413 evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
414
415 String content = Files.readString(outputFile);
416
417
418
419 long statusKeyCount = content.lines()
420 .filter(line -> line.trim().equals("/root/@status:"))
421 .count();
422 assertEquals(1, statusKeyCount,
423 "Constraints sharing a target must group under a single location key");
424
425
426
427 assertThat(content)
428 .contains("init")
429 .contains("run")
430 .contains("done");
431 }
432
433 @Test
434 void testListAllowedValuesOverwriteRequiredForExistingFile() throws Exception {
435 Path outputFile = Paths.get("target/test-list-allowed-values-overwrite.yaml");
436
437 Files.writeString(outputFile, "pre-existing content\n");
438
439 String[] argsWithoutOverwrite = { "list-allowed-values",
440 "src/test/resources/content/schema-validation-module.xml",
441 outputFile.toString(),
442 "--show-stack-trace"
443 };
444 ExitStatus withoutOverwrite = CLI.runCli(NULL_STREAM, argsWithoutOverwrite);
445 assertEquals(ExitCode.INVALID_ARGUMENTS, withoutOverwrite.getExitCode(),
446 "Writing to an existing file without --overwrite must fail with INVALID_ARGUMENTS");
447
448 assertEquals("pre-existing content\n", Files.readString(outputFile),
449 "Failed run must not have modified the destination file");
450
451 String[] argsWithOverwrite = { "list-allowed-values",
452 "src/test/resources/content/schema-validation-module.xml",
453 outputFile.toString(),
454 "--overwrite",
455 "--show-stack-trace"
456 };
457 evaluateResult(CLI.runCli(NULL_STREAM, argsWithOverwrite), ExitCode.OK, argsWithOverwrite);
458
459
460 assertThat(Files.readString(outputFile))
461 .doesNotContain("pre-existing content")
462 .contains("locations:");
463 }
464
465 @Test
466 void testListAllowedValuesConsoleOutputYaml() {
467 try (LogCaptor captor = LogCaptor.forClass(
468 dev.metaschema.cli.commands.ListAllowedValuesCommand.class)) {
469 String[] cliArgs = { "list-allowed-values",
470 "src/test/resources/content/schema-validation-module.xml",
471 "--show-stack-trace"
472 };
473 evaluateResult(CLI.runCli(NULL_STREAM, cliArgs), ExitCode.OK, cliArgs);
474
475
476
477 assertThat(captor.getInfoLogs().toString())
478 .contains("locations:")
479 .contains("/root/optional:")
480 .contains("type: allowed-values");
481 }
482 }
483
484 @Test
485 void testValidateConstraints() {
486 try (LogCaptor captor = LogCaptor.forRoot()) {
487 String[] cliArgs = { "validate",
488 "src/test/resources/content/constraint-example.xml",
489 "-c",
490 "src/test/resources/content/constraint-constraints.xml",
491 "--disable-schema-validation",
492 };
493 CLI.runCli(NULL_STREAM, cliArgs);
494 assertThat(captor.getErrorLogs().toString())
495 .contains("This constraint SHOULD be violated if test passes.");
496 }
497 }
498
499 @Test
500 void testSarifTimingOutput() throws IOException {
501 Path sarifOutput = Paths.get("target/test-sarif-timing.sarif");
502
503
504
505 String[] cliArgs = {
506 "validate-content",
507 "-m",
508 "src/test/resources/content/timing-test-module.xml",
509 "src/test/resources/content/timing-test-content.json",
510 "--as=json",
511 "-c", "src/test/resources/content/timing-test-constraints.xml",
512 "-o", sarifOutput.toString(),
513 "--sarif-timing",
514 "--sarif-include-pass",
515 "--show-stack-trace"
516 };
517
518 ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
519 evaluateResult(status, ExitCode.OK, cliArgs);
520
521
522 assertTrue(Files.exists(sarifOutput), "SARIF output file should exist");
523
524 String sarifContent = new String(Files.readAllBytes(sarifOutput), StandardCharsets.UTF_8);
525 JSONObject sarif = new JSONObject(sarifContent);
526
527 JSONArray runs = sarif.getJSONArray("runs");
528 JSONObject run = runs.getJSONObject(0);
529
530
531 assertTrue(run.has("invocations"), "Run should have invocations when --sarif-timing is enabled");
532 JSONArray invocations = run.getJSONArray("invocations");
533 assertEquals(1, invocations.length(), "Should have exactly one invocation");
534
535 JSONObject invocation = invocations.getJSONObject(0);
536 assertTrue(invocation.has("startTimeUtc"), "Invocation should have startTimeUtc");
537 assertTrue(invocation.has("endTimeUtc"), "Invocation should have endTimeUtc");
538 assertTrue(invocation.getBoolean("executionSuccessful"), "executionSuccessful should be true");
539 assertTrue(invocation.has("toolExecutionNotifications"),
540 "Invocation should have toolExecutionNotifications for phase timing");
541
542
543 JSONArray notifications = invocation.getJSONArray("toolExecutionNotifications");
544 assertTrue(notifications.length() > 0, "Should have at least one phase timing notification");
545
546
547
548 boolean foundLetTiming = false;
549 for (int idx = 0; idx < notifications.length(); idx++) {
550 JSONObject notification = notifications.getJSONObject(idx);
551 String text = notification.getJSONObject("message").getString("text");
552 if (text.startsWith("$") && text.contains(" := ")) {
553 foundLetTiming = true;
554 break;
555 }
556 }
557 assertTrue(foundLetTiming, "Should have let-statement timing notifications");
558
559
560
561 JSONArray results = run.getJSONArray("results");
562 boolean foundPerResultTiming = false;
563 for (int idx = 0; idx < results.length(); idx++) {
564 JSONObject result = results.getJSONObject(idx);
565 if (result.has("properties")) {
566 JSONObject props = result.getJSONObject("properties");
567 if (props.has("timing")) {
568 foundPerResultTiming = true;
569 JSONObject timing = props.getJSONObject("timing");
570 assertTrue(timing.has("totalMs"), "Per-result timing should have totalMs");
571 break;
572 }
573 }
574 }
575 assertTrue(foundPerResultTiming,
576 "At least one result should have per-result timing when --sarif-timing is used");
577
578
579 Path sarifSchema = Paths.get("../databind-modules/modules/sarif/sarif-schema-2.1.0.json");
580
581 try (Reader schemaReader = Files.newBufferedReader(sarifSchema, StandardCharsets.UTF_8)) {
582 JsonNode schemaNode = new OrgJsonNode(new JSONObject(new JSONTokener(schemaReader)));
583 JsonNode instanceNode = new OrgJsonNode(new JSONObject(sarifContent));
584
585 Validator.Result result = new ValidatorFactory()
586 .withJsonNodeFactory(new OrgJsonNode.Factory())
587 .withDialect(new Dialects.Draft2020Dialect())
588 .validate(schemaNode, instanceNode);
589 StringJoiner sj = new StringJoiner("\n");
590 for (dev.harrel.jsonschema.Error finding : result.getErrors()) {
591 sj.add(String.format("[%s]%s %s for schema '%s'",
592 finding.getInstanceLocation(),
593 finding.getKeyword() == null ? "" : " " + finding.getKeyword() + ":",
594 finding.getError(),
595 finding.getSchemaLocation()));
596 }
597 assertTrue(result.isValid(),
598 () -> "SARIF timing output failed schema validation. Errors:\n" + sj.toString());
599 }
600 }
601
602 @Test
603 void testSarifAlwaysOnInvocations() throws IOException {
604 Path sarifOutput = Paths.get("target/test-sarif-always-on.sarif");
605
606
607 String[] cliArgs = {
608 "validate-content",
609 "-m",
610 "src/test/resources/content/timing-test-module.xml",
611 "src/test/resources/content/timing-test-content.json",
612 "--as=json",
613 "-o", sarifOutput.toString(),
614 "--show-stack-trace"
615 };
616
617 ExitStatus status = CLI.runCli(NULL_STREAM, cliArgs);
618 evaluateResult(status, ExitCode.OK, cliArgs);
619
620 assertTrue(Files.exists(sarifOutput), "SARIF output file should exist");
621
622 String sarifContent = new String(Files.readAllBytes(sarifOutput), StandardCharsets.UTF_8);
623 JSONObject sarif = new JSONObject(sarifContent);
624
625 JSONObject run = sarif.getJSONArray("runs").getJSONObject(0);
626
627
628 assertTrue(run.has("invocations"), "Run should always have invocations (always-on timing)");
629 JSONArray invocations = run.getJSONArray("invocations");
630 assertEquals(1, invocations.length());
631
632 JSONObject invocation = invocations.getJSONObject(0);
633 assertTrue(invocation.has("startTimeUtc"), "Invocation should always have startTimeUtc");
634 assertTrue(invocation.has("endTimeUtc"), "Invocation should always have endTimeUtc");
635 assertTrue(invocation.getBoolean("executionSuccessful"));
636
637
638 assertFalse(invocation.has("toolExecutionNotifications"),
639 "Invocation should not have timing notifications without --sarif-timing");
640 }
641 }