1
2
3
4
5
6 package dev.metaschema.model.testing;
7
8 import static org.junit.jupiter.api.Assertions.assertEquals;
9
10 import org.apache.logging.log4j.LogBuilder;
11 import org.apache.logging.log4j.LogManager;
12 import org.apache.logging.log4j.Logger;
13 import org.junit.jupiter.api.DynamicContainer;
14 import org.junit.jupiter.api.DynamicNode;
15 import org.junit.jupiter.api.DynamicTest;
16 import org.junit.platform.commons.JUnitException;
17
18 import java.io.IOException;
19 import java.io.Writer;
20 import java.net.URI;
21 import java.net.URISyntaxException;
22 import java.net.URL;
23 import java.nio.charset.StandardCharsets;
24 import java.nio.file.FileVisitResult;
25 import java.nio.file.Files;
26 import java.nio.file.OpenOption;
27 import java.nio.file.Path;
28 import java.nio.file.SimpleFileVisitor;
29 import java.nio.file.StandardOpenOption;
30 import java.nio.file.attribute.BasicFileAttributes;
31 import java.util.List;
32 import java.util.function.BiFunction;
33 import java.util.function.Function;
34 import java.util.function.Supplier;
35 import java.util.stream.Stream;
36
37 import dev.metaschema.core.model.IModule;
38 import dev.metaschema.core.model.MetaschemaException;
39 import dev.metaschema.core.model.validation.IContentValidator;
40 import dev.metaschema.core.model.validation.IValidationFinding;
41 import dev.metaschema.core.model.validation.IValidationResult;
42 import dev.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
43 import dev.metaschema.core.util.ObjectUtils;
44 import dev.metaschema.databind.IBindingContext;
45 import dev.metaschema.databind.io.Format;
46 import dev.metaschema.databind.io.ISerializer;
47 import dev.metaschema.databind.model.metaschema.IBindingModuleLoader;
48 import dev.metaschema.model.testing.testsuite.GenerateSchema;
49 import dev.metaschema.model.testing.testsuite.Metaschema;
50 import dev.metaschema.model.testing.testsuite.TestCollection;
51 import dev.metaschema.model.testing.testsuite.TestScenario;
52 import dev.metaschema.model.testing.testsuite.TestSuite;
53 import dev.metaschema.model.testing.testsuite.ValidationCase;
54 import edu.umd.cs.findbugs.annotations.NonNull;
55 import edu.umd.cs.findbugs.annotations.Nullable;
56 import nl.talsmasoftware.lazy4j.Lazy;
57
58
59
60
61
62
63
64 @SuppressWarnings({
65 "PMD.GodClass",
66 "PMD.CouplingBetweenObjects"
67 })
68 public abstract class AbstractTestSuite {
69 private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
70
71 private static final boolean DELETE_RESULTS_ON_EXIT = false;
72 private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
73 StandardOpenOption.CREATE,
74 StandardOpenOption.WRITE,
75 StandardOpenOption.TRUNCATE_EXISTING
76 };
77
78 private static final String VALID = "VALID";
79
80
81
82
83
84
85 @NonNull
86 protected abstract Format getRequiredContentFormat();
87
88
89
90
91
92
93 @NonNull
94 protected abstract URI getTestSuiteURI();
95
96
97
98
99
100
101 @NonNull
102 protected abstract Path getGenerationPath();
103
104
105
106
107
108
109
110 @NonNull
111 protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112
113
114
115
116
117
118 @Nullable
119 protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120
121
122
123
124
125
126 @NonNull
127 protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128
129
130
131
132
133
134
135
136
137 @NonNull
138 protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139 try {
140 Path generationPath = getGenerationPath();
141 if (Files.exists(generationPath)) {
142 if (!Files.isDirectory(generationPath)) {
143 throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
144 }
145 } else {
146 Files.createDirectories(generationPath);
147 }
148
149 URI testSuiteUri = getTestSuiteURI();
150 URL testSuiteUrl = testSuiteUri.toURL();
151 TestSuite testSuite = bindingContext.newBoundLoader().load(TestSuite.class, testSuiteUrl);
152 List<TestCollection> testCollections = testSuite.getTestCollections();
153 return ObjectUtils.notNull(testCollections.stream()
154 .flatMap(
155 collection -> Stream
156 .of(generateCollection(
157 ObjectUtils.notNull(collection),
158 testSuiteUri,
159 generationPath,
160 bindingContext))));
161 } catch (IOException | URISyntaxException ex) {
162 throw new JUnitException("Unable to generate tests", ex);
163 }
164 }
165
166
167
168
169
170
171
172 protected void deleteCollectionOnExit(@NonNull Path path) {
173 Runtime.getRuntime().addShutdownHook(new Thread(
174 () -> {
175 try {
176 Files.walkFileTree(path, new SimpleFileVisitor<>() {
177 @Override
178 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
179 Files.delete(file);
180 return FileVisitResult.CONTINUE;
181 }
182
183 @Override
184 public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
185 if (ex == null) {
186 Files.delete(dir);
187 return FileVisitResult.CONTINUE;
188 }
189
190 throw ex;
191 }
192 });
193 } catch (IOException ex) {
194 throw new JUnitException("Failed to delete collection: " + path, ex);
195 }
196 }));
197 }
198
199 private DynamicContainer generateCollection(
200 @NonNull TestCollection collection,
201 @NonNull URI testSuiteUri,
202 @NonNull Path generationPath,
203 @NonNull IBindingContext bindingContext) {
204 URI collectionUri = testSuiteUri.resolve(collection.getLocation());
205 assert collectionUri != null;
206
207 LOGGER.atInfo().log("Collection: " + collectionUri);
208 Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
209 Path retval;
210 try {
211 retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
212 if (DELETE_RESULTS_ON_EXIT) {
213 deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
214 }
215 } catch (IOException ex) {
216 throw new JUnitException("Unable to create collection temp directory", ex);
217 }
218 return retval;
219 }));
220
221 List<TestScenario> testScenarios = collection.getTestScenarios();
222 return DynamicContainer.dynamicContainer(
223 collection.getName(),
224 testSuiteUri,
225 testScenarios.stream()
226 .flatMap(scenario -> {
227 assert scenario != null;
228 return Stream.of(generateScenario(
229 scenario,
230 collectionUri,
231 collectionGenerationPath,
232 bindingContext));
233 })
234 .sequential());
235 }
236
237 @NonNull
238 private Path generateSchema(
239 @NonNull IModule module,
240 @NonNull Lazy<Path> scenarioGenerationPath) {
241 String schemaExtension;
242 Format requiredContentFormat = getRequiredContentFormat();
243 switch (requiredContentFormat) {
244 case JSON:
245 case YAML:
246 schemaExtension = ".json";
247 break;
248 case XML:
249 schemaExtension = ".xsd";
250 break;
251 default:
252 throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat));
253 }
254
255
256 Path schemaPath;
257 try {
258 schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
259 } catch (IOException ex) {
260 throw new JUnitException("Unable to create schema temp file", ex);
261 }
262 try {
263 generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
264 } catch (IOException ex) {
265 throw new IllegalStateException(ex);
266 }
267 return ObjectUtils.notNull(schemaPath);
268 }
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283 protected void generateSchema(
284 @NonNull IModule module,
285 @NonNull Path schemaPath,
286 @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
287 Path parentDir = schemaPath.getParent();
288 if (parentDir != null && !Files.exists(parentDir)) {
289 Files.createDirectories(parentDir);
290 }
291
292 try (Writer writer = Files.newBufferedWriter(
293 schemaPath,
294 StandardCharsets.UTF_8,
295 getWriteOpenOptions())) {
296 schemaProducer.apply(module, writer);
297 }
298 LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
299 }
300
301
302
303
304
305
306 @SuppressWarnings("PMD.MethodReturnsInternalArray")
307 protected OpenOption[] getWriteOpenOptions() {
308 return OPEN_OPTIONS_TRUNCATE;
309 }
310
311 @SuppressWarnings("PMD.AvoidCatchingGenericException")
312 private DynamicContainer generateScenario(
313 @NonNull TestScenario scenario,
314 @NonNull URI collectionUri,
315 @NonNull Lazy<Path> collectionGenerationPath,
316 @NonNull IBindingContext bindingContext) {
317 Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
318 Path retval;
319 try {
320 retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
321 } catch (IOException ex) {
322 throw new JUnitException("Unable to create scenario temp directory", ex);
323 }
324 return retval;
325 }));
326
327 GenerateSchema generateSchema = scenario.getGenerateSchema();
328 if (generateSchema == null) {
329 throw new JUnitException(String.format(
330 "Scenario '%s' does not have a generate-schema directive", scenario.getName()));
331 }
332 Metaschema metaschemaDirective = generateSchema.getMetaschema();
333 URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
334
335 IModule module;
336 try {
337 IBindingModuleLoader loader = bindingContext.newModuleLoader();
338
339 module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
340 } catch (IOException | MetaschemaException ex) {
341 throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
342 }
343
344 Lazy<Path> lazySchema = Lazy.of(() -> generateSchema(module, scenarioGenerationPath));
345
346 Lazy<IContentValidator> lazyContentValidator = Lazy.of(() -> {
347 Path schemaPath = lazySchema.get();
348 return getContentValidatorSupplier().apply(schemaPath);
349 });
350 assert lazyContentValidator != null;
351
352
353 DynamicTest validateSchema = DynamicTest.dynamicTest(
354 "Validate Schema",
355 () -> {
356 Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357 if (supplier != null) {
358 Path schemaPath;
359 try {
360 schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361 } catch (Exception ex) {
362 throw new JUnitException(
363 "failed to generate schema", ex);
364 }
365 validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366 }
367 });
368
369 List<ValidationCase> validationCases = scenario.getValidationCases();
370 Stream<? extends DynamicNode> contentTests = validationCases.stream()
371 .flatMap(contentCase -> {
372 assert contentCase != null;
373 DynamicTest test
374 = generateValidationCase(
375 contentCase,
376 bindingContext,
377 lazyContentValidator,
378 collectionUri,
379 ObjectUtils.notNull(scenarioGenerationPath));
380 return test == null ? Stream.empty() : Stream.of(test);
381 }).sequential();
382
383 return DynamicContainer.dynamicContainer(
384 scenario.getName(),
385 metaschemaUri,
386 Stream.concat(Stream.of(validateSchema), contentTests).sequential());
387 }
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403 protected Path convertContent(
404 @NonNull URI resource,
405 @NonNull Path generationPath,
406 @NonNull IBindingContext context)
407 throws IOException {
408 Object object;
409 try {
410 object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
411 } catch (URISyntaxException ex) {
412 throw new IOException(ex);
413 }
414
415 if (!Files.exists(generationPath)) {
416 Files.createDirectories(generationPath);
417 }
418
419 Path convertedContetPath;
420 try {
421 convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
422 } catch (IOException ex) {
423 throw new JUnitException(
424 String.format("Unable to create converted content path in location '%s'", generationPath),
425 ex);
426 }
427
428 Format toFormat = getRequiredContentFormat();
429 if (LOGGER.isInfoEnabled()) {
430 LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
431 }
432
433 ISerializer<?> serializer
434 = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
435 serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
436
437 return convertedContetPath;
438 }
439
440
441
442
443
444
445
446
447 @Nullable
448 private static Format toFormat(@Nullable String sourceFormat) {
449 if (sourceFormat == null) {
450 return null;
451 }
452 switch (sourceFormat) {
453 case "XML":
454 return Format.XML;
455 case "JSON":
456 return Format.JSON;
457 case "YAML":
458 return Format.YAML;
459 default:
460 return null;
461 }
462 }
463
464 @SuppressWarnings("PMD.AvoidCatchingGenericException")
465 private DynamicTest generateValidationCase(
466 @NonNull ValidationCase contentCase,
467 @NonNull IBindingContext bindingContext,
468 @NonNull Lazy<IContentValidator> lazyContentValidator,
469 @NonNull URI collectionUri,
470 @NonNull Lazy<Path> resourceGenerationPath) {
471
472 URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
473
474 Format format = toFormat(contentCase.getSourceFormat());
475 boolean expectedValid = isValid(contentCase.getValidationResult());
476
477 DynamicTest retval = null;
478 if (getRequiredContentFormat().equals(format)) {
479 retval = DynamicTest.dynamicTest(
480 String.format("Validate %s=%s: %s", format, expectedValid,
481 contentCase.getLocation()),
482 contentUri,
483 () -> {
484 IContentValidator contentValidator;
485 try {
486 contentValidator = lazyContentValidator.get();
487 } catch (Exception ex) {
488 throw new JUnitException(
489 "failed to produce the content validator", ex);
490 }
491
492 assertEquals(
493 expectedValid,
494 validateWithSchema(
495 ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
496 "validation did not match expectation for: " + contentUri.toASCIIString());
497 });
498 } else if (expectedValid) {
499 retval = DynamicTest.dynamicTest(
500 String.format("Convert and Validate %s=%s: %s", format, expectedValid,
501 contentCase.getLocation()),
502 contentUri,
503 () -> {
504 Path convertedContetPath;
505 try {
506 convertedContetPath = convertContent(
507 contentUri,
508 ObjectUtils.notNull(resourceGenerationPath.get()),
509 bindingContext);
510 } catch (Exception ex) {
511 throw new JUnitException("failed to convert content: " + contentUri, ex);
512 }
513
514 IContentValidator contentValidator;
515 try {
516 contentValidator = lazyContentValidator.get();
517 } catch (Exception ex) {
518 throw new JUnitException(
519 "failed to produce the content validator",
520 ex.getCause());
521 }
522
523 if (LOGGER.isInfoEnabled()) {
524 LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
525 }
526 assertEquals(expectedValid,
527 validateWithSchema(
528 ObjectUtils.notNull(contentValidator),
529 ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
530 String.format("validation of '%s' did not match expectation", convertedContetPath));
531 });
532 }
533 return retval;
534 }
535
536
537
538
539
540
541
542
543 private static boolean isValid(@Nullable String validationResult) {
544 return VALID.equals(validationResult);
545 }
546
547 private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
548 throws IOException {
549 IValidationResult schemaValidationResult;
550 try {
551 schemaValidationResult = validator.validate(target);
552 } catch (URISyntaxException ex) {
553 throw new IOException(ex);
554 }
555 return processValidationResult(schemaValidationResult);
556 }
557
558
559
560
561
562
563
564
565
566
567
568
569 protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
570 throws IOException {
571 LOGGER.atError().log("Validating: {}", target);
572 IValidationResult schemaValidationResult = validator.validate(target);
573 if (!schemaValidationResult.isPassing()) {
574 LOGGER.atError().log("Schema validation failed for: {}", target);
575 }
576 return processValidationResult(schemaValidationResult);
577 }
578
579 private static boolean processValidationResult(IValidationResult schemaValidationResult) {
580 for (IValidationFinding finding : schemaValidationResult.getFindings()) {
581 logFinding(ObjectUtils.notNull(finding));
582 }
583 return schemaValidationResult.isPassing();
584 }
585
586 private static void logFinding(@NonNull IValidationFinding finding) {
587 LogBuilder logBuilder;
588 switch (finding.getSeverity()) {
589 case CRITICAL:
590 logBuilder = LOGGER.atFatal();
591 break;
592 case ERROR:
593 logBuilder = LOGGER.atError();
594 break;
595 case WARNING:
596 logBuilder = LOGGER.atWarn();
597 break;
598 case INFORMATIONAL:
599 logBuilder = LOGGER.atInfo();
600 break;
601 case DEBUG:
602 logBuilder = LOGGER.atDebug();
603 break;
604 default:
605 throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
606 }
607
608
609
610
611
612 if (finding instanceof JsonValidationFinding) {
613 JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
614 logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
615 } else {
616 logBuilder.log("{}", finding.getMessage());
617 }
618 }
619 }