1
2
3
4
5
6 package gov.nist.secauto.metaschema.model.testing;
7
8 import static org.junit.jupiter.api.Assertions.assertEquals;
9
10 import gov.nist.secauto.metaschema.core.model.IModule;
11 import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12 import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13 import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14 import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15 import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16 import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17 import gov.nist.secauto.metaschema.databind.DefaultBindingContext;
18 import gov.nist.secauto.metaschema.databind.IBindingContext;
19 import gov.nist.secauto.metaschema.databind.io.Format;
20 import gov.nist.secauto.metaschema.databind.io.ISerializer;
21 import gov.nist.secauto.metaschema.databind.model.metaschema.BindingModuleLoader;
22 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
23 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
24 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
25 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
26 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
27 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
28
29 import org.apache.logging.log4j.LogBuilder;
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.apache.xmlbeans.XmlException;
33 import org.apache.xmlbeans.XmlOptions;
34 import org.junit.jupiter.api.DynamicContainer;
35 import org.junit.jupiter.api.DynamicNode;
36 import org.junit.jupiter.api.DynamicTest;
37 import org.junit.platform.commons.JUnitException;
38
39 import java.io.IOException;
40 import java.io.Writer;
41 import java.net.URI;
42 import java.net.URISyntaxException;
43 import java.net.URL;
44 import java.nio.charset.StandardCharsets;
45 import java.nio.file.FileVisitResult;
46 import java.nio.file.Files;
47 import java.nio.file.OpenOption;
48 import java.nio.file.Path;
49 import java.nio.file.SimpleFileVisitor;
50 import java.nio.file.StandardOpenOption;
51 import java.nio.file.attribute.BasicFileAttributes;
52 import java.util.function.BiFunction;
53 import java.util.function.Function;
54 import java.util.function.Supplier;
55 import java.util.stream.Stream;
56
57 import edu.umd.cs.findbugs.annotations.NonNull;
58 import edu.umd.cs.findbugs.annotations.Nullable;
59 import nl.talsmasoftware.lazy4j.Lazy;
60
61 public abstract class AbstractTestSuite {
62 private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
63 private static final BindingModuleLoader LOADER;
64
65 private static final boolean DELETE_RESULTS_ON_EXIT = false;
66 private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
67 StandardOpenOption.CREATE,
68 StandardOpenOption.WRITE,
69 StandardOpenOption.TRUNCATE_EXISTING
70 };
71
72 static {
73 IBindingContext bindingContext = new DefaultBindingContext();
74 LOADER = new BindingModuleLoader(bindingContext);
75 LOADER.allowEntityResolution();
76 }
77
78
79
80
81
82
83 @NonNull
84 protected abstract Format getRequiredContentFormat();
85
86
87
88
89
90
91 @NonNull
92 protected abstract URI getTestSuiteURI();
93
94
95
96
97
98
99 @NonNull
100 protected abstract Path getGenerationPath();
101
102
103
104
105
106
107
108 @NonNull
109 protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
110
111
112
113
114
115
116 @Nullable
117 protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
118
119
120
121
122
123
124 @NonNull
125 protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
126
127
128
129
130
131
132 @NonNull
133 protected Stream<DynamicNode> testFactory() {
134 try {
135 XmlOptions options = new XmlOptions();
136 options.setBaseURI(null);
137 options.setLoadLineNumbers();
138
139 Path generationPath = getGenerationPath();
140 if (Files.exists(generationPath)) {
141 if (!Files.isDirectory(generationPath)) {
142 throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
143 }
144 } else {
145 Files.createDirectories(generationPath);
146 }
147
148 URI testSuiteUri = getTestSuiteURI();
149 URL testSuiteUrl = testSuiteUri.toURL();
150 TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
151 return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
152 .flatMap(
153 collection -> Stream
154 .of(generateCollection(ObjectUtils.notNull(collection), testSuiteUri, generationPath))));
155 } catch (XmlException | IOException ex) {
156 throw new JUnitException("Unable to generate tests", ex);
157 }
158 }
159
160
161
162
163
164
165
166 protected void deleteCollectionOnExit(@NonNull Path path) {
167 Runtime.getRuntime().addShutdownHook(new Thread(
168 () -> {
169 try {
170 Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
171 @Override
172 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
173 Files.delete(file);
174 return FileVisitResult.CONTINUE;
175 }
176
177 @Override
178 public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
179 if (ex == null) {
180 Files.delete(dir);
181 return FileVisitResult.CONTINUE;
182 }
183
184 throw ex;
185 }
186 });
187 } catch (IOException ex) {
188 throw new JUnitException("Failed to delete collection: " + path, ex);
189 }
190 }));
191 }
192
193 private DynamicContainer generateCollection(@NonNull TestCollection collection, @NonNull URI testSuiteUri,
194 @NonNull Path generationPath) {
195 URI collectionUri = testSuiteUri.resolve(collection.getLocation());
196 assert collectionUri != null;
197
198 LOGGER.atInfo().log("Collection: " + collectionUri);
199 Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
200 Path retval;
201 try {
202 retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
203 if (DELETE_RESULTS_ON_EXIT) {
204 deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
205 }
206 } catch (IOException ex) {
207 throw new JUnitException("Unable to create collection temp directory", ex);
208 }
209 return retval;
210 }));
211
212 return DynamicContainer.dynamicContainer(
213 collection.getName(),
214 testSuiteUri,
215 collection.getTestScenarioList().stream()
216 .flatMap(scenario -> {
217 assert scenario != null;
218 return Stream.of(generateScenario(scenario, collectionUri, collectionGenerationPath));
219 })
220 .sequential());
221 }
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 protected void generateSchema(
237 @NonNull IModule module,
238 @NonNull Path schemaPath,
239 @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
240 Path parentDir = schemaPath.getParent();
241 if (parentDir != null && !Files.exists(parentDir)) {
242 Files.createDirectories(parentDir);
243 }
244
245 try (Writer writer = Files.newBufferedWriter(
246 schemaPath,
247 StandardCharsets.UTF_8,
248 getWriteOpenOptions())) {
249 schemaProducer.apply(module, writer);
250 }
251 LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
252 }
253
254
255
256
257
258
259 protected OpenOption[] getWriteOpenOptions() {
260 return OPEN_OPTIONS_TRUNCATE;
261 }
262
263 private DynamicContainer generateScenario(
264 @NonNull TestScenario scenario,
265 @NonNull URI collectionUri,
266 @NonNull Lazy<Path> collectionGenerationPath) {
267 Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
268 Path retval;
269 try {
270 retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
271 } catch (IOException ex) {
272 throw new JUnitException("Unable to create scenario temp directory", ex);
273 }
274 return retval;
275 });
276
277
278
279
280
281
282
283
284
285 GenerateSchema generateSchema = scenario.getGenerateSchema();
286 MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
287 URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
288
289 Lazy<IModule> lazyModule = Lazy.lazy(() -> {
290 IModule module;
291 try {
292 module = LOADER.load(ObjectUtils.notNull(metaschemaUri.toURL()));
293 } catch (IOException | MetaschemaException ex) {
294 throw new JUnitException("Unable to generate schema for Module: " + metaschemaUri, ex);
295 }
296 return module;
297 });
298
299 Lazy<Path> lazySchema = Lazy.lazy(() -> {
300 Path schemaPath;
301
302 String schemaExtension;
303 Format requiredContentFormat = getRequiredContentFormat();
304 switch (requiredContentFormat) {
305 case JSON:
306 case YAML:
307 schemaExtension = ".json";
308 break;
309 case XML:
310 schemaExtension = ".xsd";
311 break;
312 default:
313 throw new IllegalStateException();
314 }
315
316
317 try {
318 schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
319 } catch (IOException ex) {
320 throw new JUnitException("Unable to create schema temp file", ex);
321 }
322 IModule module = lazyModule.get();
323 try {
324 generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
325 } catch (IOException ex) {
326 throw new IllegalStateException(ex);
327 }
328 return schemaPath;
329 });
330
331 Lazy<IBindingContext> lazyDynamicBindingContext = Lazy.lazy(() -> {
332 IModule module = lazyModule.get();
333 IBindingContext context;
334 try {
335 context = new DefaultBindingContext();
336 context.registerModule(
337 ObjectUtils.notNull(module),
338 ObjectUtils.notNull(scenarioGenerationPath.get()));
339 } catch (Exception ex) {
340 throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
341 }
342 return context;
343 });
344 assert lazyDynamicBindingContext != null;
345
346 Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
347 Path schemaPath = lazySchema.get();
348 return getContentValidatorSupplier().apply(schemaPath);
349 });
350 assert lazyContentValidator != null;
351
352
353 DynamicTest validateSchema = DynamicTest.dynamicTest(
354 "Validate Schema",
355 () -> {
356 Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357 if (supplier != null) {
358 Path schemaPath;
359 try {
360 schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361 } catch (Exception ex) {
362 throw new JUnitException(
363 "failed to generate schema", ex.getCause());
364 }
365 validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366 }
367 });
368
369 Stream<? extends DynamicNode> contentTests;
370 {
371 contentTests = scenario.getValidationCaseList().stream()
372 .flatMap(contentCase -> {
373 assert contentCase != null;
374 DynamicTest test
375 = generateValidationCase(
376 contentCase,
377 lazyDynamicBindingContext,
378 lazyContentValidator,
379 collectionUri,
380 ObjectUtils.notNull(scenarioGenerationPath));
381 return test == null ? Stream.empty() : Stream.of(test);
382 }).sequential();
383 }
384
385 return DynamicContainer.dynamicContainer(
386 scenario.getName(),
387 metaschemaUri,
388 Stream.concat(Stream.of(validateSchema), contentTests).sequential());
389 }
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405 protected Path convertContent(
406 @NonNull URI resource,
407 @NonNull Path generationPath,
408 @NonNull IBindingContext context)
409 throws IOException {
410 Object object;
411 try {
412 object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
413 } catch (URISyntaxException ex) {
414 throw new IOException(ex);
415 }
416
417 if (!Files.exists(generationPath)) {
418 Files.createDirectories(generationPath);
419 }
420
421 Path convertedContetPath;
422 try {
423 convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
424 } catch (IOException ex) {
425 throw new JUnitException("Unable to create schema temp file", ex);
426 }
427
428 @SuppressWarnings("rawtypes") ISerializer serializer
429 = context.newSerializer(getRequiredContentFormat(), ObjectUtils.asType(object.getClass()));
430 serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
431
432 return convertedContetPath;
433 }
434
435 private DynamicTest generateValidationCase(
436 @NonNull ContentCaseType contentCase,
437 @NonNull Lazy<IBindingContext> lazyBindingContext,
438 @NonNull Lazy<IContentValidator> lazyContentValidator,
439 @NonNull URI collectionUri,
440 @NonNull Lazy<Path> generationPath) {
441
442 URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
443
444 Format format = contentCase.getSourceFormat();
445 DynamicTest retval = null;
446 if (getRequiredContentFormat().equals(format)) {
447 retval = DynamicTest.dynamicTest(
448 String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
449 contentCase.getLocation()),
450 contentUri,
451 () -> {
452 IContentValidator contentValidator;
453 try {
454 contentValidator = lazyContentValidator.get();
455 } catch (Exception ex) {
456 throw new JUnitException(
457 "failed to produce the content validator", ex.getCause());
458 }
459
460 assertEquals(
461 contentCase.getValidationResult(),
462 validateWithSchema(
463 ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
464 "validation did not match expectation for: " + contentUri.toASCIIString());
465 });
466 } else if (contentCase.getValidationResult()) {
467 retval = DynamicTest.dynamicTest(
468 String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
469 contentCase.getLocation()),
470 contentUri,
471 () -> {
472 IBindingContext context;
473 try {
474 context = lazyBindingContext.get();
475 } catch (Exception ex) {
476 throw new JUnitException(
477 "failed to produce the content validator", ex.getCause());
478 }
479 assert context != null;
480
481 Path convertedContetPath;
482 try {
483 convertedContetPath = convertContent(contentUri, ObjectUtils.notNull(generationPath.get()), context);
484 } catch (Exception ex) {
485 throw new JUnitException("failed to convert content: " + contentUri, ex);
486 }
487
488 IContentValidator contentValidator;
489 try {
490 contentValidator = lazyContentValidator.get();
491 } catch (Exception ex) {
492 throw new JUnitException(
493 "failed to produce the content validator",
494 ex.getCause());
495 }
496 assertEquals(contentCase.getValidationResult(),
497 validateWithSchema(
498 ObjectUtils.notNull(contentValidator),
499 ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
500 String.format("validation of '%s' did not match expectation", convertedContetPath));
501 });
502 }
503 return retval;
504 }
505
506 private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
507 throws IOException {
508 IValidationResult schemaValidationResult;
509 try {
510 schemaValidationResult = validator.validate(target);
511 } catch (URISyntaxException ex) {
512 throw new IOException(ex);
513 }
514 return processValidationResult(schemaValidationResult);
515 }
516
517
518
519
520
521
522
523
524
525
526
527
528 protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
529 throws IOException {
530 IValidationResult schemaValidationResult = validator.validate(target);
531 if (!schemaValidationResult.isPassing()) {
532 LOGGER.atError().log("Schema validation failed for: {}", target);
533 }
534 return processValidationResult(schemaValidationResult);
535 }
536
537 private static boolean processValidationResult(IValidationResult schemaValidationResult) {
538 for (IValidationFinding finding : schemaValidationResult.getFindings()) {
539 logFinding(ObjectUtils.notNull(finding));
540 }
541 return schemaValidationResult.isPassing();
542 }
543
544 private static void logFinding(@NonNull IValidationFinding finding) {
545 LogBuilder logBuilder;
546 switch (finding.getSeverity()) {
547 case CRITICAL:
548 logBuilder = LOGGER.atFatal();
549 break;
550 case ERROR:
551 logBuilder = LOGGER.atError();
552 break;
553 case WARNING:
554 logBuilder = LOGGER.atWarn();
555 break;
556 case INFORMATIONAL:
557 logBuilder = LOGGER.atInfo();
558 break;
559 case DEBUG:
560 logBuilder = LOGGER.atDebug();
561 break;
562 default:
563 throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
564 }
565
566
567
568
569
570 if (finding instanceof JsonValidationFinding) {
571 JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
572 logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
573 } else {
574 logBuilder.log("{}", finding.getMessage());
575 }
576 }
577 }