1
2
3
4
5
6 package gov.nist.secauto.metaschema.model.testing;
7
8 import static org.junit.jupiter.api.Assertions.assertEquals;
9
10 import gov.nist.secauto.metaschema.core.model.IModule;
11 import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12 import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13 import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14 import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15 import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16 import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17 import gov.nist.secauto.metaschema.databind.IBindingContext;
18 import gov.nist.secauto.metaschema.databind.io.Format;
19 import gov.nist.secauto.metaschema.databind.io.ISerializer;
20 import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
21 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
22 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
23 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
24 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
25 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
26 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
27
28 import org.apache.logging.log4j.LogBuilder;
29 import org.apache.logging.log4j.LogManager;
30 import org.apache.logging.log4j.Logger;
31 import org.apache.xmlbeans.XmlException;
32 import org.apache.xmlbeans.XmlOptions;
33 import org.junit.jupiter.api.DynamicContainer;
34 import org.junit.jupiter.api.DynamicNode;
35 import org.junit.jupiter.api.DynamicTest;
36 import org.junit.platform.commons.JUnitException;
37
38 import java.io.IOException;
39 import java.io.Writer;
40 import java.net.URI;
41 import java.net.URISyntaxException;
42 import java.net.URL;
43 import java.nio.charset.StandardCharsets;
44 import java.nio.file.FileVisitResult;
45 import java.nio.file.Files;
46 import java.nio.file.OpenOption;
47 import java.nio.file.Path;
48 import java.nio.file.SimpleFileVisitor;
49 import java.nio.file.StandardOpenOption;
50 import java.nio.file.attribute.BasicFileAttributes;
51 import java.util.function.BiFunction;
52 import java.util.function.Function;
53 import java.util.function.Supplier;
54 import java.util.stream.Stream;
55
56 import edu.umd.cs.findbugs.annotations.NonNull;
57 import edu.umd.cs.findbugs.annotations.Nullable;
58 import nl.talsmasoftware.lazy4j.Lazy;
59
60 public abstract class AbstractTestSuite {
61 private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
62
63 private static final boolean DELETE_RESULTS_ON_EXIT = false;
64 private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
65 StandardOpenOption.CREATE,
66 StandardOpenOption.WRITE,
67 StandardOpenOption.TRUNCATE_EXISTING
68 };
69
70
71
72
73
74
75 @NonNull
76 protected abstract Format getRequiredContentFormat();
77
78
79
80
81
82
83 @NonNull
84 protected abstract URI getTestSuiteURI();
85
86
87
88
89
90
91 @NonNull
92 protected abstract Path getGenerationPath();
93
94
95
96
97
98
99
100 @NonNull
101 protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
102
103
104
105
106
107
108 @Nullable
109 protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
110
111
112
113
114
115
116 @NonNull
117 protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
118
119
120
121
122
123
124 @NonNull
125 protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
126 try {
127 XmlOptions options = new XmlOptions();
128 options.setBaseURI(null);
129 options.setLoadLineNumbers();
130
131 Path generationPath = getGenerationPath();
132 if (Files.exists(generationPath)) {
133 if (!Files.isDirectory(generationPath)) {
134 throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
135 }
136 } else {
137 Files.createDirectories(generationPath);
138 }
139
140 URI testSuiteUri = getTestSuiteURI();
141 URL testSuiteUrl = testSuiteUri.toURL();
142 TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
143 return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
144 .flatMap(
145 collection -> Stream
146 .of(generateCollection(
147 ObjectUtils.notNull(collection),
148 testSuiteUri,
149 generationPath,
150 bindingContext))));
151 } catch (XmlException | IOException ex) {
152 throw new JUnitException("Unable to generate tests", ex);
153 }
154 }
155
156
157
158
159
160
161
162 protected void deleteCollectionOnExit(@NonNull Path path) {
163 Runtime.getRuntime().addShutdownHook(new Thread(
164 () -> {
165 try {
166 Files.walkFileTree(path, new SimpleFileVisitor<>() {
167 @Override
168 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
169 Files.delete(file);
170 return FileVisitResult.CONTINUE;
171 }
172
173 @Override
174 public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
175 if (ex == null) {
176 Files.delete(dir);
177 return FileVisitResult.CONTINUE;
178 }
179
180 throw ex;
181 }
182 });
183 } catch (IOException ex) {
184 throw new JUnitException("Failed to delete collection: " + path, ex);
185 }
186 }));
187 }
188
189 private DynamicContainer generateCollection(
190 @NonNull TestCollection collection,
191 @NonNull URI testSuiteUri,
192 @NonNull Path generationPath,
193 @NonNull IBindingContext bindingContext) {
194 URI collectionUri = testSuiteUri.resolve(collection.getLocation());
195 assert collectionUri != null;
196
197 LOGGER.atInfo().log("Collection: " + collectionUri);
198 Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
199 Path retval;
200 try {
201 retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
202 if (DELETE_RESULTS_ON_EXIT) {
203 deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
204 }
205 } catch (IOException ex) {
206 throw new JUnitException("Unable to create collection temp directory", ex);
207 }
208 return retval;
209 }));
210
211 return DynamicContainer.dynamicContainer(
212 collection.getName(),
213 testSuiteUri,
214 collection.getTestScenarioList().stream()
215 .flatMap(scenario -> {
216 assert scenario != null;
217 return Stream.of(generateScenario(
218 scenario,
219 collectionUri,
220 collectionGenerationPath,
221 bindingContext));
222 })
223 .sequential());
224 }
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239 protected void generateSchema(
240 @NonNull IModule module,
241 @NonNull Path schemaPath,
242 @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
243 Path parentDir = schemaPath.getParent();
244 if (parentDir != null && !Files.exists(parentDir)) {
245 Files.createDirectories(parentDir);
246 }
247
248 try (Writer writer = Files.newBufferedWriter(
249 schemaPath,
250 StandardCharsets.UTF_8,
251 getWriteOpenOptions())) {
252 schemaProducer.apply(module, writer);
253 }
254 LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
255 }
256
257
258
259
260
261
262 @SuppressWarnings("PMD.MethodReturnsInternalArray")
263 protected OpenOption[] getWriteOpenOptions() {
264 return OPEN_OPTIONS_TRUNCATE;
265 }
266
267 @SuppressWarnings("PMD.AvoidCatchingGenericException")
268 private DynamicContainer generateScenario(
269 @NonNull TestScenario scenario,
270 @NonNull URI collectionUri,
271 @NonNull Lazy<Path> collectionGenerationPath,
272 @NonNull IBindingContext bindingContext) {
273 Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
274 Path retval;
275 try {
276 retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
277 } catch (IOException ex) {
278 throw new JUnitException("Unable to create scenario temp directory", ex);
279 }
280 return retval;
281 });
282
283
284
285
286
287
288
289
290
291 GenerateSchema generateSchema = scenario.getGenerateSchema();
292 MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
293 URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
294
295 IModule module;
296 try {
297 IBindingModuleLoader loader = bindingContext.newModuleLoader();
298
299 module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
300 } catch (IOException | MetaschemaException ex) {
301 throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
302 }
303
304 Lazy<Path> lazySchema = Lazy.lazy(() -> {
305 String schemaExtension;
306 Format requiredContentFormat = getRequiredContentFormat();
307 switch (requiredContentFormat) {
308 case JSON:
309 case YAML:
310 schemaExtension = ".json";
311 break;
312 case XML:
313 schemaExtension = ".xsd";
314 break;
315 default:
316 throw new IllegalStateException();
317 }
318
319
320 Path schemaPath;
321 try {
322 schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
323 } catch (IOException ex) {
324 throw new JUnitException("Unable to create schema temp file", ex);
325 }
326 try {
327 generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
328 } catch (IOException ex) {
329 throw new IllegalStateException(ex);
330 }
331 return schemaPath;
332 });
333
334 Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
335 Path schemaPath = lazySchema.get();
336 return getContentValidatorSupplier().apply(schemaPath);
337 });
338 assert lazyContentValidator != null;
339
340
341 DynamicTest validateSchema = DynamicTest.dynamicTest(
342 "Validate Schema",
343 () -> {
344 Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
345 if (supplier != null) {
346 Path schemaPath;
347 try {
348 schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
349 } catch (Exception ex) {
350 throw new JUnitException(
351 "failed to generate schema", ex);
352 }
353 validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
354 }
355 });
356
357 Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
358 .flatMap(contentCase -> {
359 assert contentCase != null;
360 DynamicTest test
361 = generateValidationCase(
362 contentCase,
363 bindingContext,
364 lazyContentValidator,
365 collectionUri,
366 ObjectUtils.notNull(scenarioGenerationPath));
367 return test == null ? Stream.empty() : Stream.of(test);
368 }).sequential();
369
370 return DynamicContainer.dynamicContainer(
371 scenario.getName(),
372 metaschemaUri,
373 Stream.concat(Stream.of(validateSchema), contentTests).sequential());
374 }
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390 protected Path convertContent(
391 @NonNull URI resource,
392 @NonNull Path generationPath,
393 @NonNull IBindingContext context)
394 throws IOException {
395 Object object;
396 try {
397 object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
398 } catch (URISyntaxException ex) {
399 throw new IOException(ex);
400 }
401
402 if (!Files.exists(generationPath)) {
403 Files.createDirectories(generationPath);
404 }
405
406 Path convertedContetPath;
407 try {
408 convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
409 } catch (IOException ex) {
410 throw new JUnitException(
411 String.format("Unable to create converted content path in location '%s'", generationPath),
412 ex);
413 }
414
415 Format toFormat = getRequiredContentFormat();
416 if (LOGGER.isInfoEnabled()) {
417 LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
418 }
419
420 ISerializer<?> serializer
421 = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
422 serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
423
424 return convertedContetPath;
425 }
426
427 @SuppressWarnings("PMD.AvoidCatchingGenericException")
428 private DynamicTest generateValidationCase(
429 @NonNull ContentCaseType contentCase,
430 @NonNull IBindingContext bindingContext,
431 @NonNull Lazy<IContentValidator> lazyContentValidator,
432 @NonNull URI collectionUri,
433 @NonNull Lazy<Path> resourceGenerationPath) {
434
435 URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
436
437 Format format = contentCase.getSourceFormat();
438 DynamicTest retval = null;
439 if (getRequiredContentFormat().equals(format)) {
440 retval = DynamicTest.dynamicTest(
441 String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
442 contentCase.getLocation()),
443 contentUri,
444 () -> {
445 IContentValidator contentValidator;
446 try {
447 contentValidator = lazyContentValidator.get();
448 } catch (Exception ex) {
449 throw new JUnitException(
450 "failed to produce the content validator", ex.getCause());
451 }
452
453 assertEquals(
454 contentCase.getValidationResult(),
455 validateWithSchema(
456 ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
457 "validation did not match expectation for: " + contentUri.toASCIIString());
458 });
459 } else if (contentCase.getValidationResult()) {
460 retval = DynamicTest.dynamicTest(
461 String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
462 contentCase.getLocation()),
463 contentUri,
464 () -> {
465 Path convertedContetPath;
466 try {
467 convertedContetPath = convertContent(
468 contentUri,
469 ObjectUtils.notNull(resourceGenerationPath.get()),
470 bindingContext);
471 } catch (Exception ex) {
472 throw new JUnitException("failed to convert content: " + contentUri, ex);
473 }
474
475 IContentValidator contentValidator;
476 try {
477 contentValidator = lazyContentValidator.get();
478 } catch (Exception ex) {
479 throw new JUnitException(
480 "failed to produce the content validator",
481 ex.getCause());
482 }
483
484 if (LOGGER.isInfoEnabled()) {
485 LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
486 }
487 assertEquals(contentCase.getValidationResult(),
488 validateWithSchema(
489 ObjectUtils.notNull(contentValidator),
490 ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
491 String.format("validation of '%s' did not match expectation", convertedContetPath));
492 });
493 }
494 return retval;
495 }
496
497 private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
498 throws IOException {
499 IValidationResult schemaValidationResult;
500 try {
501 schemaValidationResult = validator.validate(target);
502 } catch (URISyntaxException ex) {
503 throw new IOException(ex);
504 }
505 return processValidationResult(schemaValidationResult);
506 }
507
508
509
510
511
512
513
514
515
516
517
518
519 protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
520 throws IOException {
521 IValidationResult schemaValidationResult = validator.validate(target);
522 if (!schemaValidationResult.isPassing()) {
523 LOGGER.atError().log("Schema validation failed for: {}", target);
524 }
525 return processValidationResult(schemaValidationResult);
526 }
527
528 private static boolean processValidationResult(IValidationResult schemaValidationResult) {
529 for (IValidationFinding finding : schemaValidationResult.getFindings()) {
530 logFinding(ObjectUtils.notNull(finding));
531 }
532 return schemaValidationResult.isPassing();
533 }
534
535 private static void logFinding(@NonNull IValidationFinding finding) {
536 LogBuilder logBuilder;
537 switch (finding.getSeverity()) {
538 case CRITICAL:
539 logBuilder = LOGGER.atFatal();
540 break;
541 case ERROR:
542 logBuilder = LOGGER.atError();
543 break;
544 case WARNING:
545 logBuilder = LOGGER.atWarn();
546 break;
547 case INFORMATIONAL:
548 logBuilder = LOGGER.atInfo();
549 break;
550 case DEBUG:
551 logBuilder = LOGGER.atDebug();
552 break;
553 default:
554 throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
555 }
556
557
558
559
560
561 if (finding instanceof JsonValidationFinding) {
562 JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
563 logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
564 } else {
565 logBuilder.log("{}", finding.getMessage());
566 }
567 }
568 }