1
2
3
4
5
6 package gov.nist.secauto.metaschema.model.testing;
7
8 import static org.junit.jupiter.api.Assertions.assertEquals;
9
10 import gov.nist.secauto.metaschema.core.model.IModule;
11 import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12 import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13 import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14 import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15 import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16 import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17 import gov.nist.secauto.metaschema.databind.IBindingContext;
18 import gov.nist.secauto.metaschema.databind.io.Format;
19 import gov.nist.secauto.metaschema.databind.io.ISerializer;
20 import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
21 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
22 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
23 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
24 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
25 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
26 import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
27
28 import org.apache.logging.log4j.LogBuilder;
29 import org.apache.logging.log4j.LogManager;
30 import org.apache.logging.log4j.Logger;
31 import org.apache.xmlbeans.XmlException;
32 import org.apache.xmlbeans.XmlOptions;
33 import org.junit.jupiter.api.DynamicContainer;
34 import org.junit.jupiter.api.DynamicNode;
35 import org.junit.jupiter.api.DynamicTest;
36 import org.junit.platform.commons.JUnitException;
37
38 import java.io.IOException;
39 import java.io.Writer;
40 import java.net.URI;
41 import java.net.URISyntaxException;
42 import java.net.URL;
43 import java.nio.charset.StandardCharsets;
44 import java.nio.file.FileVisitResult;
45 import java.nio.file.Files;
46 import java.nio.file.OpenOption;
47 import java.nio.file.Path;
48 import java.nio.file.SimpleFileVisitor;
49 import java.nio.file.StandardOpenOption;
50 import java.nio.file.attribute.BasicFileAttributes;
51 import java.util.function.BiFunction;
52 import java.util.function.Function;
53 import java.util.function.Supplier;
54 import java.util.stream.Stream;
55
56 import edu.umd.cs.findbugs.annotations.NonNull;
57 import edu.umd.cs.findbugs.annotations.Nullable;
58 import nl.talsmasoftware.lazy4j.Lazy;
59
60
61
62
63
64
65
66 @SuppressWarnings({
67 "PMD.GodClass",
68 "PMD.CouplingBetweenObjects"
69 })
70 public abstract class AbstractTestSuite {
71 private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
72
73 private static final boolean DELETE_RESULTS_ON_EXIT = false;
74 private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
75 StandardOpenOption.CREATE,
76 StandardOpenOption.WRITE,
77 StandardOpenOption.TRUNCATE_EXISTING
78 };
79
80
81
82
83
84
85 @NonNull
86 protected abstract Format getRequiredContentFormat();
87
88
89
90
91
92
93 @NonNull
94 protected abstract URI getTestSuiteURI();
95
96
97
98
99
100
101 @NonNull
102 protected abstract Path getGenerationPath();
103
104
105
106
107
108
109
110 @NonNull
111 protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112
113
114
115
116
117
118 @Nullable
119 protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120
121
122
123
124
125
126 @NonNull
127 protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128
129
130
131
132
133
134
135
136
137 @NonNull
138 protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139 try {
140 XmlOptions options = new XmlOptions();
141 options.setBaseURI(null);
142 options.setLoadLineNumbers();
143
144 Path generationPath = getGenerationPath();
145 if (Files.exists(generationPath)) {
146 if (!Files.isDirectory(generationPath)) {
147 throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
148 }
149 } else {
150 Files.createDirectories(generationPath);
151 }
152
153 URI testSuiteUri = getTestSuiteURI();
154 URL testSuiteUrl = testSuiteUri.toURL();
155 TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
156 return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
157 .flatMap(
158 collection -> Stream
159 .of(generateCollection(
160 ObjectUtils.notNull(collection),
161 testSuiteUri,
162 generationPath,
163 bindingContext))));
164 } catch (XmlException | IOException ex) {
165 throw new JUnitException("Unable to generate tests", ex);
166 }
167 }
168
169
170
171
172
173
174
175 protected void deleteCollectionOnExit(@NonNull Path path) {
176 Runtime.getRuntime().addShutdownHook(new Thread(
177 () -> {
178 try {
179 Files.walkFileTree(path, new SimpleFileVisitor<>() {
180 @Override
181 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
182 Files.delete(file);
183 return FileVisitResult.CONTINUE;
184 }
185
186 @Override
187 public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
188 if (ex == null) {
189 Files.delete(dir);
190 return FileVisitResult.CONTINUE;
191 }
192
193 throw ex;
194 }
195 });
196 } catch (IOException ex) {
197 throw new JUnitException("Failed to delete collection: " + path, ex);
198 }
199 }));
200 }
201
202 private DynamicContainer generateCollection(
203 @NonNull TestCollection collection,
204 @NonNull URI testSuiteUri,
205 @NonNull Path generationPath,
206 @NonNull IBindingContext bindingContext) {
207 URI collectionUri = testSuiteUri.resolve(collection.getLocation());
208 assert collectionUri != null;
209
210 LOGGER.atInfo().log("Collection: " + collectionUri);
211 Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
212 Path retval;
213 try {
214 retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
215 if (DELETE_RESULTS_ON_EXIT) {
216 deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
217 }
218 } catch (IOException ex) {
219 throw new JUnitException("Unable to create collection temp directory", ex);
220 }
221 return retval;
222 }));
223
224 return DynamicContainer.dynamicContainer(
225 collection.getName(),
226 testSuiteUri,
227 collection.getTestScenarioList().stream()
228 .flatMap(scenario -> {
229 assert scenario != null;
230 return Stream.of(generateScenario(
231 scenario,
232 collectionUri,
233 collectionGenerationPath,
234 bindingContext));
235 })
236 .sequential());
237 }
238
239 @NonNull
240 private Path generateSchema(
241 @NonNull IModule module,
242 @NonNull Lazy<Path> scenarioGenerationPath) {
243 String schemaExtension;
244 Format requiredContentFormat = getRequiredContentFormat();
245 switch (requiredContentFormat) {
246 case JSON:
247 case YAML:
248 schemaExtension = ".json";
249 break;
250 case XML:
251 schemaExtension = ".xsd";
252 break;
253 default:
254 throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat));
255 }
256
257
258 Path schemaPath;
259 try {
260 schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
261 } catch (IOException ex) {
262 throw new JUnitException("Unable to create schema temp file", ex);
263 }
264 try {
265 generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
266 } catch (IOException ex) {
267 throw new IllegalStateException(ex);
268 }
269 return ObjectUtils.notNull(schemaPath);
270 }
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285 protected void generateSchema(
286 @NonNull IModule module,
287 @NonNull Path schemaPath,
288 @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
289 Path parentDir = schemaPath.getParent();
290 if (parentDir != null && !Files.exists(parentDir)) {
291 Files.createDirectories(parentDir);
292 }
293
294 try (Writer writer = Files.newBufferedWriter(
295 schemaPath,
296 StandardCharsets.UTF_8,
297 getWriteOpenOptions())) {
298 schemaProducer.apply(module, writer);
299 }
300 LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
301 }
302
303
304
305
306
307
308 @SuppressWarnings("PMD.MethodReturnsInternalArray")
309 protected OpenOption[] getWriteOpenOptions() {
310 return OPEN_OPTIONS_TRUNCATE;
311 }
312
313 @SuppressWarnings("PMD.AvoidCatchingGenericException")
314 private DynamicContainer generateScenario(
315 @NonNull TestScenario scenario,
316 @NonNull URI collectionUri,
317 @NonNull Lazy<Path> collectionGenerationPath,
318 @NonNull IBindingContext bindingContext) {
319 Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
320 Path retval;
321 try {
322 retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
323 } catch (IOException ex) {
324 throw new JUnitException("Unable to create scenario temp directory", ex);
325 }
326 return retval;
327 }));
328
329 GenerateSchema generateSchema = scenario.getGenerateSchema();
330 MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
331 URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
332
333 IModule module;
334 try {
335 IBindingModuleLoader loader = bindingContext.newModuleLoader();
336
337 module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
338 } catch (IOException | MetaschemaException ex) {
339 throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
340 }
341
342 Lazy<Path> lazySchema = Lazy.lazy(() -> generateSchema(module, scenarioGenerationPath));
343
344 Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
345 Path schemaPath = lazySchema.get();
346 return getContentValidatorSupplier().apply(schemaPath);
347 });
348 assert lazyContentValidator != null;
349
350
351 DynamicTest validateSchema = DynamicTest.dynamicTest(
352 "Validate Schema",
353 () -> {
354 Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
355 if (supplier != null) {
356 Path schemaPath;
357 try {
358 schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
359 } catch (Exception ex) {
360 throw new JUnitException(
361 "failed to generate schema", ex);
362 }
363 validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
364 }
365 });
366
367 Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
368 .flatMap(contentCase -> {
369 assert contentCase != null;
370 DynamicTest test
371 = generateValidationCase(
372 contentCase,
373 bindingContext,
374 lazyContentValidator,
375 collectionUri,
376 ObjectUtils.notNull(scenarioGenerationPath));
377 return test == null ? Stream.empty() : Stream.of(test);
378 }).sequential();
379
380 return DynamicContainer.dynamicContainer(
381 scenario.getName(),
382 metaschemaUri,
383 Stream.concat(Stream.of(validateSchema), contentTests).sequential());
384 }
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400 protected Path convertContent(
401 @NonNull URI resource,
402 @NonNull Path generationPath,
403 @NonNull IBindingContext context)
404 throws IOException {
405 Object object;
406 try {
407 object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
408 } catch (URISyntaxException ex) {
409 throw new IOException(ex);
410 }
411
412 if (!Files.exists(generationPath)) {
413 Files.createDirectories(generationPath);
414 }
415
416 Path convertedContetPath;
417 try {
418 convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
419 } catch (IOException ex) {
420 throw new JUnitException(
421 String.format("Unable to create converted content path in location '%s'", generationPath),
422 ex);
423 }
424
425 Format toFormat = getRequiredContentFormat();
426 if (LOGGER.isInfoEnabled()) {
427 LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
428 }
429
430 ISerializer<?> serializer
431 = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
432 serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
433
434 return convertedContetPath;
435 }
436
437 @SuppressWarnings("PMD.AvoidCatchingGenericException")
438 private DynamicTest generateValidationCase(
439 @NonNull ContentCaseType contentCase,
440 @NonNull IBindingContext bindingContext,
441 @NonNull Lazy<IContentValidator> lazyContentValidator,
442 @NonNull URI collectionUri,
443 @NonNull Lazy<Path> resourceGenerationPath) {
444
445 URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
446
447 Format format = contentCase.getSourceFormat();
448 DynamicTest retval = null;
449 if (getRequiredContentFormat().equals(format)) {
450 retval = DynamicTest.dynamicTest(
451 String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
452 contentCase.getLocation()),
453 contentUri,
454 () -> {
455 IContentValidator contentValidator;
456 try {
457 contentValidator = lazyContentValidator.get();
458 } catch (Exception ex) {
459 throw new JUnitException(
460 "failed to produce the content validator", ex);
461 }
462
463 assertEquals(
464 contentCase.getValidationResult(),
465 validateWithSchema(
466 ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
467 "validation did not match expectation for: " + contentUri.toASCIIString());
468 });
469 } else if (contentCase.getValidationResult()) {
470 retval = DynamicTest.dynamicTest(
471 String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
472 contentCase.getLocation()),
473 contentUri,
474 () -> {
475 Path convertedContetPath;
476 try {
477 convertedContetPath = convertContent(
478 contentUri,
479 ObjectUtils.notNull(resourceGenerationPath.get()),
480 bindingContext);
481 } catch (Exception ex) {
482 throw new JUnitException("failed to convert content: " + contentUri, ex);
483 }
484
485 IContentValidator contentValidator;
486 try {
487 contentValidator = lazyContentValidator.get();
488 } catch (Exception ex) {
489 throw new JUnitException(
490 "failed to produce the content validator",
491 ex.getCause());
492 }
493
494 if (LOGGER.isInfoEnabled()) {
495 LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
496 }
497 assertEquals(contentCase.getValidationResult(),
498 validateWithSchema(
499 ObjectUtils.notNull(contentValidator),
500 ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
501 String.format("validation of '%s' did not match expectation", convertedContetPath));
502 });
503 }
504 return retval;
505 }
506
507 private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
508 throws IOException {
509 IValidationResult schemaValidationResult;
510 try {
511 schemaValidationResult = validator.validate(target);
512 } catch (URISyntaxException ex) {
513 throw new IOException(ex);
514 }
515 return processValidationResult(schemaValidationResult);
516 }
517
518
519
520
521
522
523
524
525
526
527
528
529 protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
530 throws IOException {
531 LOGGER.atError().log("Validating: {}", target);
532 IValidationResult schemaValidationResult = validator.validate(target);
533 if (!schemaValidationResult.isPassing()) {
534 LOGGER.atError().log("Schema validation failed for: {}", target);
535 }
536 return processValidationResult(schemaValidationResult);
537 }
538
539 private static boolean processValidationResult(IValidationResult schemaValidationResult) {
540 for (IValidationFinding finding : schemaValidationResult.getFindings()) {
541 logFinding(ObjectUtils.notNull(finding));
542 }
543 return schemaValidationResult.isPassing();
544 }
545
546 private static void logFinding(@NonNull IValidationFinding finding) {
547 LogBuilder logBuilder;
548 switch (finding.getSeverity()) {
549 case CRITICAL:
550 logBuilder = LOGGER.atFatal();
551 break;
552 case ERROR:
553 logBuilder = LOGGER.atError();
554 break;
555 case WARNING:
556 logBuilder = LOGGER.atWarn();
557 break;
558 case INFORMATIONAL:
559 logBuilder = LOGGER.atInfo();
560 break;
561 case DEBUG:
562 logBuilder = LOGGER.atDebug();
563 break;
564 default:
565 throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
566 }
567
568
569
570
571
572 if (finding instanceof JsonValidationFinding) {
573 JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
574 logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
575 } else {
576 logBuilder.log("{}", finding.getMessage());
577 }
578 }
579 }