1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package gov.nist.secauto.metaschema.model.testing;
7   
8   import static org.junit.jupiter.api.Assertions.assertEquals;
9   
10  import gov.nist.secauto.metaschema.core.model.IModule;
11  import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12  import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13  import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14  import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15  import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16  import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17  import gov.nist.secauto.metaschema.databind.DefaultBindingContext;
18  import gov.nist.secauto.metaschema.databind.IBindingContext;
19  import gov.nist.secauto.metaschema.databind.io.Format;
20  import gov.nist.secauto.metaschema.databind.io.ISerializer;
21  import gov.nist.secauto.metaschema.databind.model.metaschema.BindingModuleLoader;
22  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
23  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
24  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
25  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
26  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
27  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
28  
29  import org.apache.logging.log4j.LogBuilder;
30  import org.apache.logging.log4j.LogManager;
31  import org.apache.logging.log4j.Logger;
32  import org.apache.xmlbeans.XmlException;
33  import org.apache.xmlbeans.XmlOptions;
34  import org.junit.jupiter.api.DynamicContainer;
35  import org.junit.jupiter.api.DynamicNode;
36  import org.junit.jupiter.api.DynamicTest;
37  import org.junit.platform.commons.JUnitException;
38  
39  import java.io.IOException;
40  import java.io.Writer;
41  import java.net.URI;
42  import java.net.URISyntaxException;
43  import java.net.URL;
44  import java.nio.charset.StandardCharsets;
45  import java.nio.file.FileVisitResult;
46  import java.nio.file.Files;
47  import java.nio.file.OpenOption;
48  import java.nio.file.Path;
49  import java.nio.file.SimpleFileVisitor;
50  import java.nio.file.StandardOpenOption;
51  import java.nio.file.attribute.BasicFileAttributes;
52  import java.util.function.BiFunction;
53  import java.util.function.Function;
54  import java.util.function.Supplier;
55  import java.util.stream.Stream;
56  
57  import edu.umd.cs.findbugs.annotations.NonNull;
58  import edu.umd.cs.findbugs.annotations.Nullable;
59  import nl.talsmasoftware.lazy4j.Lazy;
60  
61  public abstract class AbstractTestSuite {
62    private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
63    private static final BindingModuleLoader LOADER;
64  
65    private static final boolean DELETE_RESULTS_ON_EXIT = false;
66    private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
67        StandardOpenOption.CREATE,
68        StandardOpenOption.WRITE,
69        StandardOpenOption.TRUNCATE_EXISTING
70    };
71  
72    static {
73      IBindingContext bindingContext = new DefaultBindingContext();
74      LOADER = new BindingModuleLoader(bindingContext);
75      LOADER.allowEntityResolution();
76    }
77  
78    /**
79     * Get the content format used by the test suite.
80     *
81     * @return the format
82     */
83    @NonNull
84    protected abstract Format getRequiredContentFormat();
85  
86    /**
87     * Get the resource describing the tests to execute.
88     *
89     * @return the resource
90     */
91    @NonNull
92    protected abstract URI getTestSuiteURI();
93  
94    /**
95     * Get the filesystem location to use for generating content.
96     *
97     * @return the filesystem path
98     */
99    @NonNull
100   protected abstract Path getGenerationPath();
101 
102   /**
103    * Get the method used to generate a schema using a given Metaschema module and
104    * writer.
105    *
106    * @return the schema generator supplier
107    */
108   @NonNull
109   protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
110 
111   /**
112    * Get the method used to provide a schema validator.
113    *
114    * @return the method as a supplier
115    */
116   @Nullable
117   protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
118 
119   /**
120    * Get the method used to provide a content validator.
121    *
122    * @return the method as a supplier
123    */
124   @NonNull
125   protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
126 
127   /**
128    * Dynamically generate the unit tests.
129    *
130    * @return the steam of unit tests
131    */
132   @NonNull
133   protected Stream<DynamicNode> testFactory() {
134     try {
135       XmlOptions options = new XmlOptions();
136       options.setBaseURI(null);
137       options.setLoadLineNumbers();
138 
139       Path generationPath = getGenerationPath();
140       if (Files.exists(generationPath)) {
141         if (!Files.isDirectory(generationPath)) {
142           throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
143         }
144       } else {
145         Files.createDirectories(generationPath);
146       }
147 
148       URI testSuiteUri = getTestSuiteURI();
149       URL testSuiteUrl = testSuiteUri.toURL();
150       TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
151       return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
152           .flatMap(
153               collection -> Stream
154                   .of(generateCollection(ObjectUtils.notNull(collection), testSuiteUri, generationPath))));
155     } catch (XmlException | IOException ex) {
156       throw new JUnitException("Unable to generate tests", ex);
157     }
158   }
159 
160   /**
161    * Configure removal of the provided directory after test execution.
162    *
163    * @param path
164    *          the directory to configure for removal
165    */
166   protected void deleteCollectionOnExit(@NonNull Path path) {
167     Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
168         () -> {
169           try {
170             Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
171               @Override
172               public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
173                 Files.delete(file);
174                 return FileVisitResult.CONTINUE;
175               }
176 
177               @Override
178               public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
179                 if (ex == null) {
180                   Files.delete(dir);
181                   return FileVisitResult.CONTINUE;
182                 }
183                 // directory iteration failed for some reason
184                 throw ex;
185               }
186             });
187           } catch (IOException ex) {
188             throw new JUnitException("Failed to delete collection: " + path, ex);
189           }
190         }));
191   }
192 
193   private DynamicContainer generateCollection(@NonNull TestCollection collection, @NonNull URI testSuiteUri,
194       @NonNull Path generationPath) {
195     URI collectionUri = testSuiteUri.resolve(collection.getLocation());
196     assert collectionUri != null;
197 
198     LOGGER.atInfo().log("Collection: " + collectionUri);
199     Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
200       Path retval;
201       try {
202         retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
203         if (DELETE_RESULTS_ON_EXIT) {
204           deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
205         }
206       } catch (IOException ex) {
207         throw new JUnitException("Unable to create collection temp directory", ex);
208       }
209       return retval;
210     }));
211 
212     return DynamicContainer.dynamicContainer(
213         collection.getName(),
214         testSuiteUri,
215         collection.getTestScenarioList().stream()
216             .flatMap(scenario -> {
217               assert scenario != null;
218               return Stream.of(generateScenario(scenario, collectionUri, collectionGenerationPath));
219             })
220             .sequential());
221   }
222 
223   /**
224    * Generate a schema for the provided module using the provided schema
225    * generator.
226    *
227    * @param module
228    *          the Metaschema module to generate the schema for
229    * @param schemaPath
230    *          the location to generate the schema
231    * @param schemaProducer
232    *          the method callback to use to generate the schema
233    * @throws IOException
234    *           if an error occurred while writing the schema
235    */
236   protected void generateSchema(
237       @NonNull IModule module,
238       @NonNull Path schemaPath,
239       @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
240     Path parentDir = schemaPath.getParent();
241     if (parentDir != null && !Files.exists(parentDir)) {
242       Files.createDirectories(parentDir);
243     }
244 
245     try (Writer writer = Files.newBufferedWriter(
246         schemaPath,
247         StandardCharsets.UTF_8,
248         getWriteOpenOptions())) {
249       schemaProducer.apply(module, writer);
250     }
251     LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
252   }
253 
254   /**
255    * The the options for writing generated content.
256    *
257    * @return the options
258    */
259   protected OpenOption[] getWriteOpenOptions() {
260     return OPEN_OPTIONS_TRUNCATE;
261   }
262 
263   private DynamicContainer generateScenario(
264       @NonNull TestScenario scenario,
265       @NonNull URI collectionUri,
266       @NonNull Lazy<Path> collectionGenerationPath) {
267     Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
268       Path retval;
269       try {
270         retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
271       } catch (IOException ex) {
272         throw new JUnitException("Unable to create scenario temp directory", ex);
273       }
274       return retval;
275     });
276 
277     // try {
278     // // create the directories the schema will be stored in
279     // Files.createDirectories(scenarioGenerationPath);
280     // } catch (IOException ex) {
281     // throw new JUnitException("Unable to create test directories for path: " +
282     // scenarioGenerationPath, ex);
283     // }
284 
285     GenerateSchema generateSchema = scenario.getGenerateSchema();
286     MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
287     URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
288 
289     Lazy<IModule> lazyModule = Lazy.lazy(() -> {
290       IModule module;
291       try {
292         module = LOADER.load(ObjectUtils.notNull(metaschemaUri.toURL()));
293       } catch (IOException | MetaschemaException ex) {
294         throw new JUnitException("Unable to generate schema for Module: " + metaschemaUri, ex);
295       }
296       return module;
297     });
298 
299     Lazy<Path> lazySchema = Lazy.lazy(() -> {
300       Path schemaPath;
301 
302       String schemaExtension;
303       Format requiredContentFormat = getRequiredContentFormat();
304       switch (requiredContentFormat) {
305       case JSON:
306       case YAML:
307         schemaExtension = ".json";
308         break;
309       case XML:
310         schemaExtension = ".xsd";
311         break;
312       default:
313         throw new IllegalStateException();
314       }
315 
316       // determine what file to use for the schema
317       try {
318         schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
319       } catch (IOException ex) {
320         throw new JUnitException("Unable to create schema temp file", ex);
321       }
322       IModule module = lazyModule.get();
323       try {
324         generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
325       } catch (IOException ex) {
326         throw new IllegalStateException(ex);
327       }
328       return schemaPath;
329     });
330 
331     Lazy<IBindingContext> lazyDynamicBindingContext = Lazy.lazy(() -> {
332       IModule module = lazyModule.get();
333       IBindingContext context;
334       try {
335         context = new DefaultBindingContext();
336         context.registerModule(
337             ObjectUtils.notNull(module),
338             ObjectUtils.notNull(scenarioGenerationPath.get()));
339       } catch (Exception ex) { // NOPMD - intentional
340         throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
341       }
342       return context;
343     });
344     assert lazyDynamicBindingContext != null;
345 
346     Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
347       Path schemaPath = lazySchema.get();
348       return getContentValidatorSupplier().apply(schemaPath);
349     });
350     assert lazyContentValidator != null;
351 
352     // build a test container for the generate and validate steps
353     DynamicTest validateSchema = DynamicTest.dynamicTest(
354         "Validate Schema",
355         () -> {
356           Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357           if (supplier != null) {
358             Path schemaPath;
359             try {
360               schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361             } catch (Exception ex) {
362               throw new JUnitException( // NOPMD - cause is relevant, exception is not
363                   "failed to generate schema", ex.getCause());
364             }
365             validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366           }
367         });
368 
369     Stream<? extends DynamicNode> contentTests;
370     {
371       contentTests = scenario.getValidationCaseList().stream()
372           .flatMap(contentCase -> {
373             assert contentCase != null;
374             DynamicTest test
375                 = generateValidationCase(
376                     contentCase,
377                     lazyDynamicBindingContext,
378                     lazyContentValidator,
379                     collectionUri,
380                     ObjectUtils.notNull(scenarioGenerationPath));
381             return test == null ? Stream.empty() : Stream.of(test);
382           }).sequential();
383     }
384 
385     return DynamicContainer.dynamicContainer(
386         scenario.getName(),
387         metaschemaUri,
388         Stream.concat(Stream.of(validateSchema), contentTests).sequential());
389   }
390 
391   /**
392    * Perform content conversion.
393    *
394    * @param resource
395    *          the resource to convert
396    * @param generationPath
397    *          the path to write the converted resource to
398    * @param context
399    *          the Metaschema binding context
400    * @return the location of the converted content
401    * @throws IOException
402    *           if an error occurred while reading or writing content
403    * @see #getRequiredContentFormat()
404    */
405   protected Path convertContent(
406       @NonNull URI resource,
407       @NonNull Path generationPath,
408       @NonNull IBindingContext context)
409       throws IOException {
410     Object object;
411     try {
412       object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
413     } catch (URISyntaxException ex) {
414       throw new IOException(ex);
415     }
416 
417     if (!Files.exists(generationPath)) {
418       Files.createDirectories(generationPath);
419     }
420 
421     Path convertedContetPath;
422     try {
423       convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
424     } catch (IOException ex) {
425       throw new JUnitException("Unable to create schema temp file", ex);
426     }
427 
428     @SuppressWarnings("rawtypes") ISerializer serializer
429         = context.newSerializer(getRequiredContentFormat(), ObjectUtils.asType(object.getClass()));
430     serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
431 
432     return convertedContetPath;
433   }
434 
435   private DynamicTest generateValidationCase(
436       @NonNull ContentCaseType contentCase,
437       @NonNull Lazy<IBindingContext> lazyBindingContext,
438       @NonNull Lazy<IContentValidator> lazyContentValidator,
439       @NonNull URI collectionUri,
440       @NonNull Lazy<Path> generationPath) {
441 
442     URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
443 
444     Format format = contentCase.getSourceFormat();
445     DynamicTest retval = null;
446     if (getRequiredContentFormat().equals(format)) {
447       retval = DynamicTest.dynamicTest(
448           String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
449               contentCase.getLocation()),
450           contentUri,
451           () -> {
452             IContentValidator contentValidator;
453             try {
454               contentValidator = lazyContentValidator.get();
455             } catch (Exception ex) {
456               throw new JUnitException( // NOPMD - cause is relevant, exception is not
457                   "failed to produce the content validator", ex.getCause());
458             }
459 
460             assertEquals(
461                 contentCase.getValidationResult(),
462                 validateWithSchema(
463                     ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
464                 "validation did not match expectation for: " + contentUri.toASCIIString());
465           });
466     } else if (contentCase.getValidationResult()) {
467       retval = DynamicTest.dynamicTest(
468           String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
469               contentCase.getLocation()),
470           contentUri,
471           () -> {
472             IBindingContext context;
473             try {
474               context = lazyBindingContext.get();
475             } catch (Exception ex) {
476               throw new JUnitException( // NOPMD - cause is relevant, exception is not
477                   "failed to produce the content validator", ex.getCause());
478             }
479             assert context != null;
480 
481             Path convertedContetPath;
482             try {
483               convertedContetPath = convertContent(contentUri, ObjectUtils.notNull(generationPath.get()), context);
484             } catch (Exception ex) { // NOPMD - intentional
485               throw new JUnitException("failed to convert content: " + contentUri, ex);
486             }
487 
488             IContentValidator contentValidator;
489             try {
490               contentValidator = lazyContentValidator.get();
491             } catch (Exception ex) {
492               throw new JUnitException( // NOPMD - cause is relevant, exception is not
493                   "failed to produce the content validator",
494                   ex.getCause());
495             }
496             assertEquals(contentCase.getValidationResult(),
497                 validateWithSchema(
498                     ObjectUtils.notNull(contentValidator),
499                     ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
500                 String.format("validation of '%s' did not match expectation", convertedContetPath));
501           });
502     }
503     return retval;
504   }
505 
506   private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
507       throws IOException {
508     IValidationResult schemaValidationResult;
509     try {
510       schemaValidationResult = validator.validate(target);
511     } catch (URISyntaxException ex) {
512       throw new IOException(ex);
513     }
514     return processValidationResult(schemaValidationResult);
515   }
516 
517   /**
518    * Use the provided validator to validate the provided target.
519    *
520    * @param validator
521    *          the content validator to use
522    * @param target
523    *          the resource to validate
524    * @return {@code true} if the content is valid or {@code false} otherwise
525    * @throws IOException
526    *           if an error occurred while reading the content
527    */
528   protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
529       throws IOException {
530     IValidationResult schemaValidationResult = validator.validate(target);
531     if (!schemaValidationResult.isPassing()) {
532       LOGGER.atError().log("Schema validation failed for: {}", target);
533     }
534     return processValidationResult(schemaValidationResult);
535   }
536 
537   private static boolean processValidationResult(IValidationResult schemaValidationResult) {
538     for (IValidationFinding finding : schemaValidationResult.getFindings()) {
539       logFinding(ObjectUtils.notNull(finding));
540     }
541     return schemaValidationResult.isPassing();
542   }
543 
544   private static void logFinding(@NonNull IValidationFinding finding) {
545     LogBuilder logBuilder;
546     switch (finding.getSeverity()) {
547     case CRITICAL:
548       logBuilder = LOGGER.atFatal();
549       break;
550     case ERROR:
551       logBuilder = LOGGER.atError();
552       break;
553     case WARNING:
554       logBuilder = LOGGER.atWarn();
555       break;
556     case INFORMATIONAL:
557       logBuilder = LOGGER.atInfo();
558       break;
559     case DEBUG:
560       logBuilder = LOGGER.atDebug();
561       break;
562     default:
563       throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
564     }
565 
566     // if (finding.getCause() != null) {
567     // logBuilder.withThrowable(finding.getCause());
568     // }
569 
570     if (finding instanceof JsonValidationFinding) {
571       JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
572       logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
573     } else {
574       logBuilder.log("{}", finding.getMessage());
575     }
576   }
577 }