1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package gov.nist.secauto.metaschema.model.testing;
7   
8   import static org.junit.jupiter.api.Assertions.assertEquals;
9   
10  import gov.nist.secauto.metaschema.core.model.IModule;
11  import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12  import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13  import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14  import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15  import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16  import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17  import gov.nist.secauto.metaschema.databind.IBindingContext;
18  import gov.nist.secauto.metaschema.databind.io.Format;
19  import gov.nist.secauto.metaschema.databind.io.ISerializer;
20  import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
21  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
22  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
23  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
24  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
25  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
26  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
27  
28  import org.apache.logging.log4j.LogBuilder;
29  import org.apache.logging.log4j.LogManager;
30  import org.apache.logging.log4j.Logger;
31  import org.apache.xmlbeans.XmlException;
32  import org.apache.xmlbeans.XmlOptions;
33  import org.junit.jupiter.api.DynamicContainer;
34  import org.junit.jupiter.api.DynamicNode;
35  import org.junit.jupiter.api.DynamicTest;
36  import org.junit.platform.commons.JUnitException;
37  
38  import java.io.IOException;
39  import java.io.Writer;
40  import java.net.URI;
41  import java.net.URISyntaxException;
42  import java.net.URL;
43  import java.nio.charset.StandardCharsets;
44  import java.nio.file.FileVisitResult;
45  import java.nio.file.Files;
46  import java.nio.file.OpenOption;
47  import java.nio.file.Path;
48  import java.nio.file.SimpleFileVisitor;
49  import java.nio.file.StandardOpenOption;
50  import java.nio.file.attribute.BasicFileAttributes;
51  import java.util.function.BiFunction;
52  import java.util.function.Function;
53  import java.util.function.Supplier;
54  import java.util.stream.Stream;
55  
56  import edu.umd.cs.findbugs.annotations.NonNull;
57  import edu.umd.cs.findbugs.annotations.Nullable;
58  import nl.talsmasoftware.lazy4j.Lazy;
59  
60  /**
61   * This abstract implementation dynamically produces JUnit tests based on a test
62   * suite definition.
63   *
64   * @see #getTestSuiteURI()
65   */
66  @SuppressWarnings({
67      "PMD.GodClass",
68      "PMD.CouplingBetweenObjects"
69  })
70  public abstract class AbstractTestSuite {
71    private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
72  
73    private static final boolean DELETE_RESULTS_ON_EXIT = false;
74    private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
75        StandardOpenOption.CREATE,
76        StandardOpenOption.WRITE,
77        StandardOpenOption.TRUNCATE_EXISTING
78    };
79  
80    /**
81     * Get the content format used by the test suite.
82     *
83     * @return the format
84     */
85    @NonNull
86    protected abstract Format getRequiredContentFormat();
87  
88    /**
89     * Get the resource describing the tests to execute.
90     *
91     * @return the resource
92     */
93    @NonNull
94    protected abstract URI getTestSuiteURI();
95  
96    /**
97     * Get the filesystem location to use for generating content.
98     *
99     * @return the filesystem path
100    */
101   @NonNull
102   protected abstract Path getGenerationPath();
103 
104   /**
105    * Get the method used to generate a schema using a given Metaschema module and
106    * writer.
107    *
108    * @return the schema generator supplier
109    */
110   @NonNull
111   protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112 
113   /**
114    * Get the method used to provide a schema validator.
115    *
116    * @return the method as a supplier
117    */
118   @Nullable
119   protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120 
121   /**
122    * Get the method used to provide a content validator.
123    *
124    * @return the method as a supplier
125    */
126   @NonNull
127   protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128 
129   /**
130    * Dynamically generate the unit tests.
131    *
132    * @param bindingContext
133    *          the Module binding context
134    *
135    * @return the steam of unit tests
136    */
137   @NonNull
138   protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139     try {
140       XmlOptions options = new XmlOptions();
141       options.setBaseURI(null);
142       options.setLoadLineNumbers();
143 
144       Path generationPath = getGenerationPath();
145       if (Files.exists(generationPath)) {
146         if (!Files.isDirectory(generationPath)) {
147           throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
148         }
149       } else {
150         Files.createDirectories(generationPath);
151       }
152 
153       URI testSuiteUri = getTestSuiteURI();
154       URL testSuiteUrl = testSuiteUri.toURL();
155       TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
156       return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
157           .flatMap(
158               collection -> Stream
159                   .of(generateCollection(
160                       ObjectUtils.notNull(collection),
161                       testSuiteUri,
162                       generationPath,
163                       bindingContext))));
164     } catch (XmlException | IOException ex) {
165       throw new JUnitException("Unable to generate tests", ex);
166     }
167   }
168 
169   /**
170    * Configure removal of the provided directory after test execution.
171    *
172    * @param path
173    *          the directory to configure for removal
174    */
175   protected void deleteCollectionOnExit(@NonNull Path path) {
176     Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
177         () -> {
178           try {
179             Files.walkFileTree(path, new SimpleFileVisitor<>() {
180               @Override
181               public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
182                 Files.delete(file);
183                 return FileVisitResult.CONTINUE;
184               }
185 
186               @Override
187               public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
188                 if (ex == null) {
189                   Files.delete(dir);
190                   return FileVisitResult.CONTINUE;
191                 }
192                 // directory iteration failed for some reason
193                 throw ex;
194               }
195             });
196           } catch (IOException ex) {
197             throw new JUnitException("Failed to delete collection: " + path, ex);
198           }
199         }));
200   }
201 
202   private DynamicContainer generateCollection(
203       @NonNull TestCollection collection,
204       @NonNull URI testSuiteUri,
205       @NonNull Path generationPath,
206       @NonNull IBindingContext bindingContext) {
207     URI collectionUri = testSuiteUri.resolve(collection.getLocation());
208     assert collectionUri != null;
209 
210     LOGGER.atInfo().log("Collection: " + collectionUri);
211     Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
212       Path retval;
213       try {
214         retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
215         if (DELETE_RESULTS_ON_EXIT) {
216           deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
217         }
218       } catch (IOException ex) {
219         throw new JUnitException("Unable to create collection temp directory", ex);
220       }
221       return retval;
222     }));
223 
224     return DynamicContainer.dynamicContainer(
225         collection.getName(),
226         testSuiteUri,
227         collection.getTestScenarioList().stream()
228             .flatMap(scenario -> {
229               assert scenario != null;
230               return Stream.of(generateScenario(
231                   scenario,
232                   collectionUri,
233                   collectionGenerationPath,
234                   bindingContext));
235             })
236             .sequential());
237   }
238 
239   @NonNull
240   private Path generateSchema(
241       @NonNull IModule module,
242       @NonNull Lazy<Path> scenarioGenerationPath) {
243     String schemaExtension;
244     Format requiredContentFormat = getRequiredContentFormat();
245     switch (requiredContentFormat) {
246     case JSON:
247     case YAML:
248       schemaExtension = ".json";
249       break;
250     case XML:
251       schemaExtension = ".xsd";
252       break;
253     default:
254       throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat));
255     }
256 
257     // determine what file to use for the schema
258     Path schemaPath;
259     try {
260       schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
261     } catch (IOException ex) {
262       throw new JUnitException("Unable to create schema temp file", ex);
263     }
264     try {
265       generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
266     } catch (IOException ex) {
267       throw new IllegalStateException(ex);
268     }
269     return ObjectUtils.notNull(schemaPath);
270   }
271 
272   /**
273    * Generate a schema for the provided module using the provided schema
274    * generator.
275    *
276    * @param module
277    *          the Metaschema module to generate the schema for
278    * @param schemaPath
279    *          the location to generate the schema
280    * @param schemaProducer
281    *          the method callback to use to generate the schema
282    * @throws IOException
283    *           if an error occurred while writing the schema
284    */
285   protected void generateSchema(
286       @NonNull IModule module,
287       @NonNull Path schemaPath,
288       @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
289     Path parentDir = schemaPath.getParent();
290     if (parentDir != null && !Files.exists(parentDir)) {
291       Files.createDirectories(parentDir);
292     }
293 
294     try (Writer writer = Files.newBufferedWriter(
295         schemaPath,
296         StandardCharsets.UTF_8,
297         getWriteOpenOptions())) {
298       schemaProducer.apply(module, writer);
299     }
300     LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
301   }
302 
303   /**
304    * The the options for writing generated content.
305    *
306    * @return the options
307    */
308   @SuppressWarnings("PMD.MethodReturnsInternalArray")
309   protected OpenOption[] getWriteOpenOptions() {
310     return OPEN_OPTIONS_TRUNCATE;
311   }
312 
313   @SuppressWarnings("PMD.AvoidCatchingGenericException")
314   private DynamicContainer generateScenario(
315       @NonNull TestScenario scenario,
316       @NonNull URI collectionUri,
317       @NonNull Lazy<Path> collectionGenerationPath,
318       @NonNull IBindingContext bindingContext) {
319     Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
320       Path retval;
321       try {
322         retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
323       } catch (IOException ex) {
324         throw new JUnitException("Unable to create scenario temp directory", ex);
325       }
326       return retval;
327     }));
328 
329     GenerateSchema generateSchema = scenario.getGenerateSchema();
330     MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
331     URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
332 
333     IModule module;
334     try {
335       IBindingModuleLoader loader = bindingContext.newModuleLoader();
336 
337       module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
338     } catch (IOException | MetaschemaException ex) {
339       throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
340     }
341 
342     Lazy<Path> lazySchema = Lazy.lazy(() -> generateSchema(module, scenarioGenerationPath));
343 
344     Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
345       Path schemaPath = lazySchema.get();
346       return getContentValidatorSupplier().apply(schemaPath);
347     });
348     assert lazyContentValidator != null;
349 
350     // build a test container for the generate and validate steps
351     DynamicTest validateSchema = DynamicTest.dynamicTest(
352         "Validate Schema",
353         () -> {
354           Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
355           if (supplier != null) {
356             Path schemaPath;
357             try {
358               schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
359             } catch (Exception ex) {
360               throw new JUnitException(
361                   "failed to generate schema", ex);
362             }
363             validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
364           }
365         });
366 
367     Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
368         .flatMap(contentCase -> {
369           assert contentCase != null;
370           DynamicTest test
371               = generateValidationCase(
372                   contentCase,
373                   bindingContext,
374                   lazyContentValidator,
375                   collectionUri,
376                   ObjectUtils.notNull(scenarioGenerationPath));
377           return test == null ? Stream.empty() : Stream.of(test);
378         }).sequential();
379 
380     return DynamicContainer.dynamicContainer(
381         scenario.getName(),
382         metaschemaUri,
383         Stream.concat(Stream.of(validateSchema), contentTests).sequential());
384   }
385 
386   /**
387    * Perform content conversion.
388    *
389    * @param resource
390    *          the resource to convert
391    * @param generationPath
392    *          the path to write the converted resource to
393    * @param context
394    *          the Metaschema binding context
395    * @return the location of the converted content
396    * @throws IOException
397    *           if an error occurred while reading or writing content
398    * @see #getRequiredContentFormat()
399    */
400   protected Path convertContent(
401       @NonNull URI resource,
402       @NonNull Path generationPath,
403       @NonNull IBindingContext context)
404       throws IOException {
405     Object object;
406     try {
407       object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
408     } catch (URISyntaxException ex) {
409       throw new IOException(ex);
410     }
411 
412     if (!Files.exists(generationPath)) {
413       Files.createDirectories(generationPath);
414     }
415 
416     Path convertedContetPath;
417     try {
418       convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
419     } catch (IOException ex) {
420       throw new JUnitException(
421           String.format("Unable to create converted content path in location '%s'", generationPath),
422           ex);
423     }
424 
425     Format toFormat = getRequiredContentFormat();
426     if (LOGGER.isInfoEnabled()) {
427       LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
428     }
429 
430     ISerializer<?> serializer
431         = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
432     serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
433 
434     return convertedContetPath;
435   }
436 
437   @SuppressWarnings("PMD.AvoidCatchingGenericException")
438   private DynamicTest generateValidationCase(
439       @NonNull ContentCaseType contentCase,
440       @NonNull IBindingContext bindingContext,
441       @NonNull Lazy<IContentValidator> lazyContentValidator,
442       @NonNull URI collectionUri,
443       @NonNull Lazy<Path> resourceGenerationPath) {
444 
445     URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
446 
447     Format format = contentCase.getSourceFormat();
448     DynamicTest retval = null;
449     if (getRequiredContentFormat().equals(format)) {
450       retval = DynamicTest.dynamicTest(
451           String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
452               contentCase.getLocation()),
453           contentUri,
454           () -> {
455             IContentValidator contentValidator;
456             try {
457               contentValidator = lazyContentValidator.get();
458             } catch (Exception ex) {
459               throw new JUnitException( // NOPMD - cause is relevant, exception is not
460                   "failed to produce the content validator", ex);
461             }
462 
463             assertEquals(
464                 contentCase.getValidationResult(),
465                 validateWithSchema(
466                     ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
467                 "validation did not match expectation for: " + contentUri.toASCIIString());
468           });
469     } else if (contentCase.getValidationResult()) {
470       retval = DynamicTest.dynamicTest(
471           String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
472               contentCase.getLocation()),
473           contentUri,
474           () -> {
475             Path convertedContetPath;
476             try {
477               convertedContetPath = convertContent(
478                   contentUri,
479                   ObjectUtils.notNull(resourceGenerationPath.get()),
480                   bindingContext);
481             } catch (Exception ex) { // NOPMD - intentional
482               throw new JUnitException("failed to convert content: " + contentUri, ex);
483             }
484 
485             IContentValidator contentValidator;
486             try {
487               contentValidator = lazyContentValidator.get();
488             } catch (Exception ex) {
489               throw new JUnitException( // NOPMD - cause is relevant, exception is not
490                   "failed to produce the content validator",
491                   ex.getCause());
492             }
493 
494             if (LOGGER.isInfoEnabled()) {
495               LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
496             }
497             assertEquals(contentCase.getValidationResult(),
498                 validateWithSchema(
499                     ObjectUtils.notNull(contentValidator),
500                     ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
501                 String.format("validation of '%s' did not match expectation", convertedContetPath));
502           });
503     }
504     return retval;
505   }
506 
507   private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
508       throws IOException {
509     IValidationResult schemaValidationResult;
510     try {
511       schemaValidationResult = validator.validate(target);
512     } catch (URISyntaxException ex) {
513       throw new IOException(ex);
514     }
515     return processValidationResult(schemaValidationResult);
516   }
517 
518   /**
519    * Use the provided validator to validate the provided target.
520    *
521    * @param validator
522    *          the content validator to use
523    * @param target
524    *          the resource to validate
525    * @return {@code true} if the content is valid or {@code false} otherwise
526    * @throws IOException
527    *           if an error occurred while reading the content
528    */
529   protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
530       throws IOException {
531     LOGGER.atError().log("Validating: {}", target);
532     IValidationResult schemaValidationResult = validator.validate(target);
533     if (!schemaValidationResult.isPassing()) {
534       LOGGER.atError().log("Schema validation failed for: {}", target);
535     }
536     return processValidationResult(schemaValidationResult);
537   }
538 
539   private static boolean processValidationResult(IValidationResult schemaValidationResult) {
540     for (IValidationFinding finding : schemaValidationResult.getFindings()) {
541       logFinding(ObjectUtils.notNull(finding));
542     }
543     return schemaValidationResult.isPassing();
544   }
545 
546   private static void logFinding(@NonNull IValidationFinding finding) {
547     LogBuilder logBuilder;
548     switch (finding.getSeverity()) {
549     case CRITICAL:
550       logBuilder = LOGGER.atFatal();
551       break;
552     case ERROR:
553       logBuilder = LOGGER.atError();
554       break;
555     case WARNING:
556       logBuilder = LOGGER.atWarn();
557       break;
558     case INFORMATIONAL:
559       logBuilder = LOGGER.atInfo();
560       break;
561     case DEBUG:
562       logBuilder = LOGGER.atDebug();
563       break;
564     default:
565       throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
566     }
567 
568     // if (finding.getCause() != null) {
569     // logBuilder.withThrowable(finding.getCause());
570     // }
571 
572     if (finding instanceof JsonValidationFinding) {
573       JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
574       logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
575     } else {
576       logBuilder.log("{}", finding.getMessage());
577     }
578   }
579 }