1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package gov.nist.secauto.metaschema.model.testing;
7   
8   import static org.junit.jupiter.api.Assertions.assertEquals;
9   
10  import gov.nist.secauto.metaschema.core.model.IModule;
11  import gov.nist.secauto.metaschema.core.model.MetaschemaException;
12  import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
13  import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
14  import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
15  import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
16  import gov.nist.secauto.metaschema.core.util.ObjectUtils;
17  import gov.nist.secauto.metaschema.databind.IBindingContext;
18  import gov.nist.secauto.metaschema.databind.io.Format;
19  import gov.nist.secauto.metaschema.databind.io.ISerializer;
20  import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
21  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
22  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
23  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
24  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
25  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
26  import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
27  
28  import org.apache.logging.log4j.LogBuilder;
29  import org.apache.logging.log4j.LogManager;
30  import org.apache.logging.log4j.Logger;
31  import org.apache.xmlbeans.XmlException;
32  import org.apache.xmlbeans.XmlOptions;
33  import org.junit.jupiter.api.DynamicContainer;
34  import org.junit.jupiter.api.DynamicNode;
35  import org.junit.jupiter.api.DynamicTest;
36  import org.junit.platform.commons.JUnitException;
37  
38  import java.io.IOException;
39  import java.io.Writer;
40  import java.net.URI;
41  import java.net.URISyntaxException;
42  import java.net.URL;
43  import java.nio.charset.StandardCharsets;
44  import java.nio.file.FileVisitResult;
45  import java.nio.file.Files;
46  import java.nio.file.OpenOption;
47  import java.nio.file.Path;
48  import java.nio.file.SimpleFileVisitor;
49  import java.nio.file.StandardOpenOption;
50  import java.nio.file.attribute.BasicFileAttributes;
51  import java.util.function.BiFunction;
52  import java.util.function.Function;
53  import java.util.function.Supplier;
54  import java.util.stream.Stream;
55  
56  import edu.umd.cs.findbugs.annotations.NonNull;
57  import edu.umd.cs.findbugs.annotations.Nullable;
58  import nl.talsmasoftware.lazy4j.Lazy;
59  
60  public abstract class AbstractTestSuite {
61    private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
62  
63    private static final boolean DELETE_RESULTS_ON_EXIT = false;
64    private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
65        StandardOpenOption.CREATE,
66        StandardOpenOption.WRITE,
67        StandardOpenOption.TRUNCATE_EXISTING
68    };
69  
70    /**
71     * Get the content format used by the test suite.
72     *
73     * @return the format
74     */
75    @NonNull
76    protected abstract Format getRequiredContentFormat();
77  
78    /**
79     * Get the resource describing the tests to execute.
80     *
81     * @return the resource
82     */
83    @NonNull
84    protected abstract URI getTestSuiteURI();
85  
86    /**
87     * Get the filesystem location to use for generating content.
88     *
89     * @return the filesystem path
90     */
91    @NonNull
92    protected abstract Path getGenerationPath();
93  
94    /**
95     * Get the method used to generate a schema using a given Metaschema module and
96     * writer.
97     *
98     * @return the schema generator supplier
99     */
100   @NonNull
101   protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
102 
103   /**
104    * Get the method used to provide a schema validator.
105    *
106    * @return the method as a supplier
107    */
108   @Nullable
109   protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
110 
111   /**
112    * Get the method used to provide a content validator.
113    *
114    * @return the method as a supplier
115    */
116   @NonNull
117   protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
118 
119   /**
120    * Dynamically generate the unit tests.
121    *
122    * @return the steam of unit tests
123    */
124   @NonNull
125   protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
126     try {
127       XmlOptions options = new XmlOptions();
128       options.setBaseURI(null);
129       options.setLoadLineNumbers();
130 
131       Path generationPath = getGenerationPath();
132       if (Files.exists(generationPath)) {
133         if (!Files.isDirectory(generationPath)) {
134           throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
135         }
136       } else {
137         Files.createDirectories(generationPath);
138       }
139 
140       URI testSuiteUri = getTestSuiteURI();
141       URL testSuiteUrl = testSuiteUri.toURL();
142       TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
143       return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
144           .flatMap(
145               collection -> Stream
146                   .of(generateCollection(
147                       ObjectUtils.notNull(collection),
148                       testSuiteUri,
149                       generationPath,
150                       bindingContext))));
151     } catch (XmlException | IOException ex) {
152       throw new JUnitException("Unable to generate tests", ex);
153     }
154   }
155 
156   /**
157    * Configure removal of the provided directory after test execution.
158    *
159    * @param path
160    *          the directory to configure for removal
161    */
162   protected void deleteCollectionOnExit(@NonNull Path path) {
163     Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
164         () -> {
165           try {
166             Files.walkFileTree(path, new SimpleFileVisitor<>() {
167               @Override
168               public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
169                 Files.delete(file);
170                 return FileVisitResult.CONTINUE;
171               }
172 
173               @Override
174               public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
175                 if (ex == null) {
176                   Files.delete(dir);
177                   return FileVisitResult.CONTINUE;
178                 }
179                 // directory iteration failed for some reason
180                 throw ex;
181               }
182             });
183           } catch (IOException ex) {
184             throw new JUnitException("Failed to delete collection: " + path, ex);
185           }
186         }));
187   }
188 
189   private DynamicContainer generateCollection(
190       @NonNull TestCollection collection,
191       @NonNull URI testSuiteUri,
192       @NonNull Path generationPath,
193       @NonNull IBindingContext bindingContext) {
194     URI collectionUri = testSuiteUri.resolve(collection.getLocation());
195     assert collectionUri != null;
196 
197     LOGGER.atInfo().log("Collection: " + collectionUri);
198     Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
199       Path retval;
200       try {
201         retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
202         if (DELETE_RESULTS_ON_EXIT) {
203           deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
204         }
205       } catch (IOException ex) {
206         throw new JUnitException("Unable to create collection temp directory", ex);
207       }
208       return retval;
209     }));
210 
211     return DynamicContainer.dynamicContainer(
212         collection.getName(),
213         testSuiteUri,
214         collection.getTestScenarioList().stream()
215             .flatMap(scenario -> {
216               assert scenario != null;
217               return Stream.of(generateScenario(
218                   scenario,
219                   collectionUri,
220                   collectionGenerationPath,
221                   bindingContext));
222             })
223             .sequential());
224   }
225 
226   /**
227    * Generate a schema for the provided module using the provided schema
228    * generator.
229    *
230    * @param module
231    *          the Metaschema module to generate the schema for
232    * @param schemaPath
233    *          the location to generate the schema
234    * @param schemaProducer
235    *          the method callback to use to generate the schema
236    * @throws IOException
237    *           if an error occurred while writing the schema
238    */
239   protected void generateSchema(
240       @NonNull IModule module,
241       @NonNull Path schemaPath,
242       @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
243     Path parentDir = schemaPath.getParent();
244     if (parentDir != null && !Files.exists(parentDir)) {
245       Files.createDirectories(parentDir);
246     }
247 
248     try (Writer writer = Files.newBufferedWriter(
249         schemaPath,
250         StandardCharsets.UTF_8,
251         getWriteOpenOptions())) {
252       schemaProducer.apply(module, writer);
253     }
254     LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
255   }
256 
257   /**
258    * The the options for writing generated content.
259    *
260    * @return the options
261    */
262   @SuppressWarnings("PMD.MethodReturnsInternalArray")
263   protected OpenOption[] getWriteOpenOptions() {
264     return OPEN_OPTIONS_TRUNCATE;
265   }
266 
267   @SuppressWarnings("PMD.AvoidCatchingGenericException")
268   private DynamicContainer generateScenario(
269       @NonNull TestScenario scenario,
270       @NonNull URI collectionUri,
271       @NonNull Lazy<Path> collectionGenerationPath,
272       @NonNull IBindingContext bindingContext) {
273     Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
274       Path retval;
275       try {
276         retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
277       } catch (IOException ex) {
278         throw new JUnitException("Unable to create scenario temp directory", ex);
279       }
280       return retval;
281     });
282 
283     // try {
284     // // create the directories the schema will be stored in
285     // Files.createDirectories(scenarioGenerationPath);
286     // } catch (IOException ex) {
287     // throw new JUnitException("Unable to create test directories for path: " +
288     // scenarioGenerationPath, ex);
289     // }
290 
291     GenerateSchema generateSchema = scenario.getGenerateSchema();
292     MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
293     URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
294 
295     IModule module;
296     try {
297       IBindingModuleLoader loader = bindingContext.newModuleLoader();
298 
299       module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
300     } catch (IOException | MetaschemaException ex) {
301       throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
302     }
303 
304     Lazy<Path> lazySchema = Lazy.lazy(() -> {
305       String schemaExtension;
306       Format requiredContentFormat = getRequiredContentFormat();
307       switch (requiredContentFormat) {
308       case JSON:
309       case YAML:
310         schemaExtension = ".json";
311         break;
312       case XML:
313         schemaExtension = ".xsd";
314         break;
315       default:
316         throw new IllegalStateException();
317       }
318 
319       // determine what file to use for the schema
320       Path schemaPath;
321       try {
322         schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
323       } catch (IOException ex) {
324         throw new JUnitException("Unable to create schema temp file", ex);
325       }
326       try {
327         generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
328       } catch (IOException ex) {
329         throw new IllegalStateException(ex);
330       }
331       return schemaPath;
332     });
333 
334     Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
335       Path schemaPath = lazySchema.get();
336       return getContentValidatorSupplier().apply(schemaPath);
337     });
338     assert lazyContentValidator != null;
339 
340     // build a test container for the generate and validate steps
341     DynamicTest validateSchema = DynamicTest.dynamicTest(
342         "Validate Schema",
343         () -> {
344           Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
345           if (supplier != null) {
346             Path schemaPath;
347             try {
348               schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
349             } catch (Exception ex) {
350               throw new JUnitException(
351                   "failed to generate schema", ex);
352             }
353             validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
354           }
355         });
356 
357     Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
358         .flatMap(contentCase -> {
359           assert contentCase != null;
360           DynamicTest test
361               = generateValidationCase(
362                   contentCase,
363                   bindingContext,
364                   lazyContentValidator,
365                   collectionUri,
366                   ObjectUtils.notNull(scenarioGenerationPath));
367           return test == null ? Stream.empty() : Stream.of(test);
368         }).sequential();
369 
370     return DynamicContainer.dynamicContainer(
371         scenario.getName(),
372         metaschemaUri,
373         Stream.concat(Stream.of(validateSchema), contentTests).sequential());
374   }
375 
376   /**
377    * Perform content conversion.
378    *
379    * @param resource
380    *          the resource to convert
381    * @param generationPath
382    *          the path to write the converted resource to
383    * @param context
384    *          the Metaschema binding context
385    * @return the location of the converted content
386    * @throws IOException
387    *           if an error occurred while reading or writing content
388    * @see #getRequiredContentFormat()
389    */
390   protected Path convertContent(
391       @NonNull URI resource,
392       @NonNull Path generationPath,
393       @NonNull IBindingContext context)
394       throws IOException {
395     Object object;
396     try {
397       object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
398     } catch (URISyntaxException ex) {
399       throw new IOException(ex);
400     }
401 
402     if (!Files.exists(generationPath)) {
403       Files.createDirectories(generationPath);
404     }
405 
406     Path convertedContetPath;
407     try {
408       convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
409     } catch (IOException ex) {
410       throw new JUnitException(
411           String.format("Unable to create converted content path in location '%s'", generationPath),
412           ex);
413     }
414 
415     Format toFormat = getRequiredContentFormat();
416     if (LOGGER.isInfoEnabled()) {
417       LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
418     }
419 
420     ISerializer<?> serializer
421         = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
422     serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
423 
424     return convertedContetPath;
425   }
426 
427   @SuppressWarnings("PMD.AvoidCatchingGenericException")
428   private DynamicTest generateValidationCase(
429       @NonNull ContentCaseType contentCase,
430       @NonNull IBindingContext bindingContext,
431       @NonNull Lazy<IContentValidator> lazyContentValidator,
432       @NonNull URI collectionUri,
433       @NonNull Lazy<Path> resourceGenerationPath) {
434 
435     URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
436 
437     Format format = contentCase.getSourceFormat();
438     DynamicTest retval = null;
439     if (getRequiredContentFormat().equals(format)) {
440       retval = DynamicTest.dynamicTest(
441           String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
442               contentCase.getLocation()),
443           contentUri,
444           () -> {
445             IContentValidator contentValidator;
446             try {
447               contentValidator = lazyContentValidator.get();
448             } catch (Exception ex) {
449               throw new JUnitException( // NOPMD - cause is relevant, exception is not
450                   "failed to produce the content validator", ex.getCause());
451             }
452 
453             assertEquals(
454                 contentCase.getValidationResult(),
455                 validateWithSchema(
456                     ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
457                 "validation did not match expectation for: " + contentUri.toASCIIString());
458           });
459     } else if (contentCase.getValidationResult()) {
460       retval = DynamicTest.dynamicTest(
461           String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
462               contentCase.getLocation()),
463           contentUri,
464           () -> {
465             Path convertedContetPath;
466             try {
467               convertedContetPath = convertContent(
468                   contentUri,
469                   ObjectUtils.notNull(resourceGenerationPath.get()),
470                   bindingContext);
471             } catch (Exception ex) { // NOPMD - intentional
472               throw new JUnitException("failed to convert content: " + contentUri, ex);
473             }
474 
475             IContentValidator contentValidator;
476             try {
477               contentValidator = lazyContentValidator.get();
478             } catch (Exception ex) {
479               throw new JUnitException( // NOPMD - cause is relevant, exception is not
480                   "failed to produce the content validator",
481                   ex.getCause());
482             }
483 
484             if (LOGGER.isInfoEnabled()) {
485               LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
486             }
487             assertEquals(contentCase.getValidationResult(),
488                 validateWithSchema(
489                     ObjectUtils.notNull(contentValidator),
490                     ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
491                 String.format("validation of '%s' did not match expectation", convertedContetPath));
492           });
493     }
494     return retval;
495   }
496 
497   private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
498       throws IOException {
499     IValidationResult schemaValidationResult;
500     try {
501       schemaValidationResult = validator.validate(target);
502     } catch (URISyntaxException ex) {
503       throw new IOException(ex);
504     }
505     return processValidationResult(schemaValidationResult);
506   }
507 
508   /**
509    * Use the provided validator to validate the provided target.
510    *
511    * @param validator
512    *          the content validator to use
513    * @param target
514    *          the resource to validate
515    * @return {@code true} if the content is valid or {@code false} otherwise
516    * @throws IOException
517    *           if an error occurred while reading the content
518    */
519   protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
520       throws IOException {
521     IValidationResult schemaValidationResult = validator.validate(target);
522     if (!schemaValidationResult.isPassing()) {
523       LOGGER.atError().log("Schema validation failed for: {}", target);
524     }
525     return processValidationResult(schemaValidationResult);
526   }
527 
528   private static boolean processValidationResult(IValidationResult schemaValidationResult) {
529     for (IValidationFinding finding : schemaValidationResult.getFindings()) {
530       logFinding(ObjectUtils.notNull(finding));
531     }
532     return schemaValidationResult.isPassing();
533   }
534 
535   private static void logFinding(@NonNull IValidationFinding finding) {
536     LogBuilder logBuilder;
537     switch (finding.getSeverity()) {
538     case CRITICAL:
539       logBuilder = LOGGER.atFatal();
540       break;
541     case ERROR:
542       logBuilder = LOGGER.atError();
543       break;
544     case WARNING:
545       logBuilder = LOGGER.atWarn();
546       break;
547     case INFORMATIONAL:
548       logBuilder = LOGGER.atInfo();
549       break;
550     case DEBUG:
551       logBuilder = LOGGER.atDebug();
552       break;
553     default:
554       throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
555     }
556 
557     // if (finding.getCause() != null) {
558     // logBuilder.withThrowable(finding.getCause());
559     // }
560 
561     if (finding instanceof JsonValidationFinding) {
562       JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
563       logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
564     } else {
565       logBuilder.log("{}", finding.getMessage());
566     }
567   }
568 }