1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package dev.metaschema.model.testing;
7   
8   import static org.junit.jupiter.api.Assertions.assertEquals;
9   
10  import org.apache.logging.log4j.LogBuilder;
11  import org.apache.logging.log4j.LogManager;
12  import org.apache.logging.log4j.Logger;
13  import org.junit.jupiter.api.DynamicContainer;
14  import org.junit.jupiter.api.DynamicNode;
15  import org.junit.jupiter.api.DynamicTest;
16  import org.junit.platform.commons.JUnitException;
17  
18  import java.io.IOException;
19  import java.io.Writer;
20  import java.net.URI;
21  import java.net.URISyntaxException;
22  import java.net.URL;
23  import java.nio.charset.StandardCharsets;
24  import java.nio.file.FileVisitResult;
25  import java.nio.file.Files;
26  import java.nio.file.OpenOption;
27  import java.nio.file.Path;
28  import java.nio.file.SimpleFileVisitor;
29  import java.nio.file.StandardOpenOption;
30  import java.nio.file.attribute.BasicFileAttributes;
31  import java.util.List;
32  import java.util.function.BiFunction;
33  import java.util.function.Function;
34  import java.util.function.Supplier;
35  import java.util.stream.Stream;
36  
37  import dev.metaschema.core.model.IModule;
38  import dev.metaschema.core.model.MetaschemaException;
39  import dev.metaschema.core.model.validation.IContentValidator;
40  import dev.metaschema.core.model.validation.IValidationFinding;
41  import dev.metaschema.core.model.validation.IValidationResult;
42  import dev.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
43  import dev.metaschema.core.util.ObjectUtils;
44  import dev.metaschema.databind.IBindingContext;
45  import dev.metaschema.databind.io.Format;
46  import dev.metaschema.databind.io.ISerializer;
47  import dev.metaschema.databind.model.metaschema.IBindingModuleLoader;
48  import dev.metaschema.model.testing.testsuite.GenerateSchema;
49  import dev.metaschema.model.testing.testsuite.Metaschema;
50  import dev.metaschema.model.testing.testsuite.TestCollection;
51  import dev.metaschema.model.testing.testsuite.TestScenario;
52  import dev.metaschema.model.testing.testsuite.TestSuite;
53  import dev.metaschema.model.testing.testsuite.ValidationCase;
54  import edu.umd.cs.findbugs.annotations.NonNull;
55  import edu.umd.cs.findbugs.annotations.Nullable;
56  import nl.talsmasoftware.lazy4j.Lazy;
57  
58  /**
59   * This abstract implementation dynamically produces JUnit tests based on a test
60   * suite definition.
61   *
62   * @see #getTestSuiteURI()
63   */
64  @SuppressWarnings({
65      "PMD.GodClass",
66      "PMD.CouplingBetweenObjects"
67  })
68  public abstract class AbstractTestSuite {
69    private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
70  
71    private static final boolean DELETE_RESULTS_ON_EXIT = false;
72    private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
73        StandardOpenOption.CREATE,
74        StandardOpenOption.WRITE,
75        StandardOpenOption.TRUNCATE_EXISTING
76    };
77  
78    private static final String VALID = "VALID";
79  
80    /**
81     * Get the content format used by the test suite.
82     *
83     * @return the format
84     */
85    @NonNull
86    protected abstract Format getRequiredContentFormat();
87  
88    /**
89     * Get the resource describing the tests to execute.
90     *
91     * @return the resource
92     */
93    @NonNull
94    protected abstract URI getTestSuiteURI();
95  
96    /**
97     * Get the filesystem location to use for generating content.
98     *
99     * @return the filesystem path
100    */
101   @NonNull
102   protected abstract Path getGenerationPath();
103 
104   /**
105    * Get the method used to generate a schema using a given Metaschema module and
106    * writer.
107    *
108    * @return the schema generator supplier
109    */
110   @NonNull
111   protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112 
113   /**
114    * Get the method used to provide a schema validator.
115    *
116    * @return the method as a supplier
117    */
118   @Nullable
119   protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120 
121   /**
122    * Get the method used to provide a content validator.
123    *
124    * @return the method as a supplier
125    */
126   @NonNull
127   protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128 
129   /**
130    * Dynamically generate the unit tests.
131    *
132    * @param bindingContext
133    *          the Module binding context
134    *
135    * @return the steam of unit tests
136    */
137   @NonNull
138   protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139     try {
140       Path generationPath = getGenerationPath();
141       if (Files.exists(generationPath)) {
142         if (!Files.isDirectory(generationPath)) {
143           throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
144         }
145       } else {
146         Files.createDirectories(generationPath);
147       }
148 
149       URI testSuiteUri = getTestSuiteURI();
150       URL testSuiteUrl = testSuiteUri.toURL();
151       TestSuite testSuite = bindingContext.newBoundLoader().load(TestSuite.class, testSuiteUrl);
152       List<TestCollection> testCollections = testSuite.getTestCollections();
153       return ObjectUtils.notNull(testCollections.stream()
154           .flatMap(
155               collection -> Stream
156                   .of(generateCollection(
157                       ObjectUtils.notNull(collection),
158                       testSuiteUri,
159                       generationPath,
160                       bindingContext))));
161     } catch (IOException | URISyntaxException ex) {
162       throw new JUnitException("Unable to generate tests", ex);
163     }
164   }
165 
166   /**
167    * Configure removal of the provided directory after test execution.
168    *
169    * @param path
170    *          the directory to configure for removal
171    */
172   protected void deleteCollectionOnExit(@NonNull Path path) {
173     Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
174         () -> {
175           try {
176             Files.walkFileTree(path, new SimpleFileVisitor<>() {
177               @Override
178               public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
179                 Files.delete(file);
180                 return FileVisitResult.CONTINUE;
181               }
182 
183               @Override
184               public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
185                 if (ex == null) {
186                   Files.delete(dir);
187                   return FileVisitResult.CONTINUE;
188                 }
189                 // directory iteration failed for some reason
190                 throw ex;
191               }
192             });
193           } catch (IOException ex) {
194             throw new JUnitException("Failed to delete collection: " + path, ex);
195           }
196         }));
197   }
198 
199   private DynamicContainer generateCollection(
200       @NonNull TestCollection collection,
201       @NonNull URI testSuiteUri,
202       @NonNull Path generationPath,
203       @NonNull IBindingContext bindingContext) {
204     URI collectionUri = testSuiteUri.resolve(collection.getLocation());
205     assert collectionUri != null;
206 
207     LOGGER.atInfo().log("Collection: " + collectionUri);
208     Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
209       Path retval;
210       try {
211         retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
212         if (DELETE_RESULTS_ON_EXIT) {
213           deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
214         }
215       } catch (IOException ex) {
216         throw new JUnitException("Unable to create collection temp directory", ex);
217       }
218       return retval;
219     }));
220 
221     List<TestScenario> testScenarios = collection.getTestScenarios();
222     return DynamicContainer.dynamicContainer(
223         collection.getName(),
224         testSuiteUri,
225         testScenarios.stream()
226             .flatMap(scenario -> {
227               assert scenario != null;
228               return Stream.of(generateScenario(
229                   scenario,
230                   collectionUri,
231                   collectionGenerationPath,
232                   bindingContext));
233             })
234             .sequential());
235   }
236 
237   @NonNull
238   private Path generateSchema(
239       @NonNull IModule module,
240       @NonNull Lazy<Path> scenarioGenerationPath) {
241     String schemaExtension;
242     Format requiredContentFormat = getRequiredContentFormat();
243     switch (requiredContentFormat) {
244     case JSON:
245     case YAML:
246       schemaExtension = ".json";
247       break;
248     case XML:
249       schemaExtension = ".xsd";
250       break;
251     default:
252       throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat));
253     }
254 
255     // determine what file to use for the schema
256     Path schemaPath;
257     try {
258       schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
259     } catch (IOException ex) {
260       throw new JUnitException("Unable to create schema temp file", ex);
261     }
262     try {
263       generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
264     } catch (IOException ex) {
265       throw new IllegalStateException(ex);
266     }
267     return ObjectUtils.notNull(schemaPath);
268   }
269 
270   /**
271    * Generate a schema for the provided module using the provided schema
272    * generator.
273    *
274    * @param module
275    *          the Metaschema module to generate the schema for
276    * @param schemaPath
277    *          the location to generate the schema
278    * @param schemaProducer
279    *          the method callback to use to generate the schema
280    * @throws IOException
281    *           if an error occurred while writing the schema
282    */
283   protected void generateSchema(
284       @NonNull IModule module,
285       @NonNull Path schemaPath,
286       @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
287     Path parentDir = schemaPath.getParent();
288     if (parentDir != null && !Files.exists(parentDir)) {
289       Files.createDirectories(parentDir);
290     }
291 
292     try (Writer writer = Files.newBufferedWriter(
293         schemaPath,
294         StandardCharsets.UTF_8,
295         getWriteOpenOptions())) {
296       schemaProducer.apply(module, writer);
297     }
298     LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
299   }
300 
301   /**
302    * The the options for writing generated content.
303    *
304    * @return the options
305    */
306   @SuppressWarnings("PMD.MethodReturnsInternalArray")
307   protected OpenOption[] getWriteOpenOptions() {
308     return OPEN_OPTIONS_TRUNCATE;
309   }
310 
311   @SuppressWarnings("PMD.AvoidCatchingGenericException")
312   private DynamicContainer generateScenario(
313       @NonNull TestScenario scenario,
314       @NonNull URI collectionUri,
315       @NonNull Lazy<Path> collectionGenerationPath,
316       @NonNull IBindingContext bindingContext) {
317     Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
318       Path retval;
319       try {
320         retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
321       } catch (IOException ex) {
322         throw new JUnitException("Unable to create scenario temp directory", ex);
323       }
324       return retval;
325     }));
326 
327     GenerateSchema generateSchema = scenario.getGenerateSchema();
328     if (generateSchema == null) {
329       throw new JUnitException(String.format(
330           "Scenario '%s' does not have a generate-schema directive", scenario.getName()));
331     }
332     Metaschema metaschemaDirective = generateSchema.getMetaschema();
333     URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
334 
335     IModule module;
336     try {
337       IBindingModuleLoader loader = bindingContext.newModuleLoader();
338 
339       module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
340     } catch (IOException | MetaschemaException ex) {
341       throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
342     }
343 
344     Lazy<Path> lazySchema = Lazy.of(() -> generateSchema(module, scenarioGenerationPath));
345 
346     Lazy<IContentValidator> lazyContentValidator = Lazy.of(() -> {
347       Path schemaPath = lazySchema.get();
348       return getContentValidatorSupplier().apply(schemaPath);
349     });
350     assert lazyContentValidator != null;
351 
352     // build a test container for the generate and validate steps
353     DynamicTest validateSchema = DynamicTest.dynamicTest(
354         "Validate Schema",
355         () -> {
356           Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357           if (supplier != null) {
358             Path schemaPath;
359             try {
360               schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361             } catch (Exception ex) {
362               throw new JUnitException(
363                   "failed to generate schema", ex);
364             }
365             validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366           }
367         });
368 
369     List<ValidationCase> validationCases = scenario.getValidationCases();
370     Stream<? extends DynamicNode> contentTests = validationCases.stream()
371         .flatMap(contentCase -> {
372           assert contentCase != null;
373           DynamicTest test
374               = generateValidationCase(
375                   contentCase,
376                   bindingContext,
377                   lazyContentValidator,
378                   collectionUri,
379                   ObjectUtils.notNull(scenarioGenerationPath));
380           return test == null ? Stream.empty() : Stream.of(test);
381         }).sequential();
382 
383     return DynamicContainer.dynamicContainer(
384         scenario.getName(),
385         metaschemaUri,
386         Stream.concat(Stream.of(validateSchema), contentTests).sequential());
387   }
388 
389   /**
390    * Perform content conversion.
391    *
392    * @param resource
393    *          the resource to convert
394    * @param generationPath
395    *          the path to write the converted resource to
396    * @param context
397    *          the Metaschema binding context
398    * @return the location of the converted content
399    * @throws IOException
400    *           if an error occurred while reading or writing content
401    * @see #getRequiredContentFormat()
402    */
403   protected Path convertContent(
404       @NonNull URI resource,
405       @NonNull Path generationPath,
406       @NonNull IBindingContext context)
407       throws IOException {
408     Object object;
409     try {
410       object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
411     } catch (URISyntaxException ex) {
412       throw new IOException(ex);
413     }
414 
415     if (!Files.exists(generationPath)) {
416       Files.createDirectories(generationPath);
417     }
418 
419     Path convertedContetPath;
420     try {
421       convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
422     } catch (IOException ex) {
423       throw new JUnitException(
424           String.format("Unable to create converted content path in location '%s'", generationPath),
425           ex);
426     }
427 
428     Format toFormat = getRequiredContentFormat();
429     if (LOGGER.isInfoEnabled()) {
430       LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
431     }
432 
433     ISerializer<?> serializer
434         = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
435     serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
436 
437     return convertedContetPath;
438   }
439 
440   /**
441    * Convert a source format string to the corresponding Format enum value.
442    *
443    * @param sourceFormat
444    *          the source format string (e.g., "XML", "JSON", "YAML")
445    * @return the corresponding Format, or null if the format is not recognized
446    */
447   @Nullable
448   private static Format toFormat(@Nullable String sourceFormat) {
449     if (sourceFormat == null) {
450       return null;
451     }
452     switch (sourceFormat) {
453     case "XML":
454       return Format.XML;
455     case "JSON":
456       return Format.JSON;
457     case "YAML":
458       return Format.YAML;
459     default:
460       return null;
461     }
462   }
463 
464   @SuppressWarnings("PMD.AvoidCatchingGenericException")
465   private DynamicTest generateValidationCase(
466       @NonNull ValidationCase contentCase,
467       @NonNull IBindingContext bindingContext,
468       @NonNull Lazy<IContentValidator> lazyContentValidator,
469       @NonNull URI collectionUri,
470       @NonNull Lazy<Path> resourceGenerationPath) {
471 
472     URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
473 
474     Format format = toFormat(contentCase.getSourceFormat());
475     boolean expectedValid = isValid(contentCase.getValidationResult());
476 
477     DynamicTest retval = null;
478     if (getRequiredContentFormat().equals(format)) {
479       retval = DynamicTest.dynamicTest(
480           String.format("Validate %s=%s: %s", format, expectedValid,
481               contentCase.getLocation()),
482           contentUri,
483           () -> {
484             IContentValidator contentValidator;
485             try {
486               contentValidator = lazyContentValidator.get();
487             } catch (Exception ex) {
488               throw new JUnitException(
489                   "failed to produce the content validator", ex);
490             }
491 
492             assertEquals(
493                 expectedValid,
494                 validateWithSchema(
495                     ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
496                 "validation did not match expectation for: " + contentUri.toASCIIString());
497           });
498     } else if (expectedValid) {
499       retval = DynamicTest.dynamicTest(
500           String.format("Convert and Validate %s=%s: %s", format, expectedValid,
501               contentCase.getLocation()),
502           contentUri,
503           () -> {
504             Path convertedContetPath;
505             try {
506               convertedContetPath = convertContent(
507                   contentUri,
508                   ObjectUtils.notNull(resourceGenerationPath.get()),
509                   bindingContext);
510             } catch (Exception ex) {
511               throw new JUnitException("failed to convert content: " + contentUri, ex);
512             }
513 
514             IContentValidator contentValidator;
515             try {
516               contentValidator = lazyContentValidator.get();
517             } catch (Exception ex) {
518               throw new JUnitException( // NOPMD - cause is relevant, exception is not
519                   "failed to produce the content validator",
520                   ex.getCause());
521             }
522 
523             if (LOGGER.isInfoEnabled()) {
524               LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
525             }
526             assertEquals(expectedValid,
527                 validateWithSchema(
528                     ObjectUtils.notNull(contentValidator),
529                     ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
530                 String.format("validation of '%s' did not match expectation", convertedContetPath));
531           });
532     }
533     return retval;
534   }
535 
536   /**
537    * Check if the validation result string indicates a valid result.
538    *
539    * @param validationResult
540    *          the validation result string
541    * @return {@code true} if the result indicates valid, {@code false} otherwise
542    */
543   private static boolean isValid(@Nullable String validationResult) {
544     return VALID.equals(validationResult);
545   }
546 
547   private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
548       throws IOException {
549     IValidationResult schemaValidationResult;
550     try {
551       schemaValidationResult = validator.validate(target);
552     } catch (URISyntaxException ex) {
553       throw new IOException(ex);
554     }
555     return processValidationResult(schemaValidationResult);
556   }
557 
558   /**
559    * Use the provided validator to validate the provided target.
560    *
561    * @param validator
562    *          the content validator to use
563    * @param target
564    *          the resource to validate
565    * @return {@code true} if the content is valid or {@code false} otherwise
566    * @throws IOException
567    *           if an error occurred while reading the content
568    */
569   protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
570       throws IOException {
571     LOGGER.atError().log("Validating: {}", target);
572     IValidationResult schemaValidationResult = validator.validate(target);
573     if (!schemaValidationResult.isPassing()) {
574       LOGGER.atError().log("Schema validation failed for: {}", target);
575     }
576     return processValidationResult(schemaValidationResult);
577   }
578 
579   private static boolean processValidationResult(IValidationResult schemaValidationResult) {
580     for (IValidationFinding finding : schemaValidationResult.getFindings()) {
581       logFinding(ObjectUtils.notNull(finding));
582     }
583     return schemaValidationResult.isPassing();
584   }
585 
586   private static void logFinding(@NonNull IValidationFinding finding) {
587     LogBuilder logBuilder;
588     switch (finding.getSeverity()) {
589     case CRITICAL:
590       logBuilder = LOGGER.atFatal();
591       break;
592     case ERROR:
593       logBuilder = LOGGER.atError();
594       break;
595     case WARNING:
596       logBuilder = LOGGER.atWarn();
597       break;
598     case INFORMATIONAL:
599       logBuilder = LOGGER.atInfo();
600       break;
601     case DEBUG:
602       logBuilder = LOGGER.atDebug();
603       break;
604     default:
605       throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
606     }
607 
608     // if (finding.getCause() != null) {
609     // logBuilder.withThrowable(finding.getCause());
610     // }
611 
612     if (finding instanceof JsonValidationFinding) {
613       JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
614       logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
615     } else {
616       logBuilder.log("{}", finding.getMessage());
617     }
618   }
619 }