001/*
002 * SPDX-FileCopyrightText: none
003 * SPDX-License-Identifier: CC0-1.0
004 */
005
006package dev.metaschema.model.testing;
007
008import static org.junit.jupiter.api.Assertions.assertEquals;
009
010import org.apache.logging.log4j.LogBuilder;
011import org.apache.logging.log4j.LogManager;
012import org.apache.logging.log4j.Logger;
013import org.junit.jupiter.api.DynamicContainer;
014import org.junit.jupiter.api.DynamicNode;
015import org.junit.jupiter.api.DynamicTest;
016import org.junit.platform.commons.JUnitException;
017
018import java.io.IOException;
019import java.io.Writer;
020import java.net.URI;
021import java.net.URISyntaxException;
022import java.net.URL;
023import java.nio.charset.StandardCharsets;
024import java.nio.file.FileVisitResult;
025import java.nio.file.Files;
026import java.nio.file.OpenOption;
027import java.nio.file.Path;
028import java.nio.file.SimpleFileVisitor;
029import java.nio.file.StandardOpenOption;
030import java.nio.file.attribute.BasicFileAttributes;
031import java.util.List;
032import java.util.function.BiFunction;
033import java.util.function.Function;
034import java.util.function.Supplier;
035import java.util.stream.Stream;
036
037import dev.metaschema.core.model.IModule;
038import dev.metaschema.core.model.MetaschemaException;
039import dev.metaschema.core.model.validation.IContentValidator;
040import dev.metaschema.core.model.validation.IValidationFinding;
041import dev.metaschema.core.model.validation.IValidationResult;
042import dev.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
043import dev.metaschema.core.util.ObjectUtils;
044import dev.metaschema.databind.IBindingContext;
045import dev.metaschema.databind.io.Format;
046import dev.metaschema.databind.io.ISerializer;
047import dev.metaschema.databind.model.metaschema.IBindingModuleLoader;
048import dev.metaschema.model.testing.testsuite.GenerateSchema;
049import dev.metaschema.model.testing.testsuite.Metaschema;
050import dev.metaschema.model.testing.testsuite.TestCollection;
051import dev.metaschema.model.testing.testsuite.TestScenario;
052import dev.metaschema.model.testing.testsuite.TestSuite;
053import dev.metaschema.model.testing.testsuite.ValidationCase;
054import edu.umd.cs.findbugs.annotations.NonNull;
055import edu.umd.cs.findbugs.annotations.Nullable;
056import nl.talsmasoftware.lazy4j.Lazy;
057
058/**
059 * This abstract implementation dynamically produces JUnit tests based on a test
060 * suite definition.
061 *
062 * @see #getTestSuiteURI()
063 */
064@SuppressWarnings({
065    "PMD.GodClass",
066    "PMD.CouplingBetweenObjects"
067})
068public abstract class AbstractTestSuite {
069  private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
070
071  private static final boolean DELETE_RESULTS_ON_EXIT = false;
072  private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
073      StandardOpenOption.CREATE,
074      StandardOpenOption.WRITE,
075      StandardOpenOption.TRUNCATE_EXISTING
076  };
077
078  private static final String VALID = "VALID";
079
080  /**
081   * Get the content format used by the test suite.
082   *
083   * @return the format
084   */
085  @NonNull
086  protected abstract Format getRequiredContentFormat();
087
088  /**
089   * Get the resource describing the tests to execute.
090   *
091   * @return the resource
092   */
093  @NonNull
094  protected abstract URI getTestSuiteURI();
095
096  /**
097   * Get the filesystem location to use for generating content.
098   *
099   * @return the filesystem path
100   */
101  @NonNull
102  protected abstract Path getGenerationPath();
103
104  /**
105   * Get the method used to generate a schema using a given Metaschema module and
106   * writer.
107   *
108   * @return the schema generator supplier
109   */
110  @NonNull
111  protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112
113  /**
114   * Get the method used to provide a schema validator.
115   *
116   * @return the method as a supplier
117   */
118  @Nullable
119  protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120
121  /**
122   * Get the method used to provide a content validator.
123   *
124   * @return the method as a supplier
125   */
126  @NonNull
127  protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128
129  /**
130   * Dynamically generate the unit tests.
131   *
132   * @param bindingContext
133   *          the Module binding context
134   *
135   * @return the steam of unit tests
136   */
137  @NonNull
138  protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139    try {
140      Path generationPath = getGenerationPath();
141      if (Files.exists(generationPath)) {
142        if (!Files.isDirectory(generationPath)) {
143          throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
144        }
145      } else {
146        Files.createDirectories(generationPath);
147      }
148
149      URI testSuiteUri = getTestSuiteURI();
150      URL testSuiteUrl = testSuiteUri.toURL();
151      TestSuite testSuite = bindingContext.newBoundLoader().load(TestSuite.class, testSuiteUrl);
152      List<TestCollection> testCollections = testSuite.getTestCollections();
153      return ObjectUtils.notNull(testCollections.stream()
154          .flatMap(
155              collection -> Stream
156                  .of(generateCollection(
157                      ObjectUtils.notNull(collection),
158                      testSuiteUri,
159                      generationPath,
160                      bindingContext))));
161    } catch (IOException | URISyntaxException ex) {
162      throw new JUnitException("Unable to generate tests", ex);
163    }
164  }
165
166  /**
167   * Configure removal of the provided directory after test execution.
168   *
169   * @param path
170   *          the directory to configure for removal
171   */
172  protected void deleteCollectionOnExit(@NonNull Path path) {
173    Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
174        () -> {
175          try {
176            Files.walkFileTree(path, new SimpleFileVisitor<>() {
177              @Override
178              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
179                Files.delete(file);
180                return FileVisitResult.CONTINUE;
181              }
182
183              @Override
184              public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
185                if (ex == null) {
186                  Files.delete(dir);
187                  return FileVisitResult.CONTINUE;
188                }
189                // directory iteration failed for some reason
190                throw ex;
191              }
192            });
193          } catch (IOException ex) {
194            throw new JUnitException("Failed to delete collection: " + path, ex);
195          }
196        }));
197  }
198
199  private DynamicContainer generateCollection(
200      @NonNull TestCollection collection,
201      @NonNull URI testSuiteUri,
202      @NonNull Path generationPath,
203      @NonNull IBindingContext bindingContext) {
204    URI collectionUri = testSuiteUri.resolve(collection.getLocation());
205    assert collectionUri != null;
206
207    LOGGER.atInfo().log("Collection: " + collectionUri);
208    Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
209      Path retval;
210      try {
211        retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
212        if (DELETE_RESULTS_ON_EXIT) {
213          deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
214        }
215      } catch (IOException ex) {
216        throw new JUnitException("Unable to create collection temp directory", ex);
217      }
218      return retval;
219    }));
220
221    List<TestScenario> testScenarios = collection.getTestScenarios();
222    return DynamicContainer.dynamicContainer(
223        collection.getName(),
224        testSuiteUri,
225        testScenarios.stream()
226            .flatMap(scenario -> {
227              assert scenario != null;
228              return Stream.of(generateScenario(
229                  scenario,
230                  collectionUri,
231                  collectionGenerationPath,
232                  bindingContext));
233            })
234            .sequential());
235  }
236
237  @NonNull
238  private Path generateSchema(
239      @NonNull IModule module,
240      @NonNull Lazy<Path> scenarioGenerationPath) {
241    String schemaExtension;
242    Format requiredContentFormat = getRequiredContentFormat();
243    switch (requiredContentFormat) {
244    case JSON:
245    case YAML:
246      schemaExtension = ".json";
247      break;
248    case XML:
249      schemaExtension = ".xsd";
250      break;
251    default:
252      throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat));
253    }
254
255    // determine what file to use for the schema
256    Path schemaPath;
257    try {
258      schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
259    } catch (IOException ex) {
260      throw new JUnitException("Unable to create schema temp file", ex);
261    }
262    try {
263      generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
264    } catch (IOException ex) {
265      throw new IllegalStateException(ex);
266    }
267    return ObjectUtils.notNull(schemaPath);
268  }
269
270  /**
271   * Generate a schema for the provided module using the provided schema
272   * generator.
273   *
274   * @param module
275   *          the Metaschema module to generate the schema for
276   * @param schemaPath
277   *          the location to generate the schema
278   * @param schemaProducer
279   *          the method callback to use to generate the schema
280   * @throws IOException
281   *           if an error occurred while writing the schema
282   */
283  protected void generateSchema(
284      @NonNull IModule module,
285      @NonNull Path schemaPath,
286      @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
287    Path parentDir = schemaPath.getParent();
288    if (parentDir != null && !Files.exists(parentDir)) {
289      Files.createDirectories(parentDir);
290    }
291
292    try (Writer writer = Files.newBufferedWriter(
293        schemaPath,
294        StandardCharsets.UTF_8,
295        getWriteOpenOptions())) {
296      schemaProducer.apply(module, writer);
297    }
298    LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
299  }
300
301  /**
302   * The the options for writing generated content.
303   *
304   * @return the options
305   */
306  @SuppressWarnings("PMD.MethodReturnsInternalArray")
307  protected OpenOption[] getWriteOpenOptions() {
308    return OPEN_OPTIONS_TRUNCATE;
309  }
310
311  @SuppressWarnings("PMD.AvoidCatchingGenericException")
312  private DynamicContainer generateScenario(
313      @NonNull TestScenario scenario,
314      @NonNull URI collectionUri,
315      @NonNull Lazy<Path> collectionGenerationPath,
316      @NonNull IBindingContext bindingContext) {
317    Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.of(() -> {
318      Path retval;
319      try {
320        retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
321      } catch (IOException ex) {
322        throw new JUnitException("Unable to create scenario temp directory", ex);
323      }
324      return retval;
325    }));
326
327    GenerateSchema generateSchema = scenario.getGenerateSchema();
328    if (generateSchema == null) {
329      throw new JUnitException(String.format(
330          "Scenario '%s' does not have a generate-schema directive", scenario.getName()));
331    }
332    Metaschema metaschemaDirective = generateSchema.getMetaschema();
333    URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
334
335    IModule module;
336    try {
337      IBindingModuleLoader loader = bindingContext.newModuleLoader();
338
339      module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
340    } catch (IOException | MetaschemaException ex) {
341      throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
342    }
343
344    Lazy<Path> lazySchema = Lazy.of(() -> generateSchema(module, scenarioGenerationPath));
345
346    Lazy<IContentValidator> lazyContentValidator = Lazy.of(() -> {
347      Path schemaPath = lazySchema.get();
348      return getContentValidatorSupplier().apply(schemaPath);
349    });
350    assert lazyContentValidator != null;
351
352    // build a test container for the generate and validate steps
353    DynamicTest validateSchema = DynamicTest.dynamicTest(
354        "Validate Schema",
355        () -> {
356          Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357          if (supplier != null) {
358            Path schemaPath;
359            try {
360              schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361            } catch (Exception ex) {
362              throw new JUnitException(
363                  "failed to generate schema", ex);
364            }
365            validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366          }
367        });
368
369    List<ValidationCase> validationCases = scenario.getValidationCases();
370    Stream<? extends DynamicNode> contentTests = validationCases.stream()
371        .flatMap(contentCase -> {
372          assert contentCase != null;
373          DynamicTest test
374              = generateValidationCase(
375                  contentCase,
376                  bindingContext,
377                  lazyContentValidator,
378                  collectionUri,
379                  ObjectUtils.notNull(scenarioGenerationPath));
380          return test == null ? Stream.empty() : Stream.of(test);
381        }).sequential();
382
383    return DynamicContainer.dynamicContainer(
384        scenario.getName(),
385        metaschemaUri,
386        Stream.concat(Stream.of(validateSchema), contentTests).sequential());
387  }
388
389  /**
390   * Perform content conversion.
391   *
392   * @param resource
393   *          the resource to convert
394   * @param generationPath
395   *          the path to write the converted resource to
396   * @param context
397   *          the Metaschema binding context
398   * @return the location of the converted content
399   * @throws IOException
400   *           if an error occurred while reading or writing content
401   * @see #getRequiredContentFormat()
402   */
403  protected Path convertContent(
404      @NonNull URI resource,
405      @NonNull Path generationPath,
406      @NonNull IBindingContext context)
407      throws IOException {
408    Object object;
409    try {
410      object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
411    } catch (URISyntaxException ex) {
412      throw new IOException(ex);
413    }
414
415    if (!Files.exists(generationPath)) {
416      Files.createDirectories(generationPath);
417    }
418
419    Path convertedContetPath;
420    try {
421      convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
422    } catch (IOException ex) {
423      throw new JUnitException(
424          String.format("Unable to create converted content path in location '%s'", generationPath),
425          ex);
426    }
427
428    Format toFormat = getRequiredContentFormat();
429    if (LOGGER.isInfoEnabled()) {
430      LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
431    }
432
433    ISerializer<?> serializer
434        = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
435    serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
436
437    return convertedContetPath;
438  }
439
440  /**
441   * Convert a source format string to the corresponding Format enum value.
442   *
443   * @param sourceFormat
444   *          the source format string (e.g., "XML", "JSON", "YAML")
445   * @return the corresponding Format, or null if the format is not recognized
446   */
447  @Nullable
448  private static Format toFormat(@Nullable String sourceFormat) {
449    if (sourceFormat == null) {
450      return null;
451    }
452    switch (sourceFormat) {
453    case "XML":
454      return Format.XML;
455    case "JSON":
456      return Format.JSON;
457    case "YAML":
458      return Format.YAML;
459    default:
460      return null;
461    }
462  }
463
464  @SuppressWarnings("PMD.AvoidCatchingGenericException")
465  private DynamicTest generateValidationCase(
466      @NonNull ValidationCase contentCase,
467      @NonNull IBindingContext bindingContext,
468      @NonNull Lazy<IContentValidator> lazyContentValidator,
469      @NonNull URI collectionUri,
470      @NonNull Lazy<Path> resourceGenerationPath) {
471
472    URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
473
474    Format format = toFormat(contentCase.getSourceFormat());
475    boolean expectedValid = isValid(contentCase.getValidationResult());
476
477    DynamicTest retval = null;
478    if (getRequiredContentFormat().equals(format)) {
479      retval = DynamicTest.dynamicTest(
480          String.format("Validate %s=%s: %s", format, expectedValid,
481              contentCase.getLocation()),
482          contentUri,
483          () -> {
484            IContentValidator contentValidator;
485            try {
486              contentValidator = lazyContentValidator.get();
487            } catch (Exception ex) {
488              throw new JUnitException(
489                  "failed to produce the content validator", ex);
490            }
491
492            assertEquals(
493                expectedValid,
494                validateWithSchema(
495                    ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
496                "validation did not match expectation for: " + contentUri.toASCIIString());
497          });
498    } else if (expectedValid) {
499      retval = DynamicTest.dynamicTest(
500          String.format("Convert and Validate %s=%s: %s", format, expectedValid,
501              contentCase.getLocation()),
502          contentUri,
503          () -> {
504            Path convertedContetPath;
505            try {
506              convertedContetPath = convertContent(
507                  contentUri,
508                  ObjectUtils.notNull(resourceGenerationPath.get()),
509                  bindingContext);
510            } catch (Exception ex) {
511              throw new JUnitException("failed to convert content: " + contentUri, ex);
512            }
513
514            IContentValidator contentValidator;
515            try {
516              contentValidator = lazyContentValidator.get();
517            } catch (Exception ex) {
518              throw new JUnitException( // NOPMD - cause is relevant, exception is not
519                  "failed to produce the content validator",
520                  ex.getCause());
521            }
522
523            if (LOGGER.isInfoEnabled()) {
524              LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
525            }
526            assertEquals(expectedValid,
527                validateWithSchema(
528                    ObjectUtils.notNull(contentValidator),
529                    ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
530                String.format("validation of '%s' did not match expectation", convertedContetPath));
531          });
532    }
533    return retval;
534  }
535
536  /**
537   * Check if the validation result string indicates a valid result.
538   *
539   * @param validationResult
540   *          the validation result string
541   * @return {@code true} if the result indicates valid, {@code false} otherwise
542   */
543  private static boolean isValid(@Nullable String validationResult) {
544    return VALID.equals(validationResult);
545  }
546
547  private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
548      throws IOException {
549    IValidationResult schemaValidationResult;
550    try {
551      schemaValidationResult = validator.validate(target);
552    } catch (URISyntaxException ex) {
553      throw new IOException(ex);
554    }
555    return processValidationResult(schemaValidationResult);
556  }
557
558  /**
559   * Use the provided validator to validate the provided target.
560   *
561   * @param validator
562   *          the content validator to use
563   * @param target
564   *          the resource to validate
565   * @return {@code true} if the content is valid or {@code false} otherwise
566   * @throws IOException
567   *           if an error occurred while reading the content
568   */
569  protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
570      throws IOException {
571    LOGGER.atError().log("Validating: {}", target);
572    IValidationResult schemaValidationResult = validator.validate(target);
573    if (!schemaValidationResult.isPassing()) {
574      LOGGER.atError().log("Schema validation failed for: {}", target);
575    }
576    return processValidationResult(schemaValidationResult);
577  }
578
579  private static boolean processValidationResult(IValidationResult schemaValidationResult) {
580    for (IValidationFinding finding : schemaValidationResult.getFindings()) {
581      logFinding(ObjectUtils.notNull(finding));
582    }
583    return schemaValidationResult.isPassing();
584  }
585
586  private static void logFinding(@NonNull IValidationFinding finding) {
587    LogBuilder logBuilder;
588    switch (finding.getSeverity()) {
589    case CRITICAL:
590      logBuilder = LOGGER.atFatal();
591      break;
592    case ERROR:
593      logBuilder = LOGGER.atError();
594      break;
595    case WARNING:
596      logBuilder = LOGGER.atWarn();
597      break;
598    case INFORMATIONAL:
599      logBuilder = LOGGER.atInfo();
600      break;
601    case DEBUG:
602      logBuilder = LOGGER.atDebug();
603      break;
604    default:
605      throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
606    }
607
608    // if (finding.getCause() != null) {
609    // logBuilder.withThrowable(finding.getCause());
610    // }
611
612    if (finding instanceof JsonValidationFinding) {
613      JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
614      logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
615    } else {
616      logBuilder.log("{}", finding.getMessage());
617    }
618  }
619}