Source code

001/*
002 * SPDX-FileCopyrightText: none
003 * SPDX-License-Identifier: CC0-1.0
004 */
005
006package gov.nist.secauto.metaschema.model.testing;
007
008import static org.junit.jupiter.api.Assertions.assertEquals;
009
010import gov.nist.secauto.metaschema.core.model.IModule;
011import gov.nist.secauto.metaschema.core.model.MetaschemaException;
012import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
013import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
014import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
015import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
016import gov.nist.secauto.metaschema.core.util.ObjectUtils;
017import gov.nist.secauto.metaschema.databind.IBindingContext;
018import gov.nist.secauto.metaschema.databind.io.Format;
019import gov.nist.secauto.metaschema.databind.io.ISerializer;
020import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
021import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
022import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
023import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
024import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
025import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
026import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
027
028import org.apache.logging.log4j.LogBuilder;
029import org.apache.logging.log4j.LogManager;
030import org.apache.logging.log4j.Logger;
031import org.apache.xmlbeans.XmlException;
032import org.apache.xmlbeans.XmlOptions;
033import org.junit.jupiter.api.DynamicContainer;
034import org.junit.jupiter.api.DynamicNode;
035import org.junit.jupiter.api.DynamicTest;
036import org.junit.platform.commons.JUnitException;
037
038import java.io.IOException;
039import java.io.Writer;
040import java.net.URI;
041import java.net.URISyntaxException;
042import java.net.URL;
043import java.nio.charset.StandardCharsets;
044import java.nio.file.FileVisitResult;
045import java.nio.file.Files;
046import java.nio.file.OpenOption;
047import java.nio.file.Path;
048import java.nio.file.SimpleFileVisitor;
049import java.nio.file.StandardOpenOption;
050import java.nio.file.attribute.BasicFileAttributes;
051import java.util.function.BiFunction;
052import java.util.function.Function;
053import java.util.function.Supplier;
054import java.util.stream.Stream;
055
056import edu.umd.cs.findbugs.annotations.NonNull;
057import edu.umd.cs.findbugs.annotations.Nullable;
058import nl.talsmasoftware.lazy4j.Lazy;
059
060/**
061 * This abstract implementation dynamically produces JUnit tests based on a test
062 * suite definition.
063 *
064 * @see #getTestSuiteURI()
065 */
066@SuppressWarnings({
067    "PMD.GodClass",
068    "PMD.CouplingBetweenObjects"
069})
070public abstract class AbstractTestSuite {
071  private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
072
073  private static final boolean DELETE_RESULTS_ON_EXIT = false;
074  private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
075      StandardOpenOption.CREATE,
076      StandardOpenOption.WRITE,
077      StandardOpenOption.TRUNCATE_EXISTING
078  };
079
080  /**
081   * Get the content format used by the test suite.
082   *
083   * @return the format
084   */
085  @NonNull
086  protected abstract Format getRequiredContentFormat();
087
088  /**
089   * Get the resource describing the tests to execute.
090   *
091   * @return the resource
092   */
093  @NonNull
094  protected abstract URI getTestSuiteURI();
095
096  /**
097   * Get the filesystem location to use for generating content.
098   *
099   * @return the filesystem path
100   */
101  @NonNull
102  protected abstract Path getGenerationPath();
103
104  /**
105   * Get the method used to generate a schema using a given Metaschema module and
106   * writer.
107   *
108   * @return the schema generator supplier
109   */
110  @NonNull
111  protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
112
113  /**
114   * Get the method used to provide a schema validator.
115   *
116   * @return the method as a supplier
117   */
118  @Nullable
119  protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
120
121  /**
122   * Get the method used to provide a content validator.
123   *
124   * @return the method as a supplier
125   */
126  @NonNull
127  protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
128
129  /**
130   * Dynamically generate the unit tests.
131   *
132   * @param bindingContext
133   *          the Module binding context
134   *
135   * @return the steam of unit tests
136   */
137  @NonNull
138  protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
139    try {
140      XmlOptions options = new XmlOptions();
141      options.setBaseURI(null);
142      options.setLoadLineNumbers();
143
144      Path generationPath = getGenerationPath();
145      if (Files.exists(generationPath)) {
146        if (!Files.isDirectory(generationPath)) {
147          throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
148        }
149      } else {
150        Files.createDirectories(generationPath);
151      }
152
153      URI testSuiteUri = getTestSuiteURI();
154      URL testSuiteUrl = testSuiteUri.toURL();
155      TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
156      return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
157          .flatMap(
158              collection -> Stream
159                  .of(generateCollection(
160                      ObjectUtils.notNull(collection),
161                      testSuiteUri,
162                      generationPath,
163                      bindingContext))));
164    } catch (XmlException | IOException ex) {
165      throw new JUnitException("Unable to generate tests", ex);
166    }
167  }
168
169  /**
170   * Configure removal of the provided directory after test execution.
171   *
172   * @param path
173   *          the directory to configure for removal
174   */
175  protected void deleteCollectionOnExit(@NonNull Path path) {
176    Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
177        () -> {
178          try {
179            Files.walkFileTree(path, new SimpleFileVisitor<>() {
180              @Override
181              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
182                Files.delete(file);
183                return FileVisitResult.CONTINUE;
184              }
185
186              @Override
187              public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
188                if (ex == null) {
189                  Files.delete(dir);
190                  return FileVisitResult.CONTINUE;
191                }
192                // directory iteration failed for some reason
193                throw ex;
194              }
195            });
196          } catch (IOException ex) {
197            throw new JUnitException("Failed to delete collection: " + path, ex);
198          }
199        }));
200  }
201
202  private DynamicContainer generateCollection(
203      @NonNull TestCollection collection,
204      @NonNull URI testSuiteUri,
205      @NonNull Path generationPath,
206      @NonNull IBindingContext bindingContext) {
207    URI collectionUri = testSuiteUri.resolve(collection.getLocation());
208    assert collectionUri != null;
209
210    LOGGER.atInfo().log("Collection: " + collectionUri);
211    Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
212      Path retval;
213      try {
214        retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
215        if (DELETE_RESULTS_ON_EXIT) {
216          deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
217        }
218      } catch (IOException ex) {
219        throw new JUnitException("Unable to create collection temp directory", ex);
220      }
221      return retval;
222    }));
223
224    return DynamicContainer.dynamicContainer(
225        collection.getName(),
226        testSuiteUri,
227        collection.getTestScenarioList().stream()
228            .flatMap(scenario -> {
229              assert scenario != null;
230              return Stream.of(generateScenario(
231                  scenario,
232                  collectionUri,
233                  collectionGenerationPath,
234                  bindingContext));
235            })
236            .sequential());
237  }
238
239  @NonNull
240  private Path generateSchema(
241      @NonNull IModule module,
242      @NonNull Lazy<Path> scenarioGenerationPath) {
243    String schemaExtension;
244    Format requiredContentFormat = getRequiredContentFormat();
245    switch (requiredContentFormat) {
246    case JSON:
247    case YAML:
248      schemaExtension = ".json";
249      break;
250    case XML:
251      schemaExtension = ".xsd";
252      break;
253    default:
254      throw new IllegalStateException();
255    }
256
257    // determine what file to use for the schema
258    Path schemaPath;
259    try {
260      schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
261    } catch (IOException ex) {
262      throw new JUnitException("Unable to create schema temp file", ex);
263    }
264    try {
265      generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
266    } catch (IOException ex) {
267      throw new IllegalStateException(ex);
268    }
269    return ObjectUtils.notNull(schemaPath);
270  }
271
272  /**
273   * Generate a schema for the provided module using the provided schema
274   * generator.
275   *
276   * @param module
277   *          the Metaschema module to generate the schema for
278   * @param schemaPath
279   *          the location to generate the schema
280   * @param schemaProducer
281   *          the method callback to use to generate the schema
282   * @throws IOException
283   *           if an error occurred while writing the schema
284   */
285  protected void generateSchema(
286      @NonNull IModule module,
287      @NonNull Path schemaPath,
288      @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
289    Path parentDir = schemaPath.getParent();
290    if (parentDir != null && !Files.exists(parentDir)) {
291      Files.createDirectories(parentDir);
292    }
293
294    try (Writer writer = Files.newBufferedWriter(
295        schemaPath,
296        StandardCharsets.UTF_8,
297        getWriteOpenOptions())) {
298      schemaProducer.apply(module, writer);
299    }
300    LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
301  }
302
303  /**
304   * The the options for writing generated content.
305   *
306   * @return the options
307   */
308  @SuppressWarnings("PMD.MethodReturnsInternalArray")
309  protected OpenOption[] getWriteOpenOptions() {
310    return OPEN_OPTIONS_TRUNCATE;
311  }
312
313  @SuppressWarnings("PMD.AvoidCatchingGenericException")
314  private DynamicContainer generateScenario(
315      @NonNull TestScenario scenario,
316      @NonNull URI collectionUri,
317      @NonNull Lazy<Path> collectionGenerationPath,
318      @NonNull IBindingContext bindingContext) {
319    Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
320      Path retval;
321      try {
322        retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
323      } catch (IOException ex) {
324        throw new JUnitException("Unable to create scenario temp directory", ex);
325      }
326      return retval;
327    }));
328
329    GenerateSchema generateSchema = scenario.getGenerateSchema();
330    MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
331    URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
332
333    IModule module;
334    try {
335      IBindingModuleLoader loader = bindingContext.newModuleLoader();
336
337      module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
338    } catch (IOException | MetaschemaException ex) {
339      throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
340    }
341
342    Lazy<Path> lazySchema = Lazy.lazy(() -> generateSchema(module, scenarioGenerationPath));
343
344    Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
345      Path schemaPath = lazySchema.get();
346      return getContentValidatorSupplier().apply(schemaPath);
347    });
348    assert lazyContentValidator != null;
349
350    // build a test container for the generate and validate steps
351    DynamicTest validateSchema = DynamicTest.dynamicTest(
352        "Validate Schema",
353        () -> {
354          Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
355          if (supplier != null) {
356            Path schemaPath;
357            try {
358              schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
359            } catch (Exception ex) {
360              throw new JUnitException(
361                  "failed to generate schema", ex);
362            }
363            validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
364          }
365        });
366
367    Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
368        .flatMap(contentCase -> {
369          assert contentCase != null;
370          DynamicTest test
371              = generateValidationCase(
372                  contentCase,
373                  bindingContext,
374                  lazyContentValidator,
375                  collectionUri,
376                  ObjectUtils.notNull(scenarioGenerationPath));
377          return test == null ? Stream.empty() : Stream.of(test);
378        }).sequential();
379
380    return DynamicContainer.dynamicContainer(
381        scenario.getName(),
382        metaschemaUri,
383        Stream.concat(Stream.of(validateSchema), contentTests).sequential());
384  }
385
386  /**
387   * Perform content conversion.
388   *
389   * @param resource
390   *          the resource to convert
391   * @param generationPath
392   *          the path to write the converted resource to
393   * @param context
394   *          the Metaschema binding context
395   * @return the location of the converted content
396   * @throws IOException
397   *           if an error occurred while reading or writing content
398   * @see #getRequiredContentFormat()
399   */
400  protected Path convertContent(
401      @NonNull URI resource,
402      @NonNull Path generationPath,
403      @NonNull IBindingContext context)
404      throws IOException {
405    Object object;
406    try {
407      object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
408    } catch (URISyntaxException ex) {
409      throw new IOException(ex);
410    }
411
412    if (!Files.exists(generationPath)) {
413      Files.createDirectories(generationPath);
414    }
415
416    Path convertedContetPath;
417    try {
418      convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
419    } catch (IOException ex) {
420      throw new JUnitException(
421          String.format("Unable to create converted content path in location '%s'", generationPath),
422          ex);
423    }
424
425    Format toFormat = getRequiredContentFormat();
426    if (LOGGER.isInfoEnabled()) {
427      LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
428    }
429
430    ISerializer<?> serializer
431        = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
432    serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
433
434    return convertedContetPath;
435  }
436
437  @SuppressWarnings("PMD.AvoidCatchingGenericException")
438  private DynamicTest generateValidationCase(
439      @NonNull ContentCaseType contentCase,
440      @NonNull IBindingContext bindingContext,
441      @NonNull Lazy<IContentValidator> lazyContentValidator,
442      @NonNull URI collectionUri,
443      @NonNull Lazy<Path> resourceGenerationPath) {
444
445    URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
446
447    Format format = contentCase.getSourceFormat();
448    DynamicTest retval = null;
449    if (getRequiredContentFormat().equals(format)) {
450      retval = DynamicTest.dynamicTest(
451          String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
452              contentCase.getLocation()),
453          contentUri,
454          () -> {
455            IContentValidator contentValidator;
456            try {
457              contentValidator = lazyContentValidator.get();
458            } catch (Exception ex) {
459              throw new JUnitException( // NOPMD - cause is relevant, exception is not
460                  "failed to produce the content validator", ex.getCause());
461            }
462
463            assertEquals(
464                contentCase.getValidationResult(),
465                validateWithSchema(
466                    ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
467                "validation did not match expectation for: " + contentUri.toASCIIString());
468          });
469    } else if (contentCase.getValidationResult()) {
470      retval = DynamicTest.dynamicTest(
471          String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
472              contentCase.getLocation()),
473          contentUri,
474          () -> {
475            Path convertedContetPath;
476            try {
477              convertedContetPath = convertContent(
478                  contentUri,
479                  ObjectUtils.notNull(resourceGenerationPath.get()),
480                  bindingContext);
481            } catch (Exception ex) { // NOPMD - intentional
482              throw new JUnitException("failed to convert content: " + contentUri, ex);
483            }
484
485            IContentValidator contentValidator;
486            try {
487              contentValidator = lazyContentValidator.get();
488            } catch (Exception ex) {
489              throw new JUnitException( // NOPMD - cause is relevant, exception is not
490                  "failed to produce the content validator",
491                  ex.getCause());
492            }
493
494            if (LOGGER.isInfoEnabled()) {
495              LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
496            }
497            assertEquals(contentCase.getValidationResult(),
498                validateWithSchema(
499                    ObjectUtils.notNull(contentValidator),
500                    ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
501                String.format("validation of '%s' did not match expectation", convertedContetPath));
502          });
503    }
504    return retval;
505  }
506
507  private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
508      throws IOException {
509    IValidationResult schemaValidationResult;
510    try {
511      schemaValidationResult = validator.validate(target);
512    } catch (URISyntaxException ex) {
513      throw new IOException(ex);
514    }
515    return processValidationResult(schemaValidationResult);
516  }
517
518  /**
519   * Use the provided validator to validate the provided target.
520   *
521   * @param validator
522   *          the content validator to use
523   * @param target
524   *          the resource to validate
525   * @return {@code true} if the content is valid or {@code false} otherwise
526   * @throws IOException
527   *           if an error occurred while reading the content
528   */
529  protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
530      throws IOException {
531    IValidationResult schemaValidationResult = validator.validate(target);
532    if (!schemaValidationResult.isPassing()) {
533      LOGGER.atError().log("Schema validation failed for: {}", target);
534    }
535    return processValidationResult(schemaValidationResult);
536  }
537
538  private static boolean processValidationResult(IValidationResult schemaValidationResult) {
539    for (IValidationFinding finding : schemaValidationResult.getFindings()) {
540      logFinding(ObjectUtils.notNull(finding));
541    }
542    return schemaValidationResult.isPassing();
543  }
544
545  private static void logFinding(@NonNull IValidationFinding finding) {
546    LogBuilder logBuilder;
547    switch (finding.getSeverity()) {
548    case CRITICAL:
549      logBuilder = LOGGER.atFatal();
550      break;
551    case ERROR:
552      logBuilder = LOGGER.atError();
553      break;
554    case WARNING:
555      logBuilder = LOGGER.atWarn();
556      break;
557    case INFORMATIONAL:
558      logBuilder = LOGGER.atInfo();
559      break;
560    case DEBUG:
561      logBuilder = LOGGER.atDebug();
562      break;
563    default:
564      throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
565    }
566
567    // if (finding.getCause() != null) {
568    // logBuilder.withThrowable(finding.getCause());
569    // }
570
571    if (finding instanceof JsonValidationFinding) {
572      JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
573      logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
574    } else {
575      logBuilder.log("{}", finding.getMessage());
576    }
577  }
578}