Source code

001/*
002 * SPDX-FileCopyrightText: none
003 * SPDX-License-Identifier: CC0-1.0
004 */
005
006package gov.nist.secauto.metaschema.model.testing;
007
008import static org.junit.jupiter.api.Assertions.assertEquals;
009
010import gov.nist.secauto.metaschema.core.model.IModule;
011import gov.nist.secauto.metaschema.core.model.MetaschemaException;
012import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
013import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
014import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
015import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
016import gov.nist.secauto.metaschema.core.util.ObjectUtils;
017import gov.nist.secauto.metaschema.databind.DefaultBindingContext;
018import gov.nist.secauto.metaschema.databind.IBindingContext;
019import gov.nist.secauto.metaschema.databind.io.Format;
020import gov.nist.secauto.metaschema.databind.io.ISerializer;
021import gov.nist.secauto.metaschema.databind.model.metaschema.BindingModuleLoader;
022import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
023import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
024import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
025import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
026import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
027import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
028
029import org.apache.logging.log4j.LogBuilder;
030import org.apache.logging.log4j.LogManager;
031import org.apache.logging.log4j.Logger;
032import org.apache.xmlbeans.XmlException;
033import org.apache.xmlbeans.XmlOptions;
034import org.junit.jupiter.api.DynamicContainer;
035import org.junit.jupiter.api.DynamicNode;
036import org.junit.jupiter.api.DynamicTest;
037import org.junit.platform.commons.JUnitException;
038
039import java.io.IOException;
040import java.io.Writer;
041import java.net.URI;
042import java.net.URISyntaxException;
043import java.net.URL;
044import java.nio.charset.StandardCharsets;
045import java.nio.file.FileVisitResult;
046import java.nio.file.Files;
047import java.nio.file.OpenOption;
048import java.nio.file.Path;
049import java.nio.file.SimpleFileVisitor;
050import java.nio.file.StandardOpenOption;
051import java.nio.file.attribute.BasicFileAttributes;
052import java.util.function.BiFunction;
053import java.util.function.Function;
054import java.util.function.Supplier;
055import java.util.stream.Stream;
056
057import edu.umd.cs.findbugs.annotations.NonNull;
058import edu.umd.cs.findbugs.annotations.Nullable;
059import nl.talsmasoftware.lazy4j.Lazy;
060
061public abstract class AbstractTestSuite {
062  private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
063  private static final BindingModuleLoader LOADER;
064
065  private static final boolean DELETE_RESULTS_ON_EXIT = false;
066  private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
067      StandardOpenOption.CREATE,
068      StandardOpenOption.WRITE,
069      StandardOpenOption.TRUNCATE_EXISTING
070  };
071
072  static {
073    IBindingContext bindingContext = new DefaultBindingContext();
074    LOADER = new BindingModuleLoader(bindingContext);
075    LOADER.allowEntityResolution();
076  }
077
078  /**
079   * Get the content format used by the test suite.
080   *
081   * @return the format
082   */
083  @NonNull
084  protected abstract Format getRequiredContentFormat();
085
086  /**
087   * Get the resource describing the tests to execute.
088   *
089   * @return the resource
090   */
091  @NonNull
092  protected abstract URI getTestSuiteURI();
093
094  /**
095   * Get the filesystem location to use for generating content.
096   *
097   * @return the filesystem path
098   */
099  @NonNull
100  protected abstract Path getGenerationPath();
101
102  /**
103   * Get the method used to generate a schema using a given Metaschema module and
104   * writer.
105   *
106   * @return the schema generator supplier
107   */
108  @NonNull
109  protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
110
111  /**
112   * Get the method used to provide a schema validator.
113   *
114   * @return the method as a supplier
115   */
116  @Nullable
117  protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
118
119  /**
120   * Get the method used to provide a content validator.
121   *
122   * @return the method as a supplier
123   */
124  @NonNull
125  protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
126
127  /**
128   * Dynamically generate the unit tests.
129   *
130   * @return the steam of unit tests
131   */
132  @NonNull
133  protected Stream<DynamicNode> testFactory() {
134    try {
135      XmlOptions options = new XmlOptions();
136      options.setBaseURI(null);
137      options.setLoadLineNumbers();
138
139      Path generationPath = getGenerationPath();
140      if (Files.exists(generationPath)) {
141        if (!Files.isDirectory(generationPath)) {
142          throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
143        }
144      } else {
145        Files.createDirectories(generationPath);
146      }
147
148      URI testSuiteUri = getTestSuiteURI();
149      URL testSuiteUrl = testSuiteUri.toURL();
150      TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
151      return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
152          .flatMap(
153              collection -> Stream
154                  .of(generateCollection(ObjectUtils.notNull(collection), testSuiteUri, generationPath))));
155    } catch (XmlException | IOException ex) {
156      throw new JUnitException("Unable to generate tests", ex);
157    }
158  }
159
160  /**
161   * Configure removal of the provided directory after test execution.
162   *
163   * @param path
164   *          the directory to configure for removal
165   */
166  protected void deleteCollectionOnExit(@NonNull Path path) {
167    Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
168        () -> {
169          try {
170            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
171              @Override
172              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
173                Files.delete(file);
174                return FileVisitResult.CONTINUE;
175              }
176
177              @Override
178              public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
179                if (ex == null) {
180                  Files.delete(dir);
181                  return FileVisitResult.CONTINUE;
182                }
183                // directory iteration failed for some reason
184                throw ex;
185              }
186            });
187          } catch (IOException ex) {
188            throw new JUnitException("Failed to delete collection: " + path, ex);
189          }
190        }));
191  }
192
193  private DynamicContainer generateCollection(@NonNull TestCollection collection, @NonNull URI testSuiteUri,
194      @NonNull Path generationPath) {
195    URI collectionUri = testSuiteUri.resolve(collection.getLocation());
196    assert collectionUri != null;
197
198    LOGGER.atInfo().log("Collection: " + collectionUri);
199    Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
200      Path retval;
201      try {
202        retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
203        if (DELETE_RESULTS_ON_EXIT) {
204          deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
205        }
206      } catch (IOException ex) {
207        throw new JUnitException("Unable to create collection temp directory", ex);
208      }
209      return retval;
210    }));
211
212    return DynamicContainer.dynamicContainer(
213        collection.getName(),
214        testSuiteUri,
215        collection.getTestScenarioList().stream()
216            .flatMap(scenario -> {
217              assert scenario != null;
218              return Stream.of(generateScenario(scenario, collectionUri, collectionGenerationPath));
219            })
220            .sequential());
221  }
222
223  /**
224   * Generate a schema for the provided module using the provided schema
225   * generator.
226   *
227   * @param module
228   *          the Metaschema module to generate the schema for
229   * @param schemaPath
230   *          the location to generate the schema
231   * @param schemaProducer
232   *          the method callback to use to generate the schema
233   * @throws IOException
234   *           if an error occurred while writing the schema
235   */
236  protected void generateSchema(
237      @NonNull IModule module,
238      @NonNull Path schemaPath,
239      @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
240    Path parentDir = schemaPath.getParent();
241    if (parentDir != null && !Files.exists(parentDir)) {
242      Files.createDirectories(parentDir);
243    }
244
245    try (Writer writer = Files.newBufferedWriter(
246        schemaPath,
247        StandardCharsets.UTF_8,
248        getWriteOpenOptions())) {
249      schemaProducer.apply(module, writer);
250    }
251    LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
252  }
253
254  /**
255   * The the options for writing generated content.
256   *
257   * @return the options
258   */
259  protected OpenOption[] getWriteOpenOptions() {
260    return OPEN_OPTIONS_TRUNCATE;
261  }
262
263  private DynamicContainer generateScenario(
264      @NonNull TestScenario scenario,
265      @NonNull URI collectionUri,
266      @NonNull Lazy<Path> collectionGenerationPath) {
267    Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
268      Path retval;
269      try {
270        retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
271      } catch (IOException ex) {
272        throw new JUnitException("Unable to create scenario temp directory", ex);
273      }
274      return retval;
275    });
276
277    // try {
278    // // create the directories the schema will be stored in
279    // Files.createDirectories(scenarioGenerationPath);
280    // } catch (IOException ex) {
281    // throw new JUnitException("Unable to create test directories for path: " +
282    // scenarioGenerationPath, ex);
283    // }
284
285    GenerateSchema generateSchema = scenario.getGenerateSchema();
286    MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
287    URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
288
289    Lazy<IModule> lazyModule = Lazy.lazy(() -> {
290      IModule module;
291      try {
292        module = LOADER.load(ObjectUtils.notNull(metaschemaUri.toURL()));
293      } catch (IOException | MetaschemaException ex) {
294        throw new JUnitException("Unable to generate schema for Module: " + metaschemaUri, ex);
295      }
296      return module;
297    });
298
299    Lazy<Path> lazySchema = Lazy.lazy(() -> {
300      Path schemaPath;
301
302      String schemaExtension;
303      Format requiredContentFormat = getRequiredContentFormat();
304      switch (requiredContentFormat) {
305      case JSON:
306      case YAML:
307        schemaExtension = ".json";
308        break;
309      case XML:
310        schemaExtension = ".xsd";
311        break;
312      default:
313        throw new IllegalStateException();
314      }
315
316      // determine what file to use for the schema
317      try {
318        schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
319      } catch (IOException ex) {
320        throw new JUnitException("Unable to create schema temp file", ex);
321      }
322      IModule module = lazyModule.get();
323      try {
324        generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
325      } catch (IOException ex) {
326        throw new IllegalStateException(ex);
327      }
328      return schemaPath;
329    });
330
331    Lazy<IBindingContext> lazyDynamicBindingContext = Lazy.lazy(() -> {
332      IModule module = lazyModule.get();
333      IBindingContext context;
334      try {
335        context = new DefaultBindingContext();
336        context.registerModule(
337            ObjectUtils.notNull(module),
338            ObjectUtils.notNull(scenarioGenerationPath.get()));
339      } catch (Exception ex) { // NOPMD - intentional
340        throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
341      }
342      return context;
343    });
344    assert lazyDynamicBindingContext != null;
345
346    Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
347      Path schemaPath = lazySchema.get();
348      return getContentValidatorSupplier().apply(schemaPath);
349    });
350    assert lazyContentValidator != null;
351
352    // build a test container for the generate and validate steps
353    DynamicTest validateSchema = DynamicTest.dynamicTest(
354        "Validate Schema",
355        () -> {
356          Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
357          if (supplier != null) {
358            Path schemaPath;
359            try {
360              schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
361            } catch (Exception ex) {
362              throw new JUnitException( // NOPMD - cause is relevant, exception is not
363                  "failed to generate schema", ex.getCause());
364            }
365            validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
366          }
367        });
368
369    Stream<? extends DynamicNode> contentTests;
370    {
371      contentTests = scenario.getValidationCaseList().stream()
372          .flatMap(contentCase -> {
373            assert contentCase != null;
374            DynamicTest test
375                = generateValidationCase(
376                    contentCase,
377                    lazyDynamicBindingContext,
378                    lazyContentValidator,
379                    collectionUri,
380                    ObjectUtils.notNull(scenarioGenerationPath));
381            return test == null ? Stream.empty() : Stream.of(test);
382          }).sequential();
383    }
384
385    return DynamicContainer.dynamicContainer(
386        scenario.getName(),
387        metaschemaUri,
388        Stream.concat(Stream.of(validateSchema), contentTests).sequential());
389  }
390
391  /**
392   * Perform content conversion.
393   *
394   * @param resource
395   *          the resource to convert
396   * @param generationPath
397   *          the path to write the converted resource to
398   * @param context
399   *          the Metaschema binding context
400   * @return the location of the converted content
401   * @throws IOException
402   *           if an error occurred while reading or writing content
403   * @see #getRequiredContentFormat()
404   */
405  protected Path convertContent(
406      @NonNull URI resource,
407      @NonNull Path generationPath,
408      @NonNull IBindingContext context)
409      throws IOException {
410    Object object;
411    try {
412      object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
413    } catch (URISyntaxException ex) {
414      throw new IOException(ex);
415    }
416
417    if (!Files.exists(generationPath)) {
418      Files.createDirectories(generationPath);
419    }
420
421    Path convertedContetPath;
422    try {
423      convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
424    } catch (IOException ex) {
425      throw new JUnitException("Unable to create schema temp file", ex);
426    }
427
428    @SuppressWarnings("rawtypes") ISerializer serializer
429        = context.newSerializer(getRequiredContentFormat(), ObjectUtils.asType(object.getClass()));
430    serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
431
432    return convertedContetPath;
433  }
434
435  private DynamicTest generateValidationCase(
436      @NonNull ContentCaseType contentCase,
437      @NonNull Lazy<IBindingContext> lazyBindingContext,
438      @NonNull Lazy<IContentValidator> lazyContentValidator,
439      @NonNull URI collectionUri,
440      @NonNull Lazy<Path> generationPath) {
441
442    URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
443
444    Format format = contentCase.getSourceFormat();
445    DynamicTest retval = null;
446    if (getRequiredContentFormat().equals(format)) {
447      retval = DynamicTest.dynamicTest(
448          String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
449              contentCase.getLocation()),
450          contentUri,
451          () -> {
452            IContentValidator contentValidator;
453            try {
454              contentValidator = lazyContentValidator.get();
455            } catch (Exception ex) {
456              throw new JUnitException( // NOPMD - cause is relevant, exception is not
457                  "failed to produce the content validator", ex.getCause());
458            }
459
460            assertEquals(
461                contentCase.getValidationResult(),
462                validateWithSchema(
463                    ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
464                "validation did not match expectation for: " + contentUri.toASCIIString());
465          });
466    } else if (contentCase.getValidationResult()) {
467      retval = DynamicTest.dynamicTest(
468          String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
469              contentCase.getLocation()),
470          contentUri,
471          () -> {
472            IBindingContext context;
473            try {
474              context = lazyBindingContext.get();
475            } catch (Exception ex) {
476              throw new JUnitException( // NOPMD - cause is relevant, exception is not
477                  "failed to produce the content validator", ex.getCause());
478            }
479            assert context != null;
480
481            Path convertedContetPath;
482            try {
483              convertedContetPath = convertContent(contentUri, ObjectUtils.notNull(generationPath.get()), context);
484            } catch (Exception ex) { // NOPMD - intentional
485              throw new JUnitException("failed to convert content: " + contentUri, ex);
486            }
487
488            IContentValidator contentValidator;
489            try {
490              contentValidator = lazyContentValidator.get();
491            } catch (Exception ex) {
492              throw new JUnitException( // NOPMD - cause is relevant, exception is not
493                  "failed to produce the content validator",
494                  ex.getCause());
495            }
496            assertEquals(contentCase.getValidationResult(),
497                validateWithSchema(
498                    ObjectUtils.notNull(contentValidator),
499                    ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
500                String.format("validation of '%s' did not match expectation", convertedContetPath));
501          });
502    }
503    return retval;
504  }
505
506  private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
507      throws IOException {
508    IValidationResult schemaValidationResult;
509    try {
510      schemaValidationResult = validator.validate(target);
511    } catch (URISyntaxException ex) {
512      throw new IOException(ex);
513    }
514    return processValidationResult(schemaValidationResult);
515  }
516
517  /**
518   * Use the provided validator to validate the provided target.
519   *
520   * @param validator
521   *          the content validator to use
522   * @param target
523   *          the resource to validate
524   * @return {@code true} if the content is valid or {@code false} otherwise
525   * @throws IOException
526   *           if an error occurred while reading the content
527   */
528  protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
529      throws IOException {
530    IValidationResult schemaValidationResult = validator.validate(target);
531    if (!schemaValidationResult.isPassing()) {
532      LOGGER.atError().log("Schema validation failed for: {}", target);
533    }
534    return processValidationResult(schemaValidationResult);
535  }
536
537  private static boolean processValidationResult(IValidationResult schemaValidationResult) {
538    for (IValidationFinding finding : schemaValidationResult.getFindings()) {
539      logFinding(ObjectUtils.notNull(finding));
540    }
541    return schemaValidationResult.isPassing();
542  }
543
544  private static void logFinding(@NonNull IValidationFinding finding) {
545    LogBuilder logBuilder;
546    switch (finding.getSeverity()) {
547    case CRITICAL:
548      logBuilder = LOGGER.atFatal();
549      break;
550    case ERROR:
551      logBuilder = LOGGER.atError();
552      break;
553    case WARNING:
554      logBuilder = LOGGER.atWarn();
555      break;
556    case INFORMATIONAL:
557      logBuilder = LOGGER.atInfo();
558      break;
559    case DEBUG:
560      logBuilder = LOGGER.atDebug();
561      break;
562    default:
563      throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
564    }
565
566    // if (finding.getCause() != null) {
567    // logBuilder.withThrowable(finding.getCause());
568    // }
569
570    if (finding instanceof JsonValidationFinding) {
571      JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
572      logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
573    } else {
574      logBuilder.log("{}", finding.getMessage());
575    }
576  }
577}