Source code

001/*
002 * SPDX-FileCopyrightText: none
003 * SPDX-License-Identifier: CC0-1.0
004 */
005
006package gov.nist.secauto.metaschema.model.testing;
007
008import static org.junit.jupiter.api.Assertions.assertEquals;
009
010import gov.nist.secauto.metaschema.core.model.IModule;
011import gov.nist.secauto.metaschema.core.model.MetaschemaException;
012import gov.nist.secauto.metaschema.core.model.validation.IContentValidator;
013import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
014import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
015import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
016import gov.nist.secauto.metaschema.core.util.ObjectUtils;
017import gov.nist.secauto.metaschema.databind.IBindingContext;
018import gov.nist.secauto.metaschema.databind.io.Format;
019import gov.nist.secauto.metaschema.databind.io.ISerializer;
020import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
021import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.ContentCaseType;
022import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.GenerateSchemaDocument.GenerateSchema;
023import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.MetaschemaDocument;
024import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestCollectionDocument.TestCollection;
025import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestScenarioDocument.TestScenario;
026import gov.nist.secauto.metaschema.model.testing.xml.xmlbeans.TestSuiteDocument;
027
028import org.apache.logging.log4j.LogBuilder;
029import org.apache.logging.log4j.LogManager;
030import org.apache.logging.log4j.Logger;
031import org.apache.xmlbeans.XmlException;
032import org.apache.xmlbeans.XmlOptions;
033import org.junit.jupiter.api.DynamicContainer;
034import org.junit.jupiter.api.DynamicNode;
035import org.junit.jupiter.api.DynamicTest;
036import org.junit.platform.commons.JUnitException;
037
038import java.io.IOException;
039import java.io.Writer;
040import java.net.URI;
041import java.net.URISyntaxException;
042import java.net.URL;
043import java.nio.charset.StandardCharsets;
044import java.nio.file.FileVisitResult;
045import java.nio.file.Files;
046import java.nio.file.OpenOption;
047import java.nio.file.Path;
048import java.nio.file.SimpleFileVisitor;
049import java.nio.file.StandardOpenOption;
050import java.nio.file.attribute.BasicFileAttributes;
051import java.util.function.BiFunction;
052import java.util.function.Function;
053import java.util.function.Supplier;
054import java.util.stream.Stream;
055
056import edu.umd.cs.findbugs.annotations.NonNull;
057import edu.umd.cs.findbugs.annotations.Nullable;
058import nl.talsmasoftware.lazy4j.Lazy;
059
060public abstract class AbstractTestSuite {
061  private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class);
062
063  private static final boolean DELETE_RESULTS_ON_EXIT = false;
064  private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = {
065      StandardOpenOption.CREATE,
066      StandardOpenOption.WRITE,
067      StandardOpenOption.TRUNCATE_EXISTING
068  };
069
070  /**
071   * Get the content format used by the test suite.
072   *
073   * @return the format
074   */
075  @NonNull
076  protected abstract Format getRequiredContentFormat();
077
078  /**
079   * Get the resource describing the tests to execute.
080   *
081   * @return the resource
082   */
083  @NonNull
084  protected abstract URI getTestSuiteURI();
085
086  /**
087   * Get the filesystem location to use for generating content.
088   *
089   * @return the filesystem path
090   */
091  @NonNull
092  protected abstract Path getGenerationPath();
093
094  /**
095   * Get the method used to generate a schema using a given Metaschema module and
096   * writer.
097   *
098   * @return the schema generator supplier
099   */
100  @NonNull
101  protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier();
102
103  /**
104   * Get the method used to provide a schema validator.
105   *
106   * @return the method as a supplier
107   */
108  @Nullable
109  protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier();
110
111  /**
112   * Get the method used to provide a content validator.
113   *
114   * @return the method as a supplier
115   */
116  @NonNull
117  protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier();
118
119  /**
120   * Dynamically generate the unit tests.
121   *
122   * @return the steam of unit tests
123   */
124  @NonNull
125  protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) {
126    try {
127      XmlOptions options = new XmlOptions();
128      options.setBaseURI(null);
129      options.setLoadLineNumbers();
130
131      Path generationPath = getGenerationPath();
132      if (Files.exists(generationPath)) {
133        if (!Files.isDirectory(generationPath)) {
134          throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath));
135        }
136      } else {
137        Files.createDirectories(generationPath);
138      }
139
140      URI testSuiteUri = getTestSuiteURI();
141      URL testSuiteUrl = testSuiteUri.toURL();
142      TestSuiteDocument directive = TestSuiteDocument.Factory.parse(testSuiteUrl, options);
143      return ObjectUtils.notNull(directive.getTestSuite().getTestCollectionList().stream()
144          .flatMap(
145              collection -> Stream
146                  .of(generateCollection(
147                      ObjectUtils.notNull(collection),
148                      testSuiteUri,
149                      generationPath,
150                      bindingContext))));
151    } catch (XmlException | IOException ex) {
152      throw new JUnitException("Unable to generate tests", ex);
153    }
154  }
155
156  /**
157   * Configure removal of the provided directory after test execution.
158   *
159   * @param path
160   *          the directory to configure for removal
161   */
162  protected void deleteCollectionOnExit(@NonNull Path path) {
163    Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp
164        () -> {
165          try {
166            Files.walkFileTree(path, new SimpleFileVisitor<>() {
167              @Override
168              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
169                Files.delete(file);
170                return FileVisitResult.CONTINUE;
171              }
172
173              @Override
174              public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException {
175                if (ex == null) {
176                  Files.delete(dir);
177                  return FileVisitResult.CONTINUE;
178                }
179                // directory iteration failed for some reason
180                throw ex;
181              }
182            });
183          } catch (IOException ex) {
184            throw new JUnitException("Failed to delete collection: " + path, ex);
185          }
186        }));
187  }
188
189  private DynamicContainer generateCollection(
190      @NonNull TestCollection collection,
191      @NonNull URI testSuiteUri,
192      @NonNull Path generationPath,
193      @NonNull IBindingContext bindingContext) {
194    URI collectionUri = testSuiteUri.resolve(collection.getLocation());
195    assert collectionUri != null;
196
197    LOGGER.atInfo().log("Collection: " + collectionUri);
198    Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.lazy(() -> {
199      Path retval;
200      try {
201        retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-"));
202        if (DELETE_RESULTS_ON_EXIT) {
203          deleteCollectionOnExit(ObjectUtils.requireNonNull(retval));
204        }
205      } catch (IOException ex) {
206        throw new JUnitException("Unable to create collection temp directory", ex);
207      }
208      return retval;
209    }));
210
211    return DynamicContainer.dynamicContainer(
212        collection.getName(),
213        testSuiteUri,
214        collection.getTestScenarioList().stream()
215            .flatMap(scenario -> {
216              assert scenario != null;
217              return Stream.of(generateScenario(
218                  scenario,
219                  collectionUri,
220                  collectionGenerationPath,
221                  bindingContext));
222            })
223            .sequential());
224  }
225
226  /**
227   * Generate a schema for the provided module using the provided schema
228   * generator.
229   *
230   * @param module
231   *          the Metaschema module to generate the schema for
232   * @param schemaPath
233   *          the location to generate the schema
234   * @param schemaProducer
235   *          the method callback to use to generate the schema
236   * @throws IOException
237   *           if an error occurred while writing the schema
238   */
239  protected void generateSchema(
240      @NonNull IModule module,
241      @NonNull Path schemaPath,
242      @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException {
243    Path parentDir = schemaPath.getParent();
244    if (parentDir != null && !Files.exists(parentDir)) {
245      Files.createDirectories(parentDir);
246    }
247
248    try (Writer writer = Files.newBufferedWriter(
249        schemaPath,
250        StandardCharsets.UTF_8,
251        getWriteOpenOptions())) {
252      schemaProducer.apply(module, writer);
253    }
254    LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation());
255  }
256
257  /**
258   * The the options for writing generated content.
259   *
260   * @return the options
261   */
262  @SuppressWarnings("PMD.MethodReturnsInternalArray")
263  protected OpenOption[] getWriteOpenOptions() {
264    return OPEN_OPTIONS_TRUNCATE;
265  }
266
267  @SuppressWarnings("PMD.AvoidCatchingGenericException")
268  private DynamicContainer generateScenario(
269      @NonNull TestScenario scenario,
270      @NonNull URI collectionUri,
271      @NonNull Lazy<Path> collectionGenerationPath,
272      @NonNull IBindingContext bindingContext) {
273    Lazy<Path> scenarioGenerationPath = Lazy.lazy(() -> {
274      Path retval;
275      try {
276        retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-");
277      } catch (IOException ex) {
278        throw new JUnitException("Unable to create scenario temp directory", ex);
279      }
280      return retval;
281    });
282
283    // try {
284    // // create the directories the schema will be stored in
285    // Files.createDirectories(scenarioGenerationPath);
286    // } catch (IOException ex) {
287    // throw new JUnitException("Unable to create test directories for path: " +
288    // scenarioGenerationPath, ex);
289    // }
290
291    GenerateSchema generateSchema = scenario.getGenerateSchema();
292    MetaschemaDocument.Metaschema metaschemaDirective = generateSchema.getMetaschema();
293    URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation());
294
295    IModule module;
296    try {
297      IBindingModuleLoader loader = bindingContext.newModuleLoader();
298
299      module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL()));
300    } catch (IOException | MetaschemaException ex) {
301      throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex);
302    }
303
304    Lazy<Path> lazySchema = Lazy.lazy(() -> {
305      String schemaExtension;
306      Format requiredContentFormat = getRequiredContentFormat();
307      switch (requiredContentFormat) {
308      case JSON:
309      case YAML:
310        schemaExtension = ".json";
311        break;
312      case XML:
313        schemaExtension = ".xsd";
314        break;
315      default:
316        throw new IllegalStateException();
317      }
318
319      // determine what file to use for the schema
320      Path schemaPath;
321      try {
322        schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension);
323      } catch (IOException ex) {
324        throw new JUnitException("Unable to create schema temp file", ex);
325      }
326      try {
327        generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier());
328      } catch (IOException ex) {
329        throw new IllegalStateException(ex);
330      }
331      return schemaPath;
332    });
333
334    Lazy<IContentValidator> lazyContentValidator = Lazy.lazy(() -> {
335      Path schemaPath = lazySchema.get();
336      return getContentValidatorSupplier().apply(schemaPath);
337    });
338    assert lazyContentValidator != null;
339
340    // build a test container for the generate and validate steps
341    DynamicTest validateSchema = DynamicTest.dynamicTest(
342        "Validate Schema",
343        () -> {
344          Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier();
345          if (supplier != null) {
346            Path schemaPath;
347            try {
348              schemaPath = ObjectUtils.requireNonNull(lazySchema.get());
349            } catch (Exception ex) {
350              throw new JUnitException(
351                  "failed to generate schema", ex);
352            }
353            validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath);
354          }
355        });
356
357    Stream<? extends DynamicNode> contentTests = scenario.getValidationCaseList().stream()
358        .flatMap(contentCase -> {
359          assert contentCase != null;
360          DynamicTest test
361              = generateValidationCase(
362                  contentCase,
363                  bindingContext,
364                  lazyContentValidator,
365                  collectionUri,
366                  ObjectUtils.notNull(scenarioGenerationPath));
367          return test == null ? Stream.empty() : Stream.of(test);
368        }).sequential();
369
370    return DynamicContainer.dynamicContainer(
371        scenario.getName(),
372        metaschemaUri,
373        Stream.concat(Stream.of(validateSchema), contentTests).sequential());
374  }
375
376  /**
377   * Perform content conversion.
378   *
379   * @param resource
380   *          the resource to convert
381   * @param generationPath
382   *          the path to write the converted resource to
383   * @param context
384   *          the Metaschema binding context
385   * @return the location of the converted content
386   * @throws IOException
387   *           if an error occurred while reading or writing content
388   * @see #getRequiredContentFormat()
389   */
390  protected Path convertContent(
391      @NonNull URI resource,
392      @NonNull Path generationPath,
393      @NonNull IBindingContext context)
394      throws IOException {
395    Object object;
396    try {
397      object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL()));
398    } catch (URISyntaxException ex) {
399      throw new IOException(ex);
400    }
401
402    if (!Files.exists(generationPath)) {
403      Files.createDirectories(generationPath);
404    }
405
406    Path convertedContetPath;
407    try {
408      convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content"));
409    } catch (IOException ex) {
410      throw new JUnitException(
411          String.format("Unable to create converted content path in location '%s'", generationPath),
412          ex);
413    }
414
415    Format toFormat = getRequiredContentFormat();
416    if (LOGGER.isInfoEnabled()) {
417      LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath);
418    }
419
420    ISerializer<?> serializer
421        = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass()));
422    serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions());
423
424    return convertedContetPath;
425  }
426
427  @SuppressWarnings("PMD.AvoidCatchingGenericException")
428  private DynamicTest generateValidationCase(
429      @NonNull ContentCaseType contentCase,
430      @NonNull IBindingContext bindingContext,
431      @NonNull Lazy<IContentValidator> lazyContentValidator,
432      @NonNull URI collectionUri,
433      @NonNull Lazy<Path> resourceGenerationPath) {
434
435    URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation()));
436
437    Format format = contentCase.getSourceFormat();
438    DynamicTest retval = null;
439    if (getRequiredContentFormat().equals(format)) {
440      retval = DynamicTest.dynamicTest(
441          String.format("Validate %s=%s: %s", format, contentCase.getValidationResult(),
442              contentCase.getLocation()),
443          contentUri,
444          () -> {
445            IContentValidator contentValidator;
446            try {
447              contentValidator = lazyContentValidator.get();
448            } catch (Exception ex) {
449              throw new JUnitException( // NOPMD - cause is relevant, exception is not
450                  "failed to produce the content validator", ex.getCause());
451            }
452
453            assertEquals(
454                contentCase.getValidationResult(),
455                validateWithSchema(
456                    ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())),
457                "validation did not match expectation for: " + contentUri.toASCIIString());
458          });
459    } else if (contentCase.getValidationResult()) {
460      retval = DynamicTest.dynamicTest(
461          String.format("Convert and Validate %s=%s: %s", format, contentCase.getValidationResult(),
462              contentCase.getLocation()),
463          contentUri,
464          () -> {
465            Path convertedContetPath;
466            try {
467              convertedContetPath = convertContent(
468                  contentUri,
469                  ObjectUtils.notNull(resourceGenerationPath.get()),
470                  bindingContext);
471            } catch (Exception ex) { // NOPMD - intentional
472              throw new JUnitException("failed to convert content: " + contentUri, ex);
473            }
474
475            IContentValidator contentValidator;
476            try {
477              contentValidator = lazyContentValidator.get();
478            } catch (Exception ex) {
479              throw new JUnitException( // NOPMD - cause is relevant, exception is not
480                  "failed to produce the content validator",
481                  ex.getCause());
482            }
483
484            if (LOGGER.isInfoEnabled()) {
485              LOGGER.atInfo().log("Validating content '{}'", convertedContetPath);
486            }
487            assertEquals(contentCase.getValidationResult(),
488                validateWithSchema(
489                    ObjectUtils.notNull(contentValidator),
490                    ObjectUtils.notNull(convertedContetPath.toUri().toURL())),
491                String.format("validation of '%s' did not match expectation", convertedContetPath));
492          });
493    }
494    return retval;
495  }
496
497  private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target)
498      throws IOException {
499    IValidationResult schemaValidationResult;
500    try {
501      schemaValidationResult = validator.validate(target);
502    } catch (URISyntaxException ex) {
503      throw new IOException(ex);
504    }
505    return processValidationResult(schemaValidationResult);
506  }
507
508  /**
509   * Use the provided validator to validate the provided target.
510   *
511   * @param validator
512   *          the content validator to use
513   * @param target
514   *          the resource to validate
515   * @return {@code true} if the content is valid or {@code false} otherwise
516   * @throws IOException
517   *           if an error occurred while reading the content
518   */
519  protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target)
520      throws IOException {
521    IValidationResult schemaValidationResult = validator.validate(target);
522    if (!schemaValidationResult.isPassing()) {
523      LOGGER.atError().log("Schema validation failed for: {}", target);
524    }
525    return processValidationResult(schemaValidationResult);
526  }
527
528  private static boolean processValidationResult(IValidationResult schemaValidationResult) {
529    for (IValidationFinding finding : schemaValidationResult.getFindings()) {
530      logFinding(ObjectUtils.notNull(finding));
531    }
532    return schemaValidationResult.isPassing();
533  }
534
535  private static void logFinding(@NonNull IValidationFinding finding) {
536    LogBuilder logBuilder;
537    switch (finding.getSeverity()) {
538    case CRITICAL:
539      logBuilder = LOGGER.atFatal();
540      break;
541    case ERROR:
542      logBuilder = LOGGER.atError();
543      break;
544    case WARNING:
545      logBuilder = LOGGER.atWarn();
546      break;
547    case INFORMATIONAL:
548      logBuilder = LOGGER.atInfo();
549      break;
550    case DEBUG:
551      logBuilder = LOGGER.atDebug();
552      break;
553    default:
554      throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name());
555    }
556
557    // if (finding.getCause() != null) {
558    // logBuilder.withThrowable(finding.getCause());
559    // }
560
561    if (finding instanceof JsonValidationFinding) {
562      JsonValidationFinding jsonFinding = (JsonValidationFinding) finding;
563      logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage());
564    } else {
565      logBuilder.log("{}", finding.getMessage());
566    }
567  }
568}