001/* 002 * SPDX-FileCopyrightText: none 003 * SPDX-License-Identifier: CC0-1.0 004 */ 005 006package dev.metaschema.model.testing; 007 008import static org.junit.jupiter.api.Assertions.assertEquals; 009 010import org.apache.logging.log4j.LogBuilder; 011import org.apache.logging.log4j.LogManager; 012import org.apache.logging.log4j.Logger; 013import org.junit.jupiter.api.DynamicContainer; 014import org.junit.jupiter.api.DynamicNode; 015import org.junit.jupiter.api.DynamicTest; 016import org.junit.platform.commons.JUnitException; 017 018import java.io.IOException; 019import java.io.Writer; 020import java.net.URI; 021import java.net.URISyntaxException; 022import java.net.URL; 023import java.nio.charset.StandardCharsets; 024import java.nio.file.FileVisitResult; 025import java.nio.file.Files; 026import java.nio.file.OpenOption; 027import java.nio.file.Path; 028import java.nio.file.SimpleFileVisitor; 029import java.nio.file.StandardOpenOption; 030import java.nio.file.attribute.BasicFileAttributes; 031import java.util.List; 032import java.util.function.BiFunction; 033import java.util.function.Function; 034import java.util.function.Supplier; 035import java.util.stream.Stream; 036 037import dev.metaschema.core.model.IModule; 038import dev.metaschema.core.model.MetaschemaException; 039import dev.metaschema.core.model.validation.IContentValidator; 040import dev.metaschema.core.model.validation.IValidationFinding; 041import dev.metaschema.core.model.validation.IValidationResult; 042import dev.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding; 043import dev.metaschema.core.util.ObjectUtils; 044import dev.metaschema.databind.IBindingContext; 045import dev.metaschema.databind.io.Format; 046import dev.metaschema.databind.io.ISerializer; 047import dev.metaschema.databind.model.metaschema.IBindingModuleLoader; 048import dev.metaschema.model.testing.testsuite.GenerateSchema; 049import dev.metaschema.model.testing.testsuite.Metaschema; 050import dev.metaschema.model.testing.testsuite.TestCollection; 051import dev.metaschema.model.testing.testsuite.TestScenario; 052import dev.metaschema.model.testing.testsuite.TestSuite; 053import dev.metaschema.model.testing.testsuite.ValidationCase; 054import edu.umd.cs.findbugs.annotations.NonNull; 055import edu.umd.cs.findbugs.annotations.Nullable; 056import nl.talsmasoftware.lazy4j.Lazy; 057 058/** 059 * This abstract implementation dynamically produces JUnit tests based on a test 060 * suite definition. 061 * 062 * @see #getTestSuiteURI() 063 */ 064@SuppressWarnings({ 065 "PMD.GodClass", 066 "PMD.CouplingBetweenObjects" 067}) 068public abstract class AbstractTestSuite { 069 private static final Logger LOGGER = LogManager.getLogger(AbstractTestSuite.class); 070 071 private static final boolean DELETE_RESULTS_ON_EXIT = false; 072 private static final OpenOption[] OPEN_OPTIONS_TRUNCATE = { 073 StandardOpenOption.CREATE, 074 StandardOpenOption.WRITE, 075 StandardOpenOption.TRUNCATE_EXISTING 076 }; 077 078 private static final String VALID = "VALID"; 079 080 /** 081 * Get the content format used by the test suite. 082 * 083 * @return the format 084 */ 085 @NonNull 086 protected abstract Format getRequiredContentFormat(); 087 088 /** 089 * Get the resource describing the tests to execute. 090 * 091 * @return the resource 092 */ 093 @NonNull 094 protected abstract URI getTestSuiteURI(); 095 096 /** 097 * Get the filesystem location to use for generating content. 098 * 099 * @return the filesystem path 100 */ 101 @NonNull 102 protected abstract Path getGenerationPath(); 103 104 /** 105 * Get the method used to generate a schema using a given Metaschema module and 106 * writer. 107 * 108 * @return the schema generator supplier 109 */ 110 @NonNull 111 protected abstract BiFunction<IModule, Writer, Void> getSchemaGeneratorSupplier(); 112 113 /** 114 * Get the method used to provide a schema validator. 115 * 116 * @return the method as a supplier 117 */ 118 @Nullable 119 protected abstract Supplier<? extends IContentValidator> getSchemaValidatorSupplier(); 120 121 /** 122 * Get the method used to provide a content validator. 123 * 124 * @return the method as a supplier 125 */ 126 @NonNull 127 protected abstract Function<Path, ? extends IContentValidator> getContentValidatorSupplier(); 128 129 /** 130 * Dynamically generate the unit tests. 131 * 132 * @param bindingContext 133 * the Module binding context 134 * 135 * @return the steam of unit tests 136 */ 137 @NonNull 138 protected Stream<DynamicNode> testFactory(@NonNull IBindingContext bindingContext) { 139 try { 140 Path generationPath = getGenerationPath(); 141 if (Files.exists(generationPath)) { 142 if (!Files.isDirectory(generationPath)) { 143 throw new JUnitException(String.format("Generation path '%s' exists and is not a directory", generationPath)); 144 } 145 } else { 146 Files.createDirectories(generationPath); 147 } 148 149 URI testSuiteUri = getTestSuiteURI(); 150 URL testSuiteUrl = testSuiteUri.toURL(); 151 TestSuite testSuite = bindingContext.newBoundLoader().load(TestSuite.class, testSuiteUrl); 152 List<TestCollection> testCollections = testSuite.getTestCollections(); 153 return ObjectUtils.notNull(testCollections.stream() 154 .flatMap( 155 collection -> Stream 156 .of(generateCollection( 157 ObjectUtils.notNull(collection), 158 testSuiteUri, 159 generationPath, 160 bindingContext)))); 161 } catch (IOException | URISyntaxException ex) { 162 throw new JUnitException("Unable to generate tests", ex); 163 } 164 } 165 166 /** 167 * Configure removal of the provided directory after test execution. 168 * 169 * @param path 170 * the directory to configure for removal 171 */ 172 protected void deleteCollectionOnExit(@NonNull Path path) { 173 Runtime.getRuntime().addShutdownHook(new Thread( // NOPMD - this is not a webapp 174 () -> { 175 try { 176 Files.walkFileTree(path, new SimpleFileVisitor<>() { 177 @Override 178 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { 179 Files.delete(file); 180 return FileVisitResult.CONTINUE; 181 } 182 183 @Override 184 public FileVisitResult postVisitDirectory(Path dir, IOException ex) throws IOException { 185 if (ex == null) { 186 Files.delete(dir); 187 return FileVisitResult.CONTINUE; 188 } 189 // directory iteration failed for some reason 190 throw ex; 191 } 192 }); 193 } catch (IOException ex) { 194 throw new JUnitException("Failed to delete collection: " + path, ex); 195 } 196 })); 197 } 198 199 private DynamicContainer generateCollection( 200 @NonNull TestCollection collection, 201 @NonNull URI testSuiteUri, 202 @NonNull Path generationPath, 203 @NonNull IBindingContext bindingContext) { 204 URI collectionUri = testSuiteUri.resolve(collection.getLocation()); 205 assert collectionUri != null; 206 207 LOGGER.atInfo().log("Collection: " + collectionUri); 208 Lazy<Path> collectionGenerationPath = ObjectUtils.notNull(Lazy.of(() -> { 209 Path retval; 210 try { 211 retval = ObjectUtils.requireNonNull(Files.createTempDirectory(generationPath, "collection-")); 212 if (DELETE_RESULTS_ON_EXIT) { 213 deleteCollectionOnExit(ObjectUtils.requireNonNull(retval)); 214 } 215 } catch (IOException ex) { 216 throw new JUnitException("Unable to create collection temp directory", ex); 217 } 218 return retval; 219 })); 220 221 List<TestScenario> testScenarios = collection.getTestScenarios(); 222 return DynamicContainer.dynamicContainer( 223 collection.getName(), 224 testSuiteUri, 225 testScenarios.stream() 226 .flatMap(scenario -> { 227 assert scenario != null; 228 return Stream.of(generateScenario( 229 scenario, 230 collectionUri, 231 collectionGenerationPath, 232 bindingContext)); 233 }) 234 .sequential()); 235 } 236 237 @NonNull 238 private Path generateSchema( 239 @NonNull IModule module, 240 @NonNull Lazy<Path> scenarioGenerationPath) { 241 String schemaExtension; 242 Format requiredContentFormat = getRequiredContentFormat(); 243 switch (requiredContentFormat) { 244 case JSON: 245 case YAML: 246 schemaExtension = ".json"; 247 break; 248 case XML: 249 schemaExtension = ".xsd"; 250 break; 251 default: 252 throw new IllegalStateException(String.format("Unhandled content format '%s'.", requiredContentFormat)); 253 } 254 255 // determine what file to use for the schema 256 Path schemaPath; 257 try { 258 schemaPath = Files.createTempFile(scenarioGenerationPath.get(), "", "-schema" + schemaExtension); 259 } catch (IOException ex) { 260 throw new JUnitException("Unable to create schema temp file", ex); 261 } 262 try { 263 generateSchema(ObjectUtils.notNull(module), ObjectUtils.notNull(schemaPath), getSchemaGeneratorSupplier()); 264 } catch (IOException ex) { 265 throw new IllegalStateException(ex); 266 } 267 return ObjectUtils.notNull(schemaPath); 268 } 269 270 /** 271 * Generate a schema for the provided module using the provided schema 272 * generator. 273 * 274 * @param module 275 * the Metaschema module to generate the schema for 276 * @param schemaPath 277 * the location to generate the schema 278 * @param schemaProducer 279 * the method callback to use to generate the schema 280 * @throws IOException 281 * if an error occurred while writing the schema 282 */ 283 protected void generateSchema( 284 @NonNull IModule module, 285 @NonNull Path schemaPath, 286 @NonNull BiFunction<IModule, Writer, Void> schemaProducer) throws IOException { 287 Path parentDir = schemaPath.getParent(); 288 if (parentDir != null && !Files.exists(parentDir)) { 289 Files.createDirectories(parentDir); 290 } 291 292 try (Writer writer = Files.newBufferedWriter( 293 schemaPath, 294 StandardCharsets.UTF_8, 295 getWriteOpenOptions())) { 296 schemaProducer.apply(module, writer); 297 } 298 LOGGER.atInfo().log("Produced schema '{}' for module '{}'", schemaPath, module.getLocation()); 299 } 300 301 /** 302 * The the options for writing generated content. 303 * 304 * @return the options 305 */ 306 @SuppressWarnings("PMD.MethodReturnsInternalArray") 307 protected OpenOption[] getWriteOpenOptions() { 308 return OPEN_OPTIONS_TRUNCATE; 309 } 310 311 @SuppressWarnings("PMD.AvoidCatchingGenericException") 312 private DynamicContainer generateScenario( 313 @NonNull TestScenario scenario, 314 @NonNull URI collectionUri, 315 @NonNull Lazy<Path> collectionGenerationPath, 316 @NonNull IBindingContext bindingContext) { 317 Lazy<Path> scenarioGenerationPath = ObjectUtils.notNull(Lazy.of(() -> { 318 Path retval; 319 try { 320 retval = Files.createTempDirectory(collectionGenerationPath.get(), "scenario-"); 321 } catch (IOException ex) { 322 throw new JUnitException("Unable to create scenario temp directory", ex); 323 } 324 return retval; 325 })); 326 327 GenerateSchema generateSchema = scenario.getGenerateSchema(); 328 if (generateSchema == null) { 329 throw new JUnitException(String.format( 330 "Scenario '%s' does not have a generate-schema directive", scenario.getName())); 331 } 332 Metaschema metaschemaDirective = generateSchema.getMetaschema(); 333 URI metaschemaUri = collectionUri.resolve(metaschemaDirective.getLocation()); 334 335 IModule module; 336 try { 337 IBindingModuleLoader loader = bindingContext.newModuleLoader(); 338 339 module = loader.load(ObjectUtils.notNull(metaschemaUri.toURL())); 340 } catch (IOException | MetaschemaException ex) { 341 throw new JUnitException("Unable to generate classes for metaschema: " + metaschemaUri, ex); 342 } 343 344 Lazy<Path> lazySchema = Lazy.of(() -> generateSchema(module, scenarioGenerationPath)); 345 346 Lazy<IContentValidator> lazyContentValidator = Lazy.of(() -> { 347 Path schemaPath = lazySchema.get(); 348 return getContentValidatorSupplier().apply(schemaPath); 349 }); 350 assert lazyContentValidator != null; 351 352 // build a test container for the generate and validate steps 353 DynamicTest validateSchema = DynamicTest.dynamicTest( 354 "Validate Schema", 355 () -> { 356 Supplier<? extends IContentValidator> supplier = getSchemaValidatorSupplier(); 357 if (supplier != null) { 358 Path schemaPath; 359 try { 360 schemaPath = ObjectUtils.requireNonNull(lazySchema.get()); 361 } catch (Exception ex) { 362 throw new JUnitException( 363 "failed to generate schema", ex); 364 } 365 validateWithSchema(ObjectUtils.requireNonNull(supplier.get()), schemaPath); 366 } 367 }); 368 369 List<ValidationCase> validationCases = scenario.getValidationCases(); 370 Stream<? extends DynamicNode> contentTests = validationCases.stream() 371 .flatMap(contentCase -> { 372 assert contentCase != null; 373 DynamicTest test 374 = generateValidationCase( 375 contentCase, 376 bindingContext, 377 lazyContentValidator, 378 collectionUri, 379 ObjectUtils.notNull(scenarioGenerationPath)); 380 return test == null ? Stream.empty() : Stream.of(test); 381 }).sequential(); 382 383 return DynamicContainer.dynamicContainer( 384 scenario.getName(), 385 metaschemaUri, 386 Stream.concat(Stream.of(validateSchema), contentTests).sequential()); 387 } 388 389 /** 390 * Perform content conversion. 391 * 392 * @param resource 393 * the resource to convert 394 * @param generationPath 395 * the path to write the converted resource to 396 * @param context 397 * the Metaschema binding context 398 * @return the location of the converted content 399 * @throws IOException 400 * if an error occurred while reading or writing content 401 * @see #getRequiredContentFormat() 402 */ 403 protected Path convertContent( 404 @NonNull URI resource, 405 @NonNull Path generationPath, 406 @NonNull IBindingContext context) 407 throws IOException { 408 Object object; 409 try { 410 object = context.newBoundLoader().load(ObjectUtils.notNull(resource.toURL())); 411 } catch (URISyntaxException ex) { 412 throw new IOException(ex); 413 } 414 415 if (!Files.exists(generationPath)) { 416 Files.createDirectories(generationPath); 417 } 418 419 Path convertedContetPath; 420 try { 421 convertedContetPath = ObjectUtils.notNull(Files.createTempFile(generationPath, "", "-content")); 422 } catch (IOException ex) { 423 throw new JUnitException( 424 String.format("Unable to create converted content path in location '%s'", generationPath), 425 ex); 426 } 427 428 Format toFormat = getRequiredContentFormat(); 429 if (LOGGER.isInfoEnabled()) { 430 LOGGER.atInfo().log("Converting content '{}' to {} as '{}'", resource, toFormat, convertedContetPath); 431 } 432 433 ISerializer<?> serializer 434 = context.newSerializer(toFormat, ObjectUtils.asType(object.getClass())); 435 serializer.serialize(ObjectUtils.asType(object), convertedContetPath, getWriteOpenOptions()); 436 437 return convertedContetPath; 438 } 439 440 /** 441 * Convert a source format string to the corresponding Format enum value. 442 * 443 * @param sourceFormat 444 * the source format string (e.g., "XML", "JSON", "YAML") 445 * @return the corresponding Format, or null if the format is not recognized 446 */ 447 @Nullable 448 private static Format toFormat(@Nullable String sourceFormat) { 449 if (sourceFormat == null) { 450 return null; 451 } 452 switch (sourceFormat) { 453 case "XML": 454 return Format.XML; 455 case "JSON": 456 return Format.JSON; 457 case "YAML": 458 return Format.YAML; 459 default: 460 return null; 461 } 462 } 463 464 @SuppressWarnings("PMD.AvoidCatchingGenericException") 465 private DynamicTest generateValidationCase( 466 @NonNull ValidationCase contentCase, 467 @NonNull IBindingContext bindingContext, 468 @NonNull Lazy<IContentValidator> lazyContentValidator, 469 @NonNull URI collectionUri, 470 @NonNull Lazy<Path> resourceGenerationPath) { 471 472 URI contentUri = ObjectUtils.notNull(collectionUri.resolve(contentCase.getLocation())); 473 474 Format format = toFormat(contentCase.getSourceFormat()); 475 boolean expectedValid = isValid(contentCase.getValidationResult()); 476 477 DynamicTest retval = null; 478 if (getRequiredContentFormat().equals(format)) { 479 retval = DynamicTest.dynamicTest( 480 String.format("Validate %s=%s: %s", format, expectedValid, 481 contentCase.getLocation()), 482 contentUri, 483 () -> { 484 IContentValidator contentValidator; 485 try { 486 contentValidator = lazyContentValidator.get(); 487 } catch (Exception ex) { 488 throw new JUnitException( 489 "failed to produce the content validator", ex); 490 } 491 492 assertEquals( 493 expectedValid, 494 validateWithSchema( 495 ObjectUtils.notNull(contentValidator), ObjectUtils.notNull(contentUri.toURL())), 496 "validation did not match expectation for: " + contentUri.toASCIIString()); 497 }); 498 } else if (expectedValid) { 499 retval = DynamicTest.dynamicTest( 500 String.format("Convert and Validate %s=%s: %s", format, expectedValid, 501 contentCase.getLocation()), 502 contentUri, 503 () -> { 504 Path convertedContetPath; 505 try { 506 convertedContetPath = convertContent( 507 contentUri, 508 ObjectUtils.notNull(resourceGenerationPath.get()), 509 bindingContext); 510 } catch (Exception ex) { 511 throw new JUnitException("failed to convert content: " + contentUri, ex); 512 } 513 514 IContentValidator contentValidator; 515 try { 516 contentValidator = lazyContentValidator.get(); 517 } catch (Exception ex) { 518 throw new JUnitException( // NOPMD - cause is relevant, exception is not 519 "failed to produce the content validator", 520 ex.getCause()); 521 } 522 523 if (LOGGER.isInfoEnabled()) { 524 LOGGER.atInfo().log("Validating content '{}'", convertedContetPath); 525 } 526 assertEquals(expectedValid, 527 validateWithSchema( 528 ObjectUtils.notNull(contentValidator), 529 ObjectUtils.notNull(convertedContetPath.toUri().toURL())), 530 String.format("validation of '%s' did not match expectation", convertedContetPath)); 531 }); 532 } 533 return retval; 534 } 535 536 /** 537 * Check if the validation result string indicates a valid result. 538 * 539 * @param validationResult 540 * the validation result string 541 * @return {@code true} if the result indicates valid, {@code false} otherwise 542 */ 543 private static boolean isValid(@Nullable String validationResult) { 544 return VALID.equals(validationResult); 545 } 546 547 private static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull URL target) 548 throws IOException { 549 IValidationResult schemaValidationResult; 550 try { 551 schemaValidationResult = validator.validate(target); 552 } catch (URISyntaxException ex) { 553 throw new IOException(ex); 554 } 555 return processValidationResult(schemaValidationResult); 556 } 557 558 /** 559 * Use the provided validator to validate the provided target. 560 * 561 * @param validator 562 * the content validator to use 563 * @param target 564 * the resource to validate 565 * @return {@code true} if the content is valid or {@code false} otherwise 566 * @throws IOException 567 * if an error occurred while reading the content 568 */ 569 protected static boolean validateWithSchema(@NonNull IContentValidator validator, @NonNull Path target) 570 throws IOException { 571 LOGGER.atError().log("Validating: {}", target); 572 IValidationResult schemaValidationResult = validator.validate(target); 573 if (!schemaValidationResult.isPassing()) { 574 LOGGER.atError().log("Schema validation failed for: {}", target); 575 } 576 return processValidationResult(schemaValidationResult); 577 } 578 579 private static boolean processValidationResult(IValidationResult schemaValidationResult) { 580 for (IValidationFinding finding : schemaValidationResult.getFindings()) { 581 logFinding(ObjectUtils.notNull(finding)); 582 } 583 return schemaValidationResult.isPassing(); 584 } 585 586 private static void logFinding(@NonNull IValidationFinding finding) { 587 LogBuilder logBuilder; 588 switch (finding.getSeverity()) { 589 case CRITICAL: 590 logBuilder = LOGGER.atFatal(); 591 break; 592 case ERROR: 593 logBuilder = LOGGER.atError(); 594 break; 595 case WARNING: 596 logBuilder = LOGGER.atWarn(); 597 break; 598 case INFORMATIONAL: 599 logBuilder = LOGGER.atInfo(); 600 break; 601 case DEBUG: 602 logBuilder = LOGGER.atDebug(); 603 break; 604 default: 605 throw new IllegalArgumentException("Unknown level: " + finding.getSeverity().name()); 606 } 607 608 // if (finding.getCause() != null) { 609 // logBuilder.withThrowable(finding.getCause()); 610 // } 611 612 if (finding instanceof JsonValidationFinding) { 613 JsonValidationFinding jsonFinding = (JsonValidationFinding) finding; 614 logBuilder.log("[{}] {}", jsonFinding.getCause().getPointerToViolation(), finding.getMessage()); 615 } else { 616 logBuilder.log("{}", finding.getMessage()); 617 } 618 } 619}