1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package gov.nist.secauto.metaschema.maven.plugin;
7   
8   import gov.nist.secauto.metaschema.core.metapath.MetapathException;
9   import gov.nist.secauto.metaschema.core.model.IConstraintLoader;
10  import gov.nist.secauto.metaschema.core.model.IModule;
11  import gov.nist.secauto.metaschema.core.model.IResourceLocation;
12  import gov.nist.secauto.metaschema.core.model.MetaschemaException;
13  import gov.nist.secauto.metaschema.core.model.constraint.ConstraintValidationFinding;
14  import gov.nist.secauto.metaschema.core.model.constraint.ExternalConstraintsModulePostProcessor;
15  import gov.nist.secauto.metaschema.core.model.constraint.IConstraintSet;
16  import gov.nist.secauto.metaschema.core.model.validation.AbstractValidationResultProcessor;
17  import gov.nist.secauto.metaschema.core.model.validation.IValidationFinding;
18  import gov.nist.secauto.metaschema.core.model.validation.IValidationResult;
19  import gov.nist.secauto.metaschema.core.model.validation.JsonSchemaContentValidator.JsonValidationFinding;
20  import gov.nist.secauto.metaschema.core.model.validation.XmlSchemaContentValidator.XmlValidationFinding;
21  import gov.nist.secauto.metaschema.core.util.CollectionUtil;
22  import gov.nist.secauto.metaschema.core.util.ObjectUtils;
23  import gov.nist.secauto.metaschema.databind.DefaultBindingContext;
24  import gov.nist.secauto.metaschema.databind.IBindingContext;
25  import gov.nist.secauto.metaschema.databind.PostProcessingModuleLoaderStrategy;
26  import gov.nist.secauto.metaschema.databind.SimpleModuleLoaderStrategy;
27  import gov.nist.secauto.metaschema.databind.codegen.IGeneratedClass;
28  import gov.nist.secauto.metaschema.databind.codegen.IGeneratedModuleClass;
29  import gov.nist.secauto.metaschema.databind.codegen.IModuleBindingGenerator;
30  import gov.nist.secauto.metaschema.databind.codegen.IProduction;
31  import gov.nist.secauto.metaschema.databind.codegen.JavaCompilerSupport;
32  import gov.nist.secauto.metaschema.databind.codegen.JavaGenerator;
33  import gov.nist.secauto.metaschema.databind.codegen.ModuleCompilerHelper;
34  import gov.nist.secauto.metaschema.databind.codegen.config.DefaultBindingConfiguration;
35  import gov.nist.secauto.metaschema.databind.codegen.config.IBindingConfiguration;
36  import gov.nist.secauto.metaschema.databind.model.IBoundModule;
37  import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingMetaschemaModule;
38  import gov.nist.secauto.metaschema.databind.model.metaschema.IBindingModuleLoader;
39  import gov.nist.secauto.metaschema.databind.model.metaschema.binding.MetaschemaModelModule;
40  
41  import org.apache.maven.artifact.Artifact;
42  import org.apache.maven.artifact.DependencyResolutionRequiredException;
43  import org.apache.maven.plugin.AbstractMojo;
44  import org.apache.maven.plugin.MojoExecution;
45  import org.apache.maven.plugin.MojoExecutionException;
46  import org.apache.maven.plugin.logging.Log;
47  import org.apache.maven.plugins.annotations.Component;
48  import org.apache.maven.plugins.annotations.Parameter;
49  import org.apache.maven.project.MavenProject;
50  import org.codehaus.plexus.util.DirectoryScanner;
51  import org.sonatype.plexus.build.incremental.BuildContext;
52  import org.xml.sax.SAXParseException;
53  
54  import java.io.File;
55  import java.io.IOException;
56  import java.io.OutputStream;
57  import java.net.URI;
58  import java.nio.charset.Charset;
59  import java.nio.file.Files;
60  import java.nio.file.Path;
61  import java.nio.file.Paths;
62  import java.nio.file.StandardOpenOption;
63  import java.util.ArrayList;
64  import java.util.Collection;
65  import java.util.HashSet;
66  import java.util.LinkedHashSet;
67  import java.util.List;
68  import java.util.Objects;
69  import java.util.Set;
70  import java.util.function.Function;
71  import java.util.stream.Collectors;
72  import java.util.stream.Stream;
73  
74  import javax.tools.DiagnosticCollector;
75  
76  import edu.umd.cs.findbugs.annotations.NonNull;
77  import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
78  
79  public abstract class AbstractMetaschemaMojo
80      extends AbstractMojo {
81    private static final String[] DEFAULT_INCLUDES = { "**/*.xml" };
82  
83    /**
84     * The Maven project context.
85     *
86     * @required
87     * @readonly
88     */
89    @Parameter(defaultValue = "${project}", required = true, readonly = true)
90    MavenProject mavenProject;
91  
92    /**
93     * This will be injected if this plugin is executed as part of the standard
94     * Maven lifecycle. If the mojo is directly invoked, this parameter will not be
95     * injected.
96     */
97    @Parameter(defaultValue = "${mojoExecution}", readonly = true)
98    private MojoExecution mojoExecution;
99  
100   @Component
101   private BuildContext buildContext;
102 
103   @Parameter(defaultValue = "${plugin.artifacts}", readonly = true, required = true)
104   private List<Artifact> pluginArtifacts;
105 
106   /**
107    * <p>
108    * The directory where the staleFile is found. The staleFile is used to
109    * determine if re-generation of generated Java classes is needed, by recording
110    * when the last build occurred.
111    * </p>
112    * <p>
113    * This directory is expected to be located within the
114    * <code>${project.build.directory}</code>, to ensure that code (re)generation
115    * occurs after cleaning the project.
116    * </p>
117    */
118   @Parameter(defaultValue = "${project.build.directory}/metaschema", readonly = true, required = true)
119   protected File staleFileDirectory;
120 
121   /**
122    * <p>
123    * Defines the encoding used for generating Java Source files.
124    * </p>
125    * <p>
126    * The algorithm for finding the encoding to use is as follows (where the first
127    * non-null value found is used for encoding):
128    * <ol>
129    * <li>If the configuration property is explicitly given within the plugin's
130    * configuration, use that value.</li>
131    * <li>If the Maven property <code>project.build.sourceEncoding</code> is
132    * defined, use its value.</li>
133    * <li>Otherwise use the value from the system property
134    * <code>file.encoding</code>.</li>
135    * </ol>
136    * </p>
137    *
138    * @see #getEncoding()
139    * @since 2.0
140    */
141   @Parameter(defaultValue = "${project.build.sourceEncoding}")
142   private String encoding;
143 
144   /**
145    * Location to generate Java source files in.
146    */
147   @Parameter(
148       defaultValue = "${project.build.directory}/generated-sources/metaschema",
149       required = true,
150       property = "outputDirectory")
151   private File outputDirectory;
152 
153   /**
154    * The directory to read source metaschema from.
155    */
156   @Parameter(defaultValue = "${basedir}/src/main/metaschema")
157   private File metaschemaDir;
158 
159   /**
160    * A list of <code>files</code> containing Metaschema module constraints files.
161    */
162   @Parameter(property = "constraints")
163   private File[] constraints;
164 
165   /**
166    * A set of inclusion patterns used to select which Metaschema modules are to be
167    * processed. By default, all files are processed.
168    */
169   @Parameter
170   protected String[] includes;
171 
172   /**
173    * A set of exclusion patterns used to prevent certain files from being
174    * processed. By default, this set is empty such that no files are excluded.
175    */
176   @Parameter
177   protected String[] excludes;
178 
179   /**
180    * Indicate if the execution should be skipped.
181    */
182   @Parameter(property = "metaschema.skip", defaultValue = "false")
183   private boolean skip;
184 
185   /**
186    * The BuildContext is used to identify which files or directories were modified
187    * since last build. This is used to determine if Module-based generation must
188    * be performed again.
189    *
190    * @return the active Plexus BuildContext.
191    */
192   protected final BuildContext getBuildContext() {
193     return buildContext;
194   }
195 
196   /**
197    * Retrieve the Maven project context.
198    *
199    * @return The active MavenProject.
200    */
201   protected final MavenProject getMavenProject() {
202     return mavenProject;
203   }
204 
205   protected final List<Artifact> getPluginArtifacts() {
206     return pluginArtifacts;
207   }
208 
209   /**
210    * Retrieve the mojo execution context.
211    *
212    * @return The active MojoExecution.
213    */
214   @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "this is a data holder")
215   public MojoExecution getMojoExecution() {
216     return mojoExecution;
217   }
218 
219   /**
220    * Retrieve the directory where generated classes will be stored.
221    *
222    * @return the directory
223    */
224   protected File getOutputDirectory() {
225     return outputDirectory;
226   }
227 
228   /**
229    * Set the directory where generated classes will be stored.
230    *
231    * @param outputDirectory
232    *          the directory to use
233    */
234   protected void setOutputDirectory(File outputDirectory) {
235     Objects.requireNonNull(outputDirectory, "outputDirectory");
236     this.outputDirectory = outputDirectory;
237   }
238 
239   /**
240    * Gets the file encoding to use for generated classes.
241    * <p>
242    * The algorithm for finding the encoding to use is as follows (where the first
243    * non-null value found is used for encoding):
244    * </p>
245    * <ol>
246    * <li>If the configuration property is explicitly given within the plugin's
247    * configuration, use that value.</li>
248    * <li>If the Maven property <code>project.build.sourceEncoding</code> is
249    * defined, use its value.</li>
250    * <li>Otherwise use the value from the system property
251    * <code>file.encoding</code>.</li>
252    * </ol>
253    *
254    * @return The encoding to be used by this AbstractJaxbMojo and its tools.
255    */
256   protected final String getEncoding() {
257     String encoding;
258     if (this.encoding != null) {
259       // first try to use the provided encoding
260       encoding = this.encoding;
261       if (getLog().isDebugEnabled()) {
262         getLog().debug(String.format("Using configured encoding [%s].", encoding));
263       }
264     } else {
265       encoding = Charset.defaultCharset().displayName();
266       if (getLog().isWarnEnabled()) {
267         getLog().warn(String.format("Using system encoding [%s]. This build is platform dependent!", encoding));
268       }
269     }
270     return encoding;
271   }
272 
273   /**
274    * Retrieve a stream of Module file sources.
275    *
276    * @return the stream
277    */
278   protected Stream<File> getModuleSources() {
279     DirectoryScanner ds = new DirectoryScanner();
280     ds.setBasedir(metaschemaDir);
281     ds.setIncludes(includes != null && includes.length > 0 ? includes : DEFAULT_INCLUDES);
282     ds.setExcludes(excludes != null && excludes.length > 0 ? excludes : null);
283     ds.addDefaultExcludes();
284     ds.setCaseSensitive(true);
285     ds.setFollowSymlinks(false);
286     ds.scan();
287     return Stream.of(ds.getIncludedFiles()).map(filename -> new File(metaschemaDir, filename)).distinct();
288   }
289 
290   @NonNull
291   protected IBindingContext newBindingContext() throws IOException, MetaschemaException {
292     List<IConstraintSet> constraints = getConstraints();
293 
294     // generate Java sources based on provided metaschema sources
295     return new DefaultBindingContext(
296         new PostProcessingModuleLoaderStrategy(
297             // ensure that the external constraints do not apply to the built in module
298             CollectionUtil.singletonList(new LimitedExternalConstraintsModulePostProcessor(constraints)),
299             new SimpleModuleLoaderStrategy(
300                 // this is used instead of the default generator to ensure that plugin classpath
301                 // entries are used for compilation
302                 new ModuleBindingGenerator(
303                     ObjectUtils.notNull(Files.createDirectories(Paths.get("target/metaschema-codegen-modules"))),
304                     new DefaultBindingConfiguration()))));
305   }
306 
307   /**
308    * Get the configured collection of constraints.
309    *
310    * @return the loaded constraints
311    * @throws MetaschemaException
312    *           if a binding exception occurred while loading the constraints
313    * @throws IOException
314    *           if an error occurred while reading the constraints
315    */
316   @NonNull
317   protected List<IConstraintSet> getConstraints()
318       throws MetaschemaException, IOException {
319     IConstraintLoader loader = IBindingContext.getConstraintLoader();
320     List<IConstraintSet> constraintSets = new ArrayList<>(constraints.length);
321     for (File constraint : this.constraints) {
322       constraintSets.addAll(loader.load(ObjectUtils.notNull(constraint)));
323     }
324     return CollectionUtil.unmodifiableList(constraintSets);
325   }
326 
327   /**
328    * Determine if the execution of this mojo should be skipped.
329    *
330    * @return {@code true} if the mojo execution should be skipped, or
331    *         {@code false} otherwise
332    */
333   protected boolean shouldExecutionBeSkipped() {
334     return skip;
335   }
336 
337   /**
338    * Get the name of the file that is used to detect staleness.
339    *
340    * @return the name
341    */
342   protected abstract String getStaleFileName();
343 
344   /**
345    * Gets the staleFile for this execution.
346    *
347    * @return the staleFile
348    */
349   protected final File getStaleFile() {
350     StringBuilder builder = new StringBuilder();
351     if (getMojoExecution() != null) {
352       builder.append(getMojoExecution().getExecutionId()).append('-');
353     }
354     builder.append(getStaleFileName());
355     return new File(staleFileDirectory, builder.toString());
356   }
357 
358   /**
359    * Determine if code generation is required. This is done by comparing the last
360    * modified time of each Module source file against the stale file managed by
361    * this plugin.
362    *
363    * @return {@code true} if the code generation is needed, or {@code false}
364    *         otherwise
365    */
366   protected boolean isGenerationRequired() {
367     final File staleFile = getStaleFile();
368     boolean generate = !staleFile.exists();
369     if (generate) {
370       if (getLog().isInfoEnabled()) {
371         getLog().info(String.format("Stale file '%s' doesn't exist! Generating source files.", staleFile.getPath()));
372       }
373       generate = true;
374     } else {
375       generate = false;
376       // check for staleness
377       long staleLastModified = staleFile.lastModified();
378 
379       BuildContext buildContext = getBuildContext();
380       URI metaschemaDirRelative = getMavenProject().getBasedir().toURI().relativize(metaschemaDir.toURI());
381 
382       if (buildContext.isIncremental() && buildContext.hasDelta(metaschemaDirRelative.toString())) {
383         if (getLog().isInfoEnabled()) {
384           getLog().info("metaschemaDirRelative: " + metaschemaDirRelative.toString());
385         }
386         generate = true;
387       }
388 
389       if (!generate) {
390         for (File sourceFile : getModuleSources().collect(Collectors.toList())) {
391           if (getLog().isInfoEnabled()) {
392             getLog().info("Source file: " + sourceFile.getPath());
393           }
394           if (sourceFile.lastModified() > staleLastModified) {
395             generate = true;
396           }
397         }
398       }
399     }
400     return generate;
401   }
402 
403   protected Set<String> getClassPath() throws DependencyResolutionRequiredException {
404     Set<String> pathElements;
405     try {
406       pathElements = new LinkedHashSet<>(getMavenProject().getCompileClasspathElements());
407     } catch (DependencyResolutionRequiredException ex) {
408       getLog().warn("exception calling getCompileClasspathElements", ex);
409       throw ex;
410     }
411 
412     if (pluginArtifacts != null) {
413       for (Artifact a : getPluginArtifacts()) {
414         if (a.getFile() != null) {
415           pathElements.add(a.getFile().getAbsolutePath());
416         }
417       }
418     }
419     return pathElements;
420   }
421 
422   @NonNull
423   protected Set<IModule> getModulesToGenerateFor(@NonNull IBindingContext bindingContext)
424       throws MetaschemaException, IOException {
425     IBindingModuleLoader loader = bindingContext.newModuleLoader();
426     loader.allowEntityResolution();
427 
428     LoggingValidationHandler validationHandler = new LoggingValidationHandler();
429 
430     Set<IModule> modules = new HashSet<>();
431     for (File source : getModuleSources().collect(Collectors.toList())) {
432       assert source != null;
433       if (getLog().isInfoEnabled()) {
434         getLog().info("Using metaschema source: " + source.getPath());
435       }
436       IBindingMetaschemaModule module = loader.load(source);
437 
438       IValidationResult result = bindingContext.validate(
439           module.getSourceNodeItem(),
440           loader.getBindingContext().newBoundLoader(),
441           null);
442 
443       validationHandler.handleResults(result);
444 
445       modules.add(module);
446     }
447     return modules;
448   }
449 
450   protected void createStaleFile(@NonNull File staleFile) throws MojoExecutionException {
451     // create the stale file
452     if (!staleFileDirectory.exists() && !staleFileDirectory.mkdirs()) {
453       throw new MojoExecutionException("Unable to create output directory: " + staleFileDirectory);
454     }
455     try (OutputStream os
456         = Files.newOutputStream(staleFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE,
457             StandardOpenOption.TRUNCATE_EXISTING)) {
458       os.close();
459       if (getLog().isInfoEnabled()) {
460         getLog().info("Created stale file: " + staleFile);
461       }
462     } catch (IOException ex) {
463       throw new MojoExecutionException("Failed to write stale file: " + staleFile.getPath(), ex);
464     }
465   }
466 
467   protected final class LoggingValidationHandler
468       extends AbstractValidationResultProcessor {
469 
470     private <T extends IValidationFinding> void handleFinding(
471         @NonNull T finding,
472         @NonNull Function<T, CharSequence> formatter) {
473 
474       Log log = getLog();
475 
476       switch (finding.getSeverity()) {
477       case CRITICAL:
478       case ERROR:
479         if (log.isErrorEnabled()) {
480           log.error(formatter.apply(finding), finding.getCause());
481         }
482         break;
483       case WARNING:
484         if (log.isWarnEnabled()) {
485           getLog().warn(formatter.apply(finding), finding.getCause());
486         }
487         break;
488       case INFORMATIONAL:
489         if (log.isInfoEnabled()) {
490           getLog().info(formatter.apply(finding), finding.getCause());
491         }
492         break;
493       default:
494         if (log.isDebugEnabled()) {
495           getLog().debug(formatter.apply(finding), finding.getCause());
496         }
497         break;
498       }
499     }
500 
501     @Override
502     protected void handleJsonValidationFinding(JsonValidationFinding finding) {
503       handleFinding(finding, this::getMessage);
504     }
505 
506     @Override
507     protected void handleXmlValidationFinding(XmlValidationFinding finding) {
508       handleFinding(finding, this::getMessage);
509     }
510 
511     @Override
512     protected void handleConstraintValidationFinding(ConstraintValidationFinding finding) {
513       handleFinding(finding, this::getMessage);
514     }
515 
516     @NonNull
517     private CharSequence getMessage(JsonValidationFinding finding) {
518       StringBuilder builder = new StringBuilder();
519       builder.append('[')
520           .append(finding.getCause().getPointerToViolation())
521           .append("] ")
522           .append(finding.getMessage());
523 
524       URI documentUri = finding.getDocumentUri();
525       if (documentUri != null) {
526         builder.append(" [")
527             .append(documentUri.toString())
528             .append(']');
529       }
530       return builder;
531     }
532 
533     @NonNull
534     private CharSequence getMessage(XmlValidationFinding finding) {
535       StringBuilder builder = new StringBuilder();
536 
537       builder.append(finding.getMessage())
538           .append(" [");
539 
540       URI documentUri = finding.getDocumentUri();
541       if (documentUri != null) {
542         builder.append(documentUri.toString());
543       }
544 
545       SAXParseException ex = finding.getCause();
546       builder.append(finding.getMessage())
547           .append('{')
548           .append(ex.getLineNumber())
549           .append(',')
550           .append(ex.getColumnNumber())
551           .append("}]");
552       return builder;
553     }
554 
555     @NonNull
556     private CharSequence getMessage(@NonNull ConstraintValidationFinding finding) {
557       StringBuilder builder = new StringBuilder();
558       builder.append('[')
559           .append(finding.getTarget().getMetapath())
560           .append(']');
561 
562       String id = finding.getIdentifier();
563       if (id != null) {
564         builder.append(' ')
565             .append(id);
566       }
567 
568       builder.append(' ')
569           .append(finding.getMessage());
570 
571       URI documentUri = finding.getTarget().getBaseUri();
572       IResourceLocation location = finding.getLocation();
573       if (documentUri != null || location != null) {
574         builder.append(" [");
575       }
576 
577       if (documentUri != null) {
578         builder.append(documentUri.toString());
579       }
580 
581       if (location != null) {
582         builder.append('{')
583             .append(location.getLine())
584             .append(',')
585             .append(location.getColumn())
586             .append('}');
587       }
588       if (documentUri != null || location != null) {
589         builder.append(']');
590       }
591       return builder;
592     }
593   }
594 
595   public class ModuleBindingGenerator implements IModuleBindingGenerator {
596     @NonNull
597     private final Path compilePath;
598     @NonNull
599     private final ClassLoader classLoader;
600     @NonNull
601     private final IBindingConfiguration bindingConfiguration;
602 
603     public ModuleBindingGenerator(
604         @NonNull Path compilePath,
605         @NonNull IBindingConfiguration bindingConfiguration) {
606       this.compilePath = compilePath;
607       this.classLoader = ModuleCompilerHelper.newClassLoader(
608           compilePath,
609           ObjectUtils.notNull(Thread.currentThread().getContextClassLoader()));
610       this.bindingConfiguration = bindingConfiguration;
611     }
612 
613     @NonNull
614     public IProduction generateClasses(@NonNull IModule module) {
615       IProduction production;
616       try {
617         production = JavaGenerator.generate(module, compilePath, bindingConfiguration);
618       } catch (IOException ex) {
619         throw new MetapathException(
620             String.format("Unable to generate and compile classes for module '%s'.", module.getLocation()),
621             ex);
622       }
623       return production;
624     }
625 
626     private void compileClasses(@NonNull IProduction production, @NonNull Path classDir)
627         throws IOException, DependencyResolutionRequiredException {
628       List<IGeneratedClass> classesToCompile = production.getGeneratedClasses().collect(Collectors.toList());
629 
630       List<Path> classes = ObjectUtils.notNull(classesToCompile.stream()
631           .map(IGeneratedClass::getClassFile)
632           .collect(Collectors.toUnmodifiableList()));
633 
634       JavaCompilerSupport compiler = new JavaCompilerSupport(classDir);
635 
636       getClassPath().forEach(compiler::addToClassPath);
637 
638       JavaCompilerSupport.CompilationResult result = compiler.compile(classes, null);
639 
640       if (!result.isSuccessful()) {
641         DiagnosticCollector<?> diagnostics = new DiagnosticCollector<>();
642         if (getLog().isErrorEnabled()) {
643           getLog().error("diagnostics: " + diagnostics.getDiagnostics().toString());
644         }
645         throw new IllegalStateException(String.format("failed to compile classes: %s",
646             classesToCompile.stream()
647                 .map(clazz -> clazz.getClassName().canonicalName())
648                 .collect(Collectors.joining(","))));
649       }
650     }
651 
652     @Override
653     public Class<? extends IBoundModule> generate(IModule module) {
654       IProduction production = generateClasses(module);
655       try {
656         compileClasses(production, compilePath);
657       } catch (IOException | DependencyResolutionRequiredException ex) {
658         throw new IllegalStateException("failed to compile classes", ex);
659       }
660       IGeneratedModuleClass moduleClass = ObjectUtils.requireNonNull(production.getModuleProduction(module));
661 
662       try {
663         return moduleClass.load(classLoader);
664       } catch (ClassNotFoundException ex) {
665         throw new IllegalStateException(ex);
666       }
667     }
668   }
669 
670   private static class LimitedExternalConstraintsModulePostProcessor
671       extends ExternalConstraintsModulePostProcessor {
672 
673     public LimitedExternalConstraintsModulePostProcessor(
674         @NonNull Collection<IConstraintSet> additionalConstraintSets) {
675       super(additionalConstraintSets);
676     }
677 
678     /**
679      * This method ensures that constraints are not applied to the built-in
680      * Metaschema module module twice, when this module is selected as the source
681      * for generation.
682      */
683     @Override
684     public void processModule(IModule module) {
685       if (!(module instanceof MetaschemaModelModule)) {
686         super.processModule(module);
687       }
688     }
689   }
690 }