1   /*
2    * SPDX-FileCopyrightText: none
3    * SPDX-License-Identifier: CC0-1.0
4    */
5   
6   package dev.metaschema.cli.commands;
7   
8   import org.apache.commons.cli.CommandLine;
9   import org.apache.commons.cli.Option;
10  import org.apache.logging.log4j.LogManager;
11  import org.apache.logging.log4j.Logger;
12  
13  import java.io.File;
14  import java.io.FileNotFoundException;
15  import java.io.IOException;
16  import java.net.URI;
17  import java.net.UnknownHostException;
18  import java.nio.file.Path;
19  import java.nio.file.Paths;
20  import java.util.Collection;
21  import java.util.List;
22  import java.util.Locale;
23  import java.util.Set;
24  
25  import dev.metaschema.cli.processor.CLIProcessor;
26  import dev.metaschema.cli.processor.CallingContext;
27  import dev.metaschema.cli.processor.ExitCode;
28  import dev.metaschema.cli.processor.command.AbstractCommandExecutor;
29  import dev.metaschema.cli.processor.command.AbstractTerminalCommand;
30  import dev.metaschema.cli.processor.command.CommandExecutionException;
31  import dev.metaschema.cli.processor.command.ExtraArgument;
32  import dev.metaschema.cli.util.LoggingValidationHandler;
33  import dev.metaschema.core.configuration.DefaultConfiguration;
34  import dev.metaschema.core.configuration.IMutableConfiguration;
35  import dev.metaschema.core.metapath.MetapathException;
36  import dev.metaschema.core.metapath.format.IPathFormatter;
37  import dev.metaschema.core.metapath.format.PathFormatSelection;
38  import dev.metaschema.core.model.IModule;
39  import dev.metaschema.core.model.MetaschemaException;
40  import dev.metaschema.core.model.constraint.ConstraintValidationException;
41  import dev.metaschema.core.model.constraint.IConstraintSet;
42  import dev.metaschema.core.model.constraint.ValidationFeature;
43  import dev.metaschema.core.model.validation.AggregateValidationResult;
44  import dev.metaschema.core.model.validation.IValidationResult;
45  import dev.metaschema.core.util.IVersionInfo;
46  import dev.metaschema.core.util.ObjectUtils;
47  import dev.metaschema.databind.IBindingContext;
48  import dev.metaschema.databind.IBindingContext.ISchemaValidationProvider;
49  import dev.metaschema.databind.io.Format;
50  import dev.metaschema.databind.io.IBoundLoader;
51  import dev.metaschema.modules.sarif.SarifValidationHandler;
52  import edu.umd.cs.findbugs.annotations.NonNull;
53  import edu.umd.cs.findbugs.annotations.Nullable;
54  
55  /**
56   * Used by implementing classes to provide a content validation command.
57   */
58  public abstract class AbstractValidateContentCommand
59      extends AbstractTerminalCommand {
60    private static final Logger LOGGER = LogManager.getLogger(AbstractValidateContentCommand.class);
61    @NonNull
62    private static final String COMMAND = "validate";
63    @NonNull
64    private static final List<ExtraArgument> EXTRA_ARGUMENTS = ObjectUtils.notNull(List.of(
65        ExtraArgument.newInstance("file-or-URI-to-validate", true, URI.class)));
66  
67    @NonNull
68    private static final Option CONSTRAINTS_OPTION = ObjectUtils.notNull(
69        Option.builder("c")
70            .hasArgs()
71            .argName("URL")
72            .type(URI.class)
73            .desc("additional constraint definitions")
74            .get());
75    @NonNull
76    private static final Option SARIF_OUTPUT_FILE_OPTION = ObjectUtils.notNull(
77        Option.builder("o")
78            .hasArg()
79            .argName("FILE")
80            .type(File.class)
81            .desc("write SARIF results to the provided FILE")
82            .numberOfArgs(1)
83            .get());
84    @NonNull
85    private static final Option SARIF_INCLUDE_PASS_OPTION = ObjectUtils.notNull(
86        Option.builder()
87            .longOpt("sarif-include-pass")
88            .desc("include pass results in SARIF")
89            .get());
90    @NonNull
91    private static final Option NO_SCHEMA_VALIDATION_OPTION = ObjectUtils.notNull(
92        Option.builder()
93            .longOpt("disable-schema-validation")
94            .desc("do not perform schema validation")
95            .get());
96    @NonNull
97    private static final Option NO_CONSTRAINT_VALIDATION_OPTION = ObjectUtils.notNull(
98        Option.builder()
99            .longOpt("disable-constraint-validation")
100           .desc("do not perform constraint validation")
101           .get());
102   @NonNull
103   private static final Option PATH_FORMAT_OPTION = ObjectUtils.notNull(
104       Option.builder()
105           .longOpt("path-format")
106           .hasArg()
107           .argName("FORMAT")
108           .type(PathFormatSelection.class)
109           .desc("path format in validation output: auto (default, selects based on document format), "
110               + "metapath, xpath, jsonpointer")
111           .get());
112   @NonNull
113   private static final Option PARALLEL_THREADS_OPTION = ObjectUtils.notNull(
114       Option.builder()
115           .longOpt("threads")
116           .hasArg()
117           .argName("count")
118           .type(Number.class)
119           .desc("number of threads for parallel constraint validation (default: 1, experimental)")
120           .get());
121 
122   @Override
123   public String getName() {
124     return COMMAND;
125   }
126 
127   @SuppressWarnings("null")
128   @Override
129   public Collection<? extends Option> gatherOptions() {
130     return List.of(
131         MetaschemaCommands.AS_FORMAT_OPTION,
132         CONSTRAINTS_OPTION,
133         SARIF_OUTPUT_FILE_OPTION,
134         SARIF_INCLUDE_PASS_OPTION,
135         NO_SCHEMA_VALIDATION_OPTION,
136         NO_CONSTRAINT_VALIDATION_OPTION,
137         PATH_FORMAT_OPTION,
138         PARALLEL_THREADS_OPTION);
139   }
140 
141   @Override
142   public List<ExtraArgument> getExtraArguments() {
143     return EXTRA_ARGUMENTS;
144   }
145 
146   /**
147    * Drives the validation execution.
148    */
149   protected abstract class AbstractValidationCommandExecutor
150       extends AbstractCommandExecutor {
151 
152     /**
153      * Construct a new command executor.
154      *
155      * @param callingContext
156      *          the context of the command execution
157      * @param commandLine
158      *          the parsed command line details
159      */
160     public AbstractValidationCommandExecutor(
161         @NonNull CallingContext callingContext,
162         @NonNull CommandLine commandLine) {
163       super(callingContext, commandLine);
164     }
165 
166     /**
167      * Get the binding context to use for data processing.
168      *
169      * @param constraintSets
170      *          the constraints to configure in the resulting binding context
171      * @return the context
172      * @throws CommandExecutionException
173      *           if a error occurred while getting the binding context
174      */
175     @NonNull
176     protected abstract IBindingContext getBindingContext(@NonNull Set<IConstraintSet> constraintSets)
177         throws CommandExecutionException;
178 
179     /**
180      * Get the module to use for validation.
181      * <p>
182      * This module is used to generate schemas and as a source of built-in
183      * constraints.
184      *
185      * @param commandLine
186      *          the provided command line argument information
187      * @param bindingContext
188      *          the context used to access Metaschema module information based on
189      *          Java class bindings
190      * @return the loaded Metaschema module
191      * @throws CommandExecutionException
192      *           if an error occurred while loading the module
193      */
194     @NonNull
195     protected abstract IModule getModule(
196         @NonNull CommandLine commandLine,
197         @NonNull IBindingContext bindingContext)
198         throws CommandExecutionException;
199 
200     /**
201      * Get the schema validation implementation requested based on the provided
202      * command line arguments.
203      * <p>
204      * It is typical for this call to result in the dynamic generation of a schema
205      * to use for validation.
206      *
207      * @param module
208      *          the Metaschema module to generate the schema from
209      * @param commandLine
210      *          the provided command line argument information
211      * @param bindingContext
212      *          the context used to access Metaschema module information based on
213      *          Java class bindings
214      * @return the provider
215      */
216     @NonNull
217     protected abstract ISchemaValidationProvider getSchemaValidationProvider(
218         @NonNull IModule module,
219         @NonNull CommandLine commandLine,
220         @NonNull IBindingContext bindingContext);
221 
222     /**
223      * Execute the validation operation.
224      */
225     @Override
226     public void execute() throws CommandExecutionException {
227       CommandLine cmdLine = getCommandLine();
228       @SuppressWarnings("synthetic-access")
229       URI currentWorkingDirectory = ObjectUtils.notNull(getCurrentWorkingDirectory().toUri());
230 
231       Set<IConstraintSet> constraintSets = MetaschemaCommands.loadConstraintSets(
232           cmdLine,
233           CONSTRAINTS_OPTION,
234           currentWorkingDirectory);
235 
236       List<String> extraArgs = cmdLine.getArgList();
237 
238       URI source = MetaschemaCommands.handleSource(
239           ObjectUtils.requireNonNull(extraArgs.get(0)),
240           currentWorkingDirectory);
241 
242       IBindingContext bindingContext = getBindingContext(constraintSets);
243       IBoundLoader loader = bindingContext.newBoundLoader();
244       Format asFormat = MetaschemaCommands.determineSourceFormat(
245           cmdLine,
246           MetaschemaCommands.AS_FORMAT_OPTION,
247           loader,
248           source);
249 
250       IValidationResult validationResult = validate(source, asFormat, cmdLine, bindingContext);
251       handleOutput(source, validationResult, asFormat, cmdLine, bindingContext);
252 
253       if (validationResult == null || validationResult.isPassing()) {
254         if (LOGGER.isInfoEnabled()) {
255           LOGGER.info("The file '{}' is valid.", source);
256         }
257       } else if (LOGGER.isErrorEnabled()) {
258         LOGGER.error("The file '{}' is invalid.", source);
259       }
260 
261       if (validationResult != null && !validationResult.isPassing()) {
262         throw new CommandExecutionException(ExitCode.FAIL);
263       }
264     }
265 
266     @SuppressWarnings("PMD.CyclomaticComplexity")
267     @Nullable
268     private IValidationResult validate(
269         @NonNull URI source,
270         @NonNull Format asFormat,
271         @NonNull CommandLine commandLine,
272         @NonNull IBindingContext bindingContext) throws CommandExecutionException {
273 
274       if (LOGGER.isInfoEnabled()) {
275         LOGGER.info("Validating '{}' as {}.", source, asFormat.name());
276       }
277 
278       IValidationResult validationResult = null;
279       try {
280         // get the module, but don't register it
281         IModule module = getModule(commandLine, bindingContext);
282         if (!commandLine.hasOption(NO_SCHEMA_VALIDATION_OPTION)) {
283           // perform schema validation
284           validationResult = getSchemaValidationProvider(module, commandLine, bindingContext)
285               .validateWithSchema(source, asFormat, bindingContext);
286         }
287 
288         if (!commandLine.hasOption(NO_CONSTRAINT_VALIDATION_OPTION)) {
289           IMutableConfiguration<ValidationFeature<?>> configuration = new DefaultConfiguration<>();
290           if (commandLine.hasOption(SARIF_OUTPUT_FILE_OPTION) && commandLine.hasOption(SARIF_INCLUDE_PASS_OPTION)) {
291             configuration.enableFeature(ValidationFeature.VALIDATE_GENERATE_PASS_FINDINGS);
292           }
293 
294           // Configure parallel validation if requested
295           if (commandLine.hasOption(PARALLEL_THREADS_OPTION)) {
296             String threadValue = commandLine.getOptionValue(PARALLEL_THREADS_OPTION);
297             int threadCount;
298             try {
299               threadCount = Integer.parseInt(threadValue);
300             } catch (NumberFormatException ex) {
301               throw new CommandExecutionException(
302                   ExitCode.INVALID_ARGUMENTS,
303                   String.format("Invalid thread count '%s': must be a positive integer", threadValue),
304                   ex);
305             }
306             if (threadCount < 1) {
307               throw new CommandExecutionException(
308                   ExitCode.INVALID_ARGUMENTS,
309                   String.format("Thread count must be at least 1, got: %d", threadCount));
310             }
311             if (threadCount > 1) {
312               if (LOGGER.isWarnEnabled()) {
313                 LOGGER.warn("Parallel constraint validation is an experimental feature. "
314                     + "Using {} threads.", threadCount);
315               }
316               configuration.set(ValidationFeature.PARALLEL_THREADS, threadCount);
317             }
318           }
319 
320           // perform constraint validation
321           bindingContext.registerModule(module); // ensure the module is registered
322           IValidationResult constraintValidationResult = bindingContext.validateWithConstraints(source, configuration);
323           validationResult = validationResult == null
324               ? constraintValidationResult
325               : AggregateValidationResult.aggregate(validationResult, constraintValidationResult);
326         }
327       } catch (FileNotFoundException ex) {
328         throw new CommandExecutionException(
329             ExitCode.IO_ERROR,
330             String.format("Resource not found at '%s'", source),
331             ex);
332       } catch (UnknownHostException ex) {
333         throw new CommandExecutionException(
334             ExitCode.IO_ERROR,
335             String.format("Unknown host for '%s'.", source),
336             ex);
337       } catch (IOException ex) {
338         throw new CommandExecutionException(ExitCode.IO_ERROR, ex.getLocalizedMessage(), ex);
339       } catch (MetapathException | MetaschemaException | ConstraintValidationException ex) {
340         throw new CommandExecutionException(ExitCode.PROCESSING_ERROR, ex.getLocalizedMessage(), ex);
341       }
342       return validationResult;
343     }
344 
345     private void handleOutput(
346         @NonNull URI source,
347         @Nullable IValidationResult validationResult,
348         @NonNull Format asFormat,
349         @NonNull CommandLine commandLine,
350         @NonNull IBindingContext bindingContext) throws CommandExecutionException {
351       if (commandLine.hasOption(SARIF_OUTPUT_FILE_OPTION)) {
352         Path sarifFile = ObjectUtils.notNull(Paths.get(commandLine.getOptionValue(SARIF_OUTPUT_FILE_OPTION)));
353 
354         IVersionInfo version
355             = getCallingContext().getCLIProcessor().getVersionInfos().get(CLIProcessor.COMMAND_VERSION);
356 
357         try {
358           SarifValidationHandler sarifHandler = new SarifValidationHandler(source, version);
359           if (validationResult != null) {
360             sarifHandler.addFindings(validationResult.getFindings());
361           }
362           sarifHandler.write(sarifFile, bindingContext);
363         } catch (IOException ex) {
364           throw new CommandExecutionException(ExitCode.IO_ERROR, ex.getLocalizedMessage(), ex);
365         }
366       } else if (validationResult != null && !validationResult.getFindings().isEmpty()) {
367         LOGGER.info("Validation identified the following issues:");
368         IPathFormatter pathFormatter = resolvePathFormatter(commandLine, asFormat);
369         LoggingValidationHandler.withPathFormatter(pathFormatter).handleResults(validationResult);
370       }
371 
372     }
373 
374     /**
375      * Resolve the path formatter based on command line option and document format.
376      *
377      * @param commandLine
378      *          the parsed command line
379      * @param asFormat
380      *          the document format
381      * @return the resolved path formatter
382      */
383     @NonNull
384     private IPathFormatter resolvePathFormatter(
385         @NonNull CommandLine commandLine,
386         @NonNull Format asFormat) {
387       PathFormatSelection selection = PathFormatSelection.AUTO;
388 
389       if (commandLine.hasOption(PATH_FORMAT_OPTION)) {
390         String value = commandLine.getOptionValue(PATH_FORMAT_OPTION);
391         if (value != null) {
392           selection = parsePathFormatSelection(value);
393         }
394       }
395 
396       return Format.resolvePathFormatter(selection, asFormat);
397     }
398 
399     /**
400      * Parse the path format selection from a string value.
401      *
402      * @param value
403      *          the string value from the command line
404      * @return the parsed selection, defaults to AUTO if unrecognized
405      */
406     @NonNull
407     private PathFormatSelection parsePathFormatSelection(@NonNull String value) {
408       switch (value.toLowerCase(Locale.ROOT)) {
409       case "auto":
410         return PathFormatSelection.AUTO;
411       case "metapath":
412         return PathFormatSelection.METAPATH;
413       case "xpath":
414         return PathFormatSelection.XPATH;
415       case "jsonpointer":
416       case "json-pointer":
417         return PathFormatSelection.JSON_POINTER;
418       default:
419         LOGGER.warn("Unrecognized path format '{}', using auto", value);
420         return PathFormatSelection.AUTO;
421       }
422     }
423   }
424 }