scripts: add configuration validation script
Add a script to validate configurations against the schema. Use
https://github.com/Julian/jsonschema to perform the json validation.
The script is intended to be run from a continuous integration
environment or by configuration/schema developers. A key
assumption/feature of the script is that its users will always prefer to
resolve relative references to the local filesystem. As such, the
script computes a base URI that points to the filesystem and instructs
the validator to use that in place of whatever base_uri it derives from
the global $id attribute. For additional reading see:
https://json-schema.org/understanding-json-schema/structuring.html#the-id-property
https://github.com/Julian/jsonschema/issues/98
Without any options the script assumes it is being run from an
entity-manager source distribution and attempts to find the schema and
configuration files relative to the location of the script.
Alternatively, the script can validate arbitrary json files against
arbitrary schema:
./validate-configs.py -s foo.schema.json -c test1.json -c test2.json
By default the validation stops as soon as a configuration does not
validate. Use -k to override this behavior and validate as many
configurations as possible.
Provide an option to instruct the script to ignore a list of
configurations that are expected to fail validation to be used in
continuous integration setups - similar in concept to xfail mechanisms
provided by most build systems with unit test support.
Change-Id: I7d67a54993a6d5e00daf552d9d350c80411b997b
Signed-off-by: Brad Bishop <bradleyb@fuzziesquirrel.com>
diff --git a/scripts/validate-configs.py b/scripts/validate-configs.py
new file mode 100755
index 0000000..9af771c
--- /dev/null
+++ b/scripts/validate-configs.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""
+A tool for validating entity manager configurations.
+"""
+import argparse
+import json
+import jsonschema.validators
+import os
+import sys
+
+DEFAULT_SCHEMA_FILENAME = "global.json"
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Entity manager configuration validator",
+ )
+ parser.add_argument(
+ "-s", "--schema", help=(
+ "Use the specified schema file instead of the default "
+ "(__file__/../../schemas/global.json)"))
+ parser.add_argument(
+ "-c", "--config", action='append', help=(
+ "Validate the specified configuration files (can be "
+ "specified more than once) instead of the default "
+ "(__file__/../../configurations/**.json)"))
+ parser.add_argument(
+ "-e", "--expected-fails", help=(
+ "A file with a list of configurations to ignore should "
+ "they fail to validate"))
+ parser.add_argument(
+ "-k", "--continue", action='store_true', help=(
+ "keep validating after a failure"))
+ parser.add_argument(
+ "-v", "--verbose", action='store_true', help=(
+ "be noisy"))
+ args = parser.parse_args()
+
+ schema_file = args.schema
+ if schema_file is None:
+ try:
+ source_dir = os.path.realpath(__file__).split(os.sep)[:-2]
+ schema_file = os.sep + os.path.join(
+ *source_dir, 'schemas', DEFAULT_SCHEMA_FILENAME)
+ except Exception as e:
+ sys.stderr.write(
+ "Could not guess location of {}\n".format(
+ DEFAULT_SCHEMA_FILENAME))
+ sys.exit(2)
+
+ schema = {}
+ try:
+ with open(schema_file) as fd:
+ schema = json.load(fd)
+ except FileNotFoundError as e:
+ sys.stderr.write(
+ "Could not read schema file '{}'\n".format(schema_file))
+ sys.exit(2)
+
+ config_files = args.config or []
+ if len(config_files) == 0:
+ try:
+ source_dir = os.path.realpath(__file__).split(os.sep)[:-2]
+ configs_dir = os.sep + os.path.join(*source_dir, 'configurations')
+ data = os.walk(configs_dir)
+ for root, _, files in data:
+ for f in files:
+ if f.endswith('.json'):
+ config_files.append(os.path.join(root, f))
+ except Exception as e:
+ sys.stderr.write(
+ "Could not guess location of configurations\n")
+ sys.exit(2)
+
+ configs = []
+ for config_file in config_files:
+ try:
+ with open(config_file) as fd:
+ configs.append(json.load(fd))
+ except FileNotFoundError as e:
+ sys.stderr.write(
+ "Could not parse config file '{}'\n".format(config_file))
+ sys.exit(2)
+
+ expected_fails = []
+ if args.expected_fails:
+ try:
+ with open(args.expected_fails) as fd:
+ for line in fd:
+ expected_fails.append(line.strip())
+ except Exception as e:
+ sys.stderr.write(
+ "Could not read expected fails file '{}'\n".format(
+ args.expected_fails))
+ sys.exit(2)
+
+ base_uri = "file://{}/".format(
+ os.path.split(os.path.realpath(schema_file))[0])
+ resolver = jsonschema.RefResolver(base_uri, schema)
+ validator = jsonschema.Draft7Validator(schema, resolver=resolver)
+
+ results = {
+ "invalid": [],
+ "unexpected_pass": [],
+ }
+ for config_file, config in zip(config_files, configs):
+ name = os.path.split(config_file)[1]
+ expect_fail = name in expected_fails
+ try:
+ validator.validate(config)
+ if expect_fail:
+ results["unexpected_pass"].append(name)
+ if not getattr(args, "continue"):
+ break
+ except jsonschema.exceptions.ValidationError as e:
+ if not expect_fail:
+ results["invalid"].append(name)
+ if args.verbose:
+ print(e)
+ if expect_fail or getattr(args, "continue"):
+ continue
+ break
+
+ exit_status = 0
+ if len(results["invalid"]) + len(results["unexpected_pass"]):
+ exit_status = 1
+ unexpected_pass_suffix = " **"
+ show_suffix_explanation = False
+ print("results:")
+ for f in config_files:
+ if any([x in f for x in results["unexpected_pass"]]):
+ show_suffix_explanation = True
+ print(" '{}' passed!{}".format(f, unexpected_pass_suffix))
+ if any([x in f for x in results["invalid"]]):
+ print(" '{}' failed!".format(f))
+
+ if show_suffix_explanation:
+ print("\n** configuration expected to fail")
+
+ sys.exit(exit_status)
+
+
+if __name__ == "__main__":
+ main()