From 7c9608016591581ab5dfd9e00ebe4e0f215efb03 Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sun, 26 Feb 2023 15:08:01 +0000 Subject: [PATCH 1/7] Schema Imports v1 (#56) * Updated the grammar to support a very basic import. Updated the parser and types to include some logic to parse this * Updated the grammer to enclose import paths in quotes. Added a basic system to read the contents of other schemas with imports * Moved the parser.py code into a parser package * Moved the initial logic to load an additional schema to the parser/loaders.py module. Added a YamlatorSchema class to replace the original dictionary * Refined some of the function and type names. Seperated the schema into objects. A basic yamlator schema and a loaded one that will contain all the extra data that is loaded from the schema file that needs to be processed * Updated the existing test suite with the changes to the schema generation process * Updated the changelog and added a basic example of the importing system --- changelog.md | 4 + example/imports/common.ys | 5 + example/imports/import.yaml | 2 + example/imports/import.ys | 7 + tests/files/valid/base.ys | 15 ++ tests/files/valid/with_imports.ys | 8 + tests/parser/loaders/__init__.py | 0 .../parser/loaders/test_fetch_schema_path.py | 46 ++++ .../loaders/test_load_schema_imports.py | 87 ++++++++ .../loaders/test_parse_yamlator_schema.py | 50 +++++ .../loaders/test_resolve_unknown_types.py | 70 ++++++ tests/parser/test_parse_schema.py | 7 +- tests/parser/test_schema_transformer.py | 27 ++- tests/types/test_import_statements.py | 31 +++ tests/types/test_yamlator_schema.py | 27 +++ tests/validators/test_validate_yaml.py | 88 ++++---- yamlator/cmd/core.py | 14 +- yamlator/grammar/grammar.lark | 3 + yamlator/parser/__init__.py | 23 ++ yamlator/{parser.py => parser/core.py} | 72 ++++-- yamlator/parser/loaders.py | 198 +++++++++++++++++ yamlator/types.py | 209 +++++++++++++++++- yamlator/validators/core.py | 18 +- 23 files changed, 924 insertions(+), 87 deletions(-) create mode 100644 example/imports/common.ys create mode 100644 example/imports/import.yaml create mode 100644 example/imports/import.ys create mode 100644 tests/files/valid/base.ys create mode 100644 tests/files/valid/with_imports.ys create mode 100644 tests/parser/loaders/__init__.py create mode 100644 tests/parser/loaders/test_fetch_schema_path.py create mode 100644 tests/parser/loaders/test_load_schema_imports.py create mode 100644 tests/parser/loaders/test_parse_yamlator_schema.py create mode 100644 tests/parser/loaders/test_resolve_unknown_types.py create mode 100644 tests/types/test_import_statements.py create mode 100644 tests/types/test_yamlator_schema.py create mode 100644 yamlator/parser/__init__.py rename yamlator/{parser.py => parser/core.py} (83%) create mode 100644 yamlator/parser/loaders.py diff --git a/changelog.md b/changelog.md index 601c2ef..cb6faf2 100644 --- a/changelog.md +++ b/changelog.md @@ -53,3 +53,7 @@ * Added top level validation to Yamlator to support data structures that may not be an object or a map * Minor docstring and structure improvements in the `tests/` module * Improvements to the codebase docstrings + +## v4.0.0 (TBC) + +* Added import statements to the Yamlator schema syntax diff --git a/example/imports/common.ys b/example/imports/common.ys new file mode 100644 index 0000000..56b942b --- /dev/null +++ b/example/imports/common.ys @@ -0,0 +1,5 @@ +import Project from "../strict_mode/strict.ys" + +enum Values { + TEST = 1 +} \ No newline at end of file diff --git a/example/imports/import.yaml b/example/imports/import.yaml new file mode 100644 index 0000000..f0be9f8 --- /dev/null +++ b/example/imports/import.yaml @@ -0,0 +1,2 @@ +user: {} +value: 1 \ No newline at end of file diff --git a/example/imports/import.ys b/example/imports/import.ys new file mode 100644 index 0000000..548c797 --- /dev/null +++ b/example/imports/import.ys @@ -0,0 +1,7 @@ +import Project from "../lists/lists.ys" +import Values from "common.ys" + +schema { + user Project + value Values +} \ No newline at end of file diff --git a/tests/files/valid/base.ys b/tests/files/valid/base.ys new file mode 100644 index 0000000..7c648c1 --- /dev/null +++ b/tests/files/valid/base.ys @@ -0,0 +1,15 @@ +ruleset User { + firstName str + surname str + age int optional +} + +enum Status { + NEW_HIRE = 0 + ON_LEAVE = 1 +} + +ruleset Employee { + person User + status Status +} diff --git a/tests/files/valid/with_imports.ys b/tests/files/valid/with_imports.ys new file mode 100644 index 0000000..d0397f5 --- /dev/null +++ b/tests/files/valid/with_imports.ys @@ -0,0 +1,8 @@ +import Employee from "base.ys" +import User from "base.ys" +import Status from "base.ys" + +schema { + employees list(Employee) + status Status +} diff --git a/tests/parser/loaders/__init__.py b/tests/parser/loaders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/parser/loaders/test_fetch_schema_path.py b/tests/parser/loaders/test_fetch_schema_path.py new file mode 100644 index 0000000..49a389c --- /dev/null +++ b/tests/parser/loaders/test_fetch_schema_path.py @@ -0,0 +1,46 @@ +"""Test cases for the fetch_schema_path function""" + + +import unittest + +from typing import Any +from parameterized import parameterized + +from yamlator.parser.loaders import fetch_schema_path + + +class TestFetchSchemaPath(unittest.TestCase): + """Test cases for the fetch_schema_path function""" + + @parameterized.expand([ + ('with_none_str', None, ValueError), + ('with_empty_str', '', ValueError), + ('with_none_str_path', ['./path/file.ys'], TypeError), + ]) + def test_fetch_path_with_invalid_paths(self, name: str, schema_path: Any, + expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + fetch_schema_path(schema_path) + + @parameterized.expand([ + ('with_backslash_path', '\\awesome\\path\\test.ys', 'awesome/path'), + ('with_forward_slash_path', '/awesome/path/test.ys', 'awesome/path'), + ('with_schema_only', 'test.ys', '.'), + ('with_parent_directory', '../test.ys', '..'), + ('with_parent_directory_mixed_in_path', 'awesome/test/../wow.ys', + 'awesome/test/..') + ]) + def test_fetch_path_with_valid_paths(self, name: str, schema_path: str, + expected_path: str): + # Unused by test case, however is required by the parameterized library + del name + + actual_path = fetch_schema_path(schema_path) + self.assertEqual(expected_path, actual_path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/loaders/test_load_schema_imports.py b/tests/parser/loaders/test_load_schema_imports.py new file mode 100644 index 0000000..52c44c6 --- /dev/null +++ b/tests/parser/loaders/test_load_schema_imports.py @@ -0,0 +1,87 @@ +"""Test cases for the load_schema_imports function""" + +import unittest + +from parameterized import parameterized + +from yamlator.types import Rule +from yamlator.types import RuleType +from yamlator.types import SchemaTypes +from yamlator.types import ImportStatement +from yamlator.types import YamlatorRuleset +from yamlator.types import PartiallyLoadedYamlatorSchema +from yamlator.parser.loaders import load_schema_imports + + +def create_basic_loaded_schema(): + root = YamlatorRuleset('main', []) + return PartiallyLoadedYamlatorSchema( + root=root, + rulesets={}, + enums={}, + imports=[], + unknowns=[] + ) + + +BASIC_SCHEMA = create_basic_loaded_schema() + + +class TestLoadSchemaImports(unittest.TestCase): + """Test cases for the load_schema_imports function""" + + @parameterized.expand([ + ('with_none_schema', None, './path/test.ys', ValueError), + ('with_wrong_schema_type', YamlatorRuleset('main', []), + './path/test.ys', TypeError), + ('with_none_schema_path', BASIC_SCHEMA, None, ValueError), + ('with_wrong_schema_path_type_', BASIC_SCHEMA, ['test.ys'], TypeError), + ('with_empty_schema_path_string', BASIC_SCHEMA, '', ValueError) + ]) + def test_load_schema_imports_with_invalid_params( + self, name: str, + loaded_schema: PartiallyLoadedYamlatorSchema, + schema_path: str, expected_exception: Exception): + + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + load_schema_imports(loaded_schema, schema_path) + + def test_load_schema_imports(self): + schema_path = './tests/files/valid' + unknown_types = [ + RuleType(SchemaTypes.UNKNOWN, lookup='Employee'), + RuleType(SchemaTypes.UNKNOWN, lookup='Status') + ] + + loaded_schema = PartiallyLoadedYamlatorSchema( + root=YamlatorRuleset('main', [ + Rule('employees', + RuleType( + SchemaTypes.LIST, sub_type=unknown_types[0] + ), + True), + Rule('status', unknown_types[1], True) + ]), + rulesets={}, + enums={}, + imports=[ + ImportStatement('Employee', 'base.ys'), + ImportStatement('User', 'base.ys'), + ImportStatement('Status', 'base.ys'), + ], + unknowns=unknown_types + ) + + expected_ruleset_count = 2 + expected_enum_count = 1 + + schema = load_schema_imports(loaded_schema, schema_path) + self.assertEqual(expected_ruleset_count, len(schema.rulesets)) + self.assertEqual(expected_enum_count, len(schema.enums)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/loaders/test_parse_yamlator_schema.py b/tests/parser/loaders/test_parse_yamlator_schema.py new file mode 100644 index 0000000..bb55b77 --- /dev/null +++ b/tests/parser/loaders/test_parse_yamlator_schema.py @@ -0,0 +1,50 @@ +"""Test cases for the parse_yamlator_schema function""" + +import unittest +import typing + +from parameterized import parameterized +from yamlator.exceptions import InvalidSchemaFilenameError +from yamlator.exceptions import SchemaParseError +from yamlator.parser import SchemaSyntaxError +from yamlator.parser import parse_yamlator_schema + + +class TestParseYamlatorSchema(unittest.TestCase): + """Test cases for the parse_yamlator_schema function""" + + @parameterized.expand([ + ('with_none_path', None, ValueError), + ('with_empty_string_path', '', ValueError), + ('with_non_string_path', ['./path'], ValueError), + ('with_none_ys_extension', 'schema.yaml', InvalidSchemaFilenameError), + ('with_syntax_error', + './tests/files/invalid_files/invalid_ruleset_name.ys', + SchemaSyntaxError), + ('with_schema_nested_union', + './tests/files/invalid_files/nested_union.ys', + SchemaParseError), + ]) + def test_with_invalid_schema_paths(self, name: str, + schema_path: typing.Any, + expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + parse_yamlator_schema(schema_path) + + @parameterized.expand([ + ('without_any_imports', './tests/files/valid/valid.ys'), + ('with_imports', './tests/files/valid/with_imports.ys') + ]) + def test_with_valid_schema_paths(self, name, schema_path): + # Unused by test case, however is required by the parameterized library + del name + + schema = parse_yamlator_schema(schema_path) + self.assertIsNotNone(schema) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/loaders/test_resolve_unknown_types.py b/tests/parser/loaders/test_resolve_unknown_types.py new file mode 100644 index 0000000..245934b --- /dev/null +++ b/tests/parser/loaders/test_resolve_unknown_types.py @@ -0,0 +1,70 @@ +"""Test cases for the resolve_unknown_types function""" + +import unittest + +from parameterized import parameterized + +from yamlator.types import RuleType +from yamlator.types import SchemaTypes +from yamlator.types import YamlatorEnum +from yamlator.types import YamlatorRuleset +from yamlator.parser.loaders import resolve_unknown_types +from yamlator.exceptions import ConstructNotFoundError + + +class TestResolveUnknownTypes(unittest.TestCase): + """Test cases for the resolve_unknown_types function""" + + @parameterized.expand([ + ('with_unknown_types_is_none', None, {}, {}, ValueError), + ('with_wrong_type_for_unknown_types', {}, {}, {}, TypeError), + ('with_ruleset_is_none', [], None, {}, ValueError), + ('with_enum_is_none', [], {}, None, ValueError), + ('with_ruleset_and_enum_is_none', [], None, None, ValueError), + ('with_ruleset_wrong_type', [], [], {}, TypeError), + ('with_enum_wrong_type', [], {}, [], TypeError), + ('with_ruleset_and_enum_wrong_type', [], [], [], TypeError), + ]) + def test_resolve_unknown_types_invalid_parameters( + self, name: str, unknown_types: list, + rulesets: dict, enums: dict, expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + resolve_unknown_types(unknown_types, rulesets, enums) + + def test_resolve_unknown_types_unknown_construct_type(self): + unknown_types = [ + RuleType(SchemaTypes.UNKNOWN, lookup='Hello') + ] + + with self.assertRaises(ConstructNotFoundError): + resolve_unknown_types(unknown_types, {}, {}) + + def test_resolve_unknown_types(self): + unknown_types = [ + RuleType(SchemaTypes.UNKNOWN, lookup='Hello'), + RuleType(SchemaTypes.UNKNOWN, lookup='Status'), + RuleType(SchemaTypes.UNKNOWN, lookup='Users') + ] + + rulesets = { + 'Hello': YamlatorRuleset('Hello', []), + 'Users': YamlatorRuleset('Users', []) + } + + enums = { + 'Status': YamlatorEnum('Status', {}) + } + + resolve_unknown_types(unknown_types.copy(), rulesets, enums) + + # Check that the unknown types have been resolved + self.assertEqual(SchemaTypes.RULESET, unknown_types[0].schema_type) + self.assertEqual(SchemaTypes.ENUM, unknown_types[1].schema_type) + self.assertEqual(SchemaTypes.RULESET, unknown_types[2].schema_type) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/test_parse_schema.py b/tests/parser/test_parse_schema.py index e45411c..648fbb6 100644 --- a/tests/parser/test_parse_schema.py +++ b/tests/parser/test_parse_schema.py @@ -50,7 +50,7 @@ def test_parse_with_valid_content(self, name: str, schema_path: str, schema_content = load_schema(schema_path) instructions = parse_schema(schema_content) - main = instructions.get('main') + main = instructions.root self.assertIsNotNone(instructions) self.assertIsNotNone(main) @@ -78,11 +78,6 @@ def test_parse_with_valid_content(self, name: str, schema_path: str, './tests/files/invalid_files/invalid_ruleset_name.ys', MalformedRulesetNameError ), - ( - 'with_ruleset_not_defined', - './tests/files/invalid_files/missing_defined_ruleset.ys', - SchemaParseError - ), ( 'union_with_nested_union', './tests/files/invalid_files/nested_union.ys', diff --git a/tests/parser/test_schema_transformer.py b/tests/parser/test_schema_transformer.py index 9f0b45b..5bf4578 100644 --- a/tests/parser/test_schema_transformer.py +++ b/tests/parser/test_schema_transformer.py @@ -6,12 +6,15 @@ import lark from parameterized import parameterized -from yamlator.exceptions import ConstructNotFoundError from yamlator.exceptions import NestedUnionError from yamlator.parser import SchemaTransformer -from yamlator.types import EnumItem, Rule, RuleType -from yamlator.types import YamlatorEnum, YamlatorRuleset, SchemaTypes +from yamlator.types import EnumItem +from yamlator.types import Rule +from yamlator.types import RuleType +from yamlator.types import YamlatorEnum +from yamlator.types import YamlatorRuleset +from yamlator.types import SchemaTypes class TestSchemaTransformer(unittest.TestCase): @@ -38,6 +41,7 @@ def setUp(self): def test_rule_name(self, name: str, rule_name: str, expected: str): # Unused by test case, however is required by the parameterized library del name + token = lark.Token('TOKEN', value=rule_name) processed_token = self.transformer.rule_name([token]) self.assertEqual(expected, processed_token.value) @@ -90,11 +94,11 @@ def test_start(self): ]) ] - ruleset_items = self.transformer.start(instructions) - rulesets = ruleset_items.get('rules') - enums = ruleset_items.get('enums') + schema = self.transformer.start(instructions) + rulesets = schema.rulesets + enums = schema.enums - self.assertIsNotNone(ruleset_items.get('main')) + self.assertIsNotNone(schema.root) self.assertEqual(expected_enum_count, len(enums)) self.assertEqual(expected_ruleset_count, len(rulesets)) @@ -169,10 +173,13 @@ def test_container_type(self): ]) def test_container_type_construct_does_not_exist(self, name: str, seen_constructs: dict): - token = lark.Token(type_='TOKEN', value=name) + # Unused by test case, however is required by the parameterized library + del name + + token = lark.Token(type_='TOKEN', value='Foo') self.transformer.seen_constructs = seen_constructs - with self.assertRaises(ConstructNotFoundError): - self.transformer.container_type(token) + rule = self.transformer.container_type(token) + self.assertEqual(SchemaTypes.UNKNOWN, rule.schema_type) def test_regex_type(self): token = 'test{1}' diff --git a/tests/types/test_import_statements.py b/tests/types/test_import_statements.py new file mode 100644 index 0000000..29e2ad7 --- /dev/null +++ b/tests/types/test_import_statements.py @@ -0,0 +1,31 @@ +"""Test cases for the ImportStatement class""" + +import unittest + +from parameterized import parameterized +from yamlator.types import ImportStatement + + +class TestImportStatement(unittest.TestCase): + """Test cases for the ImportStatement class""" + + @parameterized.expand([ + ('with_none_item', None, './test', ValueError), + ('with_item_as_an_empty_string', '', './test', ValueError), + ('with_item_wrong_type', ['Test'], './test.ys', TypeError), + ('with_none_path', 'Test', None, ValueError), + ('with_path_as_an_empty_string', 'Test', '', ValueError), + ('with_item_wrong_type', 'Test', ['./test.ys'], TypeError), + ]) + def test_import_statements_invalid_params(self, name: str, item: str, + path: str, + expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + ImportStatement(item, path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/types/test_yamlator_schema.py b/tests/types/test_yamlator_schema.py new file mode 100644 index 0000000..8e2e81b --- /dev/null +++ b/tests/types/test_yamlator_schema.py @@ -0,0 +1,27 @@ +"""Test cases for the YamlatorSchema class""" + +import unittest + +from yamlator.types import YamlatorSchema + + +class TestYamlatorSchema(unittest.TestCase): + """Test cases for the YamlatorSchema class""" + + def test_schema_with_none_parameters(self): + expected_rule_count = 0 + expected_rulesets = {} + expected_enums = {} + + schema = YamlatorSchema(root=None, rulesets=None, enums=None) + + root = schema.root + self.assertIsNotNone(root) + self.assertEqual(expected_rule_count, len(root.rules)) + + self.assertIsNotNone(expected_rulesets, schema.rulesets) + self.assertIsNotNone(expected_enums, schema.enums) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/validators/test_validate_yaml.py b/tests/validators/test_validate_yaml.py index 2a81c9b..4cfb391 100644 --- a/tests/validators/test_validate_yaml.py +++ b/tests/validators/test_validate_yaml.py @@ -18,22 +18,33 @@ from yamlator.types import EnumItem from yamlator.types import YamlatorEnum from yamlator.types import YamlatorRuleset +from yamlator.types import YamlatorSchema from yamlator.types import SchemaTypes from yamlator.validators.core import validate_yaml -def create_flat_schema(): +def create_empty_schema() -> YamlatorSchema: + return YamlatorSchema( + root=YamlatorRuleset('main', []), + rulesets={}, + enums={} + ) + + +def create_flat_schema() -> YamlatorSchema: rules = [ Rule('message', RuleType(schema_type=SchemaTypes.STR), True), Rule('number', RuleType(schema_type=SchemaTypes.INT), False), ] - return { - 'main': YamlatorRuleset('main', rules), - 'rules': {} - } + + return YamlatorSchema( + root=YamlatorRuleset('main', rules), + rulesets={}, + enums={}, + ) -def create_complex_schema(): +def create_complex_schema() -> YamlatorSchema: person_ruleset = YamlatorRuleset('ruleset', [ Rule('name', RuleType(schema_type=SchemaTypes.STR), True), Rule('age', RuleType(schema_type=SchemaTypes.INT), False) @@ -66,22 +77,23 @@ def create_complex_schema(): RuleType(schema_type=SchemaTypes.ENUM, lookup='Status'), False), ]) - return { - 'main': main_ruleset, - 'rules': {'person': person_ruleset}, - 'enums': {'Status': status_enum} - } + return YamlatorSchema( + root=main_ruleset, + rulesets={'person': person_ruleset}, + enums={'Status': status_enum} + ) -FLAT_RULESET = create_flat_schema() -COMPLEX_RULESET = create_complex_schema() +EMPTY_SCHEMA = create_empty_schema() +FLAT_SCHEMA = create_flat_schema() +COMPLEX_SCHEMA = create_complex_schema() class TestWrangleData(unittest.TestCase): """Test cases for the validate_yaml function""" @parameterized.expand([ - ('none_data', None, FLAT_RULESET), + ('none_data', None, FLAT_SCHEMA), ('none_instructions', {'message': 'hello'}, None), ('none_data_and_instructions', None, None), ]) @@ -94,93 +106,93 @@ def test_validator_invalid_parameters(self, name: str, data: Data, validate_yaml(data, instructions) @parameterized.expand([ - ('empty_data_and_schema', {}, {}, 0), - ('empty_schema', {}, {'message': 'hello'}, 0), - ('primitive_data_schema', FLAT_RULESET, { + ('empty_data_and_schema', EMPTY_SCHEMA, {}, 0), + ('empty_schema', EMPTY_SCHEMA, {'message': 'hello'}, 0), + ('primitive_data_schema', FLAT_SCHEMA, { 'message': 'hello', 'number': 1 }, 0), - ('primitive_data_invalid_data', FLAT_RULESET, { + ('primitive_data_invalid_data', FLAT_SCHEMA, { 'message': 12, 'number': [] }, 2), - ('primitive_data_missing_required', FLAT_RULESET, { + ('primitive_data_missing_required', FLAT_SCHEMA, { 'number': 2 }, 1), - ('primitive_data_missing_optional', FLAT_RULESET, { + ('primitive_data_missing_optional', FLAT_SCHEMA, { 'message': 'hello' }, 0), - ('int_list', COMPLEX_RULESET, { + ('int_list', COMPLEX_SCHEMA, { 'num_lists': [[0, 1, 2], [3, 4, 5]] }, 0), - ('invalid_list_type', COMPLEX_RULESET, { + ('invalid_list_type', COMPLEX_SCHEMA, { 'num_lists': [ ['hello', 'world'] ] }, 2), - ('list_ruleset', COMPLEX_RULESET, { + ('list_ruleset', COMPLEX_SCHEMA, { 'personList': [ {'name': 'hello', 'age': 2}, {'name': 'world'} ] }, 0), - ('list_ruleset_invalid_type', COMPLEX_RULESET, { + ('list_ruleset_invalid_type', COMPLEX_SCHEMA, { 'personList': [ {'name': 0}, {'age': 2} ] }, 2), - ('valid_ruleset_type', COMPLEX_RULESET, { + ('valid_ruleset_type', COMPLEX_SCHEMA, { 'person': { 'name': 'Test', 'age': 100 } }, 0), - ('valid_ruleset_missing_optional', COMPLEX_RULESET, { + ('valid_ruleset_missing_optional', COMPLEX_SCHEMA, { 'person': { 'name': 'Test' } }, 0), - ('invald_ruleset_type', COMPLEX_RULESET, { + ('invald_ruleset_type', COMPLEX_SCHEMA, { 'person': 3 }, 1), - ('invalid_list_ruleset_type', COMPLEX_RULESET, { + ('invalid_list_ruleset_type', COMPLEX_SCHEMA, { 'personList': [0, 2, 3] }, 3), - ('valid_dict_type', COMPLEX_RULESET, { + ('valid_dict_type', COMPLEX_SCHEMA, { 'my_map': { 'val1': 'Hello', 'val2': 'World' } }, 0), - ('invalid_subtype_map_type', COMPLEX_RULESET, { + ('invalid_subtype_map_type', COMPLEX_SCHEMA, { 'my_map': { 'val1': 1, 'val2': [] } }, 2), - ('invalid_map_type', COMPLEX_RULESET, { + ('invalid_map_type', COMPLEX_SCHEMA, { 'my_map': [] }, 1), - ('valid_empty_map_type', COMPLEX_RULESET, { + ('valid_empty_map_type', COMPLEX_SCHEMA, { 'my_map': {} }, 0), - ('valid_any_type_list', COMPLEX_RULESET, { + ('valid_any_type_list', COMPLEX_SCHEMA, { 'my_any_list': [1, 2, None, 'hello'] }, 0), - ('enum_value_matches_rule', COMPLEX_RULESET, { + ('enum_value_matches_rule', COMPLEX_SCHEMA, { 'status': 'error', }, 0), - ('enum_value_does_not_matche_rule', COMPLEX_RULESET, { + ('enum_value_does_not_matche_rule', COMPLEX_SCHEMA, { 'status': 'not_found', }, 1), - ('enum_value_wrong_type', COMPLEX_RULESET, { + ('enum_value_wrong_type', COMPLEX_SCHEMA, { 'status': [], }, 1), ]) - def test_validator(self, name, ruleset, data, expected_violations_count): + def test_validator(self, name, schema, data, expected_violations_count): # Unused by test case, however is required by the parameterized library del name - violations = validate_yaml(data, ruleset) + violations = validate_yaml(data, schema) self.assertEqual(expected_violations_count, len(violations)) diff --git a/yamlator/cmd/core.py b/yamlator/cmd/core.py index 9dee3c6..f701103 100644 --- a/yamlator/cmd/core.py +++ b/yamlator/cmd/core.py @@ -6,12 +6,12 @@ from typing import Iterator from yamlator.utils import load_yaml_file -from yamlator.utils import load_schema from yamlator.parser import SchemaSyntaxError -from yamlator.parser import parse_schema +from yamlator.parser import parse_yamlator_schema from yamlator.validators.core import validate_yaml from yamlator.exceptions import SchemaParseError +from yamlator.exceptions import ConstructNotFoundError from yamlator.exceptions import InvalidSchemaFilenameError from yamlator.violations import Violation @@ -94,11 +94,15 @@ def validate_yaml_data_from_file(yaml_filepath: str, FileNotFoundError: If either argument cannot be found on the file system InvalidSchemaFilenameError: If `schema_filepath` does not have a valid filename that ends with the `.ys` extension. + SchemaParseError: If there was an error parsing the schema, e.g + syntax error or a type that was not found """ yaml_data = load_yaml_file(yaml_filepath) - ruleset_data = load_schema(schema_filepath) - - instructions = parse_schema(ruleset_data) + instructions = None + try: + instructions = parse_yamlator_schema(schema_filepath) + except ConstructNotFoundError as ex: + raise SchemaParseError(ex) from ex return validate_yaml(yaml_data, instructions) diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index e835697..0170fb5 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -7,6 +7,7 @@ start: instructions* | strict_ruleset | schema_entry | strict_schema_entry + | import_statement ?values: INT -> integer | FLOAT -> float @@ -15,6 +16,8 @@ start: instructions* schema_entry: "schema" "{" rule+ "}" strict_schema_entry: "strict schema" "{" rule+ "}" +import_statement: "import" /[A-Z]{1}[a-zA-Z0-9_]+/ "from" /\"(?:[\S]+\/)*[\S]+.ys\"/ + ruleset: "ruleset" /[A-Z]{1}[a-zA-Z0-9_]+/ "{" rule+ "}" strict_ruleset: "strict ruleset" /[A-Z]{1}[a-zA-Z0-9_]+/ "{" rule+ "}" diff --git a/yamlator/parser/__init__.py b/yamlator/parser/__init__.py new file mode 100644 index 0000000..5c907f8 --- /dev/null +++ b/yamlator/parser/__init__.py @@ -0,0 +1,23 @@ +"""Module that contains utility functions and exceptions for parsing a +Yamlator schemas +""" + +from .core import parse_schema +from .core import SchemaTransformer +from .core import SchemaSyntaxError +from .core import MissingRulesError +from .core import MalformedRulesetNameError +from .core import MalformedEnumNameError +from .core import SchemaParseError +from .loaders import parse_yamlator_schema + +__all__ = [ + 'parse_schema', + 'SchemaTransformer', + 'SchemaSyntaxError', + 'MissingRulesError', + 'SchemaParseError', + 'MalformedRulesetNameError', + 'MalformedEnumNameError', + 'parse_yamlator_schema' +] diff --git a/yamlator/parser.py b/yamlator/parser/core.py similarity index 83% rename from yamlator/parser.py rename to yamlator/parser/core.py index 0554a9c..759ea76 100644 --- a/yamlator/parser.py +++ b/yamlator/parser/core.py @@ -19,22 +19,23 @@ from yamlator.types import YamlatorRuleset from yamlator.types import YamlatorEnum from yamlator.types import YamlatorType +from yamlator.types import PartiallyLoadedYamlatorSchema from yamlator.types import RuleType from yamlator.types import UnionRuleType from yamlator.types import EnumItem from yamlator.types import SchemaTypes +from yamlator.types import ImportStatement from yamlator.exceptions import NestedUnionError -from yamlator.exceptions import ConstructNotFoundError from yamlator.exceptions import SchemaParseError -_package_dir = Path(__file__).parent.absolute() +_package_dir = Path(__file__).parent.parent.absolute() _GRAMMAR_FILE = os.path.join(_package_dir, 'grammar/grammar.lark') _QUOTES_REGEX = re.compile(r'\"|\'') -def parse_schema(schema_content: str) -> dict: +def parse_schema(schema_content: str) -> PartiallyLoadedYamlatorSchema: """Parses a schema into a set of instructions that can be used to validate a YAML file. @@ -81,9 +82,13 @@ class SchemaTransformer(Transformer): | rule_name type NEW_LINES """ - # Used to track previously seen enums or rulesets to dynamically - # determine the type of the rule if a enum or ruleset is used - seen_constructs = {} + def __init__(self, visit_tokens: bool = True) -> None: + super().__init__(visit_tokens) + + # Used to track previously seen enums or rulesets to dynamically + # determine the type of the rule is a enum or ruleset + self.seen_constructs = {} + self.unknown_types = [] def rule_name(self, tokens: 'list[Token]') -> Token: """Processes the rule name by removing any quotes""" @@ -118,15 +123,19 @@ def strict_ruleset(self, tokens: 'list[Token]') -> YamlatorRuleset: self.seen_constructs[name] = SchemaTypes.RULESET return YamlatorRuleset(name, rules, is_strict=True) - def start(self, instructions: Iterator[YamlatorType]) -> dict: + def start(self, instructions: Iterator[YamlatorType]) \ + -> PartiallyLoadedYamlatorSchema: """Transforms the instructions into a dict that sorts the rulesets, enums and entry point to validate the YAML data""" root = None rules = {} enums = {} + imports = [] + enum_handler = _EnumInstructionHandler(enums) handler_chain = _RulesetInstructionHandler(rules) - handler_chain.set_next_handler(_EnumInstructionHandler(enums)) + handler_chain.set_next_handler(enum_handler) + enum_handler.set_next_handler(_ImportInstructionHandler(imports)) for instruction in instructions: handler_chain.handle(instruction) @@ -135,11 +144,8 @@ def start(self, instructions: Iterator[YamlatorType]) -> dict: if root is not None: del rules['main'] - return { - 'main': root, - 'rules': rules, - 'enums': enums - } + return PartiallyLoadedYamlatorSchema(root, rules, enums, + imports, self.unknown_types) def str_type(self, _: 'list[Token]') -> RuleType: """Transforms a string type token into a RuleType object""" @@ -194,10 +200,11 @@ def container_type(self, token: 'list[Token]') -> RuleType: enum or ruleset cannot be found """ name = token[0] - schema_type = self.seen_constructs.get(name) - if schema_type is None: - raise ConstructNotFoundError(name) - return RuleType(schema_type=schema_type, lookup=name) + schema_type = self.seen_constructs.get(name, SchemaTypes.UNKNOWN) + rule_type = RuleType(schema_type=schema_type, lookup=name) + if schema_type == SchemaTypes.UNKNOWN: + self.unknown_types.append(rule_type) + return rule_type def regex_type(self, tokens: 'list[Token]') -> RuleType: """Transforms a regex type token into a RuleType object""" @@ -253,6 +260,15 @@ def string(self, token: str) -> str: """ return _QUOTES_REGEX.sub('', token) + def import_statement(self, tokens: 'list[Token]') -> ImportStatement: + """Transforms an import statement into a + `yamlator.types.ImportStatement` object + """ + item = tokens[0] + path = tokens[1] + path = _QUOTES_REGEX.sub('', path.value) + return ImportStatement(item.value, path) + class _InstructionHandler: """Base handle for dealing with Yamlator types""" @@ -310,6 +326,28 @@ def handle(self, instruction: YamlatorType) -> None: self._rulesets[instruction.name] = instruction +class _ImportInstructionHandler(_InstructionHandler): + """Import statement handler for putting all the + import statements into a single data structure + """ + + def __init__(self, imports: list): + """_ImportInstructionHandler init + + imports (list): Reference to a list that will store all the + import statements that were referenced in the Yamlator schema + """ + super().__init__() + self.imports = imports + + def handle(self, instruction: YamlatorType) -> None: + if instruction.container_type != ContainerTypes.IMPORT: + super().handle(instruction) + return + + self.imports.append(instruction) + + class SchemaSyntaxError(SyntaxError): """A generic syntax error in the schema content""" diff --git a/yamlator/parser/loaders.py b/yamlator/parser/loaders.py new file mode 100644 index 0000000..71d18e7 --- /dev/null +++ b/yamlator/parser/loaders.py @@ -0,0 +1,198 @@ +"""Contains functions to load """ +import re +import os + +from yamlator.utils import load_schema +from yamlator.types import RuleType +from yamlator.types import YamlatorSchema +from yamlator.types import YamlatorRuleset +from yamlator.types import YamlatorEnum +from yamlator.types import SchemaTypes +from yamlator.types import PartiallyLoadedYamlatorSchema +from yamlator.parser.core import parse_schema +from yamlator.exceptions import ConstructNotFoundError + + +_SLASHES_REGEX = re.compile(r'(?:\\{1}|\/{1})') + + +def parse_yamlator_schema(schema_path: str) -> YamlatorSchema: + """Parses a Yamlator schema from a given path on the file system + + Args: + schema_path (str): The file path to the schema file + + Returns: + A `yamlator.types.YamlatorSchema` object that contains + the contents of the schema file in a format that can + be processed by Yamlator + + Raises: + ValueError: If the schema path is `None`, not a string + or is an empty string + + yamlator.exceptions.InvalidSchemaFilenameError: If the filename + does not match a file with a `.ys` extension + + yamlator.exceptions.SchemaParseError: Raised when the parsing + process is interrupted + + yamlator.parser.SchemaSyntaxError: Raised when a syntax error + is detected in the schema + """ + if (schema_path is None) or (not isinstance(schema_path, str)): + raise ValueError('Expected parameter schema_path to be a string') + + schema_content = load_schema(schema_path) + schema = parse_schema(schema_content) + + context = fetch_schema_path(schema_path) + schema = load_schema_imports(schema, context) + return schema + + +def fetch_schema_path(schema_path: str) -> str: + """Fetches the current path for a schema file + + Args: + schema_path (str): The path to the Yamlator schema file + + Returns: + A string of the path that hosts the schema file + + Raises: + ValueError: If the parameter `schema_path` is `None` or not + a string + """ + if not schema_path: + raise ValueError( + 'Expected parameter schema_path to be a non-empty string') + + if not isinstance(schema_path, str): + raise TypeError('Expected parameter schema_path to be a string') + + context = _SLASHES_REGEX.split(schema_path)[:-1] + if not context: + return '.' + return _SLASHES_REGEX.sub('/', os.path.join(*context)) + + +def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, + schema_path: str) -> YamlatorSchema: + """Loads all import statements that have been defined in a Yamlator + schema file. This function will automatically load any import + statements from child schema files + + Args: + loaded_schema (yamlator.types.PartiallyLoadedYamlatorSchema): A schema + that has been partially loaded by the Lark transformer but has + not had all the imports resolved + + context (str): The path that contains the Yamlator schema file + + Returns: + A `yamlator.types.YamlatorSchema` object that has all the types + resolved + + Raises: + ValueError: If the `schema_path` is None, not a string or + `loaded_schema` is `None` + + yamlator.exceptions.InvalidSchemaFilenameError: If the filename + does not match a file with a `.ys` extension + + yamlator.exceptions.SchemaParseError: Raised when the parsing + process is interrupted + + yamlator.parser.SchemaSyntaxError: Raised when a syntax error + is detected in the schema + """ + if loaded_schema is None: + raise ValueError('Parameter loaded_schema should not None') + + if not schema_path: + raise ValueError( + 'Expected parameter schema_path to be a non-empty string') + + if not isinstance(schema_path, str): + raise TypeError('Expected parameter schema_path to be a string') + + if not isinstance(loaded_schema, PartiallyLoadedYamlatorSchema): + raise TypeError('Expected schema to be yamlator.types.PartiallyLoadedYamlatorSchema') # nopep8 pylint: disable=C0301 + + import_statements = loaded_schema.imports + root_rulesets = loaded_schema.rulesets + root_enums = loaded_schema.enums + + for path, resource_type in import_statements.items(): + full_path = os.path.join(schema_path, path) + schema = parse_yamlator_schema(full_path) + + imported_rulesets = schema.rulesets + imported_enums = schema.enums + + for resource in resource_type: + ruleset: YamlatorRuleset = imported_rulesets.get(resource) + if ruleset is not None: + root_rulesets[ruleset.name] = ruleset + continue + + enum: YamlatorEnum = imported_enums.get(resource) + if enum is not None: + root_enums[enum.name] = enum + continue + + unknown_types = loaded_schema.unknowns_rule_types + resolve_unknown_types(unknown_types, root_rulesets, root_enums) + return YamlatorSchema(loaded_schema.root, root_rulesets, root_enums) + + +def resolve_unknown_types(unknown_types: 'list[RuleType]', + rulesets: dict, enums: dict) -> bool: + """Resolves any types that are marked as unknown since the ruleset + or enum was imported into the schema. This function will go through + each unknown type and populate with the relevant rule type + + Args: + unknown_types (list[yamlator.types.RuleType]): A list of types that + have a `schema_type` as `SchemaType.UNKNOWN` + + rulesets (dict): A dictionary of rulesets that have been loaded from + the import statement defined in the schema + + enums (dict): A dictionary of enums that have been loaded from the + import statements defined in the schema + + Returns: + A boolean (true) to indicate it has executed successfully + + Raises: + yamlator.exceptions.ConstructNotFoundError: If the ruleset or enum + type was not found + """ + if unknown_types is None: + raise ValueError('Expected parameter unknown_types to not be None') + + if not isinstance(unknown_types, list): + raise TypeError('Expected unknown_types to be a list') + + if (rulesets is None) or (enums is None): + raise ValueError( + 'Expected parameters rulesets and enums to not be None') + + if (not isinstance(rulesets, dict)) or (not isinstance(enums, dict)): + raise TypeError( + 'Expected parameters rulesets and enums to be dictionaries') + + while len(unknown_types) > 0: + curr: RuleType = unknown_types.pop() + if enums.get(curr.lookup) is not None: + curr.schema_type = SchemaTypes.ENUM + continue + + if rulesets.get(curr.lookup) is not None: + curr.schema_type = SchemaTypes.RULESET + continue + + raise ConstructNotFoundError(curr.lookup) + return True diff --git a/yamlator/types.py b/yamlator/types.py index 825bdad..6e8552b 100644 --- a/yamlator/types.py +++ b/yamlator/types.py @@ -6,6 +6,7 @@ from typing import Union from typing import Iterator from collections import namedtuple +from collections import defaultdict Rule = namedtuple('Rule', ['name', 'rtype', 'is_required']) EnumItem = namedtuple('EnumItem', ['name', 'value']) @@ -15,7 +16,7 @@ class SchemaTypes(enum.Enum): - """Represents the support types that can be defined in a schema""" + """Represents the supported types that can be defined in a schema""" STR = enum.auto() INT = enum.auto() @@ -29,6 +30,10 @@ class SchemaTypes(enum.Enum): BOOL = enum.auto() UNION = enum.auto() + # Used when importing enums and ruleset + # at runtime since the actual type is not known + UNKNOWN = enum.auto() + class RuleType: """Represents a rule's data type that is defined in the Yamlator schema @@ -92,6 +97,10 @@ def __init__(self, schema_type: SchemaTypes, lookup: str = None, def schema_type(self): return self._schema_type + @schema_type.setter + def schema_type(self, value): + self._schema_type = value + @property def lookup(self) -> str: return self._lookup @@ -115,7 +124,8 @@ def __str__(self) -> str: SchemaTypes.LIST: 'list({})', SchemaTypes.MAP: 'map({})', SchemaTypes.BOOL: 'bool', - SchemaTypes.ANY: 'any' + SchemaTypes.ANY: 'any', + SchemaTypes.UNKNOWN: f'unknown({self.lookup})' } type_str = types[self.schema_type] @@ -175,6 +185,7 @@ class ContainerTypes(enum.Enum): """Enum of custom types used by Yamlator""" RULESET = 0 ENUM = 1 + IMPORT = 2 class YamlatorType: @@ -298,3 +309,197 @@ def __init__(self, name: str, items: dict): @property def items(self) -> dict: return self._items.copy() + + +class ImportStatement(YamlatorType): + """Represents a import statement in the Yamlator schema + + Attributes: + item (str): The name of the enum or ruleset that is being imported + path (str): The path to the schema file that contains the enum + or ruleset + """ + + def __init__(self, item: str, path: str): + """ImportStatement init + + Args: + item (str): The name of the enum or ruleset that is being imported + path (str): The path to the schema file that contains the enum + or ruleset + + Raises: + ValueError: If the `item` or `path` parameters are `None` + or are not a string + """ + if not item: + raise ValueError( + 'Expected parameter item to not be an empty string or None') + + if not path: + raise ValueError( + 'Expected parameter path to be an empty string or None') + + if not isinstance(item, str): + raise TypeError('Expected parameter item to be a string') + + if not isinstance(path, str): + raise TypeError('Expected parameter path to be a string') + + self._item = item + self._path = path + + name = f'({item}){path}' + super().__init__(name, ContainerTypes.IMPORT) + + @property + def item(self) -> str: + return self._item + + @property + def path(self) -> str: + return self._path + + +class YamlatorSchema: + """Maintains the rules and types that were defined in a Yamlator + schema which can then used to validate a YAML file. The schema will + keep a reference to a root ruleset, which acts as the entry point + for the validation process + + Attributes: + root (yamlator.types.YamlatorRuleset): The entry point ruleset + to start the validation process. The root will be defined + as a `schema` block in the `.ys` file and will called `main` + + rulesets (dict): A lookup to the ruleset objects that were defined + in the schema file. The key will be the ruleset name and the value + will be a `yamlator.types.YamlatorRuleset` object + + enums (dict): A lookup to the enum objects that were defined + in the schema file. The key will be the enum name and the value + will be a `yamlator.types.YamlatorEnum` object + """ + + def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict): + """YamlatorSchema init + + Args: + root (yamlator.types.YamlatorRuleset): The entry point ruleset + that is used to start the validation process + + rulesets (dict): A lookup to the ruleset objects that were + defined in the schema file. The key will be the ruleset name + and the value will be a `yamlator.types.YamlatorRuleset` object + + enums (dict): A lookup to the enum objects that were defined + in the schema file. The key will be the enum name and the + value will be a `yamlator.types.YamlatorEnum` object + """ + self._root = root + self._enums = enums + self._rulesets = rulesets + + @property + def root(self): + if self._root is None: + return YamlatorRuleset('main', []) + return self._root + + @property + def rulesets(self): + if self._rulesets is None: + return {} + return self._rulesets.copy() + + @property + def enums(self): + if self._enums is None: + return {} + return self._enums.copy() + + def __str__(self) -> str: + return str({ + 'root': self.root, + 'enums': self.enums, + 'rulesets': self.rulesets + }) + + +class PartiallyLoadedYamlatorSchema(YamlatorSchema): + """Represents a Yamlator schema that has been loaded from a file + but has not resolved all the types that have been defined. Unlike + `yamlator.types.YamlatorSchema`, this object will contain any import + statements and imported types that during the parsing process are + unknown on load + + Attributes: + root (yamlator.types.YamlatorRuleset): The entry point ruleset + to start the validation process. The root will be defined + as a `schema` block in the `.ys` file and will called `main + + rulesets (dict): A lookup to the ruleset objects that were defined + in the schema file. The key will be the ruleset name and the value + will be a `yamlator.types.YamlatorRuleset` object + + enums (dict): A lookup to the enum objects that were defined + in the schema file. The key will be the enum name and the value + will be a `yamlator.types.YamlatorEnum` object + + imports (Iterator[yamlator.types.ImportStatement]): A list of + `yamlator.types.ImportStatement` that contain all the import + statements that were defined in the schema file + + unknowns (Iterator[yamlator.types.RuleType]): A list of + `yamlator.types.RuleType` objects that have a schema type of + `yamlator.types.SchemaTypes.UNKNOWN` + """ + + def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, + imports: Iterator[ImportStatement], + unknowns: Iterator[RuleType] = None): + """PartiallyLoadedYamlatorSchema init + + Args: + root (yamlator.types.YamlatorRuleset): The entry point ruleset + that is used to start the validation process + + rulesets (dict): A lookup to the ruleset objects that were + defined in the schema file. The key will be the ruleset name + and the value will be a `yamlator.types.YamlatorRuleset` object + + enums (dict): A lookup to the enum objects that were defined + in the schema file. The key will be the enum name and the + value will be a `yamlator.types.YamlatorEnum` object + + imports (Iterator[yamlator.types.ImportStatement]): A list + `yamlator.types.ImportStatement` that contain all import + statements that were defined in the schema file + + unknowns (Iterator[yamlator.types.RuleType]): A list of + `yamlator.types.RuleType` objects that have a schema type + of `yamlator.types.SchemaTypes.UNKNOWN` + """ + super().__init__(root, rulesets, enums) + + self._unknowns = unknowns + if unknowns is None: + self._unknowns = [] + + self.__group_imports(imports) + + def __group_imports(self, imports: Iterator[ImportStatement]) -> None: + # Group imports and the requested type to prevent + # loading the same schema file multiple times + import_statements = defaultdict(list) + for state in imports: + import_statements[state.path].append(state.item) + self._imports = import_statements + + @property + def imports(self) -> dict: + return self._imports.copy() + + @property + def unknowns_rule_types(self) -> Iterator[RuleType]: + return self._unknowns.copy() diff --git a/yamlator/validators/core.py b/yamlator/validators/core.py index a1ffea3..4a850b3 100644 --- a/yamlator/validators/core.py +++ b/yamlator/validators/core.py @@ -3,7 +3,7 @@ """ from collections import deque -from yamlator.types import YamlatorRuleset +from yamlator.types import YamlatorSchema from yamlator.validators import AnyTypeValidator from yamlator.validators import BuiltInTypeValidator @@ -19,14 +19,14 @@ from yamlator.validators.base_validator import Validator -def validate_yaml(yaml_data: dict, instructions: dict) -> deque: +def validate_yaml(yaml_data: dict, schema: YamlatorSchema) -> deque: """Validate YAML data by comparing the data against a set of instructions. Any violations will be collected and returned in a `deque` Args: yaml_data (dict): The YAML data to validate. Assumes the YAML contains a root key - instructions (dict): Contains the enums and rulesets that will be + schema (dict): Contains the enums and rulesets that will be used to validate the YAML data Returns: @@ -38,23 +38,23 @@ def validate_yaml(yaml_data: dict, instructions: dict) -> deque: if yaml_data is None: raise ValueError('yaml_data should not be None') - if instructions is None: + if schema is None: raise ValueError('instructions should not be None') default_key = '-' violations = deque() - validators = _create_validators_chain(instructions, violations) + validators = _create_validators_chain(schema, violations) validators.validate(default_key, yaml_data, default_key, None) return violations -def _create_validators_chain(instructions: dict, +def _create_validators_chain(instructions: YamlatorSchema, violations: deque) -> Validator: - ruleset_lookups = instructions.get('rules', {}) - enum_looksups = instructions.get('enums', {}) - entry_point = instructions.get('main', YamlatorRuleset('main', [])) + ruleset_lookups = instructions.rulesets + enum_looksups = instructions.enums + entry_point = instructions.root root = EntryPointValidator(violations, entry_point) optional_validator = OptionalValidator(violations) From e388bfae0d16ff5d1014ea0e545e289fcbea2b24 Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sun, 5 Mar 2023 12:35:35 +0000 Subject: [PATCH 2/7] Import Namespaces (#57) * Created an initial implementation of namespaces in yamlator. Updated the example to use a namespace as core. Updated the grammer to make namespaces an optional component * Updated the grammar to improve the readability by adding terminals. Updated the mapping process for combining schemas * Updated the changelog --- changelog.md | 1 + example/imports/import.ys | 4 +- .../files/valid/with_import_and_namespaces.ys | 8 ++ .../loaders/test_map_imported_resource.py | 92 +++++++++++++++++++ .../loaders/test_parse_yamlator_schema.py | 4 +- yamlator/grammar/grammar.lark | 15 +-- yamlator/parser/core.py | 16 +++- yamlator/parser/loaders.py | 81 +++++++++++++--- yamlator/types.py | 20 +++- 9 files changed, 216 insertions(+), 25 deletions(-) create mode 100644 tests/files/valid/with_import_and_namespaces.ys create mode 100644 tests/parser/loaders/test_map_imported_resource.py diff --git a/changelog.md b/changelog.md index cb6faf2..60d4ceb 100644 --- a/changelog.md +++ b/changelog.md @@ -57,3 +57,4 @@ ## v4.0.0 (TBC) * Added import statements to the Yamlator schema syntax +* Added namespaces to the import statements with the `as` keyword diff --git a/example/imports/import.ys b/example/imports/import.ys index 548c797..084741d 100644 --- a/example/imports/import.ys +++ b/example/imports/import.ys @@ -1,7 +1,7 @@ import Project from "../lists/lists.ys" -import Values from "common.ys" +import Values from "common.ys" as core schema { user Project - value Values + value core.Values } \ No newline at end of file diff --git a/tests/files/valid/with_import_and_namespaces.ys b/tests/files/valid/with_import_and_namespaces.ys new file mode 100644 index 0000000..12841a7 --- /dev/null +++ b/tests/files/valid/with_import_and_namespaces.ys @@ -0,0 +1,8 @@ +import Employee from "base.ys" as e +import User from "base.ys" +import Status from "base.ys" as core + +schema { + employees list(e.Employee) + status core.Status +} diff --git a/tests/parser/loaders/test_map_imported_resource.py b/tests/parser/loaders/test_map_imported_resource.py new file mode 100644 index 0000000..1e7c7e8 --- /dev/null +++ b/tests/parser/loaders/test_map_imported_resource.py @@ -0,0 +1,92 @@ +"""Test case for the `map_imported_resource` function""" + +import unittest + +from collections import namedtuple +from parameterized import parameterized +from yamlator.types import Rule +from yamlator.types import RuleType +from yamlator.types import SchemaTypes +from yamlator.types import YamlatorRuleset +from yamlator.parser.loaders import map_imported_resource + + +Params = namedtuple('Params', [ + 'namespace', + 'resource_type', + 'resource_lookup', + 'imported_resources'] +) + + +def create_hello_ruleset(): + return YamlatorRuleset('Hello', [ + Rule('msg', RuleType(SchemaTypes.STR), False) + ]) + + +NAMESPACE = 'test' +FAKE_RESOURCE_NAME = 'Fake' +HELLO_RESOURCE_NAME = 'Hello' +HELLO_RULESET = { + 'Hello': create_hello_ruleset() +} + + +class TestMapImportedResource(unittest.TestCase): + """Test cases for the `map_imported_resource` function""" + + @parameterized.expand([ + ('all_none_params', Params(None, None, None, None), ValueError), + ('none_resource_type', + Params(None, None, {}, {'Hello': HELLO_RULESET}), ValueError), + ('none_resource_lookup', + Params(None, HELLO_RESOURCE_NAME, None, + {'Hello': HELLO_RULESET}), ValueError), + ('none_imported_resources', + Params(None, HELLO_RESOURCE_NAME, {}, None), ValueError), + ('resource_lookup_wrong_type', + Params(NAMESPACE, HELLO_RESOURCE_NAME, [], {}), TypeError), + ('imported_resources_wrong_type', + Params(NAMESPACE, HELLO_RESOURCE_NAME, {}, []), TypeError), + ]) + def test_map_imported_resource_invalid_args(self, name: str, + params: Params, + expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + map_imported_resource( + params.namespace, + params.resource_type, + params.resource_lookup, + params.imported_resources + ) + + @parameterized.expand([ + ('with_resource_that_exists_without_namespace', + Params(None, HELLO_RESOURCE_NAME, {}, HELLO_RULESET), True), + ('with_resource_that_exists_with_namespace', + Params(NAMESPACE, HELLO_RESOURCE_NAME, {}, HELLO_RULESET), True), + ('with_resource_that_does_not_exist_without_namespace', + Params(None, FAKE_RESOURCE_NAME, {}, HELLO_RULESET), False), + ('with_resource_that_does_not_exist_with_namespace', + Params(NAMESPACE, FAKE_RESOURCE_NAME, {}, HELLO_RULESET), False), + ]) + def test_map_imported_resource(self, name: str, params: Params, + expected_result: bool): + # Unused by test case, however is required by the parameterized library + del name + + result = map_imported_resource( + params.namespace, + params.resource_type, + params.resource_lookup, + params.imported_resources + ) + self.assertEqual(expected_result, result) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/loaders/test_parse_yamlator_schema.py b/tests/parser/loaders/test_parse_yamlator_schema.py index bb55b77..851cbd8 100644 --- a/tests/parser/loaders/test_parse_yamlator_schema.py +++ b/tests/parser/loaders/test_parse_yamlator_schema.py @@ -36,7 +36,9 @@ def test_with_invalid_schema_paths(self, name: str, @parameterized.expand([ ('without_any_imports', './tests/files/valid/valid.ys'), - ('with_imports', './tests/files/valid/with_imports.ys') + ('with_imports', './tests/files/valid/with_imports.ys'), + ('with_namespace_imports', + './tests/files/valid/with_import_and_namespaces.ys'), ]) def test_with_valid_schema_paths(self, name, schema_path): # Unused by test case, however is required by the parameterized library diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index 0170fb5..04c43a8 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -16,14 +16,14 @@ start: instructions* schema_entry: "schema" "{" rule+ "}" strict_schema_entry: "strict schema" "{" rule+ "}" -import_statement: "import" /[A-Z]{1}[a-zA-Z0-9_]+/ "from" /\"(?:[\S]+\/)*[\S]+.ys\"/ +import_statement: "import" CONTAINER_TYPE_NAME "from" /\"(?:[\S]+\/)*[\S]+.ys\"/ ("as" NAMESPACE)? -ruleset: "ruleset" /[A-Z]{1}[a-zA-Z0-9_]+/ "{" rule+ "}" -strict_ruleset: "strict ruleset" /[A-Z]{1}[a-zA-Z0-9_]+/ "{" rule+ "}" +ruleset: "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" +strict_ruleset: "strict ruleset" CONTAINER_TYPE_NAME" {" rule+ "}" // Enum constructs -enum: "enum" /[A-Z]{1}[a-zA-Z0-9_]+/ "{" enum_item+ "}" -enum_item: /[A-Z0-9_]+/ "=" values +enum: "enum" CONTAINER_TYPE_NAME "{" enum_item+ "}" +enum_item: ENUM_ITEM_NAME "=" values // Rule definitions ?rule: required_rule @@ -55,11 +55,14 @@ float_type: "float" bool_type: "bool" list_type: "list""(" type ")" map_type: "map""(" type ")" -container_type: /[A-Z]{1}[a-zA-Z0-9_]+/ +container_type: (NAMESPACE".")? CONTAINER_TYPE_NAME regex_type: "regex""(" string ")" union_type: "union""(" [type ("," type)+] ")" NEW_LINES: "\n"+ +NAMESPACE: /[a-z]+/ +CONTAINER_TYPE_NAME: /[A-Z]{1}[a-zA-Z0-9_]+/ +ENUM_ITEM_NAME: /[A-Z0-9_]+/ string: ESCAPED_STRING diff --git a/yamlator/parser/core.py b/yamlator/parser/core.py index 759ea76..b35a748 100644 --- a/yamlator/parser/core.py +++ b/yamlator/parser/core.py @@ -192,14 +192,17 @@ def enum(self, tokens: 'list[Token]') -> YamlatorEnum: self.seen_constructs[name] = SchemaTypes.ENUM return YamlatorEnum(name.value, enums) - def container_type(self, token: 'list[Token]') -> RuleType: + def container_type(self, tokens: 'list[Token]') -> RuleType: """Transforms a container type token into a RuleType object Raises: yamlator.exceptions.ConstructNotFoundError: Raised if the enum or ruleset cannot be found """ - name = token[0] + name = tokens[0] + if len(tokens) > 1: + name = f'{tokens[0]}.{tokens[1]}' + schema_type = self.seen_constructs.get(name, SchemaTypes.UNKNOWN) rule_type = RuleType(schema_type=schema_type, lookup=name) if schema_type == SchemaTypes.UNKNOWN: @@ -266,8 +269,15 @@ def import_statement(self, tokens: 'list[Token]') -> ImportStatement: """ item = tokens[0] path = tokens[1] + + namespace = None + try: + namespace = tokens[2].value + except IndexError: + pass + path = _QUOTES_REGEX.sub('', path.value) - return ImportStatement(item.value, path) + return ImportStatement(item.value, path, namespace) class _InstructionHandler: diff --git a/yamlator/parser/loaders.py b/yamlator/parser/loaders.py index 71d18e7..e4e5b23 100644 --- a/yamlator/parser/loaders.py +++ b/yamlator/parser/loaders.py @@ -2,11 +2,11 @@ import re import os +from typing import List from yamlator.utils import load_schema from yamlator.types import RuleType from yamlator.types import YamlatorSchema -from yamlator.types import YamlatorRuleset -from yamlator.types import YamlatorEnum +from yamlator.types import YamlatorType from yamlator.types import SchemaTypes from yamlator.types import PartiallyLoadedYamlatorSchema from yamlator.parser.core import parse_schema @@ -131,23 +131,82 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, imported_rulesets = schema.rulesets imported_enums = schema.enums - for resource in resource_type: - ruleset: YamlatorRuleset = imported_rulesets.get(resource) - if ruleset is not None: - root_rulesets[ruleset.name] = ruleset + for (resource, namespace) in resource_type: + has_mapped_rulesets = map_imported_resource(namespace, + resource, + root_rulesets, + imported_rulesets) + if has_mapped_rulesets: continue - enum: YamlatorEnum = imported_enums.get(resource) - if enum is not None: - root_enums[enum.name] = enum - continue + map_imported_resource(namespace, resource, + root_enums, imported_enums) unknown_types = loaded_schema.unknowns_rule_types resolve_unknown_types(unknown_types, root_rulesets, root_enums) return YamlatorSchema(loaded_schema.root, root_rulesets, root_enums) -def resolve_unknown_types(unknown_types: 'list[RuleType]', +def map_imported_resource(namespace: str, resource_type: str, + resource_lookup: dict, + imported_resources: dict) -> dict: + """Maps the imported resources to the `resource_lookup` dictionary so that + the resource can be used in the schema that imported the type + + Args: + namespace (str): The namespace given in the schema when importing + the resource. If the namespace is `None` then a namespace + was not used in the schema + + resource_type (str): The resource type name. E.g if we had a ruleset + or enum defined in the schema, it may be called `Foo`. + + resource_lookup (dict): A reference to a `dict` that requires the + resource to be mapped to the `resource_type` and/or `namespace + + imported_resources (dict): A `dict` containing imported resources + from an imported schema. The parameter `resource_type` is used + to find the resource in `imported_resources` + + Returns: + True if the imported type was successfully added to the + `resource_lookup` otherwise returns False to indicate + it could not find the resource in `imported_resources` + + Raises: + ValueError: If the `resource_type`, `resource_lookup` or + `imported_resources` is `None` + + TypeError: if `resource_lookup` or `imported_resources` is + not a `dict` + """ + if resource_type is None: + raise ValueError('Parameter resource_type should not be None') + + if resource_lookup is None: + raise ValueError('Parameter resource_lookup should not be None') + + if imported_resources is None: + raise ValueError('Parameter imported_resources should not be None') + + if not isinstance(resource_lookup, dict): + raise TypeError('Parameter resource_lookup should be a dictionary') + + if not isinstance(imported_resources, dict): + raise TypeError('Parameter imported_resources should be a dictionary') + + imported_type: YamlatorType = imported_resources.get(resource_type) + if imported_type is None: + return False + + if namespace is not None: + resource_type = f'{namespace}.{resource_type}' + + resource_lookup[resource_type] = imported_type + return True + + +def resolve_unknown_types(unknown_types: List[RuleType], rulesets: dict, enums: dict) -> bool: """Resolves any types that are marked as unknown since the ruleset or enum was imported into the schema. This function will go through diff --git a/yamlator/types.py b/yamlator/types.py index 6e8552b..eeb0cf4 100644 --- a/yamlator/types.py +++ b/yamlator/types.py @@ -316,18 +316,27 @@ class ImportStatement(YamlatorType): Attributes: item (str): The name of the enum or ruleset that is being imported + path (str): The path to the schema file that contains the enum or ruleset + + namespace (str): The namespace for the resource used in the import + statement. If one is not provided then defaults to `None` """ - def __init__(self, item: str, path: str): + def __init__(self, item: str, path: str, namespace: str = None): """ImportStatement init Args: item (str): The name of the enum or ruleset that is being imported + path (str): The path to the schema file that contains the enum or ruleset + namespace (str, optional): The alias for the imported resource to + make them unique if the same resource name is not unique. + If one is not present, then this will be `None` + Raises: ValueError: If the `item` or `path` parameters are `None` or are not a string @@ -348,8 +357,11 @@ def __init__(self, item: str, path: str): self._item = item self._path = path + self._namespace = namespace name = f'({item}){path}' + if namespace is not None: + name = f'({namespace}.{item}){path}' super().__init__(name, ContainerTypes.IMPORT) @property @@ -360,6 +372,10 @@ def item(self) -> str: def path(self) -> str: return self._path + @property + def namespace(self) -> str: + return self._namespace + class YamlatorSchema: """Maintains the rules and types that were defined in a Yamlator @@ -493,7 +509,7 @@ def __group_imports(self, imports: Iterator[ImportStatement]) -> None: # loading the same schema file multiple times import_statements = defaultdict(list) for state in imports: - import_statements[state.path].append(state.item) + import_statements[state.path].append((state.item, state.namespace)) self._imports = import_statements @property From e5135a95ae6f6be42e94e6e5309f5e829b213172 Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sat, 11 Mar 2023 15:27:25 +0000 Subject: [PATCH 3/7] Grammar improvements (#58) * Updated the grammar to make the strict keyword a terminal. Removed the extra strict_ruleset and strict_schema * Moved around parts of the grammar to improve readability * Updated unit tests for the schema transformer to relfect the grammar changes * Updated the changelog and fixed the setup.py to include the new parser package --- changelog.md | 1 + setup.py | 5 ++-- tests/parser/test_schema_transformer.py | 19 +++++++------ yamlator/grammar/grammar.lark | 22 +++++++-------- yamlator/parser/core.py | 37 ++++++++++++++----------- 5 files changed, 47 insertions(+), 37 deletions(-) diff --git a/changelog.md b/changelog.md index 60d4ceb..e4a7723 100644 --- a/changelog.md +++ b/changelog.md @@ -58,3 +58,4 @@ * Added import statements to the Yamlator schema syntax * Added namespaces to the import statements with the `as` keyword +* Improvements to the grammar file to include new terminals and remove duplicate constructs diff --git a/setup.py b/setup.py index e2f97a3..62587b8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import setuptools -VERSION = '0.3.1' +VERSION = '0.4.0' PACKAGE_NAME = 'yamlator' DESCRIPTION = 'Yamlator is a CLI tool that allows a YAML file to be validated using a lightweight schema language' # nopep8 @@ -45,7 +45,8 @@ def create_long_description(): 'yamlator', 'yamlator.validators', 'yamlator.cmd', - 'yamlator.cmd.outputs' + 'yamlator.cmd.outputs', + 'yamlator.parser', ]), install_requires=[ 'lark>=1.0.0', diff --git a/tests/parser/test_schema_transformer.py b/tests/parser/test_schema_transformer.py index 5bf4578..92c3299 100644 --- a/tests/parser/test_schema_transformer.py +++ b/tests/parser/test_schema_transformer.py @@ -1,4 +1,4 @@ -"""Test cases for the SchemaTransformer""" +"""Test cases for the `SchemaTransformer` class""" import re @@ -15,10 +15,11 @@ from yamlator.types import YamlatorEnum from yamlator.types import YamlatorRuleset from yamlator.types import SchemaTypes +from yamlator.parser.core import GrammarKeywords class TestSchemaTransformer(unittest.TestCase): - """Tests the Schema Transformer""" + """Tests the SchemaTransformer class""" def setUp(self): self.transformer = SchemaTransformer() @@ -69,10 +70,11 @@ def test_ruleset(self): self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) self.assertFalse(ruleset.is_strict) - def test_strict_ruleset(self): + def test_ruleset_with_strict_token(self): + strict_token = lark.Token(type_=GrammarKeywords.STRICT, value='') name = lark.Token(type_='TOKEN', value='person') - tokens = (name, *self.ruleset_rules) - ruleset = self.transformer.strict_ruleset(tokens) + tokens = (strict_token, name, *self.ruleset_rules) + ruleset = self.transformer.ruleset(tokens) self.assertEqual(name.value, ruleset.name) self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) @@ -223,10 +225,11 @@ def test_schema_entry(self): self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) self.assertFalse(ruleset.is_strict) - def test_strict_schema_entry(self): + def test_schema_entry_with_strict_token(self): expected_ruleset_name = 'main' - tokens = (*self.ruleset_rules, ) - ruleset = self.transformer.strict_schema_entry(tokens) + strict_token = lark.Token(type_=GrammarKeywords.STRICT, value='') + tokens = (strict_token, *self.ruleset_rules, ) + ruleset = self.transformer.schema_entry(tokens) self.assertEqual(expected_ruleset_name, ruleset.name) self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index 04c43a8..a7433ef 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -4,22 +4,18 @@ start: instructions* // Collection constructs ?instructions: enum | ruleset - | strict_ruleset | schema_entry - | strict_schema_entry | import_statement ?values: INT -> integer | FLOAT -> float | string -schema_entry: "schema" "{" rule+ "}" -strict_schema_entry: "strict schema" "{" rule+ "}" -import_statement: "import" CONTAINER_TYPE_NAME "from" /\"(?:[\S]+\/)*[\S]+.ys\"/ ("as" NAMESPACE)? +import_statement: "import" CONTAINER_TYPE_NAME "from" IMPORT_STATEMENT_PATH ("as" NAMESPACE)? -ruleset: "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" -strict_ruleset: "strict ruleset" CONTAINER_TYPE_NAME" {" rule+ "}" +schema_entry: STRICT_KEYWORD? "schema" "{" rule+ "}" +ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" // Enum constructs enum: "enum" CONTAINER_TYPE_NAME "{" enum_item+ "}" @@ -29,13 +25,13 @@ enum_item: ENUM_ITEM_NAME "=" values ?rule: required_rule | optional_rule -rule_name: /[\S]+/ - | /\"[\S ]+\"/ - required_rule: rule_name type "required" NEW_LINES | rule_name type NEW_LINES optional_rule: rule_name type "optional" NEW_LINES +rule_name: /[\S]+/ + | /\"[\S ]+\"/ + // Data types for rules type: int_type | float_type @@ -59,12 +55,16 @@ container_type: (NAMESPACE".")? CONTAINER_TYPE_NAME regex_type: "regex""(" string ")" union_type: "union""(" [type ("," type)+] ")" +string: ESCAPED_STRING + +// Terminals NEW_LINES: "\n"+ NAMESPACE: /[a-z]+/ CONTAINER_TYPE_NAME: /[A-Z]{1}[a-zA-Z0-9_]+/ +IMPORT_STATEMENT_PATH: /\"(?:[\S]+\/)*[\S]+.ys\"/ ENUM_ITEM_NAME: /[A-Z0-9_]+/ -string: ESCAPED_STRING +STRICT_KEYWORD: "strict" %import common.SH_COMMENT %import common.ESCAPED_STRING diff --git a/yamlator/parser/core.py b/yamlator/parser/core.py index b35a748..6abcd20 100644 --- a/yamlator/parser/core.py +++ b/yamlator/parser/core.py @@ -2,6 +2,7 @@ import re import os +import enum from pathlib import Path from typing import Iterator @@ -109,19 +110,16 @@ def optional_rule(self, tokens: 'list[Token]') -> Rule: def ruleset(self, tokens: 'list[Token]') -> YamlatorRuleset: """Transforms the ruleset tokens into a YamlatorRuleset object""" - name = tokens[0].value - rules = tokens[1:] - self.seen_constructs[name] = SchemaTypes.RULESET - return YamlatorRuleset(name, rules) - def strict_ruleset(self, tokens: 'list[Token]') -> YamlatorRuleset: - """Transforms the ruleset tokens into a YamlatorRuleset object - and marks the ruleset as being in strict mode - """ + is_strict = False + if tokens[0].type == GrammarKeywords.STRICT: + tokens = tokens[1:] + is_strict = True + name = tokens[0].value rules = tokens[1:] self.seen_constructs[name] = SchemaTypes.RULESET - return YamlatorRuleset(name, rules, is_strict=True) + return YamlatorRuleset(name, rules, is_strict) def start(self, instructions: Iterator[YamlatorType]) \ -> PartiallyLoadedYamlatorSchema: @@ -234,17 +232,20 @@ def type(self, tokens: 'list[Token]') -> Any: (t, ) = tokens return t - def schema_entry(self, rules: list) -> YamlatorRuleset: + def schema_entry(self, tokens: list) -> YamlatorRuleset: """Transforms the schema entry point token into a YamlatorRuleset called main that will act as the entry point for validating the YAML data """ - return YamlatorRuleset('main', rules) + first_token = tokens[0] - def strict_schema_entry(self, rules: list) -> YamlatorRuleset: - """Transforms the schema entry point token into a YamlatorRuleset called - main and put the ruleset into `strict` mode - """ - return YamlatorRuleset('main', rules, is_strict=True) + # If the first item is not a `lark.Token` + # then the tokens in the list are all rules + if not isinstance(first_token, Token): + return YamlatorRuleset('main', tokens) + + # If the first type is `lark.Token` then the + # first token is an indicator it is in strict mode + return YamlatorRuleset('main', tokens[1:], True) @v_args(inline=True) def integer(self, token: str) -> int: @@ -280,6 +281,10 @@ def import_statement(self, tokens: 'list[Token]') -> ImportStatement: return ImportStatement(item.value, path, namespace) +class GrammarKeywords(str, enum.Enum): + STRICT = 'STRICT_KEYWORD' + + class _InstructionHandler: """Base handle for dealing with Yamlator types""" From 3e88ca086d9831032bd0ea24bd8c49543e1c8a13 Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sat, 11 Mar 2023 17:55:08 +0000 Subject: [PATCH 4/7] Multi object imports (#59) * Updated the example and grammar to allow multiple objects to be imported in a single statement. Added new import containers to make it work with the rest of the codebase * Renamed the original import statement to import type since it now represents a single type in the statement. Updated the import container type to be the new import statement --- changelog.md | 2 +- example/imports/common.ys | 7 +++ example/imports/import.ys | 9 +++- .../files/valid/with_import_and_namespaces.ys | 10 ++-- tests/files/valid/with_imports.ys | 2 +- .../loaders/test_load_schema_imports.py | 8 +-- tests/types/test_import_statements.py | 10 ++-- yamlator/grammar/grammar.lark | 4 +- yamlator/parser/core.py | 53 ++++++++++-------- yamlator/types.py | 54 ++++++++++++++----- 10 files changed, 107 insertions(+), 52 deletions(-) diff --git a/changelog.md b/changelog.md index e4a7723..2e06df6 100644 --- a/changelog.md +++ b/changelog.md @@ -56,6 +56,6 @@ ## v4.0.0 (TBC) -* Added import statements to the Yamlator schema syntax +* Added import statements to the Yamlator schema syntax that support importing one or more types * Added namespaces to the import statements with the `as` keyword * Improvements to the grammar file to include new terminals and remove duplicate constructs diff --git a/example/imports/common.ys b/example/imports/common.ys index 56b942b..5daae00 100644 --- a/example/imports/common.ys +++ b/example/imports/common.ys @@ -2,4 +2,11 @@ import Project from "../strict_mode/strict.ys" enum Values { TEST = 1 +} + +enum Status { + SUCCESS = 0 + ERR = 1 + FAILURE = 2 + WARNING = 3 } \ No newline at end of file diff --git a/example/imports/import.ys b/example/imports/import.ys index 084741d..a19ad84 100644 --- a/example/imports/import.ys +++ b/example/imports/import.ys @@ -1,7 +1,12 @@ import Project from "../lists/lists.ys" -import Values from "common.ys" as core +import Values, Status from "common.ys" as core schema { user Project value core.Values -} \ No newline at end of file + log Log +} + +ruleset Log { + status core.Status +} diff --git a/tests/files/valid/with_import_and_namespaces.ys b/tests/files/valid/with_import_and_namespaces.ys index 12841a7..b296718 100644 --- a/tests/files/valid/with_import_and_namespaces.ys +++ b/tests/files/valid/with_import_and_namespaces.ys @@ -1,8 +1,12 @@ import Employee from "base.ys" as e -import User from "base.ys" -import Status from "base.ys" as core +import Status, User from "base.ys" as core schema { - employees list(e.Employee) + employees list(Employee) status core.Status } + +ruleset Employee { + employee e.Employee + userDetails core.User +} \ No newline at end of file diff --git a/tests/files/valid/with_imports.ys b/tests/files/valid/with_imports.ys index d0397f5..33e9f42 100644 --- a/tests/files/valid/with_imports.ys +++ b/tests/files/valid/with_imports.ys @@ -1,6 +1,6 @@ import Employee from "base.ys" import User from "base.ys" -import Status from "base.ys" +import Status, User from "base.ys" schema { employees list(Employee) diff --git a/tests/parser/loaders/test_load_schema_imports.py b/tests/parser/loaders/test_load_schema_imports.py index 52c44c6..eac1ac9 100644 --- a/tests/parser/loaders/test_load_schema_imports.py +++ b/tests/parser/loaders/test_load_schema_imports.py @@ -7,7 +7,7 @@ from yamlator.types import Rule from yamlator.types import RuleType from yamlator.types import SchemaTypes -from yamlator.types import ImportStatement +from yamlator.types import ImportedType from yamlator.types import YamlatorRuleset from yamlator.types import PartiallyLoadedYamlatorSchema from yamlator.parser.loaders import load_schema_imports @@ -68,9 +68,9 @@ def test_load_schema_imports(self): rulesets={}, enums={}, imports=[ - ImportStatement('Employee', 'base.ys'), - ImportStatement('User', 'base.ys'), - ImportStatement('Status', 'base.ys'), + ImportedType('Employee', 'base.ys'), + ImportedType('User', 'base.ys'), + ImportedType('Status', 'base.ys'), ], unknowns=unknown_types ) diff --git a/tests/types/test_import_statements.py b/tests/types/test_import_statements.py index 29e2ad7..dc4b519 100644 --- a/tests/types/test_import_statements.py +++ b/tests/types/test_import_statements.py @@ -1,13 +1,13 @@ -"""Test cases for the ImportStatement class""" +"""Test cases for the ImportedType class""" import unittest from parameterized import parameterized -from yamlator.types import ImportStatement +from yamlator.types import ImportedType -class TestImportStatement(unittest.TestCase): - """Test cases for the ImportStatement class""" +class TestImportedType(unittest.TestCase): + """Test cases for the ImportedType class""" @parameterized.expand([ ('with_none_item', None, './test', ValueError), @@ -24,7 +24,7 @@ def test_import_statements_invalid_params(self, name: str, item: str, del name with self.assertRaises(expected_exception): - ImportStatement(item, path) + ImportedType(item, path) if __name__ == '__main__': diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index a7433ef..72b4cb4 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -12,7 +12,9 @@ start: instructions* | string -import_statement: "import" CONTAINER_TYPE_NAME "from" IMPORT_STATEMENT_PATH ("as" NAMESPACE)? + +import_statement: "import" imported_types "from" IMPORT_STATEMENT_PATH ("as" NAMESPACE)? +imported_types: (CONTAINER_TYPE_NAME ("," CONTAINER_TYPE_NAME)*) schema_entry: STRICT_KEYWORD? "schema" "{" rule+ "}" ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" diff --git a/yamlator/parser/core.py b/yamlator/parser/core.py index 6abcd20..e26e656 100644 --- a/yamlator/parser/core.py +++ b/yamlator/parser/core.py @@ -25,6 +25,7 @@ from yamlator.types import UnionRuleType from yamlator.types import EnumItem from yamlator.types import SchemaTypes +from yamlator.types import ImportedType from yamlator.types import ImportStatement from yamlator.exceptions import NestedUnionError from yamlator.exceptions import SchemaParseError @@ -91,24 +92,24 @@ def __init__(self, visit_tokens: bool = True) -> None: self.seen_constructs = {} self.unknown_types = [] - def rule_name(self, tokens: 'list[Token]') -> Token: + def rule_name(self, tokens: Iterator[Token]) -> Token: """Processes the rule name by removing any quotes""" token = tokens[0] name = token.value.strip() name = _QUOTES_REGEX.sub('', name) return Token(value=name, type_=token.type) - def required_rule(self, tokens: 'list[Token]') -> Rule: + def required_rule(self, tokens: Iterator[Token]) -> Rule: """Transforms the required rule tokens in a Rule object""" (name, rtype) = tokens[0:2] return Rule(name.value, rtype, True) - def optional_rule(self, tokens: 'list[Token]') -> Rule: + def optional_rule(self, tokens: Iterator[Token]) -> Rule: """Transforms the optional rule tokens in a Rule object""" (name, rtype) = tokens[0:2] return Rule(name.value, rtype, False) - def ruleset(self, tokens: 'list[Token]') -> YamlatorRuleset: + def ruleset(self, tokens: Iterator[Token]) -> YamlatorRuleset: """Transforms the ruleset tokens into a YamlatorRuleset object""" is_strict = False @@ -145,40 +146,40 @@ def start(self, instructions: Iterator[YamlatorType]) \ return PartiallyLoadedYamlatorSchema(root, rules, enums, imports, self.unknown_types) - def str_type(self, _: 'list[Token]') -> RuleType: + def str_type(self, _: Iterator[Token]) -> RuleType: """Transforms a string type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.STR) - def int_type(self, _: 'list[Token]') -> RuleType: + def int_type(self, _: Iterator[Token]) -> RuleType: """Transforms a int type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.INT) - def float_type(self, _: 'list[Token]') -> RuleType: + def float_type(self, _: Iterator[Token]) -> RuleType: """Transforms a float type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.FLOAT) - def list_type(self, tokens: 'list[Token]') -> RuleType: + def list_type(self, tokens: Iterator[Token]) -> RuleType: """Transforms a list type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.LIST, sub_type=tokens[0]) - def map_type(self, tokens: 'list[Token]') -> RuleType: + def map_type(self, tokens: Iterator[Token]) -> RuleType: """Transforms a map type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.MAP, sub_type=tokens[0]) - def any_type(self, _: 'list[Token]') -> RuleType: + def any_type(self, _: Iterator[Token]) -> RuleType: """Transforms the any type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.ANY) - def bool_type(self, _: 'list[Token]') -> RuleType: + def bool_type(self, _: Iterator[Token]) -> RuleType: """Transforms a bool type token into a RuleType object""" return RuleType(schema_type=SchemaTypes.BOOL) - def enum_item(self, tokens: 'list[Token]') -> EnumItem: + def enum_item(self, tokens: Iterator[Token]) -> EnumItem: """Transforms a enum item token into a EnumItem object""" name, value = tokens return EnumItem(name=name, value=value) - def enum(self, tokens: 'list[Token]') -> YamlatorEnum: + def enum(self, tokens: Iterator[Token]) -> YamlatorEnum: """Transforms a enum token into a YamlatorEnum object""" enums = {} @@ -190,7 +191,7 @@ def enum(self, tokens: 'list[Token]') -> YamlatorEnum: self.seen_constructs[name] = SchemaTypes.ENUM return YamlatorEnum(name.value, enums) - def container_type(self, tokens: 'list[Token]') -> RuleType: + def container_type(self, tokens: Iterator[Token]) -> RuleType: """Transforms a container type token into a RuleType object Raises: @@ -207,7 +208,7 @@ def container_type(self, tokens: 'list[Token]') -> RuleType: self.unknown_types.append(rule_type) return rule_type - def regex_type(self, tokens: 'list[Token]') -> RuleType: + def regex_type(self, tokens: Iterator[Token]) -> RuleType: """Transforms a regex type token into a RuleType object""" (regex, ) = tokens return RuleType(schema_type=SchemaTypes.REGEX, regex=regex) @@ -225,7 +226,7 @@ def union_type(self, tokens: 'list[RuleType]'): raise NestedUnionError() return UnionRuleType(sub_types=tokens) - def type(self, tokens: 'list[Token]') -> Any: + def type(self, tokens: Iterator[Token]) -> Any: """Extracts the type tokens and passes them onto the next stage in the transformer """ @@ -264,12 +265,14 @@ def string(self, token: str) -> str: """ return _QUOTES_REGEX.sub('', token) - def import_statement(self, tokens: 'list[Token]') -> ImportStatement: + def import_statement(self, tokens: Iterator[Token]) -> ImportStatement: """Transforms an import statement into a `yamlator.types.ImportStatement` object """ - item = tokens[0] + items: Iterator[Token] = tokens[0] + path = tokens[1] + path = _QUOTES_REGEX.sub('', path.value) namespace = None try: @@ -277,8 +280,15 @@ def import_statement(self, tokens: 'list[Token]') -> ImportStatement: except IndexError: pass - path = _QUOTES_REGEX.sub('', path.value) - return ImportStatement(item.value, path, namespace) + statements = [] + for item in items: + statements.append(ImportedType(item.value, path, namespace)) + return ImportStatement(statements) + + def imported_types(self, tokens: Iterator[Token]) -> Iterator[Token]: + # This method is needed to prevent Lark from wrapping the tokens + # a tree object + return tokens class GrammarKeywords(str, enum.Enum): @@ -360,7 +370,8 @@ def handle(self, instruction: YamlatorType) -> None: super().handle(instruction) return - self.imports.append(instruction) + instruction: ImportStatement = instruction + self.imports.extend(instruction.imports) class SchemaSyntaxError(SyntaxError): diff --git a/yamlator/types.py b/yamlator/types.py index eeb0cf4..12938fa 100644 --- a/yamlator/types.py +++ b/yamlator/types.py @@ -2,6 +2,7 @@ import re import enum +import random from typing import Union from typing import Iterator @@ -311,8 +312,15 @@ def items(self) -> dict: return self._items.copy() -class ImportStatement(YamlatorType): - """Represents a import statement in the Yamlator schema +class ImportedType: + """Represents an imported type in the import statement. For example, + given the following import statement: + + ``` + import Project from "../lists/lists.ys" + ``` + + The `Project` in the statement will be represented by this class Attributes: item (str): The name of the enum or ruleset that is being imported @@ -325,7 +333,7 @@ class ImportStatement(YamlatorType): """ def __init__(self, item: str, path: str, namespace: str = None): - """ImportStatement init + """ImportedType init Args: item (str): The name of the enum or ruleset that is being imported @@ -359,11 +367,6 @@ def __init__(self, item: str, path: str, namespace: str = None): self._path = path self._namespace = namespace - name = f'({item}){path}' - if namespace is not None: - name = f'({namespace}.{item}){path}' - super().__init__(name, ContainerTypes.IMPORT) - @property def item(self) -> str: return self._item @@ -377,6 +380,29 @@ def namespace(self) -> str: return self._namespace +class ImportStatement(YamlatorType): + """Represents an import statement in the schema by maintaining + all imported types. + + Attributes: + imports (Iterator[yamlator.types.ImportedType]): The types that + were being imported in the Yamlator schema + """ + + def __init__(self, imports: Iterator[ImportedType]): + """ImportStatement init + + Args: + imports (Iterator[yamlator.types.ImportedType]): The types + that have been defined in the import statement + """ + self.imports = imports + + # Assigned a random number to give each container a unique name + container_number = str(random.randint(1, 10000)) + super().__init__(container_number, ContainerTypes.IMPORT) + + class YamlatorSchema: """Maintains the rules and types that were defined in a Yamlator schema which can then used to validate a YAML file. The schema will @@ -462,8 +488,8 @@ class PartiallyLoadedYamlatorSchema(YamlatorSchema): in the schema file. The key will be the enum name and the value will be a `yamlator.types.YamlatorEnum` object - imports (Iterator[yamlator.types.ImportStatement]): A list of - `yamlator.types.ImportStatement` that contain all the import + imports (Iterator[yamlator.types.ImportedType]): A list of + `yamlator.types.ImportedType` that contain all the import statements that were defined in the schema file unknowns (Iterator[yamlator.types.RuleType]): A list of @@ -472,7 +498,7 @@ class PartiallyLoadedYamlatorSchema(YamlatorSchema): """ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, - imports: Iterator[ImportStatement], + imports: Iterator[ImportedType], unknowns: Iterator[RuleType] = None): """PartiallyLoadedYamlatorSchema init @@ -488,8 +514,8 @@ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, in the schema file. The key will be the enum name and the value will be a `yamlator.types.YamlatorEnum` object - imports (Iterator[yamlator.types.ImportStatement]): A list - `yamlator.types.ImportStatement` that contain all import + imports (Iterator[yamlator.types.ImportedType]): A list + `yamlator.types.ImportedType` that contain all import statements that were defined in the schema file unknowns (Iterator[yamlator.types.RuleType]): A list of @@ -504,7 +530,7 @@ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, self.__group_imports(imports) - def __group_imports(self, imports: Iterator[ImportStatement]) -> None: + def __group_imports(self, imports: Iterator[ImportedType]) -> None: # Group imports and the requested type to prevent # loading the same schema file multiple times import_statements = defaultdict(list) From f93a0d38998ccf0401774d6b6fded80c2d9f9e2b Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sun, 12 Mar 2023 16:52:04 +0000 Subject: [PATCH 5/7] Ruleset field inheritance (#60) * Initial version of the inheritance system into Yamlator. Updated the grammer to allow single inheritance and a way to resolve at runtime * Refined the new function that handles ruleset inheritance and added a docstrings. Updated the types.py to use the List type alias instead of the iterator type * Added some additional parameter checks to resolve_ruleset_inheritance. Added unit test for resolve_ruleset_inheritance * Updated the test cases for the schema transformer to reflect the changes made to the transformer class * Fixed the import_type test case to have the correct name. Updated the changelog and the import example to show inheritance in action --- changelog.md | 1 + example/imports/common.ys | 5 + example/imports/import.yaml | 5 +- example/imports/import.ys | 13 +- example/lists/lists.ys | 2 +- tests/files/valid/inheritance.ys | 13 ++ .../loaders/test_parse_yamlator_schema.py | 2 + .../test_resolve_ruleset_inheritance.py | 134 ++++++++++++++++++ tests/parser/test_schema_transformer.py | 19 +++ ...port_statements.py => test_import_type.py} | 0 yamlator/grammar/grammar.lark | 3 +- yamlator/parser/core.py | 11 +- yamlator/parser/loaders.py | 71 ++++++++++ yamlator/types.py | 63 +++++--- 14 files changed, 310 insertions(+), 32 deletions(-) create mode 100644 tests/files/valid/inheritance.ys create mode 100644 tests/parser/loaders/test_resolve_ruleset_inheritance.py rename tests/types/{test_import_statements.py => test_import_type.py} (100%) diff --git a/changelog.md b/changelog.md index 2e06df6..7648c30 100644 --- a/changelog.md +++ b/changelog.md @@ -58,4 +58,5 @@ * Added import statements to the Yamlator schema syntax that support importing one or more types * Added namespaces to the import statements with the `as` keyword +* Added ruleset inheritance to allow a ruleset to inherit rules from a different ruleset. If the child ruleset has a rule with the same name as the parent, then the child rule name will be preserved to reflect that the parent has been overridden. This works for rulesets where the parent is in the same or different schema files * Improvements to the grammar file to include new terminals and remove duplicate constructs diff --git a/example/imports/common.ys b/example/imports/common.ys index 5daae00..91f8a28 100644 --- a/example/imports/common.ys +++ b/example/imports/common.ys @@ -9,4 +9,9 @@ enum Status { ERR = 1 FAILURE = 2 WARNING = 3 +} + +ruleset ProjectDetails { + author str + created str } \ No newline at end of file diff --git a/example/imports/import.yaml b/example/imports/import.yaml index f0be9f8..3fa76df 100644 --- a/example/imports/import.yaml +++ b/example/imports/import.yaml @@ -1,2 +1,3 @@ -user: {} -value: 1 \ No newline at end of file +project: + version: 1 + name: test diff --git a/example/imports/import.ys b/example/imports/import.ys index a19ad84..935213e 100644 --- a/example/imports/import.ys +++ b/example/imports/import.ys @@ -1,12 +1,13 @@ -import Project from "../lists/lists.ys" -import Values, Status from "common.ys" as core +import Project from "../lists/lists.ys" as base +import Values, Status, ProjectDetails from "common.ys" as core schema { - user Project + project Project value core.Values - log Log } -ruleset Log { +strict ruleset Project(base.Project) { status core.Status -} + apis list(str) + details core.ProjectDetails +} \ No newline at end of file diff --git a/example/lists/lists.ys b/example/lists/lists.ys index bc70903..2e47bed 100644 --- a/example/lists/lists.ys +++ b/example/lists/lists.ys @@ -5,7 +5,7 @@ ruleset User { strict ruleset Project { version int required name str required - users list(User) optional + users list(User) } strict schema { diff --git a/tests/files/valid/inheritance.ys b/tests/files/valid/inheritance.ys new file mode 100644 index 0000000..8eb841b --- /dev/null +++ b/tests/files/valid/inheritance.ys @@ -0,0 +1,13 @@ +import Employee from "base.ys" as base +import User from "base.ys" +import Status, User from "base.ys" + +schema { + employees list(Employee) + status Status +} + +ruleset Employee (base.Employee) { + department str + salary float +} diff --git a/tests/parser/loaders/test_parse_yamlator_schema.py b/tests/parser/loaders/test_parse_yamlator_schema.py index 851cbd8..2b31c6d 100644 --- a/tests/parser/loaders/test_parse_yamlator_schema.py +++ b/tests/parser/loaders/test_parse_yamlator_schema.py @@ -39,6 +39,8 @@ def test_with_invalid_schema_paths(self, name: str, ('with_imports', './tests/files/valid/with_imports.ys'), ('with_namespace_imports', './tests/files/valid/with_import_and_namespaces.ys'), + ('with_inheritance', + './tests/files/valid/inheritance.ys'), ]) def test_with_valid_schema_paths(self, name, schema_path): # Unused by test case, however is required by the parameterized library diff --git a/tests/parser/loaders/test_resolve_ruleset_inheritance.py b/tests/parser/loaders/test_resolve_ruleset_inheritance.py new file mode 100644 index 0000000..89603e2 --- /dev/null +++ b/tests/parser/loaders/test_resolve_ruleset_inheritance.py @@ -0,0 +1,134 @@ +"""Test cases for the `resolve_ruleset_inheritance` function""" + +import unittest + +from typing import Any +from parameterized import parameterized + +from yamlator.types import Rule +from yamlator.types import RuleType +from yamlator.types import SchemaTypes +from yamlator.types import YamlatorRuleset +from yamlator.exceptions import ConstructNotFoundError +from yamlator.parser.loaders import resolve_ruleset_inheritance + + +class TestResolveRulesetInheritance(unittest.TestCase): + """Test cases for the `resolve_ruleset_inheritance` function""" + + @parameterized.expand([ + ('with_none_rulesets', None, ValueError), + ('with_rulesets_as_a_list', [], TypeError), + ('with_rulesets_as_a_string', 'Foo', TypeError), + ('with_unknown_ruleset', { + 'Foo': YamlatorRuleset( + name='Foo', + rules=[], + is_strict=False, + parent=RuleType(SchemaTypes.RULESET, lookup='Bar') + ) + }, ConstructNotFoundError) + ]) + def test_resolve_ruleset_raises_error(self, name: str, rulesets: Any, + expected_exception: Exception): + # Unused by test case, however is required by the parameterized library + del name + + with self.assertRaises(expected_exception): + resolve_ruleset_inheritance(rulesets) + + def test_resolve_ruleset_inheritance(self): + rulesets = { + 'Foo': YamlatorRuleset( + name='Foo', + rules=[ + Rule('message', RuleType(SchemaTypes.STR), True), + Rule('name', RuleType(SchemaTypes.STR), True), + Rule('age', RuleType(SchemaTypes.INT), False) + ], + is_strict=False, + parent=RuleType(SchemaTypes.RULESET, lookup='Bar') + ), + 'Bar': YamlatorRuleset( + name='Bar', + rules=[ + Rule('first_name', RuleType(SchemaTypes.STR), True), + Rule('last_name', RuleType(SchemaTypes.STR), True) + ] + ) + } + expected_ruleset_count = 2 + expected_foo_rule_count = 5 + + updated_rules = resolve_ruleset_inheritance(rulesets) + + actual_ruleset_count = len(updated_rules) + actual_foo_rule_count = len(updated_rules['Foo'].rules) + self.assertEqual(expected_ruleset_count, actual_ruleset_count) + self.assertEqual(expected_foo_rule_count, actual_foo_rule_count) + + def test_resolve_ruleset_inheritance_parent_same_rule_name(self): + rulesets = { + 'Foo': YamlatorRuleset( + name='Foo', + rules=[ + Rule('message', RuleType(SchemaTypes.STR), True), + Rule('name', RuleType(SchemaTypes.STR), True), + ], + is_strict=False, + parent=RuleType(SchemaTypes.RULESET, lookup='Bar') + ), + 'Bar': YamlatorRuleset( + name='Bar', + rules=[ + Rule('name', RuleType(SchemaTypes.FLOAT), True), + ] + ) + } + expected_ruleset_count = 2 + expected_foo_rule_count = 2 + + updated_rules = resolve_ruleset_inheritance(rulesets) + foo_rules = updated_rules['Foo'].rules + + actual_ruleset_count = len(updated_rules) + actual_foo_rule_count = len(foo_rules) + self.assertEqual(expected_ruleset_count, actual_ruleset_count) + self.assertEqual(expected_foo_rule_count, actual_foo_rule_count) + + # Extract the rule called name that is common in both rulesets above + # and check that the type is a SchemaTypes.STR + overridden_rule = [rule for rule in foo_rules if rule.name == 'name'][0] + self.assertEqual(SchemaTypes.STR, overridden_rule.rtype.schema_type) + + def test_resolve_ruleset_inheritance_without_any_parents(self): + rulesets = { + 'Foo': YamlatorRuleset( + name='Foo', + rules=[ + Rule('message', RuleType(SchemaTypes.STR), True), + Rule('name', RuleType(SchemaTypes.STR), True), + Rule('age', RuleType(SchemaTypes.INT), False) + ], + ), + 'Bar': YamlatorRuleset( + name='Bar', + rules=[ + Rule('first_name', RuleType(SchemaTypes.STR), True), + Rule('last_name', RuleType(SchemaTypes.STR), True) + ] + ) + } + expected_ruleset_count = 2 + expected_foo_rule_count = 3 + + updated_rules = resolve_ruleset_inheritance(rulesets) + + actual_ruleset_count = len(updated_rules) + actual_foo_rule_count = len(updated_rules['Foo'].rules) + self.assertEqual(expected_ruleset_count, actual_ruleset_count) + self.assertEqual(expected_foo_rule_count, actual_foo_rule_count) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/test_schema_transformer.py b/tests/parser/test_schema_transformer.py index 92c3299..5c445fa 100644 --- a/tests/parser/test_schema_transformer.py +++ b/tests/parser/test_schema_transformer.py @@ -69,6 +69,7 @@ def test_ruleset(self): self.assertEqual(name.value, ruleset.name) self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) self.assertFalse(ruleset.is_strict) + self.assertIsNone(ruleset.parent) def test_ruleset_with_strict_token(self): strict_token = lark.Token(type_=GrammarKeywords.STRICT, value='') @@ -79,6 +80,24 @@ def test_ruleset_with_strict_token(self): self.assertEqual(name.value, ruleset.name) self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) self.assertTrue(ruleset.is_strict) + self.assertIsNone(ruleset.parent) + + def test_ruleset_with_parent(self): + strict_token = lark.Token(type_=GrammarKeywords.STRICT, value='') + name = lark.Token(type_='TOKEN', value='person') + parent = RuleType(SchemaTypes.RULESET, lookup='Foo') + tokens = (strict_token, name, parent, *self.ruleset_rules) + ruleset = self.transformer.ruleset(tokens) + + self.assertEqual(name.value, ruleset.name) + self.assertEqual(len(self.ruleset_rules), len(ruleset.rules)) + self.assertTrue(ruleset.is_strict) + self.assertIsNotNone(ruleset.parent) + + def test_ruleset_parent(self): + parent_rule_type = RuleType(SchemaTypes.RULESET, lookup='Foo') + result = self.transformer.ruleset_parent([parent_rule_type]) + self.assertEqual(parent_rule_type, result) def test_start(self): # This will be zero since main is removed from the dict diff --git a/tests/types/test_import_statements.py b/tests/types/test_import_type.py similarity index 100% rename from tests/types/test_import_statements.py rename to tests/types/test_import_type.py diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index 72b4cb4..68c3748 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -17,7 +17,8 @@ import_statement: "import" imported_types "from" IMPORT_STATEMENT_PATH ("as" NAM imported_types: (CONTAINER_TYPE_NAME ("," CONTAINER_TYPE_NAME)*) schema_entry: STRICT_KEYWORD? "schema" "{" rule+ "}" -ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" +ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME (ruleset_parent)? "{" rule+ "}" +ruleset_parent: "(" container_type ")" // Enum constructs enum: "enum" CONTAINER_TYPE_NAME "{" enum_item+ "}" diff --git a/yamlator/parser/core.py b/yamlator/parser/core.py index e26e656..e2266ef 100644 --- a/yamlator/parser/core.py +++ b/yamlator/parser/core.py @@ -111,7 +111,6 @@ def optional_rule(self, tokens: Iterator[Token]) -> Rule: def ruleset(self, tokens: Iterator[Token]) -> YamlatorRuleset: """Transforms the ruleset tokens into a YamlatorRuleset object""" - is_strict = False if tokens[0].type == GrammarKeywords.STRICT: tokens = tokens[1:] @@ -119,9 +118,19 @@ def ruleset(self, tokens: Iterator[Token]) -> YamlatorRuleset: name = tokens[0].value rules = tokens[1:] + self.seen_constructs[name] = SchemaTypes.RULESET + parent_token = tokens[1] + if isinstance(parent_token, RuleType): + return YamlatorRuleset(name, tokens[2:], is_strict, parent_token) return YamlatorRuleset(name, rules, is_strict) + def ruleset_parent(self, tokens: Iterator[RuleType]) -> RuleType: + """Extracts the ruleset parent from the token list""" + # This method is needed to prevent Lark from wrapping the tokens + # a tree object + return tokens[0] + def start(self, instructions: Iterator[YamlatorType]) \ -> PartiallyLoadedYamlatorSchema: """Transforms the instructions into a dict that sorts the rulesets, diff --git a/yamlator/parser/loaders.py b/yamlator/parser/loaders.py index e4e5b23..82f854e 100644 --- a/yamlator/parser/loaders.py +++ b/yamlator/parser/loaders.py @@ -2,10 +2,12 @@ import re import os +from typing import Dict from typing import List from yamlator.utils import load_schema from yamlator.types import RuleType from yamlator.types import YamlatorSchema +from yamlator.types import YamlatorRuleset from yamlator.types import YamlatorType from yamlator.types import SchemaTypes from yamlator.types import PartiallyLoadedYamlatorSchema @@ -144,6 +146,8 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, unknown_types = loaded_schema.unknowns_rule_types resolve_unknown_types(unknown_types, root_rulesets, root_enums) + + root_rulesets = resolve_ruleset_inheritance(root_rulesets) return YamlatorSchema(loaded_schema.root, root_rulesets, root_enums) @@ -255,3 +259,70 @@ def resolve_unknown_types(unknown_types: List[RuleType], raise ConstructNotFoundError(curr.lookup) return True + + +def resolve_ruleset_inheritance(rulesets: Dict[str, YamlatorRuleset]) -> dict: + """Resolves any rulesets that have a parent ruleset defined in the schema. + For example: + + ```text + ruleset Foo(Bar) { + ... + } + ``` + + This function will extract all the rules defined in parent ruleset (`Bar`) + and include them in the `Foo` ruleset. If both parent and child have a rule + with the same name, the child rule will been used. + + Args: + rulesets (dict): The rulesets defined in the schema where the key is the + type and the value is a `yamlator.types.YamlatorRuleset` + + Returns: + A dictionary where all the rulesets parent dependencies have been + resolved by merging the parent rules in the child rules + + Raises: + ValueError: If the ruleset parameter is `None` + TypeError: If the ruleset parameter is not a `dict` + yamlator.exceptions.ConstructNotFoundError: If the parent of the ruleset + cannot be found in the `rulesets` parameter + """ + if rulesets is None: + raise ValueError('Parameter rulesets cannot be None') + + if not isinstance(rulesets, dict): + raise TypeError( + 'Parameter rulesets cannot be None should be a dictionary') + + updated_rulesets = {} + + for key, ruleset in rulesets.items(): + if ruleset.parent is None: + updated_rulesets[key] = ruleset + continue + + parent_name = ruleset.parent.lookup + parent = rulesets.get(parent_name) + if parent is None: + raise ConstructNotFoundError(parent) + + base_rules = ruleset.rules.copy() + parent_rules = parent.rules.copy() + + # Index the rules in the base and parent rulesets to make it + # easier to merge the different rules together + base_rules_index = {rule.name: rule for rule in base_rules} + parent_rules_index = {rule.name: rule for rule in parent_rules} + + # Merged the 2 rule lists together. If a rule name is present + # in both the base rules will be prioritised since it assumed + # it is being overridden + merged_rules = list({**parent_rules_index, **base_rules_index}.values()) + updated_rulesets[key] = YamlatorRuleset( + name=ruleset.name, + rules=merged_rules, + is_strict=ruleset.is_strict + ) + return updated_rulesets diff --git a/yamlator/types.py b/yamlator/types.py index 12938fa..b00268e 100644 --- a/yamlator/types.py +++ b/yamlator/types.py @@ -5,7 +5,7 @@ import random from typing import Union -from typing import Iterator +from typing import List from collections import namedtuple from collections import defaultdict @@ -144,6 +144,12 @@ def __repr__(self) -> str: self.lookup, self.sub_type) + if self.schema_type == SchemaTypes.UNKNOWN: + repr_template = '{}(type={}, lookup={})' + return repr_template.format(self.__class__.__name__, + self.schema_type, + self.lookup) + repr_template = '{}(type={}, sub_type={})' return repr_template.format(self.__class__.__name__, self.schema_type, @@ -154,24 +160,24 @@ class UnionRuleType(RuleType): """Represents a Union data type that is defined in the Yamlator schema Attributes: - sub_types (Iterator[yamlator.types.RuleType]): A list of sub types + sub_types (List[yamlator.types.RuleType]): A list of sub types that are considered valid types when validating the data. For example, given `Union(int, str)`, a rule type will be created for the `int` and `str` """ - def __init__(self, sub_types: 'Iterator[RuleType]') -> None: + def __init__(self, sub_types: List[RuleType]) -> None: """UnionRuleType init Args: - sub_types (Iterator[yamlator.types.RuleType]): An iterable of rule + sub_types (List[yamlator.types.RuleType]): An iterable of rule types that the union will compare """ super().__init__(SchemaTypes.UNION) self._sub_types = sub_types @property - def sub_types(self) -> 'Iterator[RuleType]': + def sub_types(self) -> List[RuleType]: return self._sub_types def __str__(self) -> str: @@ -238,36 +244,51 @@ class YamlatorRuleset(YamlatorType): representation of the current type. This will always be `ContainerTypes.RULESET` - rules (Iterator[yamlator.types.Rule]): The list of rules in the ruleset + rules (List[yamlator.types.Rule]): The list of rules in the ruleset - is_strict (bool, optional): If the ruleset is in strict mode. When + is_strict (bool): If the ruleset is in strict mode. When enabled any additional keys that are not part of the ruleset will raise a strict mode violation + + parent (yamlator.types.RuleType): The parent ruleset which + this ruleset should inherit additional rules from. If a parent + is not specified this defaults to `None` """ - def __init__(self, name: str, rules: Iterator[Rule], - is_strict: bool = False): + def __init__(self, name: str, rules: List[Rule], + is_strict: bool = False, parent: RuleType = None): """YamlatorRuleset init Args: name (str): The name of the ruleset + rules (list): A list of rules for the ruleset + is_strict (bool, optional): Sets the ruleset to be in strict mode. When used with the validators, it will check to ensure any extra fields are raised as a violation + + parent (yamlator.types.RuleType, optional): The parent ruleset + which this ruleset should inherit additional rules from. + To indicate this ruleset does not have a parent, set to `None` """ super().__init__(name, ContainerTypes.RULESET) self._rules = rules self._is_strict = is_strict + self._parent = parent @property - def rules(self) -> Iterator[Rule]: + def rules(self) -> List[Rule]: return self._rules @property def is_strict(self) -> bool: return self._is_strict + @property + def parent(self) -> RuleType: + return self._parent + class YamlatorEnum(YamlatorType): """Represents a Yamlator Enum Type @@ -385,15 +406,15 @@ class ImportStatement(YamlatorType): all imported types. Attributes: - imports (Iterator[yamlator.types.ImportedType]): The types that + imports (List[yamlator.types.ImportedType]): The types that were being imported in the Yamlator schema """ - def __init__(self, imports: Iterator[ImportedType]): + def __init__(self, imports: List[ImportedType]): """ImportStatement init Args: - imports (Iterator[yamlator.types.ImportedType]): The types + imports (List[yamlator.types.ImportedType]): The types that have been defined in the import statement """ self.imports = imports @@ -488,18 +509,18 @@ class PartiallyLoadedYamlatorSchema(YamlatorSchema): in the schema file. The key will be the enum name and the value will be a `yamlator.types.YamlatorEnum` object - imports (Iterator[yamlator.types.ImportedType]): A list of + imports (List[yamlator.types.ImportedType]): A list of `yamlator.types.ImportedType` that contain all the import statements that were defined in the schema file - unknowns (Iterator[yamlator.types.RuleType]): A list of + unknowns (List[yamlator.types.RuleType]): A list of `yamlator.types.RuleType` objects that have a schema type of `yamlator.types.SchemaTypes.UNKNOWN` """ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, - imports: Iterator[ImportedType], - unknowns: Iterator[RuleType] = None): + imports: List[ImportedType], + unknowns: List[RuleType] = None): """PartiallyLoadedYamlatorSchema init Args: @@ -514,11 +535,11 @@ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, in the schema file. The key will be the enum name and the value will be a `yamlator.types.YamlatorEnum` object - imports (Iterator[yamlator.types.ImportedType]): A list + imports (List[yamlator.types.ImportedType]): A list `yamlator.types.ImportedType` that contain all import statements that were defined in the schema file - unknowns (Iterator[yamlator.types.RuleType]): A list of + unknowns (List[yamlator.types.RuleType]): A list of `yamlator.types.RuleType` objects that have a schema type of `yamlator.types.SchemaTypes.UNKNOWN` """ @@ -530,7 +551,7 @@ def __init__(self, root: YamlatorRuleset, rulesets: dict, enums: dict, self.__group_imports(imports) - def __group_imports(self, imports: Iterator[ImportedType]) -> None: + def __group_imports(self, imports: List[ImportedType]) -> None: # Group imports and the requested type to prevent # loading the same schema file multiple times import_statements = defaultdict(list) @@ -543,5 +564,5 @@ def imports(self) -> dict: return self._imports.copy() @property - def unknowns_rule_types(self) -> Iterator[RuleType]: + def unknowns_rule_types(self) -> List[RuleType]: return self._unknowns.copy() From b025cba62849410f01a47b14a09030ba7afd5006 Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Sun, 26 Mar 2023 14:51:01 +0100 Subject: [PATCH 6/7] Import dependency management (#61) * Added a basic dfs algorithm to perform cycle detections on the graph * Refined the dependency manager to not need any additional data structures to handle the mapping between md5 hashes and nodes. Added additional test case for a self cycling node * Added a new cycle detection exception. Added this exception to the load_import_schema function * Added a test case to test_main to check for a cycle * Renmaed the invalid file path from cycle_dependencies to cycles. Added missing docstrings to the test_dependency_manager * Added a constant file to the test/cmd package to reduce common strings being used in test cases. Moved the invalid.yaml file from the valid path to invalid * Added 2 new test cases to load_schema_imports to test for cycles in the import statements --- example/imports/common.ys | 1 + tests/cmd/constants.py | 26 +++++ tests/cmd/test_main.py | 46 ++++---- .../cmd/test_validate_yaml_data_from_file.py | 50 ++++---- tests/files/invalid_files/cycles/common.ys | 17 +++ tests/files/invalid_files/cycles/root.ys | 12 ++ .../files/invalid_files/cycles/self_cycle.ys | 10 ++ .../{valid => invalid_files}/invalid.yaml | 0 .../invalid_files/ruleset_missing_rules.ys | 6 +- .../loaders/test_load_schema_imports.py | 110 +++++++++++++++++- tests/parser/test_dependency_manager.py | 68 +++++++++++ tests/parser/test_parse_schema.py | 22 ++-- tests/utils/test_load_schema.py | 6 +- tests/utils/test_load_yaml_file.py | 7 +- yamlator/cmd/core.py | 4 + yamlator/exceptions.py | 7 ++ yamlator/parser/dependency.py | 75 ++++++++++++ yamlator/parser/loaders.py | 73 +++++++++--- 18 files changed, 464 insertions(+), 76 deletions(-) create mode 100644 tests/cmd/constants.py create mode 100644 tests/files/invalid_files/cycles/common.ys create mode 100644 tests/files/invalid_files/cycles/root.ys create mode 100644 tests/files/invalid_files/cycles/self_cycle.ys rename tests/files/{valid => invalid_files}/invalid.yaml (100%) create mode 100644 tests/parser/test_dependency_manager.py create mode 100644 yamlator/parser/dependency.py diff --git a/example/imports/common.ys b/example/imports/common.ys index 91f8a28..65eb2f7 100644 --- a/example/imports/common.ys +++ b/example/imports/common.ys @@ -1,4 +1,5 @@ import Project from "../strict_mode/strict.ys" +import Project from "import.ys" enum Values { TEST = 1 diff --git a/tests/cmd/constants.py b/tests/cmd/constants.py new file mode 100644 index 0000000..ea07985 --- /dev/null +++ b/tests/cmd/constants.py @@ -0,0 +1,26 @@ +"""Contains constants for the test files paths""" + +_BASE_INVALID_PATH = './tests/files/invalid_files' + +INVALID_ENUM_NAME_SCHEMA = f'{_BASE_INVALID_PATH}/invalid_enum_name.ys' +INVALID_RULESET_NAME_SCHEMA = f'{_BASE_INVALID_PATH}/invalid_ruleset_name.ys' +INVALID_SYNTAX_SCHEMA = f'{_BASE_INVALID_PATH}/invalid_syntax.ys' +MISSING_RULESET_DEF_SCHEMA = f'{_BASE_INVALID_PATH}/missing_defined_ruleset.ys' +NESTED_UNION_SCHEMA = f'{_BASE_INVALID_PATH}/nested_union.ys' +MISSING_RULESET_RULES_SCHEMA = f'{_BASE_INVALID_PATH}/ruleset_missing_rules.ys' +MISSING_SCHEMA_RULES_SCHEMA = f'{_BASE_INVALID_PATH}/schema_missing_rules.ys' +SELF_CYCLE_SCHEMA = f'{_BASE_INVALID_PATH}/cycles/self_cycle.ys' +INVALID_YAML_DATA = f'{_BASE_INVALID_PATH}/invalid.yaml' + +_BASE_VALID_PATH = './tests/files/valid' +VALID_YAML_DATA = f'{_BASE_VALID_PATH}/valid.yaml' +VALID_SCHEMA = f'{_BASE_VALID_PATH}/valid.ys' +VALID_KEYLESS_DIRECTIVE_SCHEMA = f'{_BASE_VALID_PATH}/keyless_directive.ys' +VALID_KEYLESS_RULES_SCHEMA = f'{_BASE_VALID_PATH}/keyless_and_standard_rules.ys' + +# These files don't exists and are used to force specific errors in the tests +NONE_PATH = None +EMPTY_PATH = '' +NOT_FOUND_SCHEMA = 'not_found.ys' +NOT_FOUND_YAML_DATA = 'not_found.yaml' +INVALID_SCHEMA_EXTENSION = './tests/files/hello.ruleset' diff --git a/tests/cmd/test_main.py b/tests/cmd/test_main.py index 9cae899..2ce385a 100644 --- a/tests/cmd/test_main.py +++ b/tests/cmd/test_main.py @@ -19,9 +19,8 @@ from yamlator.cmd import DisplayMethod from yamlator.cmd.outputs import SuccessCode -HELLO_YAML_FILE_PATH = './tests/files/valid/valid.yaml' -HELLO_RULESET_FILE_PATH = './tests/files/valid/valid.ys' -INVALID_HELLO_YAML_FILE_PATH = './tests/files/valid/invalid.yaml' +from tests.cmd import constants + ValidateArgs = namedtuple('ValidateArgs', ['file', 'ruleset_schema', 'output']) @@ -31,48 +30,53 @@ class TestMain(unittest.TestCase): @parameterized.expand([ ('with_yaml_matching_ruleset', ValidateArgs( - HELLO_YAML_FILE_PATH, - HELLO_RULESET_FILE_PATH, + constants.VALID_YAML_DATA, + constants.VALID_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.SUCCESS), ('with_yaml_containing_ruleset_violations', ValidateArgs( - INVALID_HELLO_YAML_FILE_PATH, - HELLO_RULESET_FILE_PATH, + constants.INVALID_YAML_DATA, + constants.VALID_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_ruleset_file_not_found', ValidateArgs( - HELLO_YAML_FILE_PATH, - '/test/files/not_found.ys', + constants.VALID_YAML_DATA, + constants.NOT_FOUND_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_yaml_data_not_found', ValidateArgs( - './tests/files/not_found.yaml', - HELLO_RULESET_FILE_PATH, + constants.NOT_FOUND_YAML_DATA, + constants.VALID_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_empty_yaml_file_path', ValidateArgs( - '', - HELLO_RULESET_FILE_PATH, + constants.EMPTY_PATH, + constants.VALID_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_empty_ruleset_path', ValidateArgs( - HELLO_YAML_FILE_PATH, - '', + constants.VALID_YAML_DATA, + constants.EMPTY_PATH, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_invalid_ruleset_extension', ValidateArgs( - HELLO_YAML_FILE_PATH, - './tests/files/hello.ruleset', + constants.VALID_YAML_DATA, + constants.INVALID_SCHEMA_EXTENSION, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_syntax_errors', ValidateArgs( - HELLO_YAML_FILE_PATH, - './tests/files/invalid_files/invalid_enum_name.ys', + constants.VALID_YAML_DATA, + constants.INVALID_ENUM_NAME_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR), ('with_ruleset_not_defined', ValidateArgs( - HELLO_YAML_FILE_PATH, - './tests/files/invalid_files/missing_defined_ruleset.ys', + constants.VALID_YAML_DATA, + constants.MISSING_RULESET_DEF_SCHEMA, + DisplayMethod.TABLE.value + ), SuccessCode.ERR), + ('with_self_cycle_in_ruleset', ValidateArgs( + constants.VALID_YAML_DATA, + constants.SELF_CYCLE_SCHEMA, DisplayMethod.TABLE.value ), SuccessCode.ERR) ]) diff --git a/tests/cmd/test_validate_yaml_data_from_file.py b/tests/cmd/test_validate_yaml_data_from_file.py index 8fa77fe..33087db 100644 --- a/tests/cmd/test_validate_yaml_data_from_file.py +++ b/tests/cmd/test_validate_yaml_data_from_file.py @@ -17,9 +17,8 @@ from yamlator.cmd import validate_yaml_data_from_file from yamlator.exceptions import InvalidSchemaFilenameError -EMPTY_STR = '' -VALID_YAML_DATA_FILE_PATH = './tests/files/valid/valid.yaml' -VALID_SCHEMA_FILE_PATH = './tests/files/valid/valid.ys' +from tests.cmd import constants + ValidateArgs = namedtuple('ValidateArgs', ['yaml_filepath', 'schema_filepath']) @@ -31,34 +30,41 @@ class TestValidateYamlDataFromFile(unittest.TestCase): """ @parameterized.expand([ - ('none_yaml_path', - ValidateArgs(None, VALID_SCHEMA_FILE_PATH), ValueError), - ('none_schema_path', - ValidateArgs(VALID_YAML_DATA_FILE_PATH, None), ValueError), - ('none_yaml_and_schema_path', - ValidateArgs(None, None), ValueError), + ('none_yaml_path', ValidateArgs( + constants.NONE_PATH, + constants.VALID_SCHEMA + ), ValueError), + ('none_schema_path', ValidateArgs( + constants.VALID_YAML_DATA, + constants.NONE_PATH + ), ValueError), + ('none_yaml_and_schema_path', ValidateArgs( + constants.NONE_PATH, + constants.NONE_PATH + ), ValueError), ('empty_yaml_path_str', ValidateArgs( - EMPTY_STR, VALID_SCHEMA_FILE_PATH + constants.EMPTY_PATH, + constants.VALID_SCHEMA ), ValueError), ('empty_schema_path_str', ValidateArgs( - VALID_YAML_DATA_FILE_PATH, - EMPTY_STR + constants.VALID_YAML_DATA, + constants.EMPTY_PATH ), ValueError), ('empty_yaml_and_path_str', ValidateArgs( - EMPTY_STR, - EMPTY_STR + constants.EMPTY_PATH, + constants.EMPTY_PATH ), ValueError), ('yaml_data_file_not_found', ValidateArgs( - 'not_found.yaml', - VALID_SCHEMA_FILE_PATH + constants.NOT_FOUND_YAML_DATA, + constants.VALID_SCHEMA ), FileNotFoundError), ('schema_file_not_found', ValidateArgs( - VALID_YAML_DATA_FILE_PATH, - 'not_found.ys' + constants.VALID_YAML_DATA, + constants.NOT_FOUND_SCHEMA ), FileNotFoundError), ('schema_invalid_file_extension', ValidateArgs( - VALID_YAML_DATA_FILE_PATH, - './tests/files/hello.ruleset' + constants.VALID_YAML_DATA, + constants.INVALID_SCHEMA_EXTENSION ), InvalidSchemaFilenameError) ]) def test_validate_yaml_data_from_file_with_invalid_args(self, name: str, @@ -74,8 +80,8 @@ def test_validate_yaml_data_from_file_with_invalid_args(self, name: str, def test_validate_yaml_data_from_file_with_valid_data(self): expected_violation_count = 0 violations = validate_yaml_data_from_file( - yaml_filepath=VALID_YAML_DATA_FILE_PATH, - schema_filepath=VALID_SCHEMA_FILE_PATH + yaml_filepath=constants.VALID_YAML_DATA, + schema_filepath=constants.VALID_SCHEMA ) actual_violation_count = len(violations) diff --git a/tests/files/invalid_files/cycles/common.ys b/tests/files/invalid_files/cycles/common.ys new file mode 100644 index 0000000..2bc827d --- /dev/null +++ b/tests/files/invalid_files/cycles/common.ys @@ -0,0 +1,17 @@ +import Project from "root.ys" + +enum Values { + TEST = 1 +} + +enum Status { + SUCCESS = 0 + ERR = 1 + FAILURE = 2 + WARNING = 3 +} + +ruleset ProjectDetails { + author str + created str +} \ No newline at end of file diff --git a/tests/files/invalid_files/cycles/root.ys b/tests/files/invalid_files/cycles/root.ys new file mode 100644 index 0000000..a9b5e27 --- /dev/null +++ b/tests/files/invalid_files/cycles/root.ys @@ -0,0 +1,12 @@ +import Values, Status from "common.ys" as core + +ruleset Project { + version str + name str + status core.Status optional +} + +schema { + project Project + value core.Values +} diff --git a/tests/files/invalid_files/cycles/self_cycle.ys b/tests/files/invalid_files/cycles/self_cycle.ys new file mode 100644 index 0000000..1567bf5 --- /dev/null +++ b/tests/files/invalid_files/cycles/self_cycle.ys @@ -0,0 +1,10 @@ +import Test from "self_cycle.ys" + +ruleset Test { + data str + number int +} + +schema { + test Test optional +} \ No newline at end of file diff --git a/tests/files/valid/invalid.yaml b/tests/files/invalid_files/invalid.yaml similarity index 100% rename from tests/files/valid/invalid.yaml rename to tests/files/invalid_files/invalid.yaml diff --git a/tests/files/invalid_files/ruleset_missing_rules.ys b/tests/files/invalid_files/ruleset_missing_rules.ys index a44187f..95f2465 100644 --- a/tests/files/invalid_files/ruleset_missing_rules.ys +++ b/tests/files/invalid_files/ruleset_missing_rules.ys @@ -1,6 +1,6 @@ -ruleset Details {} - schema { message str details Details -} \ No newline at end of file +} + +ruleset Details {} diff --git a/tests/parser/loaders/test_load_schema_imports.py b/tests/parser/loaders/test_load_schema_imports.py index eac1ac9..79735aa 100644 --- a/tests/parser/loaders/test_load_schema_imports.py +++ b/tests/parser/loaders/test_load_schema_imports.py @@ -1,5 +1,5 @@ """Test cases for the load_schema_imports function""" - +import hashlib import unittest from parameterized import parameterized @@ -11,6 +11,9 @@ from yamlator.types import YamlatorRuleset from yamlator.types import PartiallyLoadedYamlatorSchema from yamlator.parser.loaders import load_schema_imports +from yamlator.parser.dependency import DependencyManager +from yamlator.exceptions import CycleDependencyError +from yamlator.utils import load_schema def create_basic_loaded_schema(): @@ -29,6 +32,10 @@ def create_basic_loaded_schema(): class TestLoadSchemaImports(unittest.TestCase): """Test cases for the load_schema_imports function""" + def setUp(self): + md5_digest = hashlib.md5('root'.encode('utf-8')) + self.parent_hash = md5_digest.hexdigest() + self.dependencies = DependencyManager() @parameterized.expand([ ('with_none_schema', None, './path/test.ys', ValueError), @@ -47,7 +54,8 @@ def test_load_schema_imports_with_invalid_params( del name with self.assertRaises(expected_exception): - load_schema_imports(loaded_schema, schema_path) + load_schema_imports(loaded_schema, schema_path, + self.parent_hash, self.dependencies) def test_load_schema_imports(self): schema_path = './tests/files/valid' @@ -78,10 +86,106 @@ def test_load_schema_imports(self): expected_ruleset_count = 2 expected_enum_count = 1 - schema = load_schema_imports(loaded_schema, schema_path) + schema = load_schema_imports(loaded_schema, schema_path, + self.parent_hash, self.dependencies) self.assertEqual(expected_ruleset_count, len(schema.rulesets)) self.assertEqual(expected_enum_count, len(schema.enums)) + def test_load_schema_imports_cycle_raises_error(self): + schema_path = './tests/files/invalid_files/cycles' + unknown_types = [ + RuleType(SchemaTypes.UNKNOWN, lookup='core.Value'), + RuleType(SchemaTypes.UNKNOWN, lookup='core.Status') + ] + + # This schema is a representation of the file located in + # './tests/files/invalid_files/cycles/root.ys' to help test + # the function when a cycle is present + loaded_schema = PartiallyLoadedYamlatorSchema( + root=YamlatorRuleset('main', [ + Rule('project', + RuleType( + SchemaTypes.RULESET, lookup='Project' + ), + True), + Rule('project', + RuleType( + SchemaTypes.RULESET, lookup='core.Values' + ), + True), + ]), + rulesets={ + 'Project': YamlatorRuleset( + name='Project', + rules=[ + Rule('version', RuleType(SchemaTypes.STR), True), + Rule('name', RuleType(SchemaTypes.STR), True), + Rule('status', + RuleType( + SchemaTypes.RULESET, + lookup='core.Status'), + False) + ] + ) + }, + enums={}, + imports=[ + ImportedType('Value', 'common.ys', 'core'), + ImportedType('Status', 'common.ys', 'core'), + ], + unknowns=unknown_types + ) + + # Load the actual schema file so the md5 hash can be extracted + # so subsequent load imports hashes match to detect the cycle + file_path = f'{schema_path}/root.ys' + schema = load_schema(file_path) + schema_hash = self.dependencies.add(schema) + + with self.assertRaises(CycleDependencyError): + load_schema_imports(loaded_schema, schema_path, + schema_hash, self.dependencies) + + def test_load_schema_self_cycle_raises_error(self): + schema_path = './tests/files/invalid_files/cycles' + + # This schema is a representation of the file located in + # './tests/files/invalid_files/cycles/self_cycle.ys' to help test + # the function when a cycle is present + loaded_schema = PartiallyLoadedYamlatorSchema( + root=YamlatorRuleset('main', [ + Rule('test', + RuleType( + SchemaTypes.RULESET, lookup='Test' + ), + False), + ]), + rulesets={ + 'Test': YamlatorRuleset( + name='Test', + rules=[ + Rule('data', RuleType(SchemaTypes.STR), True), + Rule('number', RuleType(SchemaTypes.INT), True), + ] + ) + }, + enums={}, + imports=[ + ImportedType('Test', 'self_cycle.ys'), + ], + unknowns=[] + ) + + # Load the actual schema file so the md5 hash can be extracted + # so subsequent load imports hashes match to detect the cycle + file_path = f'{schema_path}/self_cycle.ys' + schema = load_schema(file_path) + schema_hash = self.dependencies.add(schema) + + with self.assertRaises(CycleDependencyError): + load_schema_imports(loaded_schema, schema_path, + schema_hash, self.dependencies) + if __name__ == '__main__': unittest.main() diff --git a/tests/parser/test_dependency_manager.py b/tests/parser/test_dependency_manager.py new file mode 100644 index 0000000..5c14449 --- /dev/null +++ b/tests/parser/test_dependency_manager.py @@ -0,0 +1,68 @@ +"""Test cases for the DependencyManager class""" + +import unittest + +from yamlator.parser.dependency import DependencyManager + + +class TestDependencyManager(unittest.TestCase): + """Test cases for the DependencyManager""" + + def setUp(self): + self.dependencies = DependencyManager() + + def test_dependency_mgmr_add_returns_hash(self): + content = 'Hello world' + md5_hash = self.dependencies.add(content) + self.assertIsNotNone(md5_hash) + + def test_dependency_mgmr_add_child_returns_true(self): + parent_hash = self.dependencies.add('parent') + child_content = 'Hello World' + + result = self.dependencies.add_child(parent_hash, child_content) + self.assertTrue(result) + + def test_dependency_mgmr_add_child_without_parent_add_returns_true(self): + parent_hash = 'abcdefghijk1233456' + child_content = 'Hello World' + + result = self.dependencies.add_child(parent_hash, child_content) + self.assertTrue(result) + + def test_dependency_mgmr_has_cycle_returns_false(self): + n1 = self.dependencies.add('n1') + n2 = self.dependencies.add('n2') + n3 = self.dependencies.add('n3') + n4 = self.dependencies.add('n4') + + self.dependencies.add_child(n1, n2) + self.dependencies.add_child(n1, n3) + self.dependencies.add_child(n2, n3) + self.dependencies.add_child(n3, n4) + + has_cycyle = self.dependencies.has_cycle() + self.assertFalse(has_cycyle) + + def test_dependency_mgmr_has_cycle_returns_true(self): + n1 = self.dependencies.add('n1') + n2 = self.dependencies.add('n2') + n3 = self.dependencies.add('n3') + + self.dependencies.add_child(n1, n2) + self.dependencies.add_child(n1, n3) + self.dependencies.add_child(n2, n1) + + has_cycyle = self.dependencies.has_cycle() + self.assertTrue(has_cycyle) + + def test_dependency_mgmr_has_cycle_self_cycle_returns_true(self): + n1 = self.dependencies.add('n1') + self.dependencies.add_child(n1, n1) + + has_cycle = self.dependencies.has_cycle() + self.assertTrue(has_cycle) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/parser/test_parse_schema.py b/tests/parser/test_parse_schema.py index 648fbb6..d01334c 100644 --- a/tests/parser/test_parse_schema.py +++ b/tests/parser/test_parse_schema.py @@ -25,6 +25,8 @@ from yamlator.parser import SchemaParseError from yamlator.parser import SchemaSyntaxError +from tests.cmd import constants + class TestParseSchema(unittest.TestCase): """Tests the parse schema function""" @@ -38,10 +40,12 @@ def test_parse_with_empty_text(self): self.assertIsNotNone(instructions) @parameterized.expand([ - ('with_root_key_schema', './tests/files/valid/valid.ys', 4), - ('with_keyless_schema', './tests/files/valid/keyless_directive.ys', 1), + ('with_root_key_schema', + constants.VALID_SCHEMA, 4), ('with_keyless_schema', - './tests/files/valid/keyless_and_standard_rules.ys', 2), + constants.VALID_KEYLESS_DIRECTIVE_SCHEMA, 1), + ('with_keyless_schema_and_rules', + constants.VALID_KEYLESS_RULES_SCHEMA, 2), ]) def test_parse_with_valid_content(self, name: str, schema_path: str, expected_schema_rule_count: int): @@ -60,32 +64,32 @@ def test_parse_with_valid_content(self, name: str, schema_path: str, @parameterized.expand([ ( 'with_schema_missing_rules', - './tests/files/invalid_files/schema_missing_rules.ys', + constants.MISSING_SCHEMA_RULES_SCHEMA, MissingRulesError ), ( 'with_ruleset_missing_rules', - './tests/files/invalid_files/schema_missing_rules.ys', + constants.MISSING_RULESET_RULES_SCHEMA, MissingRulesError ), ( 'with_invalid_enum_name', - './tests/files/invalid_files/invalid_enum_name.ys', + constants.INVALID_ENUM_NAME_SCHEMA, MalformedEnumNameError ), ( 'with_invalid_ruleset_name', - './tests/files/invalid_files/invalid_ruleset_name.ys', + constants.INVALID_RULESET_NAME_SCHEMA, MalformedRulesetNameError ), ( 'union_with_nested_union', - './tests/files/invalid_files/nested_union.ys', + constants.NESTED_UNION_SCHEMA, SchemaParseError ), ( 'with_invalid_rule_syntax', - './tests/files/invalid_files/invalid_syntax.ys', + constants.INVALID_SYNTAX_SCHEMA, SchemaSyntaxError ) ]) diff --git a/tests/utils/test_load_schema.py b/tests/utils/test_load_schema.py index 346ecdc..ce266c2 100644 --- a/tests/utils/test_load_schema.py +++ b/tests/utils/test_load_schema.py @@ -14,13 +14,15 @@ from yamlator.utils import load_schema from yamlator.exceptions import InvalidSchemaFilenameError +from tests.cmd import constants + class TestLoadSchema(unittest.TestCase): """Test cases for the Load Schema function""" @parameterized.expand([ - ('with_empty_str', '', ValueError), - ('with_none', None, ValueError), + ('with_empty_str', constants.EMPTY_PATH, ValueError), + ('with_none', constants.NONE_PATH, ValueError), ('with_yaml_extension', 'test.yaml', InvalidSchemaFilenameError), ('with_txt_extension', 'test/test.txt', InvalidSchemaFilenameError), ]) diff --git a/tests/utils/test_load_yaml_file.py b/tests/utils/test_load_yaml_file.py index 0787ae6..d2b8377 100644 --- a/tests/utils/test_load_yaml_file.py +++ b/tests/utils/test_load_yaml_file.py @@ -12,14 +12,15 @@ from parameterized import parameterized from yamlator.utils import load_yaml_file +from tests.cmd import constants class TestLoadYamlFile(unittest.TestCase): """Test cases for the Load Yaml File function""" @parameterized.expand([ - ('with_empty_str', '', ValueError), - ('with_none', None, ValueError) + ('with_empty_str', constants.EMPTY_PATH, ValueError), + ('with_none_path', constants.NONE_PATH, ValueError) ]) def test_yaml_file_invalid_filename(self, name: str, filename: str, expected_exception: Type[Exception]): @@ -30,7 +31,7 @@ def test_yaml_file_invalid_filename(self, name: str, filename: str, load_yaml_file(filename) @parameterized.expand([ - ('yaml_file', 'tests/files/valid/valid.yaml') + ('with_a_valid_yaml_file', constants.VALID_YAML_DATA) ]) def test_load_yaml_file(self, name: str, filename: str): # Unused by test case, however is required by the parameterized library diff --git a/yamlator/cmd/core.py b/yamlator/cmd/core.py index f701103..da2eb87 100644 --- a/yamlator/cmd/core.py +++ b/yamlator/cmd/core.py @@ -13,6 +13,7 @@ from yamlator.exceptions import SchemaParseError from yamlator.exceptions import ConstructNotFoundError from yamlator.exceptions import InvalidSchemaFilenameError +from yamlator.exceptions import CycleDependencyError from yamlator.violations import Violation from yamlator.cmd.outputs import SuccessCode @@ -48,6 +49,9 @@ def main() -> int: except FileNotFoundError as ex: print(ex) return SuccessCode.ERR + except CycleDependencyError as ex: + print(f'Cycle Detected Error: {ex}') + return SuccessCode.ERR except ValueError as ex: print(ex) return SuccessCode.ERR diff --git a/yamlator/exceptions.py b/yamlator/exceptions.py index 62bf962..a94dedb 100644 --- a/yamlator/exceptions.py +++ b/yamlator/exceptions.py @@ -42,3 +42,10 @@ def __init__(self): message = 'Unions cannot have a union nested within it' super().__init__(message) + + +class CycleDependencyError(RuntimeError): + """Represents a cycle has been detected when checking + a dependency chain + """ + pass diff --git a/yamlator/parser/dependency.py b/yamlator/parser/dependency.py new file mode 100644 index 0000000..2edb585 --- /dev/null +++ b/yamlator/parser/dependency.py @@ -0,0 +1,75 @@ +"""Utilties for managing dependencies in Yamlator""" + +import hashlib +from collections import defaultdict + + +class DependencyManager: + """Tracks and detects dependencies between objects by representing + data as a Md5 hash. Once a node has been added, a depth first search + is executed against all nodes to detect a cycle + """ + + def __init__(self) -> None: + self._graph = defaultdict(list) + + def add(self, node: str) -> str: + """Add a new node to the graph. The contents of the parameter + `node` will be hashed with Md5 + + Args: + node (str): A string that contains the content or represents + an item that needs to be tracked for a cycle + + Return: + A Md5 hash of the content provided in the `node` parameter + """ + md5 = hashlib.md5(node.encode('utf-8')) + digest = md5.hexdigest() + + self._graph[digest] = [] + return digest + + def add_child(self, parent_hash: str, child_hash: str) -> bool: + """Adds a child node to a parent node in the dependency chain + + Args: + parent_hash (str): The Md5 hash of the parent node + child_hash (str): The Md5 hash of the child node + + Returns: + True to indicate that the function completed successfully + """ + self._graph[parent_hash].append(child_hash) + return True + + def has_cycle(self) -> bool: + """Detects a cycle against the contents the manager is representing + + Returns: + A boolean to indicate if a cycle is present. True indicates + a cycle was detected, False indicates no cycle is present + """ + visited = defaultdict(bool) + rec_stack = defaultdict(bool) + + for node in self._graph.keys(): + if not visited[node]: + if self._detect_cycle(node, visited, rec_stack): + return True + return False + + def _detect_cycle(self, curr_node: str, visited: dict, + rec_stack: dict) -> bool: + visited[curr_node] = True + rec_stack[curr_node] = True + + for child_node in self._graph[curr_node]: + if not visited[child_node]: + if self._detect_cycle(child_node, visited, rec_stack): + return True + elif rec_stack[child_node]: + return True + + rec_stack[curr_node] = False + return False diff --git a/yamlator/parser/loaders.py b/yamlator/parser/loaders.py index 82f854e..d539fe5 100644 --- a/yamlator/parser/loaders.py +++ b/yamlator/parser/loaders.py @@ -1,9 +1,13 @@ -"""Contains functions to load """ +"""Maintains utility functions that can load Yamlator schemas, +load any import statements in the schema and checks for cycles +""" + import re import os from typing import Dict from typing import List + from yamlator.utils import load_schema from yamlator.types import RuleType from yamlator.types import YamlatorSchema @@ -11,8 +15,10 @@ from yamlator.types import YamlatorType from yamlator.types import SchemaTypes from yamlator.types import PartiallyLoadedYamlatorSchema -from yamlator.parser.core import parse_schema from yamlator.exceptions import ConstructNotFoundError +from yamlator.exceptions import CycleDependencyError +from yamlator.parser.core import parse_schema +from yamlator.parser.dependency import DependencyManager _SLASHES_REGEX = re.compile(r'(?:\\{1}|\/{1})') @@ -46,10 +52,13 @@ def parse_yamlator_schema(schema_path: str) -> YamlatorSchema: raise ValueError('Expected parameter schema_path to be a string') schema_content = load_schema(schema_path) - schema = parse_schema(schema_content) + dependencies = DependencyManager() + schema_hash = dependencies.add(schema_content) + + schema = parse_schema(schema_content) context = fetch_schema_path(schema_path) - schema = load_schema_imports(schema, context) + schema = load_schema_imports(schema, context, schema_hash, dependencies) return schema @@ -80,9 +89,11 @@ def fetch_schema_path(schema_path: str) -> str: def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, - schema_path: str) -> YamlatorSchema: + schema_path: str, + parent_hash: str, + dependencies: DependencyManager) -> YamlatorSchema: """Loads all import statements that have been defined in a Yamlator - schema file. This function will automatically load any import + schema file. This function will automatically load any subsequent import statements from child schema files Args: @@ -92,6 +103,12 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, context (str): The path that contains the Yamlator schema file + parent_hash (str): A string hash of the parent of this schema + + dependencies (yamlator.parser.dependency.DependencyManager): A utility + class that represents dependencies as a graph which can + be used to detect cycles + Returns: A `yamlator.types.YamlatorSchema` object that has all the types resolved @@ -108,6 +125,9 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, yamlator.parser.SchemaSyntaxError: Raised when a syntax error is detected in the schema + + yamlator.parser.CycleDependencyError: Raised if a cycle was deteted + when loading a schema and its imported child schema files """ if loaded_schema is None: raise ValueError('Parameter loaded_schema should not None') @@ -128,21 +148,29 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, for path, resource_type in import_statements.items(): full_path = os.path.join(schema_path, path) - schema = parse_yamlator_schema(full_path) + + schema = _load_child_schema(full_path, parent_hash, dependencies) imported_rulesets = schema.rulesets imported_enums = schema.enums for (resource, namespace) in resource_type: - has_mapped_rulesets = map_imported_resource(namespace, - resource, - root_rulesets, - imported_rulesets) + has_mapped_rulesets = map_imported_resource( + namespace, + resource, + root_rulesets, + imported_rulesets + ) + if has_mapped_rulesets: continue - map_imported_resource(namespace, resource, - root_enums, imported_enums) + map_imported_resource( + namespace, + resource, + root_enums, + imported_enums + ) unknown_types = loaded_schema.unknowns_rule_types resolve_unknown_types(unknown_types, root_rulesets, root_enums) @@ -151,6 +179,25 @@ def load_schema_imports(loaded_schema: PartiallyLoadedYamlatorSchema, return YamlatorSchema(loaded_schema.root, root_rulesets, root_enums) +def _load_child_schema(schema_path: str, parent_hash: str, + dependencies: DependencyManager) -> YamlatorSchema: + schema_content = load_schema(schema_path) + schema_hash = dependencies.add(schema_content) + + dependencies.add_child(parent_hash, schema_hash) + + if dependencies.has_cycle(): + message = f'A cycle was detected when loading {schema_path}' + raise CycleDependencyError(message) + + parsed_schema = parse_schema(schema_content) + + context = fetch_schema_path(schema_path) + schema = load_schema_imports(parsed_schema, context, + parent_hash, dependencies) + return schema + + def map_imported_resource(namespace: str, resource_type: str, resource_lookup: dict, imported_resources: dict) -> dict: From a253325f31aa1b392de5f12d9d48f60fc5bf00ee Mon Sep 17 00:00:00 2001 From: Ryan Flynn <9076629+ryan95f@users.noreply.github.com> Date: Thu, 30 Mar 2023 19:42:59 +0100 Subject: [PATCH 7/7] Import documentation (#62) * Added documentation for the schema imports into schema components file * Removed inheritance from the grammer as additional work is required. Updated the changelog and removed grammer tests for the imports * Adjusted the example to remove the redundent import and made some minor changes to the schema_components section on imports --- changelog.md | 4 +- docs/schema_components.md | 53 +++++++++++++++++++ example/imports/common.ys | 1 - example/imports/import.ys | 3 +- .../loaders/test_parse_yamlator_schema.py | 2 - yamlator/grammar/grammar.lark | 2 +- 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index 7648c30..0ad0092 100644 --- a/changelog.md +++ b/changelog.md @@ -54,9 +54,9 @@ * Minor docstring and structure improvements in the `tests/` module * Improvements to the codebase docstrings -## v4.0.0 (TBC) +## v4.0.0 (30th March 2023) * Added import statements to the Yamlator schema syntax that support importing one or more types * Added namespaces to the import statements with the `as` keyword -* Added ruleset inheritance to allow a ruleset to inherit rules from a different ruleset. If the child ruleset has a rule with the same name as the parent, then the child rule name will be preserved to reflect that the parent has been overridden. This works for rulesets where the parent is in the same or different schema files +* Improvements to the loading of rulesets to allow for schema files to be less restrictive in the order resources are defined. * Improvements to the grammar file to include new terminals and remove duplicate constructs diff --git a/docs/schema_components.md b/docs/schema_components.md index 5b69a37..549d579 100644 --- a/docs/schema_components.md +++ b/docs/schema_components.md @@ -17,6 +17,8 @@ Below are the various components that can be used to construct a schema with Yam * [Ruleset Type](#ruleset-type) * [Enum Type](#enum-type) * [Union Type](#union-type) +* [Importing schemas](#importing-schemas) + * [Schema import paths](#schema-import-paths) ## Schema @@ -423,3 +425,54 @@ schema { items list(Item) } ``` + +## Importing Schemas + +As of version `0.4.0`, Yamlator now supports importing rulesets and enums from other schema files. A single import statement can import one or more resources. Imports can also be given an optional namespace to support importing multiple resources with the same name from different schemas. For example, below is an import statement structure without a namespace: + +```text +import from +``` + +Or an import statement with a namespace: + +```text +import from as +``` + +For example, given the schema called `main.ys`, which is importing resources from two different schemas: + +```text +import Api from "../web/apis.ys" +import Status, ProjectDetails from "common.ys" as core + +schema { + project Project +} + +strict ruleset Project { + status core.Status + apis list(Api) + details core.ProjectDetails +} +``` + +Any imported resources that use a namespace must have the namespace and the resource type specified in the rule type. For example, in the case of importing `ProjectDetails`, which is coming from the `core` namespace, you can see that the rule type is `core.ProjectDetails`. If the namespace is omitted for an imported resource that uses a namespace, a unknown type error will be shown and Yamlator will exit with a non-zero status code. + +__NOTE__: Importing schema blocks and using wildcards (typically seen as `*`) to import every resource are not supported in Yamlator. + +### Schema import paths + +When importing a schema, the path is the location of the schema relative to schema that is using its resources. For example, given the schema above, it would exist in the following file structure: + +```text +web/ + apis.ys +main/ + common.ys + main.ys +``` + +The `apis` will be fetched from the `web` directory and `common.ys` is located in the same directory location as the `main.ys` file. An full example of importing schemas can be found in [example folder](../example/imports/). + +__NOTE__: If an import cycle is detected, Yamlator will exit with a non-zero status code. diff --git a/example/imports/common.ys b/example/imports/common.ys index 65eb2f7..91f8a28 100644 --- a/example/imports/common.ys +++ b/example/imports/common.ys @@ -1,5 +1,4 @@ import Project from "../strict_mode/strict.ys" -import Project from "import.ys" enum Values { TEST = 1 diff --git a/example/imports/import.ys b/example/imports/import.ys index 935213e..62838b8 100644 --- a/example/imports/import.ys +++ b/example/imports/import.ys @@ -1,4 +1,3 @@ -import Project from "../lists/lists.ys" as base import Values, Status, ProjectDetails from "common.ys" as core schema { @@ -6,7 +5,7 @@ schema { value core.Values } -strict ruleset Project(base.Project) { +strict ruleset Project { status core.Status apis list(str) details core.ProjectDetails diff --git a/tests/parser/loaders/test_parse_yamlator_schema.py b/tests/parser/loaders/test_parse_yamlator_schema.py index 2b31c6d..851cbd8 100644 --- a/tests/parser/loaders/test_parse_yamlator_schema.py +++ b/tests/parser/loaders/test_parse_yamlator_schema.py @@ -39,8 +39,6 @@ def test_with_invalid_schema_paths(self, name: str, ('with_imports', './tests/files/valid/with_imports.ys'), ('with_namespace_imports', './tests/files/valid/with_import_and_namespaces.ys'), - ('with_inheritance', - './tests/files/valid/inheritance.ys'), ]) def test_with_valid_schema_paths(self, name, schema_path): # Unused by test case, however is required by the parameterized library diff --git a/yamlator/grammar/grammar.lark b/yamlator/grammar/grammar.lark index 68c3748..855c7db 100644 --- a/yamlator/grammar/grammar.lark +++ b/yamlator/grammar/grammar.lark @@ -17,7 +17,7 @@ import_statement: "import" imported_types "from" IMPORT_STATEMENT_PATH ("as" NAM imported_types: (CONTAINER_TYPE_NAME ("," CONTAINER_TYPE_NAME)*) schema_entry: STRICT_KEYWORD? "schema" "{" rule+ "}" -ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME (ruleset_parent)? "{" rule+ "}" +ruleset: STRICT_KEYWORD? "ruleset" CONTAINER_TYPE_NAME "{" rule+ "}" ruleset_parent: "(" container_type ")" // Enum constructs