Rule name Improvements (#45)

* Renamed the example directory in the test directory to valid

* Updated the grammer to allow dashes to be used in rule names

* Updated the grammer to allow for keys with spaces if they are enclosed with quotes. Updated the parsing test case schema file with different names to confirm the regex works

* Updated the parser rule_name to have test cases. Replaced the NamedTuple
Token with the actual Lark type

* Updated the documentation to include details on the update rule names. Updated the changelog with the release date and set the new version in the setup.py file

.github/workflows/publish.yaml

@@ -5,7 +5,7 @@ on:
       - 'v*' 
 jobs:
   build-and-publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.9
@@ -21,6 +21,6 @@ jobs:
         python setup.py sdist bdist_wheel
     - name: Publish to PyPI
       if: startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}

changelog.md

@@ -31,3 +31,8 @@
 * Added new example [strict_mode](./example/strict_mode/) to the `example` directory
 * Updated GitHub workloads to use the latest actions
 * Fixed bug with rule definitions where field names with similar character patterns to types would create two separate rules
+
+## v0.2.1 (15th January 2023)
+
+* Improved the Yamlator grammar to allow rule names to accept options including symbols, unicode, spaces and dashes. Any YAML key that contains spaces, must be enclosed in double quotes
+* Upgraded the Ubuntu version on the publish workflow

docs/schema_components.md

@@ -187,6 +187,20 @@ ruleset <ruleset-name> {
 
 Required rules validate that the key is present in the YAML data. If the required data is missing then a required violation is raised. If the rule is optional, then a violation is not raised when it is missing from the YAML data.
 
+Rule names support dashes, underscores and all unicode characters. If the field name in YAML contains spaces, e.g `my awesome field`, then the rule name will need to be enclosed in double quotes. For example, given the following YAML data:
+
+```yaml
+my awesome field: 42
+```
+
+The corresponding rule name in a ruleset:
+
+```text
+ruleset <ruleset-name> {
+    "my awesome field" int
+}
+```
+
 ## Rule Types
 
 For each rule, a type can be specified to validate the expected data type is present.

setup.py

@@ -2,7 +2,7 @@
 
 import setuptools
 
-VERSION = '0.2.0'
+VERSION = '0.2.1'
 PACKAGE_NAME = 'yamlator'
 DESCRIPTION = 'Yamlator is a CLI tool that allows a YAML file to be validated using a lightweight schema language'  # nopep8
 

tests/cmd/test_main.py

@@ -19,9 +19,9 @@
 from yamlator.cmd import DisplayMethod
 from yamlator.cmd.outputs import SuccessCode
 
-HELLO_YAML_FILE_PATH = './tests/files/example/example.yaml'
-HELLO_RULESET_FILE_PATH = './tests/files/example/example.ys'
-INVALID_HELLO_YAML_FILE_PATH = './tests/files/example/invalid_example.yaml'
+HELLO_YAML_FILE_PATH = './tests/files/valid/valid.yaml'
+HELLO_RULESET_FILE_PATH = './tests/files/valid/valid.ys'
+INVALID_HELLO_YAML_FILE_PATH = './tests/files/valid/invalid.yaml'
 
 ValidateArgs = namedtuple('ValidateArgs', ['file', 'ruleset_schema', 'output'])
 

tests/cmd/test_validate_yaml_data_from_file.py

@@ -19,8 +19,8 @@
 from yamlator.exceptions import InvalidSchemaFilenameError
 
 EMPTY_STR = ''
-VALID_YAML_DATA_FILE_PATH = './tests/files/example/example.yaml'
-VALID_SCHEMA_FILE_PATH = './tests/files/example/example.ys'
+VALID_YAML_DATA_FILE_PATH = './tests/files/valid/valid.yaml'
+VALID_SCHEMA_FILE_PATH = './tests/files/valid/valid.ys'
 
 ValidateArgs = namedtuple('ValidateArgs', ['yaml_filepath', 'schema_filepath'])
 

tests/files/example/example.yaml → tests/files/valid/valid.yaml

@@ -1,7 +1,8 @@
 message: hello world
 number: 42
 person:
-  name: Test Tester
+  first_name: Test
+  last-name: Tester
   age: 42
   isEmployed: true
-  department: lead
+  department: lead

tests/files/valid/valid.ys

@@ -0,0 +1,39 @@
+# Test the use of underscores
+enum Employee_department {
+    MANAGER = "manager"
+    LEAD = "lead"
+}
+
+ruleset PersonAddress {
+    houseNumber int
+    street str
+    city str
+    post_code str
+}
+
+ruleset Person {
+    first_name str  # Make sure underscores are parsed correctly
+    last-name str   # Make sure dashes are parsed correctly 
+    age int
+    address PersonAddress optional
+    isEmployed bool
+    department Employee_department
+}
+
+# Testing different field options
+ruleset FieldOptions {
+    under_scores str
+    required-under_scores str required
+    required_under-scores str optional
+    "hello world" str optional
+    ரஷ str required
+    ரஷ3 str
+    !!test[] str optional
+}
+
+schema {
+    message str
+    number int
+    person Person optional
+    options FieldOptions optional
+}

tests/parser/test_parse_schema.py

@@ -30,8 +30,8 @@ class TestParseSchema(unittest.TestCase):
     """Tests the parse schema function"""
 
     def setUp(self):
-        self.valid_schema_file = './tests/files/example/example.ys'
-        self.invalid_schema_file = './tests/files/example/example.yaml'
+        self.valid_schema_file = './tests/files/valid/valid.ys'
+        self.invalid_schema_file = './tests/files/valid/valid.yaml'
 
     def test_parse_with_none_text(self):
         with self.assertRaises(ValueError):
@@ -78,7 +78,7 @@ def test_parse_with_valid_content(self):
         ),
         (
             'with_invalid_schema_syntax',
-            './tests/files/example/example.yaml',
+            './tests/files/valid/valid.yaml',
             SchemaSyntaxError
         )
     ])

tests/parser/test_schema_transformer.py

@@ -53,8 +53,8 @@
 
 import re
 import unittest
+import lark
 
-from collections import namedtuple
 from parameterized import parameterized
 from yamlator.exceptions import ConstructNotFoundError
 
@@ -63,23 +63,34 @@
 from yamlator.types import YamlatorEnum, YamlatorRuleset, SchemaTypes
 
 
-Token = namedtuple('Token', ['value'])
-
-
 class TestSchemaTransformer(unittest.TestCase):
     """Tests the Schema Transformer"""
 
     def setUp(self):
         self.transformer = SchemaTransformer()
-        self.name_token = Token('message')
-        self.status_code_token = Token('StatusCode')
+        self.name_token = lark.Token(type_='NAME', value='message')
+        self.status_code_token = lark.Token(type_='STATUS', value='StatusCode')
 
         self.str_rtype = RuleType(schema_type=SchemaTypes.STR)
         self.ruleset_rules = [
             Rule('name', RuleType(schema_type=SchemaTypes.STR), True),
             Rule('age', RuleType(schema_type=SchemaTypes.INT), True),
         ]
 
+    @parameterized.expand([
+        ('without_quotes', 'ruleName', 'ruleName'),
+        ('with_double_quotes', '\"ruleName\"', 'ruleName'),
+        ('with_single_quotes', '\'ruleName\'', 'ruleName'),
+        ('with_spaces', 'rule name', 'rule name'),
+        ('with_padding_spaces', '    rule name    ', 'rule name'),
+    ])
+    def test_rule_name(self, name: str, rule_name: str, expected: str):
+        # Unused by test case, however is required by the parameterized library
+        del name
+        token = lark.Token('TOKEN', value=rule_name)
+        processed_token = self.transformer.rule_name([token])
+        self.assertEqual(expected, processed_token.value)
+
     def test_required_rule(self):
         tokens = (self.name_token, self.str_rtype)
 
@@ -95,7 +106,7 @@ def test_optional_rule(self):
         self.assertFalse(optional_rule.is_required)
 
     def test_ruleset(self):
-        name = Token('person')
+        name = lark.Token(type_='TOKEN', value='person')
         tokens = (name, *self.ruleset_rules)
         ruleset = self.transformer.ruleset(tokens)
 
@@ -104,7 +115,7 @@ def test_ruleset(self):
         self.assertFalse(ruleset.is_strict)
 
     def test_strict_ruleset(self):
-        name = Token('person')
+        name = lark.Token(type_='TOKEN', value='person')
         tokens = (name, *self.ruleset_rules)
         ruleset = self.transformer.strict_ruleset(tokens)
 
@@ -193,9 +204,9 @@ def test_enum(self):
         self.assertEqual(len(enum_items), len(enum.items))
 
     def test_container_type(self):
-        token = Token('Employee')
+        token = lark.Token(type_='TOKEN', value='Employee')
         self.transformer.seen_constructs = {'Employee': SchemaTypes.RULESET}
-        rule = self.transformer.container_type(token)
+        rule = self.transformer.container_type([token])
         self.assertEqual(rule.schema_type, SchemaTypes.RULESET)
 
     @parameterized.expand([
@@ -207,7 +218,7 @@ def test_container_type(self):
     ])
     def test_container_type_construct_does_not_exist(self, name: str,
                                                      seen_constructs: dict):
-        token = Token(name)
+        token = lark.Token(type_='TOKEN', value=name)
         self.transformer.seen_constructs = seen_constructs
         with self.assertRaises(ConstructNotFoundError):
             self.transformer.container_type(token)

tests/test_utils.py

@@ -37,7 +37,7 @@ def test_yaml_file_invalid_filename(self, name: str, filename: str,
             load_yaml_file(filename)
 
     @parameterized.expand([
-        ('yaml_file', 'tests/files/example/example.yaml')
+        ('yaml_file', 'tests/files/valid/valid.yaml')
     ])
     def test_load_yaml_file(self, name: str, filename: str):
         # Unused by test case, however is required by the parameterized library
@@ -65,8 +65,8 @@ def test_load_schema_with_invalid_filename(self, name: str, filename: str,
             load_schema(filename)
 
     @parameterized.expand([
-        ('with_unix_style_path', 'tests/files/example/example.ys'),
-        ('with_windows_style_path', 'tests\\files\\example\\example.ys')
+        ('with_unix_style_path', 'tests/files/valid/valid.ys'),
+        ('with_windows_style_path', 'tests\\files\\valid\\valid.ys')
     ])
     def test_successfully_load_schema(self, name, filename):
         # Unused by test case, however is required by the parameterized library

yamlator/grammar/grammar.lark

@@ -26,9 +26,12 @@ enum_item: /[A-Z0-9_]+/ "=" values
 ?rule: required_rule
      | optional_rule
 
-required_rule: /[a-zA-Z0-9_]+/ type "required" NEW_LINES
-             | /[a-zA-Z0-9_]+/ type NEW_LINES
-optional_rule: /[a-zA-Z0-9_]+/ type "optional" NEW_LINES
+rule_name: /[\S]+/
+         | /\"[\S ]+\"/
+
+required_rule: rule_name type "required" NEW_LINES
+             | rule_name type NEW_LINES
+optional_rule: rule_name type "optional" NEW_LINES
 
 // Data types for rules
 type: int_type