Many positive and negative schema tests for generators.yaml

test/generator_yaml/invalid_yaml/empty-generators.yaml

@@ -0,0 +1,2 @@
+# The smallest valid generator
+

test/generator_yaml/invalid_yaml/invalid.generators.yaml

@@ -0,0 +1,373 @@
+# This file contains multiple yaml documents. Each of them should fail to parse as a generators.yaml file.
+# empty
+---
+# missing secret
+data: {sample: {data: []}}
+---
+# missing sample:
+data: {secret: {data: []}}
+---
+# invalid testgroup below root
+data: {sample: {data: []}, secret: {data: []}, public: {data: []}}
+---
+# solution must be null or string
+solution: 0
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: 0.0
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: []
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: {}
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: true
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: false
+data: {sample: {data: []}, secret: {data: []}}
+---
+# visualizer must be null or string
+visualizer: 0
+data: {sample: {data: []}, secret: {data: []}}
+---
+# random_salt must be null or string
+random_salt: 0
+data: {sample: {data: []}, secret: {data: []}}
+---
+random_salt: 0.0
+data: {sample: {data: []}, secret: {data: []}}
+---
+random_salt: []
+data: {sample: {data: []}, secret: {data: []}}
+---
+random_salt: {}
+data: {sample: {data: []}, secret: {data: []}}
+---
+random_salt: true
+data: {sample: {data: []}, secret: {data: []}}
+---
+random_salt: false
+data: {sample: {data: []}, secret: {data: []}}
+---
+# generators must be a dictionary of non-empty list of string
+generators: str
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators: []
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators: [str]
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators:
+  g: a
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators:
+  g: 1
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators:
+  g:
+data: {sample: {data: []}, secret: {data: []}}
+---
+# TODO
+generators:
+  g: []
+data: {sample: {data: []}, secret: {data: []}}
+---
+generators:
+  g:
+    - true
+data: {sample: {data: []}, secret: {data: []}}
+---
+# data: must be a dictionary or list of dictionaries
+data: str
+---
+# Note: We could actually decide to make this work in the future, but for now this is an error.
+data:
+  - str
+---
+data: 1
+---
+data: [1]
+---
+data: {1: 2}
+---
+data: [{}]
+---
+data:
+  - true
+---
+# Unnumbered keys must not be empty.
+data:
+  sample: {data: []}
+  secret:
+    data:
+      '': str
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      - '':
+        - a
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        - a
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: []
+---
+# generate must not contain {seed} twice.
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: gen {seed} {seed}
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: gen {seed} {seed:1}
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: gen {seed:1} {seed}
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: gen {seed:1} {seed:1}
+---
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: gen {seed:1} {seed:2}
+---
+# no empty string TODO
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: ""
+---
+# no null production
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+---
+# Testcase object must not be boolean
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: true
+---
+# ... or int
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: 1
+---
+# ... or list
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a: []
+---
+# Generator invocation can't be boolean
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        generate: true
+---
+# ... or int
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        generate: 1
+---
+# ... or list
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        generate: []
+---
+# Copied cases must not have an absolute path. TODO
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        copy: /a/b/c
+---
+# Copied cases must not have an absolute path. TODO
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        copy: /testcase
+---
+# Copied cases must not have an absolute path. TODO
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        copy: /generators/a/b/c
+---
+# Generator invocations must not have an absolute path.
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        generate: /generators/gen_tree
+---
+# Generator invocations must not have an absolute path.
+data:
+  sample: {data: []}
+  secret:
+    data:
+      a:
+        generate: /generators/gen.py
+---
+# Solution and visualizer must have an absolute path:
+solution: a
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: /a/b
+data: {sample: {data: []}, secret: {data: []}}
+---
+solution: a 1 2
+data: {sample: {data: []}, secret: {data: []}}
+---
+visualizer: a
+data: {sample: {data: []}, secret: {data: []}}
+---
+visualizer: a/b
+data: {sample: {data: []}, secret: {data: []}}
+---
+visualizer: a 1 2
+data: {sample: {data: []}, secret: {data: []}}
+---
+visualizer: a {name}
+data: {sample: {data: []}, secret: {data: []}}
+---
+## No toplevel generate TODO
+#generate: xyz
+#data: {sample: {data: []}, secret: {data: []}}
+#---
+# No testgroup generate
+data:
+  sample: {data: []} 
+  secret:
+    generate: x
+    data: []
+---
+# Testcases may not have data:
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      a:
+        in: foo
+        data: []
+---
+## Keys are strings
+#data:
+#  sample: {data: []} 
+#  secret:
+#    data:
+#      1.: { in: a }
+#---
+## Keys are strings
+#data:
+#  sample: {data: []} 
+#  secret:
+#    data:
+#      true: { in: a }
+#---
+## Keys are strings; here "1" and "1" are given twice (with different content)
+#data:
+#  sample: {data: []} 
+#  secret:
+#    data:
+#      '1': a
+#      1: b
+#---
+## Keys are strings
+#data:
+#  sample: {data: []} 
+#  secret:
+#    data:
+#      01: a
+#      '1': a
+#---
+#data:
+#  sample: {data: []} 
+#  secret:
+#    data:
+#      0: a
+#---
+# Numbered cases must contain exactly one testcase
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      - {}
+---
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      - a: a
+        b: a
+---
+# Numbered testcases must contain a dict
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      - 1
+---
+# Numbered testcases must contain a dict
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      - foo
+---
+# Numbered testcases must contain a dict
+data:
+  sample: {data: []} 
+  secret:
+    data:
+      - []