Fix head command and match the whole query to the grammar (#52)

* Fix head command * Require an EOF token at the end of a query * Disable or fix tests that throw an exception now that an EOF token is required * Add issue numbers as descriptions to disabled tests * Assert that the exact exception is thrown in InvalidWhereQueryTest instead of generic * Set Integer and eval to optional in head command's grammar * Use the specific exception in HeadSyntaxTests, don't print parse trees * Return sort test payloads to how they were, new disabled test for them
teragrep · Aug 9, 2024 · cd379e2 · cd379e2
1 parent 1e4c115
commit cd379e2
Show file tree

Hide file tree

Showing 25 changed files with 428 additions and 13 deletions.
diff --git a/src/main/antlr4/com/teragrep/pth_03/antlr/DPLParser.g4 b/src/main/antlr4/com/teragrep/pth_03/antlr/DPLParser.g4
@@ -54,7 +54,7 @@ import DPLParserTransform;
 options { tokenVocab=DPLLexer; }
 
 root
-        : searchTransformationRoot? (PIPE transformStatement)?
+        : searchTransformationRoot? (PIPE transformStatement)? EOF
         ;
 
 searchTransformationRoot

diff --git a/src/main/antlr4/imports/COMMAND_HEAD_MODE.g4 b/src/main/antlr4/imports/COMMAND_HEAD_MODE.g4
@@ -68,7 +68,7 @@ COMMAND_HEAD_MODE_LIMIT : 'limit=' -> pushMode(GET_INTEGER);
 COMMAND_HEAD_MODE_NULL : 'null=' -> pushMode(GET_BOOLEAN);
 fragment MINUS: '-';
 fragment DIGIT: [0-9]+;
-COMMAND_HEAD_MODE_INTEGER: MINUS? DIGIT+ -> pushMode(COMMAND_HEAD_MODE);
+COMMAND_HEAD_MODE_INTEGER: MINUS? DIGIT+;
 
 
 COMMAND_HEAD_MODE_COMMENT: '<!--' .*? '-->' -> channel(DPLCOMMENT);
diff --git a/src/main/antlr4/imports/DPLParserTransform_head.g4 b/src/main/antlr4/imports/DPLParserTransform_head.g4
@@ -46,7 +46,8 @@
 parser grammar DPLParserTransform_head;
         //syntax = <stats-agg> (as <wc-field>)? not sure if done correctly. Also, should binSpans-s be in one bracket?
 headTransformation
-        : COMMAND_MODE_HEAD ((COMMAND_HEAD_MODE_INTEGER)|t_head_evalParameter)? (t_head_limitParameter)? (t_head_nullParameter)? (t_head_keepLastParameter)?
+        : COMMAND_MODE_HEAD (t_head_nullParameter | t_head_keepLastParameter)*? t_head_integerType? (t_head_nullParameter | t_head_keepLastParameter)*?
+        | COMMAND_MODE_HEAD (t_head_limitParameter | t_head_nullParameter | t_head_keepLastParameter)*? t_head_evalParameter? (t_head_limitParameter | t_head_nullParameter | t_head_keepLastParameter)*?
         ;
 
 t_head_keepLastParameter
@@ -62,4 +63,8 @@ t_head_nullParameter
         ;
 t_head_evalParameter
         : COMMAND_HEAD_MODE_PARENTHESIS_L evalStatement EVAL_LANGUAGE_MODE_PARENTHESIS_R
+        ;
+
+t_head_integerType
+        : COMMAND_HEAD_MODE_INTEGER
         ;
diff --git a/src/test/java/com/teragrep/pth_03/ParseTreeToXMLTest.java b/src/test/java/com/teragrep/pth_03/ParseTreeToXMLTest.java
@@ -135,6 +135,7 @@ void xmlTreeFromDplTest() throws Exception {
                 "            </whereTransformation>\n" +
                 "        </transformStatement>\n" +
                 "    </transformStatement>\n" +
+                "    <EOF/>\n" +
                 "</root>\n" ;
         CharStream inputStream = CharStreams.fromString(q);
         DPLLexer lexer = new DPLLexer(inputStream);

diff --git a/src/test/java/com/teragrep/pth_03/TicketSyntaxTests.java b/src/test/java/com/teragrep/pth_03/TicketSyntaxTests.java
@@ -46,9 +46,14 @@
 
 package com.teragrep.pth_03;
 
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
+
+import java.lang.reflect.InvocationTargetException;
+
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TicketSyntaxTests {
@@ -63,7 +68,6 @@ public class TicketSyntaxTests {
             "ticket16",
             "ticket18_g",
             "ticket_2_g",
-            "ticket17",
             "ticket18",
             "ticket19",
             "ticket21",
@@ -113,8 +117,6 @@ public class TicketSyntaxTests {
             "ticket68",
             "ticket69",
             "ticket73",
-            "tickets_71",
-            "tickets71_A",
             "ticket74_1",
             "ticket75",
             "ticket75_1",
@@ -127,13 +129,9 @@ public class TicketSyntaxTests {
             "ticket77",
             "ticket78",
             "ticket81",
-            "ticket82",
             "ticket83",
             "ticket84",
-            "ticket84_1",
             "ticket86",
-            "ticket87",
-            "ticket87_1",
             "ticket87_2",
             "ticket81_a",
             "ticket91",
@@ -165,6 +163,21 @@ public void syntaxParseTest(String arg) throws Exception {
                 = new ParserSyntaxTestingUtility(fileName, false);
         parserSyntaxTestingUtility.syntaxParseTest(arg);
     }
+
+    // TODO when enabling this test, just add the text files back to the syntaxParseTest above
+    @Disabled(value = "GH issue #49: sort doesn't support sorting with more than two fields.")
+    @ParameterizedTest(name = "{index} command=''{0}''")
+    @ValueSource(strings = {
+            "tickets_71",
+            "tickets71_A",
+    })
+    public void sortParseTest(String arg) {
+        String fileName = "src/test/resources/antlr4/tickets/"+ arg +".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
     @ParameterizedTest
     @ValueSource(strings = {
             "ticket76_correct",
@@ -303,4 +316,63 @@ void xpathTest10(String arg) throws Exception {
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "ticket17",
+    })
+    void invalidWhereQueryTest(String arg) {
+        String fileName = "src/test/resources/antlr4/tickets/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertThrows(InvocationTargetException.class, () -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @Disabled(value = "Internal issue #82: Can't parse second right parenthesis")
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "ticket82",
+    })
+    void aggregateWithEvalColumnTest(String arg) {
+        String fileName = "src/test/resources/antlr4/tickets/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @Disabled(value = "Internal issue #82: Can't parse second right parenthesis")
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "ticket84_1",
+    })
+    void aggregateWithEvalColumnTest_2(String arg) {
+        String fileName = "src/test/resources/antlr4/tickets/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @Disabled(value = "GH pth_03 #50: Not implemented. Parameter is not allowed on the left side of the FieldList yet.")
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "ticket87",
+    })
+    void dedupConsecutiveOnLeftSideOfFieldListTest(String arg) { // should work on either side of the fieldList
+        String fileName = "src/test/resources/antlr4/tickets/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @Disabled(value = "GH pth_03 #50: Not implemented. Parameter is not allowed on the left side of the FieldList yet.")
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "ticket87_1",
+    })
+    void dedupKeepemptyOnLeftSideOfFieldListTest(String arg) { // should work on either side of the fieldList
+        String fileName = "src/test/resources/antlr4/tickets/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
 }
diff --git a/src/test/java/com/teragrep/pth_03/tests/DatamodelSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/DatamodelSyntaxTests.java
@@ -47,13 +47,15 @@
 
 import com.teragrep.pth_03.ParserStructureTestingUtility;
 import com.teragrep.pth_03.ParserSyntaxTestingUtility;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class DatamodelSyntaxTests {
+    @Disabled(value = "GH pth_03 #41: Incorrect grammar specification, test fails with valid input")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "datamodel",

diff --git a/src/test/java/com/teragrep/pth_03/tests/HeadSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/HeadSyntaxTests.java
@@ -50,14 +50,20 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
+import java.lang.reflect.InvocationTargetException;
+
 public class HeadSyntaxTests {
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "head",
+            "head2",
+            "head4",
+            "head6",
     })
     public void headSyntaxParseTest(String arg) throws Exception {
         String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
@@ -135,4 +141,83 @@ void xpathTest5(String arg) throws Exception {
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "head2",
+    })
+    void sortAfterHeadTest(String arg) { // parsing should continue after head command
+        ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
+        String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
+        String integerExp = "/root/transformStatement/headTransformation/t_head_integerType/value";
+        String sortExp = "/root/transformStatement/transformStatement/sortTransformation/value";
+
+        NodeList integerNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, integerExp, false));
+        NodeList sortNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, sortExp, false));
+
+        // Check that 1 found
+        assertEquals(1, integerNode.getLength());
+        assertEquals(1, sortNode.getLength());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "head3",
+    })
+    void headWithEvalAndIntegerTest(String arg) { // should not parse, both integer and eval is not allowed
+        String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertThrows(InvocationTargetException.class, () -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "head4",
+    })
+    void headWithParametersOnBothSides(String arg) { // parameters on left and right side of an eval statement
+        ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
+        String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
+        String evalExp = "/root/transformStatement/headTransformation/t_head_evalParameter/evalStatement/evalStatement[1]/evalFieldType/value";
+        String limitExp = "/root/transformStatement/headTransformation/t_head_limitParameter/value";
+        String keepLastExp = "/root/transformStatement/headTransformation/t_head_keepLastParameter/value";
+        String nullExp = "/root/transformStatement/headTransformation/t_head_nullParameter/value";
+
+        NodeList evalNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, evalExp, false));
+        NodeList limitNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, limitExp, false));
+        NodeList keepLastNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, keepLastExp, false));
+        NodeList nullNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, nullExp, false));
+
+        // Check that 1 found
+        assertEquals(1, evalNode.getLength());
+        assertEquals(1, limitNode.getLength());
+        assertEquals(1, keepLastNode.getLength());
+        assertEquals(1, nullNode.getLength());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "head5",
+    })
+    void headWithLimitAndInteger(String arg) { // should not parse, both integer and limit is not allowed
+        String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
+        ParserSyntaxTestingUtility parserSyntaxTestingUtility
+                = new ParserSyntaxTestingUtility(fileName, false);
+        Assertions.assertThrows(InvocationTargetException.class, () -> parserSyntaxTestingUtility.syntaxParseTest(arg));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "head6",
+    })
+    void headWithoutParametersTest(String arg) { // no parameters
+        ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
+        String fileName = "src/test/resources/antlr4/commands/head/" + arg + ".txt";
+        String headCmd = "/root/transformStatement/headTransformation/value";
+
+        NodeList headNode = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, headCmd, false));
+
+        // Check that 1 found
+        assertEquals(1, headNode.getLength());
+    }
 }
diff --git a/src/test/java/com/teragrep/pth_03/tests/InputcsvSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/InputcsvSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class InputcsvSyntaxTests {
+    @Disabled(value = "GH pth_03 #42: Fails to parse after WHERE instruction, bug in logicalStatement")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "inputcsv",

diff --git a/src/test/java/com/teragrep/pth_03/tests/InputlookupSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/InputlookupSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class InputlookupSyntaxTests {
+    @Disabled(value = "GH pth_03 #42: Fails to parse after WHERE instruction, bug in logicalStatement")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "inputlookup",

diff --git a/src/test/java/com/teragrep/pth_03/tests/PredictSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/PredictSyntaxTests.java
@@ -50,13 +50,15 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class PredictSyntaxTests {
+    @Disabled(value = "Internal issue #117: lower and upper options don't parse correctly")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "predict",

diff --git a/src/test/java/com/teragrep/pth_03/tests/ReturnSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/ReturnSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class ReturnSyntaxTests {
+    @Disabled(value = "GH pth_03 #43: Grammar has a bug with the aliasParameter")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "return",

diff --git a/src/test/java/com/teragrep/pth_03/tests/SavedsearchSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/SavedsearchSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class SavedsearchSyntaxTests {
+    @Disabled(value = "GH pth_03 #44: Grammar bug, strEQstrParameter is not recognized")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "savedsearch",

diff --git a/src/test/java/com/teragrep/pth_03/tests/SetSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/SetSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class SetSyntaxTests {
+    @Disabled(value = "GH pth_03 #53: Grammar is wrong, should require two subsearchStatements")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "set",

diff --git a/src/test/java/com/teragrep/pth_03/tests/SetfieldsSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/SetfieldsSyntaxTests.java
@@ -50,11 +50,13 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
 
 public class SetfieldsSyntaxTests {
+    @Disabled(value = "GH pth_03 #45: Grammar is wrong, doesn't recognize additional setfieldsParameters")
     @ParameterizedTest(name = "{index} command=''{0}''")
     @ValueSource(strings = {
             "setfields",