From bc478b735945f26905103d911152587910d3bd18 Mon Sep 17 00:00:00 2001 From: molsonkiko <46202915+molsonkiko@users.noreply.github.com> Date: Fri, 6 Oct 2023 09:27:53 -0700 Subject: [PATCH] first ini file parser that seems to work fine still need to add more ini parser tests, integrate into main UI should decide whether to implement interpolation too --- CHANGELOG.md | 1 + JsonToolsNppPlugin/JSONTools/IniFileParser.cs | 315 ++++++++++++++---- JsonToolsNppPlugin/JSONTools/JNode.cs | 47 ++- JsonToolsNppPlugin/Properties/AssemblyInfo.cs | 4 +- .../Tests/IniFileParserTests.cs | 63 ++-- JsonToolsNppPlugin/Tests/TestRunner.cs | 15 +- JsonToolsNppPlugin/Utils/Settings.cs | 4 +- most recent errors.txt | 60 ++-- testfiles/small/example_ini.ini | 17 +- testfiles/small/example_ini.json | 15 +- testfiles/small/example_ini_reformatted.ini | 45 +++ 11 files changed, 454 insertions(+), 132 deletions(-) create mode 100644 testfiles/small/example_ini_reformatted.ini diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ab62c..3eba6f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added +1. Add parser for `.ini` files, allowing them to be reformatted and edited with the tree view. 1. [Python-style spreading of an array](/docs/RemesPath.md#spreading-function-args-to-fill-multiple-arguments-added-in-v58) with `*` to fill multiple arguments of a function. 2. New RemesPath function(s): [`at` function](/docs/RemesPath.md#non-vectorized-functions) for indexing into array or object at an index or key determined at runtime (since indexing with square braces does not support keys/indices that are functions of input). 3. Made it possible to customize newline for CSV files generated by the [JSON-to-CSV form](/docs/README.md#json-to-csv). diff --git a/JsonToolsNppPlugin/JSONTools/IniFileParser.cs b/JsonToolsNppPlugin/JSONTools/IniFileParser.cs index 83736d8..9801f66 100644 --- a/JsonToolsNppPlugin/JSONTools/IniFileParser.cs +++ b/JsonToolsNppPlugin/JSONTools/IniFileParser.cs @@ -1,32 +1,49 @@ using System.Collections.Generic; using System; - +using System.Text.RegularExpressions; +using System.Security.Principal; namespace JSON_Tools.JSON_Tools { - public class IniParserException : Exception + public class IniParserException : FormatException { - public IniParserException(string message) : base(message) { } + public new string Message { get; set; } + public char CurChar { get; set; } + public int Position { get; set; } + + public IniParserException(string Message, char c, int pos) + { + this.Message = Message; + this.CurChar = c; + this.Position = pos; + } + + public override string ToString() + { + return $"{Message} at position {Position} (char {JsonLint.CharDisplay(CurChar)})"; + } } public class IniFileParser { - /// - /// if true, ${SECTION:KEY} is replaced with the value associated with key KEY in section SECTION

- /// ${KEY} is replaced with the value associated with key KEY if KEY is in the same section - /// - public bool UseInterpolation; + ///// + ///// if true, ${SECTION:KEY} is replaced with the value associated with key KEY in section SECTION

+ ///// ${KEY} is replaced with the value associated with key KEY if KEY is in the same section + ///// + //public bool UseInterpolation; + public int utf8ExtraBytes { get; private set; } public int ii { get; private set; } + public int lineNum { get; private set; } public int utf8Pos { get { return ii + utf8ExtraBytes; } } public List comments; - public IniFileParser(bool useInterpolation) + public IniFileParser(/*bool useInterpolation*/) { ii = 0; utf8ExtraBytes = 0; comments = new List(); - UseInterpolation = useInterpolation; + //UseInterpolation = useInterpolation; } public void Reset() @@ -36,71 +53,220 @@ public void Reset() comments.Clear(); } - //public static readonly Regex INI_TOKENIZER = new Regex( - // @"^\s*\[([^\r\n]+)\]\s*$|" + // section name - // @"^\s*([^\r\n:=]+)\s*[:=](.+)$|" + // key and value within section - // @"^\s*[;#](.+)$|" + // comment - // @"^\s*(?![;#])(\S+)$" // multi-line continuation of value - //); - - public JObject Parse(string text) + public JObject Parse(string inp) { Reset(); var doc = new Dictionary(); - return new JObject(); - while (ii < text.Length) + while (ii < inp.Length) { - + ConsumeInsignificantChars(inp); + char c = inp[ii++]; + if (c == '[') + { + // section header + int startOfHeader = ii; + int headerStartUtf8 = ii - 1 + utf8ExtraBytes; + int endOfHeader = -1; + // find end of header, then keep going to start of next line, + // throwing an exception if newline comes before end of header, + // or non-whitespace comes between end of header and newline + while (ii < inp.Length) + { + c = inp[ii]; + switch (c) + { + case ']': + endOfHeader = ii; + break; + case ' ': + case '\t': + case '\r': + break; + case '\u2028': + case '\u2029': + case '\ufeff': + case '\xa0': + case '\u1680': + case '\u2000': + case '\u2001': + case '\u2002': + case '\u2003': + case '\u2004': + case '\u2005': + case '\u2006': + case '\u2007': + case '\u2008': + case '\u2009': + case '\u200A': + case '\u202F': + case '\u205F': + case '\u3000': + utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); + break; + case '\n': + ii++; + lineNum++; + if (endOfHeader < 0) + throw new IniParserException("Opening '[' and closing ']' of section header were not on the same line", c, headerStartUtf8 - 1); + else + goto headerFinished; + default: + if (endOfHeader > 0) + throw new IniParserException("Non-whitespace between end of section header and end of header line", '[', headerStartUtf8 - 1); + utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); + break; + } + ii++; + } + headerFinished: + string header = JNode.StrToString(inp.Substring(startOfHeader, endOfHeader - startOfHeader), false); + if (doc.ContainsKey(header)) + { + throw new IniParserException($"Document has duplicate section header [{header}]", c, headerStartUtf8 - 1); + } + JObject section = ParseSection(inp, header); + section.position = headerStartUtf8; + doc[header] = section; + } } + return new JObject(0, doc); } - public JObject ParseSection(string text, int sectionIndent) + public JObject ParseSection(string inp, string header) { var section = new Dictionary(); - return new JObject(); - while (ii < text.Length) + while (ii < inp.Length) { - int curIndent = ConsumeInsignificantChars(text); - char c = text[ii]; + int indent = ConsumeInsignificantChars(inp); + int keyStartUtf8 = utf8Pos; + if (ii >= inp.Length) + break; + char c = inp[ii]; + if (c == '[') + // start of the next section + break; + KeyValuePair kv = ParseKeyValuePair(inp, indent); + if (section.ContainsKey(kv.Key)) + { + throw new IniParserException($"Section [{header}] has duplicate key \"{kv.Key}\"", c, keyStartUtf8); + } + section[kv.Key] = kv.Value; } - int startOfSectionName = ii; + return new JObject(0, section); } - public static JObject Interpolate(JObject iniJson) + public KeyValuePair ParseKeyValuePair(string inp, int keyIndent) { - return iniJson; + if (ii >= inp.Length) + throw new IniParserException("EOF when expecting key-value pair", inp[inp.Length - 1], inp.Length - 1); + int startOfKey = ii; + char startC = inp[ii]; + int keyStartUtf8 = utf8Pos; + int endOfKey = ii; + bool foundKeyValueSep = false; + while (ii < inp.Length) + { + char c = inp[ii]; + switch (c) + { + case '=': + case ':': + foundKeyValueSep = true; + break; + case ' ': + case '\t': + case '\r': + break; + case '\u2028': + case '\u2029': + case '\ufeff': + case '\xa0': + case '\u1680': + case '\u2000': + case '\u2001': + case '\u2002': + case '\u2003': + case '\u2004': + case '\u2005': + case '\u2006': + case '\u2007': + case '\u2008': + case '\u2009': + case '\u200A': + case '\u202F': + case '\u205F': + case '\u3000': + utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); + break; + case '\n': + if (!foundKeyValueSep) + throw new IniParserException("No '=' or ':' on same line as key", startC, keyStartUtf8); + // key with empty value + goto exitKeyLoop; + default: + utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); + if (foundKeyValueSep) + { + // first non-whitespace char after ':' or '=' + goto exitKeyLoop; + } + // last non-whitespace seen so far before ':' or '=' + endOfKey = ii; + break; + } + ii++; + } + exitKeyLoop: + string key = JNode.StrToString(inp.Substring(startOfKey, endOfKey + 1 - startOfKey), false); + int startOfValue = ii; + int valueStartUtf8 = utf8Pos; + int endOfValue = ii; + while (ii < inp.Length) + { + int lineStart = ii; + ConsumeLine(inp); + endOfValue = JsonParser.EndOfPreviousLine(inp, ii, lineStart); + // tentatively ending value at end of last line + // we'll check to see if the current line has non-whitespace, non-comment at a greater indent than the start of the key + // if it does, the value will continue to the end of this line. + int indent = ConsumeToIndent(inp); + if (ii < inp.Length) + { + if (indent > keyIndent) + { + char c = inp[ii]; + if (c == ';' || c == '#') + // start of comment, so value is over + break; + } + else + break; // value ends with line that has indent <= keyIndent. + } + } + string valueStr = inp.Substring(startOfValue, endOfValue - startOfValue); + JNode valueNode = new JNode(valueStr, valueStartUtf8); + return new KeyValuePair(key, valueNode); } + //public static JObject Interpolate(JObject iniJson) + //{ + // return iniJson; + //} + /// - /// Consume comments and whitespace until the next character that is not - /// '#', ';', or whitespace. + /// Consume comments and whitespace until EOF or the next character that is not + /// whitespace or the beginning of a comment.

+ /// Return the indent of the current position, as described by ConsumeToIndent below. /// /// the number of characters since the last newline private int ConsumeInsignificantChars(string inp) { - bool isFirstNonWhiteSpaceCharOfLine = false; - int charsSinceLastNewline = -1; + int indent = 0; while (ii < inp.Length) { char c = inp[ii]; - charsSinceLastNewline++; switch (c) { - case '\n': - ii++; - charsSinceLastNewline = 0; - isFirstNonWhiteSpaceCharOfLine = true; - break; - case ';': - case '#': - if (!isFirstNonWhiteSpaceCharOfLine) - throw new IniParserException("JsonTools cannot parse ini file comments where the comment start character ('#' or ';') is not the first non-whitespace character of the line"); - int commentStartUtf8 = utf8Pos; - int commentContentStartII = ii + 1; - ConsumeLine(inp); - int commentContentEndII = JsonParser.EndOfPreviousLine(inp, ii, commentContentStartII); - comments.Add(new Comment(inp.Substring(commentContentStartII, commentContentEndII - commentContentStartII), false, commentStartUtf8)); - break; case ' ': case '\t': case '\r': @@ -123,16 +289,32 @@ private int ConsumeInsignificantChars(string inp) case '\u202F': case '\u205F': case '\u3000': + indent++; + ii++; utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); + break; + case '\n': + lineNum++; ii++; + indent = 0; + break; + case ';': + case '#': + int commentStartUtf8 = utf8Pos; + int commentContentStartII = ii + 1; + ConsumeLine(inp); + int commentContentEndII = JsonParser.EndOfPreviousLine(inp, ii, commentContentStartII); + comments.Add(new Comment(inp.Substring(commentContentStartII, commentContentEndII - commentContentStartII), false, commentStartUtf8)); + indent = 0; break; - default: return charsSinceLastNewline; + default: return indent; } } - return charsSinceLastNewline; + return 0; } /// + /// Move ii to the first character of the next line (after the next newline), or to EOF

/// note that both this and ConsumeLineAndGetLastNonWhitespace treat '\n' as the newline character, /// because this works for both Windows CRLF and Unix LF. However, Macintosh CR is not supported. /// @@ -143,15 +325,27 @@ private void ConsumeLine(string inp) { char c = inp[ii++]; if (c == '\n') + { + lineNum++; return; + } utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); } } - private int ConsumeLineAndGetLastNonWhitespace(string inp) + /// + /// consume characters from this line until '\n', the next non-whitespace, or EOF.

+ /// Return 0 if we stopped on a newline or EOF.

+ /// If we stopped on non-whitespace, return the number of characters between our starting ii (the beginning of the line) + /// and the current ii.

+ /// For example, if we start in position 1 of the string "\n \tf", we will get an indent of 2 because there are + /// two whitespace chars (' ' and '\t' count equally) between our starting position and the "f".

+ /// But if we start at position 1 of ' \n', we will return 0 because the line had no non-whitespace. + /// + /// + private int ConsumeToIndent(string inp) { - int lastNonWhitespace = ii; - return 0; + int indent = 0; while (ii < inp.Length) { char c = inp[ii]; @@ -160,6 +354,7 @@ private int ConsumeLineAndGetLastNonWhitespace(string inp) case ' ': case '\t': case '\r': + break; case '\u2028': case '\u2029': case '\ufeff': @@ -182,14 +377,20 @@ private int ConsumeLineAndGetLastNonWhitespace(string inp) utf8ExtraBytes += JsonParser.ExtraUTF8Bytes(c); break; case '\n': + // consumed entire line without seeing non-whitespace + // we treat a line with no non-whitespace characters as having an indent of 0 + // which terminates multi-line strings. ii++; - return lastNonWhitespace; + lineNum++; + return 0; default: - lastNonWhitespace = ii; - break; + // non-whitespace, return # chars since start of line + return indent; } ii++; + indent++; } + return 0; } } } diff --git a/JsonToolsNppPlugin/JSONTools/JNode.cs b/JsonToolsNppPlugin/JSONTools/JNode.cs index d9d0ab3..cc2c2d8 100644 --- a/JsonToolsNppPlugin/JSONTools/JNode.cs +++ b/JsonToolsNppPlugin/JSONTools/JNode.cs @@ -343,6 +343,15 @@ public static string StrToString(string s, bool quoted) return sb.ToString(); } + /// + /// mostly useful for quickly converting a JObject key (which must be escaped during parsing) into the raw string that it represents

+ /// Choose strAlreadyQuoted = true only if str is already wrapped in double quotes. + /// + public static string UnescapedJsonString(string str, bool strAlreadyQuoted) + { + return (string)new JsonParser().ParseString(strAlreadyQuoted ? str : $"\"{str}\"").value; + } + /// /// Compactly prints the JSON.

/// If sort_keys is true, the keys of objects are printed in alphabetical order.

@@ -1207,9 +1216,45 @@ public override (int comment_idx, int extra_utf8_bytes) PrettyPrintWithCommentsH return (comment_idx, extra_utf8_bytes); } + /// + /// dump this JObject as an ini file + /// + /// + /// + /// public string ToIniFile(List comments) { - return ""; + var sb = new StringBuilder(); + int positionInComments = 0; + int utf8ExtraBytes = 0; + foreach (KeyValuePair kv in children) + { + string header = kv.Key; + if (!(kv.Value is JObject section)) + { + throw new InvalidOperationException("Only objects where all children are objects with only string values can be converted to ini files"); + } + // section is treated as beginning just before the open squarebrace of the header + (positionInComments, utf8ExtraBytes) = Comment.AppendAllCommentsBeforePosition(sb, comments, positionInComments, utf8ExtraBytes, section.position, "", DocumentType.INI); + string unescapedHeader = UnescapedJsonString(header, false); + sb.Append($"[{unescapedHeader}]\r\n"); + utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(unescapedHeader, 0, unescapedHeader.Length); + foreach (KeyValuePair sectKv in section.children) + { + string key = sectKv.Key; + JNode value = sectKv.Value; + if (!(value.value is string valueStr)) + { + throw new InvalidOperationException("Only objects where all children are objects with only string values can be converted to ini files"); + } + (positionInComments, utf8ExtraBytes) = Comment.AppendAllCommentsBeforePosition(sb, comments, positionInComments, utf8ExtraBytes, value.position, "", DocumentType.INI); + string unescapedKey = UnescapedJsonString(key, false); + sb.Append($"{unescapedKey}={valueStr}\r\n"); + utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(unescapedKey, 0, unescapedKey.Length); + utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(valueStr, 0, valueStr.Length); + } + } + return sb.ToString(); } /// diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs index a0333ea..6a51c76 100644 --- a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs +++ b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("5.7.0.6")] -[assembly: AssemblyFileVersion("5.7.0.6")] +[assembly: AssemblyVersion("5.7.0.7")] +[assembly: AssemblyFileVersion("5.7.0.7")] diff --git a/JsonToolsNppPlugin/Tests/IniFileParserTests.cs b/JsonToolsNppPlugin/Tests/IniFileParserTests.cs index ffd9f03..72886f5 100644 --- a/JsonToolsNppPlugin/Tests/IniFileParserTests.cs +++ b/JsonToolsNppPlugin/Tests/IniFileParserTests.cs @@ -12,14 +12,17 @@ public static bool Test() int ii = 0; int testsFailed = 0; var exampleIniFname = @"plugins\JsonTools\testfiles\small\example_ini.ini"; + var exampleIniReformattedFname = @"plugins\JsonTools\testfiles\small\example_ini_reformatted.ini"; var exampleJsonFname = @"plugins\JsonTools\testfiles\small\example_ini.json"; string exampleJsonText = null; string exampleIniText = null; + string exampleIniReformattedText = null; try { exampleIniText = File.ReadAllText(exampleIniFname); exampleJsonText = File.ReadAllText(exampleJsonFname); + exampleIniReformattedText = File.ReadAllText(exampleIniReformattedFname); } catch { @@ -27,74 +30,89 @@ public static bool Test() Npp.AddLine("Either testfiles/small/example_ini.ini or testfiles/small/example_ini.json was not found in the plugins/JsonTools directory"); } - var testcases = new (string iniText, string correctJsonStrWithoutInterpolation, bool interpolation)[] + var testcases = new (string iniText, string correctJsonStrWithoutInterpolation, /*bool interpolation,*/ string correctReformattedIniText)[] { - (exampleIniText, exampleJsonText, false), - (exampleIniText, exampleJsonText, true), + (exampleIniText, exampleJsonText, /*false,*/ exampleIniReformattedText), + //(exampleIniText, exampleJsonText, true, exampleIniReformattedText), }; var jsonParser = new JsonParser(LoggerLevel.JSON5, false, true, true, true); - - foreach ((string iniText, string correctJsonStrWithoutInterpolation, bool interpolation) in testcases) + int testsPerLoop = 3; + foreach ((string iniText, string correctJsonStrWithoutInterpolation, /*bool interpolation,*/ string correctReformattedIniText) in testcases) { JObject correctJson; - JObject correctJsonWithInterpolation; + //JObject correctJsonWithInterpolation; string correctJsonStr; - ii += 3; - Npp.AddLine($"Parsing ini file (shown here as JSON string)\r\n{JNode.StrToString(iniText, true)}"); + ii += testsPerLoop; + bool hasShownIniFileDescription = false; + void showFileDescription() + { + if (!hasShownIniFileDescription) + { + string iniFileDescription = $"Parsing ini file (shown here as JSON string)\r\n{JNode.StrToString(iniText, true)}"; + hasShownIniFileDescription = true; + Npp.AddLine(iniFileDescription); + } + }; try { correctJson = (JObject)jsonParser.Parse(correctJsonStrWithoutInterpolation); - correctJsonWithInterpolation = IniFileParser.Interpolate(correctJson); - correctJsonStr = interpolation ? correctJsonStrWithoutInterpolation : correctJsonWithInterpolation.PrettyPrint(); - if (interpolation) - correctJson = correctJsonWithInterpolation; + //correctJsonWithInterpolation = IniFileParser.Interpolate(correctJson); + correctJsonStr = correctJson.PrettyPrintWithComments(jsonParser.comments, sort_keys:false); } catch (Exception ex) { + showFileDescription(); Npp.AddLine($"While trying to parse JSON\r\n{correctJsonStrWithoutInterpolation}\r\nGOT EXCEPTION\r\n{ex}"); - testsFailed += 3; + testsFailed += testsPerLoop; continue; } JObject gotJson; - IniFileParser iniParser = new IniFileParser(interpolation); + IniFileParser iniParser = new IniFileParser(); try { gotJson = iniParser.Parse(iniText); } catch (Exception ex) { + showFileDescription(); Npp.AddLine($"While trying to parse ini file\r\nGOT EXCEPTION\r\n{ex}"); - testsFailed += 3; + testsFailed += testsPerLoop; continue; } string gotJsonStr = gotJson.ToString(); if (gotJson.TryEquals(correctJson, out _)) { // test if comments are parsed correctly + string gotJsonWithComments = ""; try { - string gotJsonWithComments = gotJson.ToStringWithComments(iniParser.comments); + gotJsonWithComments = gotJson.PrettyPrintWithComments(iniParser.comments, sort_keys:false); if (gotJsonWithComments != correctJsonStr) { testsFailed++; - Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT JSON WITH COMMENTS\r\n{gotJsonStr}"); + showFileDescription(); + Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT JSON WITH COMMENTS\r\n{gotJsonWithComments}"); } } catch (Exception ex) { - testsFailed++; + testsFailed += 2; + showFileDescription(); Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT EXCEPTION\r\n{ex}"); + continue; } // test if dumping the JSON back to an ini file and re-parsing the dumped ini will return the same JSON string dumpedIniText = ""; try { - dumpedIniText = gotJson.ToIniFile(iniParser.comments); + JObject reParsedJson = (JObject)jsonParser.Parse(gotJsonWithComments); + dumpedIniText = reParsedJson.ToIniFile(jsonParser.comments); } catch (Exception ex) { testsFailed++; - Npp.AddLine($"When ini-dumping JSON\r\n{gotJsonStr}\r\nEXPECTED INI FILE\r\n{correctJsonStr}\r\nGOT EXCEPTION {ex}"); + showFileDescription(); + Npp.AddLine($"When ini-dumping JSON\r\n{gotJsonStr}\r\nEXPECTED INI FILE\r\n{correctReformattedIniText}\r\nGOT EXCEPTION {ex}"); continue; } JObject jsonFromDumpedIniText; @@ -104,19 +122,22 @@ public static bool Test() } catch (Exception ex) { + showFileDescription(); Npp.AddLine($"EXPECTED JSON FROM DUMPED INI FILE\r\n{correctJsonStr}\r\nGOT EXCEPTION\r\n{ex}"); testsFailed++; continue; } if (!jsonFromDumpedIniText.TryEquals(correctJson, out _)) { + showFileDescription(); testsFailed++; Npp.AddLine($"EXPECTED JSON FROM DUMPED INI FILE\r\n{correctJson.ToString()}\r\nGOT JSON\r\n{gotJson.ToString()}"); } } else { - testsFailed += 3; + testsFailed += testsPerLoop; + showFileDescription(); Npp.AddLine($"EXPECTED JSON\r\n{correctJson.ToString()}\r\nGOT JSON\r\n{gotJson.ToString()}"); } } diff --git a/JsonToolsNppPlugin/Tests/TestRunner.cs b/JsonToolsNppPlugin/Tests/TestRunner.cs index 6e44792..de934f0 100644 --- a/JsonToolsNppPlugin/Tests/TestRunner.cs +++ b/JsonToolsNppPlugin/Tests/TestRunner.cs @@ -13,6 +13,8 @@ namespace JSON_Tools.Tests { public class TestRunner { + private const string fuzzTestName = "RemesPath produces sane outputs on randomly generated queries"; + public static async Task RunAll() { Npp.notepad.FileNew(); @@ -39,7 +41,7 @@ public static async Task RunAll() (RemesParserTester.Test, "RemesPath parser and compiler", false, false), (RemesPathThrowsWhenAppropriateTester.Test, "RemesPath throws errors on bad inputs", false, false), (RemesPathAssignmentTester.Test, "RemesPath assignment operations", false, false), - (() => RemesPathFuzzTester.Test(10000, 20), "RemesPath produces sane outputs on randomly generated queries", false, false), + (() => RemesPathFuzzTester.Test(10000, 20), fuzzTestName, false, false), (RemesPathComplexQueryTester.Test, "multi-statement queries in RemesPath", false, false), (JsonSchemaMakerTester.Test, "JsonSchema generator", false, false), @@ -111,6 +113,11 @@ public static async Task RunAll() } skipped.Add(name); } + else if (Main.settings.skip_api_request_and_fuzz_tests && name == fuzzTestName) + { + Npp.AddLine("\r\nskipped RemesPath fuzz tests because settings.skip_api_request_and_fuzz_tests was set to true"); + skipped.Add(name); + } else { Npp.AddLine($@"========================= @@ -129,9 +136,9 @@ public static async Task RunAll() Testing JSON grepper's API request tool ========================= "); - if (Main.settings.skip_api_request_tests) + if (Main.settings.skip_api_request_and_fuzz_tests) { - Npp.AddLine("skipped tests because settings.skip_api_request_tests was set to true"); + Npp.AddLine("skipped tests because settings.skip_api_request_and_fuzz_tests was set to true"); skipped.Add("JSON grepper's API request tool"); } else @@ -141,6 +148,8 @@ Testing JSON grepper's API request tool } } + if (skipped.Count > 0) + Npp.editor.InsertText(header.Length + 2, "Tests skipped: " + string.Join(", ", skipped) + "\r\n"); Npp.editor.InsertText(header.Length + 2, "Tests failed: " + string.Join(", ", failures) + "\r\n"); } } diff --git a/JsonToolsNppPlugin/Utils/Settings.cs b/JsonToolsNppPlugin/Utils/Settings.cs index 5e9fa92..1d913c5 100644 --- a/JsonToolsNppPlugin/Utils/Settings.cs +++ b/JsonToolsNppPlugin/Utils/Settings.cs @@ -135,9 +135,9 @@ public class Settings : SettingsBase Category("Miscellaneous"), DefaultValue("\"[{")] public string try_parse_start_chars { get; set; } - [Description("When running tests, skip the tests that send requests to APIs"), + [Description("When running tests, skip the tests that send requests to APIs and the RemesPath fuzz tests"), Category("Miscellaneous"), DefaultValue(true)] - public bool skip_api_request_tests { get; set; } + public bool skip_api_request_and_fuzz_tests { get; set; } [Description("Which type of newline to use for generated CSV files."), Category("Miscellaneous"), DefaultValue(EndOfLine.LF)] diff --git a/most recent errors.txt b/most recent errors.txt index a9c999e..0129d65 100644 --- a/most recent errors.txt +++ b/most recent errors.txt @@ -1,6 +1,6 @@ -Test results for JsonTools v5.7.0.6 on Notepad++ 8.5.7 64bit +Test results for JsonTools v5.7.0.7 on Notepad++ 8.5.7 64bit NOTE: Ctrl-F (regular expressions *on*) for "Failed [1-9]\d*" to find all failed tests -Tests failed: YAML dumper, INI file parser +Tests failed: YAML dumper ========================= Testing JNode Copy method ========================= @@ -182,59 +182,47 @@ Passed 24 tests. Testing INI file parser ========================= -Parsing ini file (shown here as JSON string) -"# cömment before foo section\r\n[föo]\r\nbär : 草 \r\nbaz : 34\r\n\r\n;comment after foo.baz\r\n\r\nquz = a multiline \r\n response\r\n to quz\r\n#comment ending quz\r\n indented header = in unindented section\r\n \r\n ; but we need a comment at end of indented section\r\n [indented section]\r\n subsection 1 = \"5\"\r\n   \r\n subsection 2 = some more text\r\n  even on another line\r\n   \r\n ; indented　comment\r\n 　 whoa : even more indentation\t\r\n \r\n [triple indentation]\r\n \r\n subsection:1\r\n[ dedented section ]\r\nempty\tvalue\theader : \r\n# the previous section name had an empty string as value\r\nempty =\r\n\r\n[empty section ]\r\n[interpolation example]\r\ndollar sign = use $$ instead\r\nfoo header : tab\tsep\ttext\r\n\r\nbar = ${foo header}\r\n; above line should be same as foo header\r\n [ another empty section]\r\n # but with a comment inside\r\n[interpolation from other sections]\r\nbaz = ${foo reference} ${föo:baz}\r\n\r\n# to test if references to interpolated values work correctly\r\nfoo reference = ${interpolation example:foo header}\r\n[colons_and_equals_signs]\r\nequals=inside value=fine\r\ncolon=inside value : also fine\r\n :even on separate line (if indented)" -EXPECTED JSON -{" another empty section": {}, " dedented section ": {"empty": "", "empty\tvalue\theader": ""}, "colons_and_equals_signs": {"colon": "inside value : also fine\r\n :even on separate line (if indented)", "equals": "inside value=fine"}, "empty section ": {}, "föo": {"bär": "草", "baz": "34", "indented header": "in unindented section", "quz": "a multiline\r\n response to\r\n quz"}, "indented section": {"subsection 1": "\"5\"", "subsection 2": "some more text\r\n  even on another line", "whoa": "even more indentation\t"}, "interpolation example": {"bar": "${foo header}", "dollar sign": "use $$ instead", "foo header": "tab\tsep\ttext"}, "interpolation from other sections": {"baz": "${foo reference} ${föo:baz}", "foo reference": "${interpolation example:foo header}"}, "triple indentation": {"subsection": "1"}} -GOT JSON -{} -Parsing ini file (shown here as JSON string) -"# cömment before foo section\r\n[föo]\r\nbär : 草 \r\nbaz : 34\r\n\r\n;comment after foo.baz\r\n\r\nquz = a multiline \r\n response\r\n to quz\r\n#comment ending quz\r\n indented header = in unindented section\r\n \r\n ; but we need a comment at end of indented section\r\n [indented section]\r\n subsection 1 = \"5\"\r\n   \r\n subsection 2 = some more text\r\n  even on another line\r\n   \r\n ; indented　comment\r\n 　 whoa : even more indentation\t\r\n \r\n [triple indentation]\r\n \r\n subsection:1\r\n[ dedented section ]\r\nempty\tvalue\theader : \r\n# the previous section name had an empty string as value\r\nempty =\r\n\r\n[empty section ]\r\n[interpolation example]\r\ndollar sign = use $$ instead\r\nfoo header : tab\tsep\ttext\r\n\r\nbar = ${foo header}\r\n; above line should be same as foo header\r\n [ another empty section]\r\n # but with a comment inside\r\n[interpolation from other sections]\r\nbaz = ${foo reference} ${föo:baz}\r\n\r\n# to test if references to interpolated values work correctly\r\nfoo reference = ${interpolation example:foo header}\r\n[colons_and_equals_signs]\r\nequals=inside value=fine\r\ncolon=inside value : also fine\r\n :even on separate line (if indented)" -EXPECTED JSON -{" another empty section": {}, " dedented section ": {"empty": "", "empty\tvalue\theader": ""}, "colons_and_equals_signs": {"colon": "inside value : also fine\r\n :even on separate line (if indented)", "equals": "inside value=fine"}, "empty section ": {}, "föo": {"bär": "草", "baz": "34", "indented header": "in unindented section", "quz": "a multiline\r\n response to\r\n quz"}, "indented section": {"subsection 1": "\"5\"", "subsection 2": "some more text\r\n  even on another line", "whoa": "even more indentation\t"}, "interpolation example": {"bar": "${foo header}", "dollar sign": "use $$ instead", "foo header": "tab\tsep\ttext"}, "interpolation from other sections": {"baz": "${foo reference} ${föo:baz}", "foo reference": "${interpolation example:foo header}"}, "triple indentation": {"subsection": "1"}} -GOT JSON -{} -Failed 6 tests. -Passed 0 tests. +Failed 0 tests. +Passed 3 tests. ========================= Testing UI tests ========================= Failed 0 tests -Passed 199 tests +Passed 209 tests ========================= Testing JsonParser performance ========================= Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 89556 into JNode took 2.644 +/- 1.272 ms over 32 trials -Load times (ms): 1, 3, 2, 1, 1, 5, 4, 2, 2, 4, 1, 1, 2, 3, 1, 1, 4, 1, 1, 1, 2, 1, 1, 4, 1, 1, 2, 4, 2, 2, 5, 1 +To convert JSON string of size 89556 into JNode took 2.786 +/- 1.154 ms over 32 trials +Load times (ms): 4, 2, 3, 4, 4, 1, 2, 2, 3, 1, 1, 5, 1, 1, 1, 2, 1, 1, 3, 4, 1, 1, 2, 3, 2, 3, 3, 1, 2, 3, 2, 3 ========================= Performance tests for RemesPath (float arithmetic) ========================= -Compiling query "@[@[:].a * @[:].t < @[:].e]" into took 1.295 +/- 9.777 microseconds over 64 trials -To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.025 +/- 0.008 ms over 64 trials -Query times (ms): 0.089, 0.036, 0.024, 0.023, 0.024, 0.023, 0.023, 0.023, 0.023, 0.022, 0.023, 0.028, 0.024, 0.023, 0.023, 0.022, 0.023, 0.022, 0.022, 0.023, 0.022, 0.026, 0.023, 0.024, 0.023, 0.022, 0.023, 0.023, 0.023, 0.022, 0.022, 0.026, 0.023, 0.023, 0.023, 0.023, 0.023, 0.023, 0.023, 0.023, 0.023, 0.026, 0.023, 0.023, 0.023, 0.023, 0.024, 0.022, 0.022, 0.023, 0.023, 0.026, 0.023, 0.024, 0.028, 0.023, 0.023, 0.023, 0.022, 0.022, 0.023, 0.026, 0.023, 0.023 +Compiling query "@[@[:].a * @[:].t < @[:].e]" into took 1.409 +/- 10.695 microseconds over 64 trials +To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.075 +/- 0.366 ms over 64 trials +Query times (ms): 0.068, 2.95, 0.025, 0.024, 0.022, 0.022, 0.022, 0.023, 0.023, 0.022, 0.023, 0.025, 0.022, 0.022, 0.022, 0.023, 0.022, 0.022, 0.022, 0.022, 0.022, 0.023, 0.022, 0.022, 0.022, 0.022, 0.022, 0.021, 0.022, 0.022, 0.022, 0.023, 0.021, 0.022, 0.022, 0.022, 0.022, 0.022, 0.022, 0.022, 0.021, 0.022, 0.022, 0.021, 0.022, 0.022, 0.022, 0.022, 0.022, 0.022, 0.022, 0.023, 0.023, 0.029, 0.022, 0.023, 0.023, 0.023, 0.023, 0.022, 0.023, 0.417, 0.023, 0.022 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (string operations) ========================= -Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 0.939 +/- 6.862 microseconds over 64 trials -To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.054 +/- 0.01 ms over 64 trials -Query times (ms): 0.128, 0.055, 0.054, 0.07, 0.077, 0.051, 0.051, 0.051, 0.052, 0.051, 0.051, 0.053, 0.054, 0.052, 0.052, 0.051, 0.051, 0.05, 0.051, 0.051, 0.051, 0.054, 0.053, 0.053, 0.052, 0.051, 0.051, 0.051, 0.051, 0.05, 0.052, 0.051, 0.052, 0.05, 0.051, 0.051, 0.05, 0.05, 0.051, 0.051, 0.05, 0.051, 0.051, 0.072, 0.05, 0.052, 0.051, 0.051, 0.051, 0.05, 0.051, 0.052, 0.052, 0.051, 0.052, 0.051, 0.054, 0.052, 0.053, 0.051, 0.051, 0.052, 0.06, 0.054 +Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 1.136 +/- 8.475 microseconds over 64 trials +To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.069 +/- 0.045 ms over 64 trials +Query times (ms): 0.112, 0.127, 0.065, 0.052, 0.051, 0.052, 0.051, 0.052, 0.052, 0.051, 0.052, 0.051, 0.052, 0.052, 0.052, 0.051, 0.052, 0.051, 0.051, 0.052, 0.051, 0.052, 0.051, 0.052, 0.051, 0.051, 0.05, 0.051, 0.051, 0.051, 0.34, 0.054, 0.052, 0.052, 0.052, 0.119, 0.122, 0.103, 0.103, 0.152, 0.123, 0.128, 0.072, 0.063, 0.053, 0.186, 0.053, 0.052, 0.051, 0.052, 0.051, 0.052, 0.05, 0.052, 0.052, 0.05, 0.051, 0.051, 0.052, 0.051, 0.051, 0.051, 0.052, 0.051 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "NK