From 9328eb5e24fa69a4dc424ed75b30a526758e27b1 Mon Sep 17 00:00:00 2001 From: molsonkiko <46202915+molsonkiko@users.noreply.github.com> Date: Wed, 6 Dec 2023 20:00:17 -0800 Subject: [PATCH] make JSON-to-CSV, remespath to_csv use same algo - ensure that delimiter inside non-strings are escaped in JSON-to-CSV - ensure that null is represented as empty string in RemesPath to_csv - add more tests for quoted numbers in CSV --- CHANGELOG.md | 3 + JsonToolsNppPlugin/Forms/JsonToCsvForm.cs | 2 +- JsonToolsNppPlugin/Forms/TreeViewer.cs | 44 +++++++++- JsonToolsNppPlugin/JSONTools/JsonParser.cs | 30 ------- .../JSONTools/JsonTabularize.cs | 84 ++++++++++--------- .../JSONTools/RemesPathFunctions.cs | 9 +- JsonToolsNppPlugin/Properties/AssemblyInfo.cs | 4 +- .../Tests/JsonTabularizerTests.cs | 20 ++--- JsonToolsNppPlugin/Tests/RemesPathTests.cs | 6 +- .../Tests/UserInterfaceTests.cs | 15 ++-- docs/RemesPath.md | 2 + docs/json-to-csv.md | 13 ++- most recent errors.txt | 84 +++++++++---------- 13 files changed, 171 insertions(+), 145 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a6f0d..1489379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - bug with calling arg functions on projections - seems like object projections are treated as arrays when calling arg functions on them in some cases? - issue with treeview closing when a file with a treeview is moved from one view to another - `loop()` function used in `s_sub` callbacks is not thread-safe. This doesn't matter right now because RemesPath is single-threaded, but it could matter in the future. +- __Known issues with `Select This` and `Select all children` commands in tree view (FIX THESE ASAP)__: + * Empty strings are ignored by `Select this` + * Floating point numbers may not be selected correctly (JSON string may have different length from the number representation in the CSV) ## [6.0.0] - (UNRELEASED) 2023-MM-DD diff --git a/JsonToolsNppPlugin/Forms/JsonToCsvForm.cs b/JsonToolsNppPlugin/Forms/JsonToCsvForm.cs index 90d884b..38ce0da 100644 --- a/JsonToolsNppPlugin/Forms/JsonToCsvForm.cs +++ b/JsonToolsNppPlugin/Forms/JsonToCsvForm.cs @@ -64,7 +64,7 @@ Stringify iterables Dictionary schema = JsonSchemaMaker.BuildSchema(json); JNode tab = tabularizer.BuildTable(json, schema, keysep); string eol = Npp.GetEndOfLineString(eolComboBox.SelectedIndex); - csv = tabularizer.TableToCsv((JArray)tab, delim, '"', null, BoolsToIntsCheckBox.Checked, eol); + csv = tabularizer.TableToCsv((JArray)tab, delim, '"', eol, null, BoolsToIntsCheckBox.Checked); } catch (Exception ex) { diff --git a/JsonToolsNppPlugin/Forms/TreeViewer.cs b/JsonToolsNppPlugin/Forms/TreeViewer.cs index 381dd81..aeb6d2c 100644 --- a/JsonToolsNppPlugin/Forms/TreeViewer.cs +++ b/JsonToolsNppPlugin/Forms/TreeViewer.cs @@ -1003,6 +1003,44 @@ public string KeyOfTreeNode(TreeNode node, KeyStyle style) return JNode.FormatKey(node.Name, style); } + /// + /// how long we think the representation of a JNode is in regex mode.

+ /// if delim is '\x00' (not in CSV mode), this is just the length of its UTF8 repr.

+ /// Otherwise, we do some special casing. + /// + /// + /// + /// + /// + /// + public static int LengthOfStringInRegexMode(JNode jnode, int startpos, char delim, char quote) + { + string s = jnode.ValueOrToString(); + int utf8len = Encoding.UTF8.GetByteCount(s); + if (delim == 0) + return utf8len; // not a CSV document, so we assume it's the same length in the document + int quoteCount = 0; + bool delimOrNewline = false; + for (int ii = 0; ii < s.Length; ii++) + { + char c = s[ii]; + if (c == delim || c == '\r' || c == '\n') + delimOrNewline = true; + else if (c == quote) + quoteCount++; + } + if (!delimOrNewline && quoteCount == 0) + { + // in general, if a string has no delimiters or quotes or newlines, it doesn't need to be wrapped in quotes. + // but it still *could* be wrapped in quotes, and that would be equivalent, so we need to check. + // note that this assumes the quote character is ASCII, which seems eminently reasonable. + int startByte = Npp.editor.GetCharAt(startpos); + return quote == startByte ? utf8len + 2 : utf8len; + } + // it must be a quoted string, in which case it's wrapped in quotes (2 extra bytes) and every literal quote inside is doubled up (quoteCount extra bytes) + return utf8len + quoteCount + 2; + } + public void SelectTreeNodeJson(TreeNode node) { if (Main.activeFname != fname) @@ -1012,7 +1050,7 @@ public void SelectTreeNodeJson(TreeNode node) { nodeStartPos = NodePosInJsonDoc(node); if (GetDocumentType() == DocumentType.REGEX) - nodeEndPos = nodeStartPos + JsonParser.UTF8BytesInCSVRepr(jnode.ValueOrToString(), csvDelim, csvQuote); + nodeEndPos = nodeStartPos + LengthOfStringInRegexMode(jnode, nodeStartPos, csvDelim, csvQuote); else nodeEndPos = Main.EndOfJNodeAtPos(nodeStartPos, Npp.editor.GetLength()); } @@ -1050,8 +1088,8 @@ public void SelectTreeNodeJsonChildren(TreeNode node) break; } string childstr = child.ValueOrToString(); - int utf8Len = JsonParser.UTF8BytesInCSVRepr(childstr, csvDelim, csvQuote); - int startPos = child.position; + int startPos = child.position + selectionStartPos; + int utf8Len = LengthOfStringInRegexMode(child, startPos, csvDelim, csvQuote); int endPos = startPos + utf8Len; if (endPos > startPos) { diff --git a/JsonToolsNppPlugin/JSONTools/JsonParser.cs b/JsonToolsNppPlugin/JSONTools/JsonParser.cs index 710186f..b1bf04a 100644 --- a/JsonToolsNppPlugin/JSONTools/JsonParser.cs +++ b/JsonToolsNppPlugin/JSONTools/JsonParser.cs @@ -325,36 +325,6 @@ public static int ExtraUTF8BytesBetween(string inp, int start, int end) return count; } - /// - /// number of bytes in the UTF8-encoded CSV representation of string s (with delimiter delim and quote character quote)

- /// e.g. "fö\n'" would be represented as 'fö\n''' in a CSV file with '\'' as quote character, so UTF8BytesInCSVRepr("fö\n'", ',', '\'') would return 8.

- /// if delim is '\x00', return the UTF8 bytecount of s - /// - /// CSV delimiter (or '\x00' for non-CSV file) - /// CSV quote character - /// - public static int UTF8BytesInCSVRepr(string s, char delim, char quote) - { - if (delim == 0) - return Encoding.UTF8.GetByteCount(s); - int quoteCount = 0; - bool delimOrNewline = false; - int byteCount = s.Length; - for (int ii = 0; ii < s.Length; ii++) - { - char c = s[ii]; - if (c == '\r' || c == '\n' || c == delim) - delimOrNewline = true; - else if (c == quote) - quoteCount++; - else - byteCount += ExtraUTF8Bytes(c); - } - if (delimOrNewline || quoteCount > 0) - byteCount += 2 + quoteCount; // strings containing newlines, quotes or delimiters must be wrapped in quotes, and quote chars in quoted strings must be doubled up - return byteCount; - } - /// /// Set the parser's state to severity, unless the state was already higher.

/// If the severity is above the parser's loggerLevel:

diff --git a/JsonToolsNppPlugin/JSONTools/JsonTabularize.cs b/JsonToolsNppPlugin/JSONTools/JsonTabularize.cs index c762cf2..8be7051 100644 --- a/JsonToolsNppPlugin/JSONTools/JsonTabularize.cs +++ b/JsonToolsNppPlugin/JSONTools/JsonTabularize.cs @@ -802,41 +802,69 @@ public JArray BuildTable(JNode obj, Dictionary schema, string ke /// /// /// - /// + /// /// - public static void ApplyQuotesIfNeeded(StringBuilder sb, string s, char delim, char quote_char) + public static void ApplyQuotesIfNeeded(StringBuilder sb, string s, char delim, char quote) { - if (s.IndexOfAny(new char[] {delim, '\r', '\n', quote_char}) >= 0) + if (s.IndexOfAny(new char[] {delim, '\r', '\n', quote}) >= 0) { - sb.Append(quote_char); + sb.Append(quote); for (int ii = 0; ii < s.Length; ii++) { char c = s[ii]; sb.Append(c); - if (c == quote_char) - sb.Append(quote_char); + if (c == quote) + sb.Append(quote); } - sb.Append(quote_char); + sb.Append(quote); } else sb.Append(s); } - public string TableToCsv(JArray table, char delim = ',', char quote_char = '"', string[] header = null, bool bools_as_ints = false, string newline = "\n") + public static void CsvStringToSb(StringBuilder sb, JNode jnode, char delim, char quote, bool boolsAsInts) + { + string val; + switch (jnode.type) + { + case Dtype.STR: + val = (string)jnode.value; + break; // only apply quotes if internal delims, quotes, or newlines + case Dtype.DATE: + val = ((DateTime)jnode.value).ToString("yyyy-MM-dd"); + break; + case Dtype.DATETIME: + val = ((DateTime)jnode.value).ToString("yyyy-MM-dd hh:mm:ss"); + break; + case Dtype.NULL: + return; // nulls should be empty entries + case Dtype.BOOL: + sb.Append((bool)jnode.value + ? (boolsAsInts ? "1" : "true") + : (boolsAsInts ? "0" : "false")); + return; + default: + val = jnode.ToString(); + break; + } + ApplyQuotesIfNeeded(sb, val, delim, quote); + } + + public string TableToCsv(JArray table, char delim = ',', char quote = '"', string newline = "\n", string[] header = null, bool boolsAsInts = false) { // allow the user to supply their own column order. If they don't, just alphabetically sort colnames if (header == null) { - HashSet all_keys = new HashSet(); + HashSet allKeys = new HashSet(); foreach (JNode child in table.children) { foreach (string key in ((JObject)child).children.Keys) { - all_keys.Add(key); + allKeys.Add(key); } } - header = new string[all_keys.Count]; + header = new string[allKeys.Count]; int ii = 0; - foreach (string key in all_keys) + foreach (string key in allKeys) { header[ii++] = key; } @@ -846,7 +874,7 @@ public string TableToCsv(JArray table, char delim = ',', char quote_char = '"', for (int ii = 0; ii < header.Length; ii++) { string col = header[ii]; - ApplyQuotesIfNeeded(sb, col, delim, quote_char); + ApplyQuotesIfNeeded(sb, col, delim, quote); if (ii < header.Length - 1) sb.Append(delim); } sb.Append(newline); @@ -856,40 +884,14 @@ public string TableToCsv(JArray table, char delim = ',', char quote_char = '"', for (int ii = 0; ii < header.Length; ii++) { string col = header[ii]; - if (!orow.children.TryGetValue(col, out JNode val)) + if (!orow.children.TryGetValue(col, out JNode jnode)) { // sometimes the row might have a missing key. // in that case, just leave an empty entry, and add a delimiter if needed. if (ii < header.Length - 1) sb.Append(delim); continue; } - switch (val.type) - { - case Dtype.STR: - ApplyQuotesIfNeeded(sb, (string)val.value, delim, quote_char); - break; // only apply quotes if internal delim - case Dtype.DATE: - sb.Append(((DateTime)val.value).ToString("yyyy-MM-dd")); - break; - case Dtype.DATETIME: - sb.Append(((DateTime)val.value).ToString("yyyy-MM-dd hh:mm:ss")); - break; - case Dtype.NULL: - break; // nulls should be empty entries - case Dtype.BOOL: - if (bools_as_ints) - { - sb.Append((bool)val.value ? "1" : "0"); - } - else - { - sb.Append((bool)val.value ? "true" : "false"); - } - break; - default: - sb.Append(val.ToString()); // everything else gets quote chars - break; - } + CsvStringToSb(sb, jnode, delim, quote, boolsAsInts); if (ii < header.Length - 1) sb.Append(delim); } sb.Append(newline); diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs index 9f05ca7..4af4da1 100644 --- a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs +++ b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs @@ -2865,7 +2865,7 @@ public static JNode ToCsv(List args) int ii = 0; foreach (JNode ochild in o.children.Values) { - JsonTabularizer.ApplyQuotesIfNeeded(sb, ochild.ValueOrToString(), delim, quote); + JsonTabularizer.CsvStringToSb(sb, ochild, delim, quote, false); ii++; if (ii < o.Length) sb.Append(delim); @@ -2881,15 +2881,16 @@ public static JNode ToCsv(List args) { if (!(x is JArray a)) { - JsonTabularizer.ApplyQuotesIfNeeded(sb, x.ValueOrToString(), delim, quote); + JsonTabularizer.CsvStringToSb(sb, x, delim, quote, false); sb.Append(newline); nColumns = 1; return true; } - nColumns = a.children.Count; + var children = a.children; + nColumns = children.Count; for (int ii = 0; ii < nColumns; ii++) { - JsonTabularizer.ApplyQuotesIfNeeded(sb, a.children[ii].ValueOrToString(), delim, quote); + JsonTabularizer.CsvStringToSb(sb, children[ii], delim, quote, false); if (ii < a.Length - 1) sb.Append(delim); } diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs index 249724c..63f4181 100644 --- a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs +++ b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("5.8.0.14")] -[assembly: AssemblyFileVersion("5.8.0.14")] +[assembly: AssemblyVersion("5.8.0.15")] +[assembly: AssemblyFileVersion("5.8.0.15")] diff --git a/JsonToolsNppPlugin/Tests/JsonTabularizerTests.cs b/JsonToolsNppPlugin/Tests/JsonTabularizerTests.cs index 37b3471..5991b60 100644 --- a/JsonToolsNppPlugin/Tests/JsonTabularizerTests.cs +++ b/JsonToolsNppPlugin/Tests/JsonTabularizerTests.cs @@ -455,7 +455,7 @@ public static bool Test() // TEST CSV CREATION JsonParser fancyParser = new JsonParser(LoggerLevel.JSON5); - var csv_testcases = new (string inp, string desired_out, char delim, char quote_char, string[] header, bool bools_as_ints, string eol)[] + var csvTestcases = new (string inp, string desired_out, char delim, char quote_char, string[] header, bool bools_as_ints, string eol)[] { ( "[{\"a\": 1, \"b\": \"a\"}, {\"a\": 2, \"b\": \"b\"}]", "a,b\r\n1,a\r\n2,b\r\n", ',', '"', null, false, "\r\n" ), ( "[{\"a\": 1, \"b\": \"a\"}, {\"a\": 2, \"b\": \"b\"}]", "a,b\r1,a\r2,b\r", ',', '"', null, false, "\r" ), @@ -576,40 +576,40 @@ public static bool Test() ',', '"', null, false, "\r\n" ), }; - foreach ((string inp, string desired_out, char delim, char quote_char, string[] header, bool bools_as_ints, string eol) in csv_testcases) + foreach ((string inp, string desired_out, char delim, char quote, string[] header, bool boolsAsInts, string eol) in csvTestcases) { ii++; JNode table = fancyParser.Parse(inp); string result = ""; string head_str = header == null ? "null" : '[' + string.Join(", ", header) + ']'; string escapedEol = JNode.StrToString(eol, true); - string message_without_desired = $"With default strategy, expected TableToCsv({inp}, '{delim}', '{quote_char}', {head_str}, {escapedEol})\nto return\n"; - string base_message = $"{message_without_desired}{desired_out}\n"; - int msg_len = Encoding.UTF8.GetByteCount(desired_out) + 1 + message_without_desired.Length; + string messageWithoutDesired = $"With default strategy, expected TableToCsv({inp}, '{delim}', '{quote}', {head_str}, {escapedEol})\nto return\n"; + string baseMessage = $"{messageWithoutDesired}{desired_out}\n"; + int msgLen = Encoding.UTF8.GetByteCount(desired_out) + 1 + messageWithoutDesired.Length; try { - result = tabularizer.TableToCsv((JArray)table, delim, quote_char, header, bools_as_ints, eol); + result = tabularizer.TableToCsv((JArray)table, delim, quote, eol, header, boolsAsInts); int result_len = Encoding.UTF8.GetByteCount(result); try { if (!desired_out.Equals(result)) { tests_failed++; - Npp.editor.AppendText(msg_len + 17 + result_len + 1, $"{base_message}Instead returned\n{result}\n"); + Npp.editor.AppendText(msgLen + 17 + result_len + 1, $"{baseMessage}Instead returned\n{result}\n"); } } catch (Exception ex) { tests_failed++; int ex_len = ex.ToString().Length; - Npp.editor.AppendText(msg_len + 17 + result_len + 21 + ex_len + 1, - $"{base_message}Instead returned\n{result}\nand threw exception\n{ex}\n"); + Npp.editor.AppendText(msgLen + 17 + result_len + 21 + ex_len + 1, + $"{baseMessage}Instead returned\n{result}\nand threw exception\n{ex}\n"); } } catch (Exception ex) { tests_failed++; - Npp.AddLine($"{base_message}Instead threw exception\n{ex}"); + Npp.AddLine($"{baseMessage}Instead threw exception\n{ex}"); } } // TEST NO_RECURSION setting diff --git a/JsonToolsNppPlugin/Tests/RemesPathTests.cs b/JsonToolsNppPlugin/Tests/RemesPathTests.cs index eb6dc56..2b8fd4a 100644 --- a/JsonToolsNppPlugin/Tests/RemesPathTests.cs +++ b/JsonToolsNppPlugin/Tests/RemesPathTests.cs @@ -483,7 +483,7 @@ public static bool Test() new Query_DesiredResult("str(csv_regex(5, , , `'`))", JNode.StrToString(JNode.StrToString(ArgFunction.CsvRowRegex(5, quote:'\''), true), true)), new Query_DesiredResult("to_csv(@.foo)", "\"0,1,2\\r\\n3.0,4.0,5.0\\r\\n6.0,7.0,8.0\\r\\n\""), new Query_DesiredResult("to_csv(j`[{\"a\\\\tb\": 1, \"c\": null}, {\"a\\\\tb\": -7.5, \"c\": \"bar\\\\n'baz'\"}]`, `\\t`, `\\r`, `'`)", - "\"'a\\tb'\\tc\\r1\\tnull\\r-7.5\\t'bar\\n''baz'''\\r\""), + "\"'a\\tb'\\tc\\r1\\t\\r-7.5\\t'bar\\n''baz'''\\r\""), // null values are converted to empty strings new Query_DesiredResult("to_csv(@.foo[:2], `.`, `\\n`, `#`)", "\"0.1.2\\n#3.0#.#4.0#.#5.0#\\n\""), new Query_DesiredResult("to_csv(@.foo[1], `.`)", "\"\\\"3.0\\\"\\r\\n\\\"4.0\\\"\\r\\n\\\"5.0\\\"\""), new Query_DesiredResult("to_csv(@.foo[:][0],,`\\n`)", "\"0\\n3.0\\n6.0\""), @@ -581,9 +581,9 @@ public static bool Test() "1\\r" + ".3\\r" + "5\\r" + - "-7.2`" + + "$-7.2$`" + ", 1, `^`, `\\r`, `$`, d, 0)", - "[{\"foo\": 1}, {\"foo\": 0.3}, {\"foo\": 5}, {\"foo\": -7.2}]"), + "[{\"foo\": 1}, {\"foo\": 0.3}, {\"foo\": 5}, {\"foo\": \"-7.2\"}]"), // ====================== s_fa function for parsing regex search results as string arrays or arrays of arrays of strings ========= // 2 capture groups (2nd optional), parse the first group as number new Query_DesiredResult("s_fa(`1. foo boo\\r\\n" + diff --git a/JsonToolsNppPlugin/Tests/UserInterfaceTests.cs b/JsonToolsNppPlugin/Tests/UserInterfaceTests.cs index 5983ba0..e734484 100644 --- a/JsonToolsNppPlugin/Tests/UserInterfaceTests.cs +++ b/JsonToolsNppPlugin/Tests/UserInterfaceTests.cs @@ -510,7 +510,7 @@ public static bool ExecuteFileManipulation(string command, List messages // TEST PARSE INI FILE ("overwrite", new object[]{"[foồ]\r\n;a\r\nfoo=1\r\n[bar]\r\n bar=2\r\n [дaz]\r\n baz=3\r\n ;b\r\n baz2 = 7 \r\n[quz]\r\nquz=4\r\n;c"}), ("tree_open", new object[]{}), - ("tree_open", new object[]{DocumentType.INI}), + ("tree_open", new object[]{DocumentType.INI}), ("tree_query", new object[]{"@..g`z`"}), ("treenode_click", new object[]{new string[] {"0 : {2}", "baz2 : \"7 \""} }), ("compare_selections", new object[]{new string[]{"63,63" } }), @@ -637,7 +637,7 @@ public static bool ExecuteFileManipulation(string command, List messages "ab\tc\t😀\tf\r\n" + "'fo\nö'\t7\tbar\t-8.5\r\n" + "baz\t19\t''\t-4e3\r\n" + - "zorq\t\tkywq\t.75" + "'zorq'\t\tkywq\t'.75'" }), ("regex_search", new object[] { @@ -646,8 +646,8 @@ public static bool ExecuteFileManipulation(string command, List messages ("treenode_click", new object[]{new string[]{"0 : {4}", "ab : \"fo\\nö\"" } }), ("treenode_click", new object[]{new string[]{"1 : {4}", "c : 19"} }), ("treenode_click", new object[]{new string[]{"1 : {4}", "😀 : \"\"" } }), - ("treenode_click", new object[]{new string[]{"2 : {4}", "f : 0.75"} }), - ("compare_selections", new object[]{new string[] {"60,60"} }), + ("treenode_click", new object[]{new string[]{"2 : {4}", "f : \".75\""} }), + ("compare_selections", new object[]{new string[] {"62,62"} }), ("regex_search", new object[] { "{\"csv\": true, \"delim\": \"\\\\t\", \"quote\": \"'\", \"newline\": \"\\r\\n\", \"header\": 1, \"nColumns\": 4, \"numCols\": [1]}" @@ -660,7 +660,7 @@ public static bool ExecuteFileManipulation(string command, List messages ("tree_query", new object[]{"s_csv(@, 4, `\\t`, `\\r\\n`, `'`, h)[:][0]"}), ("treenode_click", new object[]{new string[]{ } }), // click root ("select_treenode_json_children", new object[]{}), - ("compare_selections", new object[]{new string[] {"0,2", "13,20", "33,36", "49,53"} }), + ("compare_selections", new object[]{new string[] {"0,2", "13,20", "33,36", "49,55"} }), ("treenode_click", new object[]{new string[]{ "1 : \"fo\\nö\"" } }), ("select_treenode_json", new object[]{}), ("compare_selections", new object[]{new string[] {"13,20"} }), @@ -671,10 +671,13 @@ public static bool ExecuteFileManipulation(string command, List messages "@ = to_csv(c,`,`, `\\n`, `\"`)"}), ("compare_text", new object[]{"ab,c,😀,f\n\"fo\nö\",1.75,bar,-8.5\nbaz,4.75,,-4e3\nzorq,,kywq,.75\n"}), // TEST REGEX SEARCH FORM (SINGLE-CAPTURE-GROUP CSV AND REGEX MODES) - ("overwrite", new object[]{" 草1\nｪ2\n◐3"}), + ("overwrite", new object[]{" 草1\nｪ2\n◐3\n'4"}), // last row has CSV quote char to make sure CSV delimiter is forgotten now that a non-csv search was made ("regex_search", new object[]{ "{\"csv\":false,\"regex\":\"^\\\\S\\\\d\\\\r?$\",\"ignoreCase\":true,\"fullMatch\":false}" }), + ("treenode_click", new object[]{new string[]{ } }), // click root + ("select_treenode_json_children", new object[]{}), + ("compare_selections", new object[]{new string[] {"6,10", "11,15", "16,18"} }), ("treenode_click", new object[]{new string[] { "1 : \"◐3\"" } }), ("compare_selections", new object[]{new string[] {"11,11"} }), ("select_treenode_json", new object[]{}), diff --git a/docs/RemesPath.md b/docs/RemesPath.md index e460175..41eac20 100644 --- a/docs/RemesPath.md +++ b/docs/RemesPath.md @@ -575,6 +575,8 @@ Returns x formatted as a CSV (RFC 4180 rules as normal), according to the follow * if x is an array of arrays, each subarray is converted to a row * if x is an array of objects, the keys of the first subobject are converted to a header row, and the values of every subobject become their own row. +See [json-to-csv.md](/docs/json-to-csv.md#how-json-nodes-are-represented-in-csv) for information on how JSON values are represented in CSVs. + --- `to_records(x: iterable, [strategy: str]) -> array[object]` diff --git a/docs/json-to-csv.md b/docs/json-to-csv.md index 10b6250..cd15d20 100644 --- a/docs/json-to-csv.md +++ b/docs/json-to-csv.md @@ -1,7 +1,14 @@ # Converting JSON to CSVs # This app uses an [algorithm](https://github.com/molsonkiko/JsonToolsNppPlugin/blob/main/JsonToolsNppPlugin/JSONTools/JsonTabularize.cs) based on analysis of a JSON iterable's [schema](https://github.com/molsonkiko/JsonToolsNppPlugin/blob/main/JsonToolsNppPlugin/JSONTools/JsonSchemaMaker.cs) (generated on the fly) to attempt to convert it into a table. -As of [v6.0](/CHANGELOG.md#600---unreleased-2023-mm-dd), the CSV files generated by JsonTools use an algorithm similar to [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt), in that all values containing the quote character, `\r`, or `\n` must be wrapped in quotes, and quote characters are escaped by doubling them up. + +## How JSON nodes are represented in CSV ## +* As of [v6.0](/CHANGELOG.md#600---unreleased-2023-mm-dd), the CSV files generated by JsonTools use an algorithm similar to [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt), in that all values containing the quote character, `\r`, or `\n` must be wrapped in quotes, and quote characters are escaped by doubling them up. + * Thus the JSON string `"foo,\n\"bar\""` would become `"foo,\n""bar""""` (with an actual newline) in a CSV file with `,` as delimiter and `"` as quote character, because the newline, quote character, and delimiter in the string all necessitate enclosing quotes, and the internal quotes must be doubled up. +* Floating point numbers are always exported with `.` as the decimal separator, as in JSON. This is not currently configurable, but may become optional in the future. +* `null` is represented as an empty string. +* Dates are represented in `YYYY-MM-DD` format (e.g., January 7, 2015 is represented as `2015-01-07`) +* Datetimes are represented in `yyyy-MM-dd hh:mm:ss` format (e.g., 4:15:35PM April 29 2021 is represented as `2021-04-29 16:15:35`) [Pre-v6.0 docs are here](https://github.com/molsonkiko/JsonToolsNppPlugin/blob/110cbb7d30c6a48cd2c7cfac3cb65534230b9c86/docs/json-to-csv.md). @@ -101,14 +108,14 @@ make a table containing all the rows. No recursive search will be done - the *en * For example, ```json -{"a": 1, "b": [1, 2, 3], "c": ["a", "b", "c"]} +{"a": 1, "b": [1, 2, 3], "c": ["a", "b", null]} ``` will be tabularized to ``` a,b,c 1,1,a 1,2,b -1,3,c +1,3, ``` * But ```json diff --git a/most recent errors.txt b/most recent errors.txt index e862a97..7034631 100644 --- a/most recent errors.txt +++ b/most recent errors.txt @@ -1,4 +1,4 @@ -Test results for JsonTools v5.8.0.14 on Notepad++ 8.5.8 64bit +Test results for JsonTools v5.8.0.15 on Notepad++ 8.5.8 64bit NOTE: Ctrl-F (regular expressions *on*) for "Failed [1-9]\d*" to find all failed tests Tests failed: YAML dumper ========================= @@ -195,40 +195,40 @@ Testing UI tests ========================= Failed 0 tests -Passed 295 tests +Passed 298 tests ========================= Testing JsonParser performance ========================= Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 89556 into JNode took 3.02 +/- 1.053 ms over 32 trials -Load times (ms): 2, 2, 2, 5, 5, 4, 4, 1, 2, 3, 3, 3, 4, 1, 1, 1, 3, 2, 3, 3, 1, 2, 3, 2, 3, 3, 4, 1, 1, 2, 3, 2 +To convert JSON string of size 89556 into JNode took 2.705 +/- 1.982 ms over 32 trials +Load times (ms): 2, 2, 1, 3, 1, 2, 5, 1, 1, 1, 3, 1, 1, 4, 1, 1, 11, 4, 1, 1, 4, 1, 1, 1, 2, 4, 1, 2, 2, 2, 2, 1 ========================= Performance tests for RemesPath (float arithmetic) ========================= -Compiling query "@[@[:].a * @[:].t < @[:].e]" into took 1.995 +/- 11.962 microseconds over 40 trials -To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.053 +/- 0.185 ms over 40 trials -Query times (ms): 0.052, 0.031, 0.024, 0.022, 0.023, 0.022, 0.023, 0.026, 0.024, 0.023, 0.023, 0.022, 0.023, 0.024, 0.023, 1.208, 0.022, 0.021, 0.021, 0.022, 0.021, 0.021, 0.021, 0.021, 0.021, 0.022, 0.022, 0.023, 0.022, 0.022, 0.022, 0.023, 0.022, 0.022, 0.023, 0.022, 0.022, 0.023, 0.023, 0.022 +Compiling query "@[@[:].a * @[:].t < @[:].e]" into took 2.482 +/- 15.103 microseconds over 40 trials +To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.029 +/- 0.013 ms over 40 trials +Query times (ms): 0.058, 0.058, 0.023, 0.023, 0.023, 0.023, 0.023, 0.025, 0.023, 0.023, 0.023, 0.023, 0.023, 0.024, 0.024, 0.023, 0.023, 0.023, 0.023, 0.024, 0.023, 0.023, 0.023, 0.022, 0.023, 0.039, 0.074, 0.04, 0.035, 0.025, 0.034, 0.075, 0.024, 0.024, 0.034, 0.023, 0.023, 0.028, 0.025, 0.024 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (string operations) ========================= -Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 1.552 +/- 9.247 microseconds over 40 trials -To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.099 +/- 0.269 ms over 40 trials -Query times (ms): 0.085, 0.055, 0.053, 0.053, 0.054, 0.053, 0.053, 0.053, 0.053, 0.053, 0.053, 0.053, 0.051, 0.053, 0.052, 0.053, 0.052, 1.78, 0.062, 0.087, 0.083, 0.056, 0.053, 0.052, 0.052, 0.052, 0.053, 0.053, 0.052, 0.052, 0.052, 0.053, 0.052, 0.053, 0.052, 0.053, 0.052, 0.052, 0.053, 0.052 +Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 2.975 +/- 17.954 microseconds over 40 trials +To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.112 +/- 0.02 ms over 40 trials +Query times (ms): 0.191, 0.165, 0.112, 0.114, 0.112, 0.116, 0.104, 0.111, 0.102, 0.107, 0.128, 0.113, 0.109, 0.111, 0.112, 0.111, 0.111, 0.113, 0.149, 0.112, 0.095, 0.098, 0.106, 0.098, 0.103, 0.107, 0.119, 0.106, 0.105, 0.103, 0.108, 0.123, 0.115, 0.116, 0.104, 0.121, 0.106, 0.111, 0.072, 0.064 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "NK nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" into took 4.095 +/- 24.308 microseconds over 40 trials +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" into took 5.272 +/- 31.406 microseconds over 40 trials To run pre-compiled query "var qmask = @[:].q; var nmax_q = max(@[qmask].n); var nmax_notq = max(@[not qmask].n); -ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.028 +/- 0.055 ms over 40 trials -Query times (ms): 0.052, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015, 0.016, 0.015, 0.016, 0.016, 0.016, 0.017, 0.015, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.368, 0.064, 0.035, 0.036, 0.026, 0.026, 0.024 +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.033 +/- 0.042 ms over 40 trials +Query times (ms): 0.088, 0.027, 0.028, 0.025, 0.018, 0.048, 0.018, 0.017, 0.017, 0.287, 0.026, 0.026, 0.024, 0.025, 0.025, 0.023, 0.024, 0.025, 0.025, 0.032, 0.026, 0.016, 0.016, 0.016, 0.021, 0.023, 0.024, 0.04, 0.025, 0.025, 0.026, 0.025, 0.024, 0.026, 0.024, 0.025, 0.025, 0.025, 0.025, 0.026 Preview of result: "when q=false, nmax= 9830935647.0" ... ========================= @@ -267,11 +267,11 @@ Performance tests for RemesPath (references to compile-time constant variables) Compiling query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" into took 2.35 +/- 14.035 microseconds over 40 trials +@[:]->at(@, X)->at(@, onetwo)" into took 3.157 +/- 19.094 microseconds over 40 trials To run pre-compiled query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.013 +/- 0.003 ms over 40 trials -Query times (ms): 0.031, 0.013, 0.013, 0.012, 0.013, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.013, 0.012, 0.013, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.013, 0.012, 0.013 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.016 +/- 0.01 ms over 40 trials +Query times (ms): 0.054, 0.026, 0.016, 0.02, 0.013, 0.012, 0.013, 0.013, 0.015, 0.05, 0.028, 0.013, 0.012, 0.013, 0.039, 0.012, 0.013, 0.012, 0.013, 0.013, 0.012, 0.013, 0.013, 0.012, 0.013, 0.012, 0.013, 0.013, 0.012, 0.013, 0.013, 0.012, 0.013, 0.012, 0.013, 0.012, 0.012, 0.013, 0.013, 0.012 Preview of result: [[1695727848, 0.287562638736685], [2126430375, 0.00767794129708177], [5310550656, 0.380769772645687], [2519183283, 0.153176220930558], [6610062385, 0.662996225870666], [987168256, 0.924410189999928], [6615003609, 0.917112691225947], [4465232046, 0.684311931851536], [8654414565, 0.631485392105992], [ ... ========================= @@ -280,29 +280,29 @@ Performance tests for RemesPath (references to variables that are not compile-ti Compiling query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" into took 2.745 +/- 16.518 microseconds over 40 trials +@[:]->at(@, X)->at(@, onetwo)" into took 4.12 +/- 24.849 microseconds over 40 trials To run pre-compiled query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.016 +/- 0.003 ms over 40 trials -Query times (ms): 0.037, 0.017, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.031 +/- 0.011 ms over 40 trials +Query times (ms): 0.091, 0.029, 0.029, 0.028, 0.028, 0.04, 0.028, 0.058, 0.028, 0.028, 0.028, 0.028, 0.027, 0.028, 0.028, 0.027, 0.028, 0.028, 0.027, 0.028, 0.028, 0.028, 0.028, 0.028, 0.027, 0.029, 0.027, 0.028, 0.028, 0.028, 0.028, 0.034, 0.028, 0.027, 0.029, 0.028, 0.028, 0.028, 0.028, 0.027 Preview of result: [[1695727848, 0.287562638736685], [2126430375, 0.00767794129708177], [5310550656, 0.380769772645687], [2519183283, 0.153176220930558], [6610062385, 0.662996225870666], [987168256, 0.924410189999928], [6615003609, 0.917112691225947], [4465232046, 0.684311931851536], [8654414565, 0.631485392105992], [ ... ========================= Performance tests for RemesPath (simple string mutations) ========================= -Compiling query "@[:].z = s_sub(@, g, B)" into took 1.168 +/- 6.939 microseconds over 40 trials -To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.015 +/- 0.008 ms over 40 trials -Query times (ms): 0.02, 0.018, 0.013, 0.009, 0.009, 0.025, 0.01, 0.01, 0.01, 0.012, 0.041, 0.015, 0.009, 0.01, 0.011, 0.013, 0.009, 0.008, 0.01, 0.037, 0.009, 0.009, 0.012, 0.011, 0.014, 0.011, 0.009, 0.01, 0.015, 0.009, 0.009, 0.011, 0.014, 0.038, 0.016, 0.012, 0.011, 0.032, 0.02, 0.024 +Compiling query "@[:].z = s_sub(@, g, B)" into took 2.535 +/- 15.319 microseconds over 40 trials +To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.025 +/- 0.007 ms over 40 trials +Query times (ms): 0.054, 0.025, 0.022, 0.022, 0.021, 0.024, 0.027, 0.026, 0.022, 0.031, 0.022, 0.02, 0.021, 0.021, 0.019, 0.024, 0.022, 0.023, 0.026, 0.023, 0.024, 0.023, 0.024, 0.03, 0.021, 0.024, 0.02, 0.02, 0.021, 0.02, 0.02, 0.035, 0.032, 0.035, 0.022, 0.028, 0.031, 0.032, 0.016, 0.015 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (simple number mutations) ========================= -Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" into took 2.15 +/- 12.786 microseconds over 40 trials -To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.041 +/- 0.012 ms over 40 trials -Query times (ms): 0.042, 0.026, 0.056, 0.051, 0.037, 0.022, 0.031, 0.029, 0.048, 0.053, 0.038, 0.038, 0.038, 0.057, 0.026, 0.022, 0.023, 0.023, 0.051, 0.041, 0.034, 0.038, 0.048, 0.057, 0.035, 0.041, 0.046, 0.041, 0.044, 0.08, 0.068, 0.043, 0.038, 0.03, 0.036, 0.05, 0.04, 0.041, 0.038, 0.037 +Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" into took 2.815 +/- 16.955 microseconds over 40 trials +To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.09 +/- 0.36 ms over 40 trials +Query times (ms): 0.044, 0.029, 0.026, 0.024, 0.025, 0.026, 0.051, 0.034, 0.04, 0.027, 0.024, 0.027, 0.026, 0.026, 0.024, 0.023, 0.024, 0.024, 0.034, 0.032, 0.066, 0.044, 0.027, 0.026, 0.026, 0.024, 0.025, 0.025, 0.023, 0.024, 0.023, 0.024, 0.03, 0.088, 2.334, 0.047, 0.034, 0.038, 0.036, 0.035 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= @@ -312,12 +312,12 @@ Performance tests for RemesPath (mutations with a for loop) Compiling query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" into took 4.322 +/- 26.193 microseconds over 40 trials +end for;" into took 6.272 +/- 38.195 microseconds over 40 trials To run pre-compiled query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" on JNode from JSON of size 89556 into took 0.079 +/- 0.157 ms over 40 trials -Query times (ms): 0.058, 0.037, 0.045, 0.036, 0.035, 0.036, 0.04, 0.034, 0.032, 0.033, 0.696, 0.097, 0.051, 0.039, 0.037, 0.043, 0.036, 0.035, 0.032, 0.039, 0.032, 0.033, 0.035, 0.044, 0.035, 0.034, 0.033, 0.816, 0.036, 0.033, 0.032, 0.038, 0.039, 0.034, 0.034, 0.044, 0.154, 0.055, 0.075, 0.035 +end for;" on JNode from JSON of size 89556 into took 0.064 +/- 0.023 ms over 40 trials +Query times (ms): 0.092, 0.101, 0.116, 0.053, 0.063, 0.059, 0.087, 0.063, 0.062, 0.063, 0.064, 0.058, 0.057, 0.034, 0.038, 0.085, 0.085, 0.067, 0.062, 0.049, 0.044, 0.043, 0.043, 0.043, 0.041, 0.055, 0.07, 0.049, 0.05, 0.051, 0.049, 0.091, 0.068, 0.062, 0.061, 0.102, 0.14, 0.052, 0.038, 0.051 Preview of result: [["bar", false], ["bar", false], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["ba ... ========================= @@ -326,18 +326,18 @@ Testing performance of JSON compression and pretty-printing Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To compress JNode from JSON string of 89556 took 4.036 +/- 0.76 ms over 64 trials (minimal whitespace, sort_keys=TRUE) -To compress JNode from JSON string of 89556 took 2.483 +/- 2.128 ms over 64 trials (minimal whitespace, sort_keys=FALSE) -To Google-style pretty-print JNode from JSON string of 89556 took 4.475 +/- 0.762 ms over 64 trials (sort_keys=true, indent=4) -To Whitesmith-style pretty-print JNode from JSON string of 89556 took 4.221 +/- 0.397 ms over 64 trials (sort_keys=true, indent=4) -To PPrint-style pretty-print JNode from JSON string of 89556 took 6.551 +/- 1.174 ms over 64 trials (sort_keys=true, indent=4) +To compress JNode from JSON string of 89556 took 3.984 +/- 0.471 ms over 64 trials (minimal whitespace, sort_keys=TRUE) +To compress JNode from JSON string of 89556 took 2.148 +/- 0.475 ms over 64 trials (minimal whitespace, sort_keys=FALSE) +To Google-style pretty-print JNode from JSON string of 89556 took 4.269 +/- 0.549 ms over 64 trials (sort_keys=true, indent=4) +To Whitesmith-style pretty-print JNode from JSON string of 89556 took 4.328 +/- 0.448 ms over 64 trials (sort_keys=true, indent=4) +To PPrint-style pretty-print JNode from JSON string of 89556 took 7.543 +/- 1.133 ms over 64 trials (sort_keys=true, indent=4) ========================= Testing performance of JsonSchemaValidator and random JSON creation ========================= -To create a random set of tweet JSON of size 144006 (15 tweets) based on the matching schema took 7.789 +/- 4.306 ms over 64 trials -To compile the tweet schema to a validation function took 0.246 +/- 0.287 ms over 64 trials -To validate tweet JSON of size 144006 (15 tweets) based on the compiled schema took 0.92 +/- 0.23 ms over 64 trials +To create a random set of tweet JSON of size 140467 (15 tweets) based on the matching schema took 6.966 +/- 3.393 ms over 64 trials +To compile the tweet schema to a validation function took 0.232 +/- 0.057 ms over 64 trials +To validate tweet JSON of size 140467 (15 tweets) based on the compiled schema took 1.13 +/- 0.276 ms over 64 trials ========================= Testing JSON grepper's API request tool =========================