Skip to content

Commit

Permalink
first ini file parser that seems to work fine
Browse files Browse the repository at this point in the history 
still need to add more ini parser tests, integrate into main UI
should decide whether to implement interpolation too
  • Loading branch information
@molsonkiko
molsonkiko committed Oct 6, 2023
1 parent 0aa2235 commit bc478b7
Show file tree
Hide file tree
Showing 11 changed files with 454 additions and 132 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).

### Added

1. Add parser for `.ini` files, allowing them to be reformatted and edited with the tree view.
1. [Python-style spreading of an array](/docs/RemesPath.md#spreading-function-args-to-fill-multiple-arguments-added-in-v58) with `*` to fill multiple arguments of a function.
2. New RemesPath function(s): [`at` function](/docs/RemesPath.md#non-vectorized-functions) for indexing into array or object at an index or key determined at runtime (since indexing with square braces does not support keys/indices that are functions of input).
3. Made it possible to customize newline for CSV files generated by the [JSON-to-CSV form](/docs/README.md#json-to-csv).
Expand Down
315 changes: 258 additions & 57 deletions JsonToolsNppPlugin/JSONTools/IniFileParser.cs
View file

Large diffs are not rendered by default.

47 changes: 46 additions & 1 deletion JsonToolsNppPlugin/JSONTools/JNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,15 @@ public static string StrToString(string s, bool quoted)
return sb.ToString();
}

/// <summary>
/// mostly useful for quickly converting a JObject key (which must be escaped during parsing) into the raw string that it represents<br></br>
/// Choose strAlreadyQuoted = true only if str is already wrapped in double quotes.
/// </summary>
public static string UnescapedJsonString(string str, bool strAlreadyQuoted)
{
return (string)new JsonParser().ParseString(strAlreadyQuoted ? str : $"\"{str}\"").value;
}

/// <summary>
/// Compactly prints the JSON.<br></br>
/// If sort_keys is true, the keys of objects are printed in alphabetical order.<br></br>
Expand Down Expand Up @@ -1207,9 +1216,45 @@ public override (int comment_idx, int extra_utf8_bytes) PrettyPrintWithCommentsH
return (comment_idx, extra_utf8_bytes);
}

/// <summary>
/// dump this JObject as an ini file
/// </summary>
/// <param name="comments"></param>
/// <param name="changePositions"></param>
/// <returns></returns>
public string ToIniFile(List<Comment> comments)
{
return "";
var sb = new StringBuilder();
int positionInComments = 0;
int utf8ExtraBytes = 0;
foreach (KeyValuePair<string, JNode> kv in children)
{
string header = kv.Key;
if (!(kv.Value is JObject section))
{
throw new InvalidOperationException("Only objects where all children are objects with only string values can be converted to ini files");
}
// section is treated as beginning just before the open squarebrace of the header
(positionInComments, utf8ExtraBytes) = Comment.AppendAllCommentsBeforePosition(sb, comments, positionInComments, utf8ExtraBytes, section.position, "", DocumentType.INI);
string unescapedHeader = UnescapedJsonString(header, false);
sb.Append($"[{unescapedHeader}]\r\n");
utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(unescapedHeader, 0, unescapedHeader.Length);
foreach (KeyValuePair<string, JNode> sectKv in section.children)
{
string key = sectKv.Key;
JNode value = sectKv.Value;
if (!(value.value is string valueStr))
{
throw new InvalidOperationException("Only objects where all children are objects with only string values can be converted to ini files");
}
(positionInComments, utf8ExtraBytes) = Comment.AppendAllCommentsBeforePosition(sb, comments, positionInComments, utf8ExtraBytes, value.position, "", DocumentType.INI);
string unescapedKey = UnescapedJsonString(key, false);
sb.Append($"{unescapedKey}={valueStr}\r\n");
utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(unescapedKey, 0, unescapedKey.Length);
utf8ExtraBytes += JsonParser.ExtraUTF8BytesBetween(valueStr, 0, valueStr.Length);
}
}
return sb.ToString();
}

/// <summary>
Expand Down
4 changes: 2 additions & 2 deletions JsonToolsNppPlugin/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,5 @@
// Build Number
// Revision
//
[assembly: AssemblyVersion("5.7.0.6")]
[assembly: AssemblyFileVersion("5.7.0.6")]
[assembly: AssemblyVersion("5.7.0.7")]
[assembly: AssemblyFileVersion("5.7.0.7")]
63 changes: 42 additions & 21 deletions JsonToolsNppPlugin/Tests/IniFileParserTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,89 +12,107 @@ public static bool Test()
int ii = 0;
int testsFailed = 0;
var exampleIniFname = @"plugins\JsonTools\testfiles\small\example_ini.ini";
var exampleIniReformattedFname = @"plugins\JsonTools\testfiles\small\example_ini_reformatted.ini";
var exampleJsonFname = @"plugins\JsonTools\testfiles\small\example_ini.json";
string exampleJsonText = null;
string exampleIniText = null;
string exampleIniReformattedText = null;

try
{
exampleIniText = File.ReadAllText(exampleIniFname);
exampleJsonText = File.ReadAllText(exampleJsonFname);
exampleIniReformattedText = File.ReadAllText(exampleIniReformattedFname);
}
catch
{
testsFailed++;
Npp.AddLine("Either testfiles/small/example_ini.ini or testfiles/small/example_ini.json was not found in the plugins/JsonTools directory");
}

var testcases = new (string iniText, string correctJsonStrWithoutInterpolation, bool interpolation)[]
var testcases = new (string iniText, string correctJsonStrWithoutInterpolation, /*bool interpolation,*/ string correctReformattedIniText)[]
{
(exampleIniText, exampleJsonText, false),
(exampleIniText, exampleJsonText, true),
(exampleIniText, exampleJsonText, /*false,*/ exampleIniReformattedText),
//(exampleIniText, exampleJsonText, true, exampleIniReformattedText),
};
var jsonParser = new JsonParser(LoggerLevel.JSON5, false, true, true, true);

foreach ((string iniText, string correctJsonStrWithoutInterpolation, bool interpolation) in testcases)
int testsPerLoop = 3;
foreach ((string iniText, string correctJsonStrWithoutInterpolation, /*bool interpolation,*/ string correctReformattedIniText) in testcases)
{
JObject correctJson;
JObject correctJsonWithInterpolation;
//JObject correctJsonWithInterpolation;
string correctJsonStr;
ii += 3;
Npp.AddLine($"Parsing ini file (shown here as JSON string)\r\n{JNode.StrToString(iniText, true)}");
ii += testsPerLoop;
bool hasShownIniFileDescription = false;
void showFileDescription()
{
if (!hasShownIniFileDescription)
{
string iniFileDescription = $"Parsing ini file (shown here as JSON string)\r\n{JNode.StrToString(iniText, true)}";
hasShownIniFileDescription = true;
Npp.AddLine(iniFileDescription);
}
};
try
{
correctJson = (JObject)jsonParser.Parse(correctJsonStrWithoutInterpolation);
correctJsonWithInterpolation = IniFileParser.Interpolate(correctJson);
correctJsonStr = interpolation ? correctJsonStrWithoutInterpolation : correctJsonWithInterpolation.PrettyPrint();
if (interpolation)
correctJson = correctJsonWithInterpolation;
//correctJsonWithInterpolation = IniFileParser.Interpolate(correctJson);
correctJsonStr = correctJson.PrettyPrintWithComments(jsonParser.comments, sort_keys:false);
}
catch (Exception ex)
{
showFileDescription();
Npp.AddLine($"While trying to parse JSON\r\n{correctJsonStrWithoutInterpolation}\r\nGOT EXCEPTION\r\n{ex}");
testsFailed += 3;
testsFailed += testsPerLoop;
continue;
}
JObject gotJson;
IniFileParser iniParser = new IniFileParser(interpolation);
IniFileParser iniParser = new IniFileParser();
try
{
gotJson = iniParser.Parse(iniText);
}
catch (Exception ex)
{
showFileDescription();
Npp.AddLine($"While trying to parse ini file\r\nGOT EXCEPTION\r\n{ex}");
testsFailed += 3;
testsFailed += testsPerLoop;
continue;
}
string gotJsonStr = gotJson.ToString();
if (gotJson.TryEquals(correctJson, out _))
{
// test if comments are parsed correctly
string gotJsonWithComments = "";
try
{
string gotJsonWithComments = gotJson.ToStringWithComments(iniParser.comments);
gotJsonWithComments = gotJson.PrettyPrintWithComments(iniParser.comments, sort_keys:false);
if (gotJsonWithComments != correctJsonStr)
{
testsFailed++;
Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT JSON WITH COMMENTS\r\n{gotJsonStr}");
showFileDescription();
Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT JSON WITH COMMENTS\r\n{gotJsonWithComments}");
}
}
catch (Exception ex)
{
testsFailed++;
testsFailed += 2;
showFileDescription();
Npp.AddLine($"EXPECTED JSON WITH COMMENTS\r\n{correctJsonStr}\r\nGOT EXCEPTION\r\n{ex}");
continue;
}
// test if dumping the JSON back to an ini file and re-parsing the dumped ini will return the same JSON
string dumpedIniText = "";
try
{
dumpedIniText = gotJson.ToIniFile(iniParser.comments);
JObject reParsedJson = (JObject)jsonParser.Parse(gotJsonWithComments);
dumpedIniText = reParsedJson.ToIniFile(jsonParser.comments);
}
catch (Exception ex)
{
testsFailed++;
Npp.AddLine($"When ini-dumping JSON\r\n{gotJsonStr}\r\nEXPECTED INI FILE\r\n{correctJsonStr}\r\nGOT EXCEPTION {ex}");
showFileDescription();
Npp.AddLine($"When ini-dumping JSON\r\n{gotJsonStr}\r\nEXPECTED INI FILE\r\n{correctReformattedIniText}\r\nGOT EXCEPTION {ex}");
continue;
}
JObject jsonFromDumpedIniText;
Expand All @@ -104,19 +122,22 @@ public static bool Test()
}
catch (Exception ex)
{
showFileDescription();
Npp.AddLine($"EXPECTED JSON FROM DUMPED INI FILE\r\n{correctJsonStr}\r\nGOT EXCEPTION\r\n{ex}");
testsFailed++;
continue;
}
if (!jsonFromDumpedIniText.TryEquals(correctJson, out _))
{
showFileDescription();
testsFailed++;
Npp.AddLine($"EXPECTED JSON FROM DUMPED INI FILE\r\n{correctJson.ToString()}\r\nGOT JSON\r\n{gotJson.ToString()}");
}
}
else
{
testsFailed += 3;
testsFailed += testsPerLoop;
showFileDescription();
Npp.AddLine($"EXPECTED JSON\r\n{correctJson.ToString()}\r\nGOT JSON\r\n{gotJson.ToString()}");
}
}
Expand Down
15 changes: 12 additions & 3 deletions JsonToolsNppPlugin/Tests/TestRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ namespace JSON_Tools.Tests
{
public class TestRunner
{
private const string fuzzTestName = "RemesPath produces sane outputs on randomly generated queries";

public static async Task RunAll()
{
Npp.notepad.FileNew();
Expand All @@ -39,7 +41,7 @@ public static async Task RunAll()
(RemesParserTester.Test, "RemesPath parser and compiler", false, false),
(RemesPathThrowsWhenAppropriateTester.Test, "RemesPath throws errors on bad inputs", false, false),
(RemesPathAssignmentTester.Test, "RemesPath assignment operations", false, false),
(() => RemesPathFuzzTester.Test(10000, 20), "RemesPath produces sane outputs on randomly generated queries", false, false),
(() => RemesPathFuzzTester.Test(10000, 20), fuzzTestName, false, false),
(RemesPathComplexQueryTester.Test, "multi-statement queries in RemesPath", false, false),

(JsonSchemaMakerTester.Test, "JsonSchema generator", false, false),
Expand Down Expand Up @@ -111,6 +113,11 @@ public static async Task RunAll()
}
skipped.Add(name);
}
else if (Main.settings.skip_api_request_and_fuzz_tests && name == fuzzTestName)
{
Npp.AddLine("\r\nskipped RemesPath fuzz tests because settings.skip_api_request_and_fuzz_tests was set to true");
skipped.Add(name);
}
else
{
Npp.AddLine($@"=========================
Expand All @@ -129,9 +136,9 @@ public static async Task RunAll()
Testing JSON grepper's API request tool
=========================
");
if (Main.settings.skip_api_request_tests)
if (Main.settings.skip_api_request_and_fuzz_tests)
{
Npp.AddLine("skipped tests because settings.skip_api_request_tests was set to true");
Npp.AddLine("skipped tests because settings.skip_api_request_and_fuzz_tests was set to true");
skipped.Add("JSON grepper's API request tool");
}
else
Expand All @@ -141,6 +148,8 @@ Testing JSON grepper's API request tool
}
}

if (skipped.Count > 0)
Npp.editor.InsertText(header.Length + 2, "Tests skipped: " + string.Join(", ", skipped) + "\r\n");
Npp.editor.InsertText(header.Length + 2, "Tests failed: " + string.Join(", ", failures) + "\r\n");
}
}
Expand Down
4 changes: 2 additions & 2 deletions JsonToolsNppPlugin/Utils/Settings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,9 @@ public class Settings : SettingsBase
Category("Miscellaneous"), DefaultValue("\"[{")]
public string try_parse_start_chars { get; set; }

[Description("When running tests, skip the tests that send requests to APIs"),
[Description("When running tests, skip the tests that send requests to APIs and the RemesPath fuzz tests"),
Category("Miscellaneous"), DefaultValue(true)]
public bool skip_api_request_tests { get; set; }
public bool skip_api_request_and_fuzz_tests { get; set; }

[Description("Which type of newline to use for generated CSV files."),
Category("Miscellaneous"), DefaultValue(EndOfLine.LF)]
Expand Down
Loading

0 comments on commit bc478b7

Please sign in to comment.