diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs index 4e72aed7..83e8cf08 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs @@ -53,5 +53,6 @@ public class Context public string CsvTimeFormat { get; set; } public bool CsvIgnoreInvalidRows { get; set; } public bool CsvRealign { get; set; } + public bool CsvRemoveDuplicatePoints { get; set; } = true; } } diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/CsvReader.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/CsvReader.cs index 1d3cabed..13815264 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/CsvReader.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/CsvReader.cs @@ -89,6 +89,36 @@ private List LoadPoints(string path) points.Add(point); } + if (Context.CsvRemoveDuplicatePoints) + { + points = points + .OrderBy(p => p.Time) + .ToList(); + + var duplicatePointCount = 0; + + for (var i = 1; i < points.Count; ++i) + { + var prevPoint = points[i - 1]; + var point = points[i]; + + if (point.Time != prevPoint.Time) + continue; + + ++duplicatePointCount; + + Log.Warn($"Discarding duplicate CSV point at {point.Time} with value {point.Value}"); + points.RemoveAt(i); + + --i; + } + + if (duplicatePointCount > 0) + { + Log.Warn($"Removed {duplicatePointCount} duplicate CSV points."); + } + } + if (Context.CsvRealign) { points = points diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Option.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/Option.cs index da785020..8ad7173c 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Option.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Option.cs @@ -24,7 +24,7 @@ public string UsageText() return $"{key} {Description}{defaultValue}"; } - private const int KeyWidth = 22; + private const int KeyWidth = 24; private static readonly string SeparatorLine = string.Empty.PadRight(KeyWidth + 1, '='); } } diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/PointsAppender.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/PointsAppender.cs index 160a106e..9158a8dd 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/PointsAppender.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/PointsAppender.cs @@ -62,21 +62,34 @@ public void AppendPoints() var numberOfPointsAppended = 0; var numberOfPointsDeleted = 0; - var batchCount = 0; var stopwatch = Stopwatch.StartNew(); - foreach (var batch in GetPointBatches()) + var pointBatches = GetPointBatches(Points).ToList(); + var isBatched = pointBatches.Count > 1; + var batchIndex = 1; + + foreach (var batch in pointBatches) { - var result = AppendPointBatch(client, timeSeries, batch.Item1, batch.Item2, isReflected, hasTimeRange); + if (isBatched) + { + var batchSummary = + $"Appending batch #{batchIndex}: {batch.Points.Count} points [{batch.Points.First().Time} to {batch.Points.Last().Time}]"; + + Log.Info( hasTimeRange + ? $"{batchSummary} within TimeRange={batch.TimeRange} ..." + : $"{batchSummary} ..."); + } + + var result = AppendPointBatch(client, timeSeries, batch.Points, batch.TimeRange, isReflected, hasTimeRange); numberOfPointsAppended += result.NumberOfPointsAppended; numberOfPointsDeleted += result.NumberOfPointsDeleted; - batchCount++; + ++batchIndex; if (result.AppendStatus != AppendStatusCode.Completed) throw new ExpectedException($"Unexpected append status={result.AppendStatus}"); } - var batchText = batchCount > 1 ? $" using {batchCount} appends" : ""; + var batchText = isBatched ? $" using {pointBatches.Count} appends" : ""; Log.Info($"Appended {numberOfPointsAppended} points (deleting {numberOfPointsDeleted} points) in {stopwatch.ElapsedMilliseconds / 1000.0:F1} seconds{batchText}."); } } @@ -131,9 +144,9 @@ private TimeSeriesAppendStatus AppendPointBatch(IAquariusClient client, TimeSeri Context.AppendTimeout); } - private IEnumerable, Interval>> GetPointBatches() + private IEnumerable<(List Points, Interval TimeRange)> GetPointBatches( + List points) { - var points = GetPoints(); var remainingTimeRange = GetTimeRange(); var index = 0; @@ -143,12 +156,12 @@ private IEnumerable, Interval>> GetPointBat var batchTimeRange = new Interval(remainingTimeRange.Start, batchPoints.Last().Time.GetValueOrDefault().PlusTicks(1)); remainingTimeRange = new Interval(batchTimeRange.End, remainingTimeRange.End); - yield return new Tuple, Interval>(batchPoints, batchTimeRange); + yield return (batchPoints, batchTimeRange); index += Context.BatchSize; } - yield return new Tuple, Interval>(points.Skip(index).ToList(), remainingTimeRange); + yield return (points.Skip(index).ToList(), remainingTimeRange); } private List GetPoints() diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs index 4cfa8593..d00cd64a 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs @@ -136,6 +136,7 @@ private static Context ParseArgs(string[] args) new Option {Key = nameof(context.CsvSkipRows), Setter = value => context.CsvSkipRows = int.Parse(value), Getter = () => context.CsvSkipRows.ToString(), Description = "Number of CSV rows to skip before parsing"}, new Option {Key = nameof(context.CsvIgnoreInvalidRows), Setter = value => context.CsvIgnoreInvalidRows = bool.Parse(value), Getter = () => context.CsvIgnoreInvalidRows.ToString(), Description = "Ignore CSV rows that can't be parsed"}, new Option {Key = nameof(context.CsvRealign), Setter = value => context.CsvRealign = bool.Parse(value), Getter = () => context.CsvRealign.ToString(), Description = $"Realign imported CSV points to the /{nameof(context.StartTime)} value"}, + new Option {Key = nameof(context.CsvRemoveDuplicatePoints), Setter = value => context.CsvRemoveDuplicatePoints = bool.Parse(value), Getter = () => context.CsvRemoveDuplicatePoints.ToString(), Description = "Remove duplicate points in the CSV before appending."}, new Option {Key = "CsvFormat", Description = "Shortcut for known CSV formats. One of 'NG' or '3X'. [default: NG]", Setter = value => { diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md b/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md index 9926020d..4e0bef4d 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md @@ -93,9 +93,6 @@ The `-csvFormat=` option supports two prefconfigured formats: - `-csvFormat=NG` is equivalent to `-csvTimeField=1 -csvValueField=3 -csvGradeField=5 -csvQualifiersField=6 -csvSkipRows=0 -csvComment="#"` - `-csvFormat=3X` is equivalent to `-csvTimeField=1 -csvValueField=2 -csvGradeField=3 -csvQualifiersField=0 -csvSkipRows=2 -csvTimeFormat="MM/dd/yyyy HH:mm:ss"` - - - ```sh $ ./PointZilla.exe -server=myserver Stage.Label@MyLocation Downloads/Stage.Historical@A001002.EntireRecord.csv @@ -208,5 +205,66 @@ $ ./PointZilla.exe -server=myserver Stage.Label@Location -TimeRange=2018-04-25T0 Like `curl`, the `PointZilla` tool has dozens of command line options, which can be a bit overwhelming. Fortunately, you'll rarely need to use all the options at once. -Try the `/Help` option to see the entire list of supported options. +Try the `/Help` option to see the entire list of supported options and read the [wiki for the @optionsFile syntax](https://github.com/AquaticInformatics/examples/wiki/Common-command-line-options). + +``` +Append points to an AQTS time-series. + +usage: PointZilla [-option=value] [@optionsFile] [command] [identifierOrGuid] [value] [csvFile] ... + +Supported -option=value settings (/option=value works too): + + -Server AQTS server name + -Username AQTS username [default: admin] + -Password AQTS password [default: admin] + -Wait Wait for the append request to complete [default: True] + -AppendTimeout Timeout period for append completion, in .NET TimeSpan format. + -BatchSize Maximum number of points to send in a single append request [default: 500000] + + ========================= Time-series options: + -TimeSeries Target time-series identifier or unique ID + -TimeRange Time-range for overwrite in ISO8061/ISO8601 (defaults to start/end points) + -Command Append operation to perform. One of Auto, Append, OverwriteAppend, Reflected, DeleteAllPoints. [default: Auto] + -GradeCode Optional grade code for all appended points + -Qualifiers Optional qualifier list for all appended points + -CreateMode Mode for creating missing time-series. One of Never, Basic, Reflected. [default: Never] + -GapTolerance Set the gap tolerance for newly-created time-series. [default: "MaxDuration"] + -UtcOffset Set the UTC offset for any created location. [default: Use system timezone] + + ========================= Copy points from another time-series: + -SourceTimeSeries Source time-series to copy. Prefix with [server2] or [server2:username2:password2] to copy from another server + -SourceQueryFrom Start time of extracted points in ISO8601 format. + -SourceQueryTo End time of extracted points + + ========================= Point-generator options: + -StartTime Start time of generated points, in ISO8601 format. [default: the current time] + -PointInterval Interval between generated points, in .NET TimeSpan format. [default: 00:01:00] + -NumberOfPoints Number of points to generate. If 0, use NumberOfPeriods [default: 0] + -NumberOfPeriods Number of waveform periods to generate. [default: 1] + -WaveformType Waveform to generate. One of Linear, SawTooth, SineWave, SquareWave. [default: SineWave] + -WaveformOffset Offset the generated waveform by this constant. [default: 0] + -WaveformPhase Phase within one waveform period [default: 0] + -WaveformScalar Scale the waveform by this amount [default: 1] + -WaveformPeriod Waveform period before repeating [default: 1440] + + ========================= CSV parsing options: + -CSV Parse the CSV file + -CsvTimeField CSV column index for timestamps [default: 1] + -CsvValueField CSV column index for values [default: 3] + -CsvGradeField CSV column index for grade codes [default: 5] + -CsvQualifiersField CSV column index for qualifiers [default: 6] + -CsvTimeFormat Format of CSV time fields (defaults to ISO8601) + -CsvComment CSV comment lines begin with this prefix [default: #] + -CsvSkipRows Number of CSV rows to skip before parsing [default: 0] + -CsvIgnoreInvalidRows Ignore CSV rows that can't be parsed [default: True] + -CsvRealign Realign imported CSV points to the /StartTime value [default: False] + -CsvRemoveDuplicatePoints Remove duplicate points in the CSV before appending. [default: True] + -CsvFormat Shortcut for known CSV formats. One of 'NG' or '3X'. [default: NG] + +Use the @optionsFile syntax to read more options from a file. + + Each line in the file is treated as a command line option. + Blank lines and leading/trailing whitespace is ignored. + Comment lines begin with a # or // marker. +```