From f53f1abeaa2ecdb7204f3437557d5be31e445732 Mon Sep 17 00:00:00 2001 From: Doug Schmidt Date: Fri, 16 Nov 2018 14:20:03 -0800 Subject: [PATCH] Issue-94 Added more command line options for time-series creation Added ability to set unit, comment, computation and sublocation identifiers, publish flag, and extended attributes when creating a time-series. When the -SourceTimeSeries option is used, use the source time-series properties as much as possible if a new time-series is being created. This brings PointZilla closer to cloning the corrected value of a time-series. Some settings are still not a perfect match, so it isn't recommend for officially migrating a time-series. But it is pretty close. PointZilla is now a 64-bit-only app so it can deal with bigger signals on modern systems. --- .../SdkExamples/PointZilla/Context.cs | 11 ++ .../PointZilla/ExternalPointsReader.cs | 122 ++++++++++++++++-- .../SdkExamples/PointZilla/PointZilla.csproj | 8 +- .../SdkExamples/PointZilla/Program.cs | 36 +++++- .../SdkExamples/PointZilla/Readme.md | 19 ++- .../PointZilla/TimeSeriesCreator.cs | 46 +++++-- 6 files changed, 210 insertions(+), 32 deletions(-) diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs index 83e8cf08..6d0972d7 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Aquarius.TimeSeries.Client.Helpers; using Aquarius.TimeSeries.Client.ServiceModels.Acquisition; +using Aquarius.TimeSeries.Client.ServiceModels.Provisioning; using NodaTime; namespace PointZilla @@ -24,6 +25,16 @@ public class Context public CreateMode CreateMode { get; set; } = CreateMode.Never; public Duration GapTolerance { get; set; } = DurationExtensions.MaxGapDuration; public Offset? UtcOffset { get; set; } + public string Unit { get; set; } + public InterpolationType? InterpolationType { get; set; } + public bool Publish { get; set; } + public string Description { get; set; } = "Created by PointZilla"; + public string Comment { get; set; } + public string Method { get; set; } + public string ComputationIdentifier { get; set; } + public string ComputationPeriodIdentifier { get; set; } + public string SubLocationIdentifier { get; set; } + public List ExtendedAttributeValues { get; set; } = new List(); public TimeSeriesIdentifier SourceTimeSeries { get; set; } public Instant? SourceQueryFrom { get; set; } diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/ExternalPointsReader.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/ExternalPointsReader.cs index 17497537..e66c59f8 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/ExternalPointsReader.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/ExternalPointsReader.cs @@ -3,11 +3,15 @@ using System.Linq; using System.Reflection; using Aquarius.TimeSeries.Client; +using Aquarius.TimeSeries.Client.Helpers; using Aquarius.TimeSeries.Client.ServiceModels.Acquisition; +using Aquarius.TimeSeries.Client.ServiceModels.Provisioning; using Aquarius.TimeSeries.Client.ServiceModels.Publish; using Get3xCorrectedData = Aquarius.TimeSeries.Client.ServiceModels.Legacy.Publish3x.TimeSeriesDataCorrectedServiceRequest; +using Get3xTimeSeriesDescription = Aquarius.TimeSeries.Client.ServiceModels.Legacy.Publish3x.TimeSeriesDescriptionServiceRequest; using NodaTime; using ServiceStack.Logging; +using InterpolationType = Aquarius.TimeSeries.Client.ServiceModels.Provisioning.InterpolationType; namespace PointZilla { @@ -44,29 +48,106 @@ private List LoadPointsFromNg(IAquariusClient client) { var timeSeriesInfo = client.GetTimeSeriesInfo(Context.SourceTimeSeries.Identifier); - var points = client.Publish.Get(new TimeAlignedDataServiceRequest - { - TimeSeriesUniqueIds = new List { timeSeriesInfo.UniqueId }, - QueryFrom = Context.SourceQueryFrom?.ToDateTimeOffset(), - QueryTo = Context.SourceQueryTo?.ToDateTimeOffset() - }) + var timeSeriesData = client.Publish.Get(new TimeSeriesDataCorrectedServiceRequest + { + TimeSeriesUniqueId = timeSeriesInfo.UniqueId, + QueryFrom = Context.SourceQueryFrom?.ToDateTimeOffset(), + QueryTo = Context.SourceQueryTo?.ToDateTimeOffset() + }); + + var points = timeSeriesData .Points .Select(p => new ReflectedTimeSeriesPoint { - Time = Instant.FromDateTimeOffset(p.Timestamp), - Value = p.NumericValue1, - GradeCode = p.GradeCode1.HasValue ? (int)p.GradeCode1 : (int?)null, - Qualifiers = QualifiersParser.Parse(p.Qualifiers1) + Time = Instant.FromDateTimeOffset(p.Timestamp.DateTimeOffset), + Value = p.Value.Numeric, + GradeCode = GetFirstMetadata(timeSeriesData.Grades, p.Timestamp.DateTimeOffset, g => int.Parse(g.GradeCode)), + Qualifiers = GetManyMetadata(timeSeriesData.Qualifiers, p.Timestamp.DateTimeOffset, q => q.Identifier).ToList() }) .ToList(); + var gapToleranceInMinutes = timeSeriesData.GapTolerances.Last().ToleranceInMinutes; + var gapTolerance = gapToleranceInMinutes.HasValue + ? Duration.FromMinutes((long) gapToleranceInMinutes.Value) + : DurationExtensions.MaxGapDuration; + var interpolationType = (InterpolationType) Enum.Parse(typeof(InterpolationType), timeSeriesData.InterpolationTypes.Last().Type, true); + + SetTimeSeriesCreationProperties( + timeSeriesInfo, + timeSeriesInfo.UtcOffset, + timeSeriesData.Methods.LastOrDefault()?.MethodCode, + gapTolerance, + interpolationType); + Log.Info($"Loaded {points.Count} points from {timeSeriesInfo.Identifier}"); return points; } + private static T GetFirstMetadata(IEnumerable items, DateTimeOffset time, Func func) + where TMetadata : TimeRange + { + var metadata = items.FirstOrDefault(i => i.StartTime <= time && time < i.EndTime); + + return metadata == null ? default(T) : func(metadata); + } + + private static IEnumerable GetManyMetadata(IEnumerable items, DateTimeOffset time, Func func) + where TMetadata : TimeRange + { + return items + .Where(i => i.StartTime <= time && time < i.EndTime) + .Select(func); + } + + private void SetTimeSeriesCreationProperties( + TimeSeries timeSeries, + Offset? utcOffset = null, + string method = null, + Duration? gapTolerance = null, + InterpolationType? interpolationType = null) + { + if (gapTolerance.HasValue) + Context.GapTolerance = gapTolerance.Value; + + if (interpolationType.HasValue && !Context.InterpolationType.HasValue) + Context.InterpolationType = interpolationType; + + if (utcOffset.HasValue && !Context.UtcOffset.HasValue) + Context.UtcOffset = utcOffset.Value; + + Context.Publish = timeSeries.Publish; + Context.Description = timeSeries.Description; + + Context.Method = Context.Method ?? method; + Context.Unit = Context.Unit ?? timeSeries.Unit; + Context.Comment = Context.Comment ?? timeSeries.Comment; + Context.ComputationIdentifier = Context.ComputationIdentifier ?? timeSeries.ComputationIdentifier; + Context.ComputationPeriodIdentifier = Context.ComputationPeriodIdentifier ?? timeSeries.ComputationPeriodIdentifier; + Context.SubLocationIdentifier = Context.SubLocationIdentifier ?? timeSeries.SubLocationIdentifier; + + foreach (var extendedAttributeValue in timeSeries.ExtendedAttributeValues) + { + if (Context.ExtendedAttributeValues.Any(a => a.ColumnIdentifier == extendedAttributeValue.ColumnIdentifier)) + continue; + + Context.ExtendedAttributeValues.Add(extendedAttributeValue); + } + } + private List LoadPointsFrom3X(IAquariusClient client) { + var timeSeriesDescription = client.Publish.Get(new Get3xTimeSeriesDescription + { + LocationIdentifier = Context.SourceTimeSeries.LocationIdentifier, + Parameter = Context.SourceTimeSeries.Parameter + }) + .TimeSeriesDescriptions + .SingleOrDefault(ts => ts.Identifier == Context.SourceTimeSeries.Identifier); + + if (timeSeriesDescription == null) + throw new ExpectedException($"Can't find '{Context.SourceTimeSeries.Identifier}' time-series in location '{Context.SourceTimeSeries.LocationIdentifier}'."); + var points = client.Publish.Get(new Get3xCorrectedData { TimeSeriesIdentifier = Context.SourceTimeSeries.Identifier, @@ -82,6 +163,27 @@ private List LoadPointsFrom3X(IAquariusClient client) }) .ToList(); + SetTimeSeriesCreationProperties(new TimeSeries + { + Parameter = timeSeriesDescription.Parameter, + Label = timeSeriesDescription.Label, + Unit = timeSeriesDescription.Unit, + Publish = timeSeriesDescription.Publish, + Description = timeSeriesDescription.Description, + Comment = timeSeriesDescription.Comment, + ComputationIdentifier = timeSeriesDescription.ComputationIdentifier, + ComputationPeriodIdentifier = timeSeriesDescription.ComputationPeriodIdentifier, + SubLocationIdentifier = timeSeriesDescription.SubLocationIdentifier, + LocationIdentifier = timeSeriesDescription.LocationIdentifier, + ExtendedAttributeValues = timeSeriesDescription.ExtendedAttributes.Select(ea => + new ExtendedAttributeValue + { + ColumnIdentifier = $"{ea.Name.ToUpperInvariant()}@TIMESERIES_EXTENSION", + Value = ea.Value.ToString() + }) + .ToList() + }); + Log.Info($"Loaded {points.Count} points from {Context.SourceTimeSeries.Identifier}"); return points; diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/PointZilla.csproj b/TimeSeries/PublicApis/SdkExamples/PointZilla/PointZilla.csproj index c16701bf..950f5077 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/PointZilla.csproj +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/PointZilla.csproj @@ -16,7 +16,7 @@ - AnyCPU + x64 true full false @@ -24,15 +24,17 @@ DEBUG;TRACE prompt 4 + false - AnyCPU + x64 pdbonly true bin\Release\ TRACE prompt 4 + false @@ -128,4 +130,4 @@ - + \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs index d00cd64a..6714340b 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs @@ -8,6 +8,7 @@ using System.Xml; using Aquarius.TimeSeries.Client; using Aquarius.TimeSeries.Client.ServiceModels.Acquisition; +using Aquarius.TimeSeries.Client.ServiceModels.Provisioning; using log4net; using NodaTime; using ServiceStack; @@ -105,9 +106,21 @@ private static Context ParseArgs(string[] args) new Option {Key = nameof(context.Command), Setter = value => context.Command = ParseEnum(value), Getter = () => context.Command.ToString(), Description = $"Append operation to perform. {EnumOptions()}"}, new Option {Key = nameof(context.GradeCode), Setter = value => context.GradeCode = int.Parse(value), Getter = () => context.GradeCode.ToString(), Description = "Optional grade code for all appended points"}, new Option {Key = nameof(context.Qualifiers), Setter = value => context.Qualifiers = QualifiersParser.Parse(value), Getter = () => string.Join(",", context.Qualifiers), Description = "Optional qualifier list for all appended points"}, - new Option {Key = nameof(context.CreateMode), Setter = value => context.CreateMode = ParseEnum(value), Getter = () => context.CreateMode.ToString(), Description = $"Mode for creating missing time-series. {EnumOptions()}"}, - new Option {Key = nameof(context.GapTolerance), Setter = value => context.GapTolerance = value.FromJson(), Getter = () => context.GapTolerance.ToJson(), Description = "Set the gap tolerance for newly-created time-series."}, - new Option {Key = nameof(context.UtcOffset), Setter = value => context.UtcOffset = value.FromJson(), Getter = () => string.Empty, Description = "Set the UTC offset for any created location. [default: Use system timezone]"}, + + new Option(), new Option {Description = "Time-series creation options:"}, + new Option {Key = nameof(context.CreateMode), Setter = value => context.CreateMode = ParseEnum(value), Getter = () => context.CreateMode.ToString(), Description = $"Mode for creating missing time-series. {EnumOptions()}"}, + new Option {Key = nameof(context.GapTolerance), Setter = value => context.GapTolerance = value.FromJson(), Getter = () => context.GapTolerance.ToJson(), Description = "Gap tolerance for newly-created time-series."}, + new Option {Key = nameof(context.UtcOffset), Setter = value => context.UtcOffset = value.FromJson(), Getter = () => string.Empty, Description = "UTC offset for any created time-series or location. [default: Use system timezone]"}, + new Option {Key = nameof(context.Unit), Setter = value => context.Unit = value, Getter = () => context.Unit, Description = "Time-series unit"}, + new Option {Key = nameof(context.InterpolationType), Setter = value => context.InterpolationType = ParseEnum(value), Getter = () => context.InterpolationType.ToString(), Description = $"Time-series interpolation type. {EnumOptions()}"}, + new Option {Key = nameof(context.Publish), Setter = value => context.Publish = bool.Parse(value), Getter = () => context.Publish.ToString(), Description = "Publish flag."}, + new Option {Key = nameof(context.Description), Setter = value => context.Description = value, Getter = () => context.Description, Description = "Time-series description"}, + new Option {Key = nameof(context.Comment), Setter = value => context.Comment = value, Getter = () => context.Comment, Description = "Time-series comment"}, + new Option {Key = nameof(context.Method), Setter = value => context.Method = value, Getter = () => context.Method, Description = "Time-series monitoring method"}, + new Option {Key = nameof(context.ComputationIdentifier), Setter = value => context.ComputationIdentifier = value, Getter = () => context.ComputationIdentifier, Description = "Time-series computation identifier"}, + new Option {Key = nameof(context.ComputationPeriodIdentifier), Setter = value => context.ComputationPeriodIdentifier = value, Getter = () => context.ComputationPeriodIdentifier, Description = "Time-series computation period identifier"}, + new Option {Key = nameof(context.SubLocationIdentifier), Setter = value => context.SubLocationIdentifier = value, Getter = () => context.SubLocationIdentifier, Description = "Time-series sub-location identifier"}, + new Option {Key = nameof(context.ExtendedAttributeValues), Setter = value => ParseExtendedAttributeValue(context, value), Getter = () => string.Empty, Description = "Extended attribute values in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value syntax. Can be set multiple times."}, new Option(), new Option {Description = "Copy points from another time-series:"}, new Option {Key = nameof(context.SourceTimeSeries), Setter = value => {if (TimeSeriesIdentifier.TryParse(value, out var tsi)) context.SourceTimeSeries = tsi; }, Getter = () => context.SourceTimeSeries?.ToString(), Description = "Source time-series to copy. Prefix with [server2] or [server2:username2:password2] to copy from another server"}, @@ -349,6 +362,23 @@ private static Interval ParseInterval(string text) components[1].FromJson()); } + private static void ParseExtendedAttributeValue(Context context, string text) + { + var components = text.Split(new[] {'='}, 2); + + if (components.Length < 2) + throw new ExpectedException($"'{text}' is not in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value format."); + + var columnIdentifier = components[0].Trim(); + var value = components[1].Trim(); + + context.ExtendedAttributeValues.Add(new ExtendedAttributeValue + { + ColumnIdentifier = columnIdentifier, + Value = value + }); + } + private readonly Context _context; public Program(Context context) diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md b/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md index 4e0bef4d..284fedf4 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md @@ -227,9 +227,22 @@ Supported -option=value settings (/option=value works too): -Command Append operation to perform. One of Auto, Append, OverwriteAppend, Reflected, DeleteAllPoints. [default: Auto] -GradeCode Optional grade code for all appended points -Qualifiers Optional qualifier list for all appended points - -CreateMode Mode for creating missing time-series. One of Never, Basic, Reflected. [default: Never] - -GapTolerance Set the gap tolerance for newly-created time-series. [default: "MaxDuration"] - -UtcOffset Set the UTC offset for any created location. [default: Use system timezone] + + ========================= Time-series creation options: + -CreateMode Mode for creating missing time-series. One of Never, Basic, Reflected. [default: Never] + -GapTolerance Gap tolerance for newly-created time-series. [default: "MaxDuration"] + -UtcOffset UTC offset for any created time-series or location. [default: Use system timezone] + -Unit Time-series unit + -InterpolationType Time-series interpolation type. One of InstantaneousValues, PrecedingConstant, PrecedingTotals, InstantaneousTotals, DiscreteValues, SucceedingConstant. + -Publish Publish flag. [default: False] + -Description Time-series description [default: Created by PointZilla] + -Comment Time-series comment + -Method Time-series monitoring method + -ComputationIdentifier Time-series computation identifier + -ComputationPeriodIdentifier Time-series computation period identifier + -SubLocationIdentifier Time-series sub-location identifier + -ExtendedAttributeValues Extended attribute values in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value syntax. Can be set multiple times. + ========================= Copy points from another time-series: -SourceTimeSeries Source time-series to copy. Prefix with [server2] or [server2:username2:password2] to copy from another server diff --git a/TimeSeries/PublicApis/SdkExamples/PointZilla/TimeSeriesCreator.cs b/TimeSeries/PublicApis/SdkExamples/PointZilla/TimeSeriesCreator.cs index 5c54d32f..fb106bd0 100644 --- a/TimeSeries/PublicApis/SdkExamples/PointZilla/TimeSeriesCreator.cs +++ b/TimeSeries/PublicApis/SdkExamples/PointZilla/TimeSeriesCreator.cs @@ -60,7 +60,7 @@ private Location CreateLocation(string locationIdentifier) Description = "Dummy location created by PointZilla", LocationPath = locationFolder.LocationFolderPath, UtcOffset = Context.UtcOffset ?? Offset.FromTicks(DateTimeOffset.Now.Offset.Ticks), - ExtendedAttributeValues = GetDefaultExtendedAttributes(locationType.ExtendedAttributeFields).ToList() + ExtendedAttributeValues = MergeExtendedAttributesWithMandatoryExtendedAttributes(locationType.ExtendedAttributeFields).ToList() }; Log.Info($"Creating location '{locationIdentifier}' ..."); @@ -89,7 +89,9 @@ private void GetOrCreateTimeSeries(Location location, string timeSeriesIdentifie if (parameter == null) throw new ExpectedException($"Parameter '{timeSeriesInfo.Parameter}' does not exist in the system."); - var gapTolerance = InterpolationTypesWithNoGaps.Contains(parameter.InterpolationType) + var interpolationType = Context.InterpolationType ?? parameter.InterpolationType; + + var gapTolerance = InterpolationTypesWithNoGaps.Contains(interpolationType) ? DurationExtensions.MaxGapDuration : Context.GapTolerance; @@ -103,7 +105,7 @@ private void GetOrCreateTimeSeries(Location location, string timeSeriesIdentifie if (Context.CreateMode == CreateMode.Reflected) { - reflectedTimeSeries = new PostReflectedTimeSeries { GapTolerance = gapTolerance }; + reflectedTimeSeries = new PostReflectedTimeSeries {GapTolerance = gapTolerance}; request = reflectedTimeSeries; } else @@ -113,14 +115,19 @@ private void GetOrCreateTimeSeries(Location location, string timeSeriesIdentifie } request.LocationUniqueId = location.UniqueId; - request.UtcOffset = location.UtcOffset; + request.UtcOffset = Context.UtcOffset ?? location.UtcOffset; request.Label = timeSeriesInfo.Label; request.Parameter = parameter.ParameterId; - request.Description = "Created by PointZilla"; - request.ExtendedAttributeValues = GetDefaultExtendedAttributes(timeSeriesExtendedAttributes).ToList(); - request.Unit = parameter.UnitIdentifier; - request.InterpolationType = parameter.InterpolationType; - request.Method = defaultMonitoringMethod.MethodCode; + request.Description = Context.Description; + request.Comment = Context.Comment; + request.Unit = Context.Unit ?? parameter.UnitIdentifier; + request.InterpolationType = interpolationType; + request.Publish = Context.Publish; + request.Method = Context.Method ?? defaultMonitoringMethod.MethodCode; + request.ComputationIdentifier = Context.ComputationIdentifier; + request.ComputationPeriodIdentifier = Context.ComputationPeriodIdentifier; + request.SubLocationIdentifier = Context.SubLocationIdentifier; + request.ExtendedAttributeValues = MergeExtendedAttributesWithMandatoryExtendedAttributes(timeSeriesExtendedAttributes).ToList(); Log.Info($"Creating '{timeSeriesIdentifier}' time-series ..."); @@ -137,11 +144,24 @@ private void GetOrCreateTimeSeries(Location location, string timeSeriesIdentifie InterpolationType.DiscreteValues, }; - private static IEnumerable GetDefaultExtendedAttributes(IList extendedAttributeFields) + private IEnumerable MergeExtendedAttributesWithMandatoryExtendedAttributes(IList extendedAttributeFields) { - return extendedAttributeFields - .Where(f => !f.CanBeEmpty) - .Select(CreateDefaultValue); + var unknownAttributes = Context + .ExtendedAttributeValues + .Where(eav => extendedAttributeFields.All(f => eav.ColumnIdentifier != f.ColumnIdentifier)) + .ToList(); + + if (unknownAttributes.Any()) + { + Log.Warn($"Ignoring {unknownAttributes.Count} unknown extended attributes: {string.Join(", ", unknownAttributes.Select(a => $"{a.ColumnIdentifier}={a.Value}"))}"); + } + + return Context + .ExtendedAttributeValues + .Where(eav => extendedAttributeFields.Any(f => eav.ColumnIdentifier == f.ColumnIdentifier)) + .Concat(extendedAttributeFields + .Where(f => Context.ExtendedAttributeValues.All(eav => eav.ColumnIdentifier != f.ColumnIdentifier) && !f.CanBeEmpty) + .Select(CreateDefaultValue)); } private static ExtendedAttributeValue CreateDefaultValue(ExtendedAttributeField field)