Merge pull request #95 from DougSchmidt-AI/feature/Issue-94-ImprovePo…

…intZillaTimeSeriesCloning

Issue-94 Added more command line options for time-series creation

TimeSeries/PublicApis/SdkExamples/PointZilla/Context.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using Aquarius.TimeSeries.Client.Helpers;
 using Aquarius.TimeSeries.Client.ServiceModels.Acquisition;
+using Aquarius.TimeSeries.Client.ServiceModels.Provisioning;
 using NodaTime;
 
 namespace PointZilla
@@ -24,6 +25,16 @@ public class Context
         public CreateMode CreateMode { get; set; } = CreateMode.Never;
         public Duration GapTolerance { get; set; } = DurationExtensions.MaxGapDuration;
         public Offset? UtcOffset { get; set; }
+        public string Unit { get; set; }
+        public InterpolationType? InterpolationType { get; set; }
+        public bool Publish { get; set; }
+        public string Description { get; set; } = "Created by PointZilla";
+        public string Comment { get; set; }
+        public string Method { get; set; }
+        public string ComputationIdentifier { get; set; }
+        public string ComputationPeriodIdentifier { get; set; }
+        public string SubLocationIdentifier { get; set; }
+        public List<ExtendedAttributeValue> ExtendedAttributeValues { get; set; } = new List<ExtendedAttributeValue>();
 
         public TimeSeriesIdentifier SourceTimeSeries { get; set; }
         public Instant? SourceQueryFrom { get; set; }

TimeSeries/PublicApis/SdkExamples/PointZilla/ExternalPointsReader.cs

@@ -3,11 +3,15 @@
 using System.Linq;
 using System.Reflection;
 using Aquarius.TimeSeries.Client;
+using Aquarius.TimeSeries.Client.Helpers;
 using Aquarius.TimeSeries.Client.ServiceModels.Acquisition;
+using Aquarius.TimeSeries.Client.ServiceModels.Provisioning;
 using Aquarius.TimeSeries.Client.ServiceModels.Publish;
 using Get3xCorrectedData = Aquarius.TimeSeries.Client.ServiceModels.Legacy.Publish3x.TimeSeriesDataCorrectedServiceRequest;
+using Get3xTimeSeriesDescription = Aquarius.TimeSeries.Client.ServiceModels.Legacy.Publish3x.TimeSeriesDescriptionServiceRequest;
 using NodaTime;
 using ServiceStack.Logging;
+using InterpolationType = Aquarius.TimeSeries.Client.ServiceModels.Provisioning.InterpolationType;
 
 namespace PointZilla
 {
@@ -44,29 +48,106 @@ private List<ReflectedTimeSeriesPoint> LoadPointsFromNg(IAquariusClient client)
         {
             var timeSeriesInfo = client.GetTimeSeriesInfo(Context.SourceTimeSeries.Identifier);
 
-            var points = client.Publish.Get(new TimeAlignedDataServiceRequest
-                {
-                    TimeSeriesUniqueIds = new List<Guid> { timeSeriesInfo.UniqueId },
-                    QueryFrom = Context.SourceQueryFrom?.ToDateTimeOffset(),
-                    QueryTo = Context.SourceQueryTo?.ToDateTimeOffset()
-                })
+            var timeSeriesData = client.Publish.Get(new TimeSeriesDataCorrectedServiceRequest
+            {
+                TimeSeriesUniqueId = timeSeriesInfo.UniqueId,
+                QueryFrom = Context.SourceQueryFrom?.ToDateTimeOffset(),
+                QueryTo = Context.SourceQueryTo?.ToDateTimeOffset()
+            });
+
+            var points = timeSeriesData
                 .Points
                 .Select(p => new ReflectedTimeSeriesPoint
                 {
-                    Time = Instant.FromDateTimeOffset(p.Timestamp),
-                    Value = p.NumericValue1,
-                    GradeCode = p.GradeCode1.HasValue ? (int)p.GradeCode1 : (int?)null,
-                    Qualifiers = QualifiersParser.Parse(p.Qualifiers1)
+                    Time = Instant.FromDateTimeOffset(p.Timestamp.DateTimeOffset),
+                    Value = p.Value.Numeric,
+                    GradeCode = GetFirstMetadata(timeSeriesData.Grades, p.Timestamp.DateTimeOffset, g => int.Parse(g.GradeCode)),
+                    Qualifiers = GetManyMetadata(timeSeriesData.Qualifiers, p.Timestamp.DateTimeOffset, q => q.Identifier).ToList()
                 })
                 .ToList();
 
+            var gapToleranceInMinutes = timeSeriesData.GapTolerances.Last().ToleranceInMinutes;
+            var gapTolerance = gapToleranceInMinutes.HasValue
+                ? Duration.FromMinutes((long) gapToleranceInMinutes.Value)
+                : DurationExtensions.MaxGapDuration;
+            var interpolationType = (InterpolationType) Enum.Parse(typeof(InterpolationType), timeSeriesData.InterpolationTypes.Last().Type, true);
+
+            SetTimeSeriesCreationProperties(
+                timeSeriesInfo,
+                timeSeriesInfo.UtcOffset,
+                timeSeriesData.Methods.LastOrDefault()?.MethodCode,
+                gapTolerance,
+                interpolationType);
+
             Log.Info($"Loaded {points.Count} points from {timeSeriesInfo.Identifier}");
 
             return points;
         }
 
+        private static T GetFirstMetadata<TMetadata, T>(IEnumerable<TMetadata> items, DateTimeOffset time, Func<TMetadata,T> func)
+            where TMetadata : TimeRange
+        {
+            var metadata = items.FirstOrDefault(i => i.StartTime <= time && time < i.EndTime);
+
+            return metadata == null ? default(T) : func(metadata);
+        }
+
+        private static IEnumerable<T> GetManyMetadata<TMetadata, T>(IEnumerable<TMetadata> items, DateTimeOffset time, Func<TMetadata, T> func)
+            where TMetadata : TimeRange
+        {
+            return items
+                .Where(i => i.StartTime <= time && time < i.EndTime)
+                .Select(func);
+        }
+
+        private void SetTimeSeriesCreationProperties(
+            TimeSeries timeSeries,
+            Offset? utcOffset = null,
+            string method = null,
+            Duration? gapTolerance = null,
+            InterpolationType? interpolationType = null)
+        {
+            if (gapTolerance.HasValue)
+                Context.GapTolerance = gapTolerance.Value;
+
+            if (interpolationType.HasValue && !Context.InterpolationType.HasValue)
+                Context.InterpolationType = interpolationType;
+
+            if (utcOffset.HasValue && !Context.UtcOffset.HasValue)
+                Context.UtcOffset = utcOffset.Value;
+
+            Context.Publish = timeSeries.Publish;
+            Context.Description = timeSeries.Description;
+
+            Context.Method = Context.Method ?? method;
+            Context.Unit = Context.Unit ?? timeSeries.Unit;
+            Context.Comment = Context.Comment ?? timeSeries.Comment;
+            Context.ComputationIdentifier = Context.ComputationIdentifier ?? timeSeries.ComputationIdentifier;
+            Context.ComputationPeriodIdentifier = Context.ComputationPeriodIdentifier ?? timeSeries.ComputationPeriodIdentifier;
+            Context.SubLocationIdentifier = Context.SubLocationIdentifier ?? timeSeries.SubLocationIdentifier;
+
+            foreach (var extendedAttributeValue in timeSeries.ExtendedAttributeValues)
+            {
+                if (Context.ExtendedAttributeValues.Any(a => a.ColumnIdentifier == extendedAttributeValue.ColumnIdentifier))
+                    continue;
+
+                Context.ExtendedAttributeValues.Add(extendedAttributeValue);
+            }
+        }
+
         private List<ReflectedTimeSeriesPoint> LoadPointsFrom3X(IAquariusClient client)
         {
+            var timeSeriesDescription = client.Publish.Get(new Get3xTimeSeriesDescription
+                {
+                    LocationIdentifier = Context.SourceTimeSeries.LocationIdentifier,
+                    Parameter = Context.SourceTimeSeries.Parameter
+                })
+                .TimeSeriesDescriptions
+                .SingleOrDefault(ts => ts.Identifier == Context.SourceTimeSeries.Identifier);
+
+            if (timeSeriesDescription == null)
+                throw new ExpectedException($"Can't find '{Context.SourceTimeSeries.Identifier}' time-series in location '{Context.SourceTimeSeries.LocationIdentifier}'.");
+
             var points = client.Publish.Get(new Get3xCorrectedData
                 {
                     TimeSeriesIdentifier = Context.SourceTimeSeries.Identifier,
@@ -82,6 +163,27 @@ private List<ReflectedTimeSeriesPoint> LoadPointsFrom3X(IAquariusClient client)
                 })
                 .ToList();
 
+            SetTimeSeriesCreationProperties(new TimeSeries
+            {
+                Parameter = timeSeriesDescription.Parameter,
+                Label = timeSeriesDescription.Label,
+                Unit = timeSeriesDescription.Unit,
+                Publish = timeSeriesDescription.Publish,
+                Description = timeSeriesDescription.Description,
+                Comment = timeSeriesDescription.Comment,
+                ComputationIdentifier = timeSeriesDescription.ComputationIdentifier,
+                ComputationPeriodIdentifier = timeSeriesDescription.ComputationPeriodIdentifier,
+                SubLocationIdentifier = timeSeriesDescription.SubLocationIdentifier,
+                LocationIdentifier = timeSeriesDescription.LocationIdentifier,
+                ExtendedAttributeValues = timeSeriesDescription.ExtendedAttributes.Select(ea =>
+                        new ExtendedAttributeValue
+                        {
+                            ColumnIdentifier = $"{ea.Name.ToUpperInvariant()}@TIMESERIES_EXTENSION",
+                            Value = ea.Value.ToString()
+                        })
+                    .ToList()
+            });
+
             Log.Info($"Loaded {points.Count} points from {Context.SourceTimeSeries.Identifier}");
 
             return points;

TimeSeries/PublicApis/SdkExamples/PointZilla/PointZilla.csproj

@@ -16,23 +16,25 @@
     <TargetFrameworkProfile />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
+    <PlatformTarget>x64</PlatformTarget>
     <DebugSymbols>true</DebugSymbols>
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
     <OutputPath>bin\Debug\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
+    <PlatformTarget>x64</PlatformTarget>
     <DebugType>pdbonly</DebugType>
     <Optimize>true</Optimize>
     <OutputPath>bin\Release\</OutputPath>
     <DefineConstants>TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="Aquarius.Client, Version=18.7.4.0, Culture=neutral, processorArchitecture=MSIL">
@@ -128,4 +130,4 @@
     <Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
     <Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
   </Target>
-</Project>
+</Project>

TimeSeries/PublicApis/SdkExamples/PointZilla/Program.cs

@@ -8,6 +8,7 @@
 using System.Xml;
 using Aquarius.TimeSeries.Client;
 using Aquarius.TimeSeries.Client.ServiceModels.Acquisition;
+using Aquarius.TimeSeries.Client.ServiceModels.Provisioning;
 using log4net;
 using NodaTime;
 using ServiceStack;
@@ -105,9 +106,21 @@ private static Context ParseArgs(string[] args)
                 new Option {Key = nameof(context.Command), Setter = value => context.Command = ParseEnum<CommandType>(value), Getter = () => context.Command.ToString(), Description = $"Append operation to perform.  {EnumOptions<CommandType>()}"},
                 new Option {Key = nameof(context.GradeCode), Setter = value => context.GradeCode = int.Parse(value), Getter = () => context.GradeCode.ToString(), Description = "Optional grade code for all appended points"},
                 new Option {Key = nameof(context.Qualifiers), Setter = value => context.Qualifiers = QualifiersParser.Parse(value), Getter = () => string.Join(",", context.Qualifiers), Description = "Optional qualifier list for all appended points"},
-                new Option {Key = nameof(context.CreateMode), Setter = value => context.CreateMode = ParseEnum<CreateMode>(value), Getter = () => context.CreateMode.ToString(), Description = $"Mode for creating missing time-series.  {EnumOptions<CreateMode>()}"},
-                new Option {Key = nameof(context.GapTolerance), Setter = value => context.GapTolerance = value.FromJson<Duration>(), Getter = () => context.GapTolerance.ToJson(), Description = "Set the gap tolerance for newly-created time-series."},
-                new Option {Key = nameof(context.UtcOffset), Setter = value => context.UtcOffset = value.FromJson<Offset>(), Getter = () => string.Empty, Description = "Set the UTC offset for any created location. [default: Use system timezone]"},
+
+                new Option(), new Option {Description = "Time-series creation options:"},
+                new Option {Key = nameof(context.CreateMode), Setter = value => context.CreateMode = ParseEnum<CreateMode>(value), Getter = () => context.CreateMode.ToString(), Description = $"Mode for creating missing time-series. {EnumOptions<CreateMode>()}"},
+                new Option {Key = nameof(context.GapTolerance), Setter = value => context.GapTolerance = value.FromJson<Duration>(), Getter = () => context.GapTolerance.ToJson(), Description = "Gap tolerance for newly-created time-series."},
+                new Option {Key = nameof(context.UtcOffset), Setter = value => context.UtcOffset = value.FromJson<Offset>(), Getter = () => string.Empty, Description = "UTC offset for any created time-series or location. [default: Use system timezone]"},
+                new Option {Key = nameof(context.Unit), Setter = value => context.Unit = value, Getter = () => context.Unit, Description = "Time-series unit"},
+                new Option {Key = nameof(context.InterpolationType), Setter = value => context.InterpolationType = ParseEnum<InterpolationType>(value), Getter = () => context.InterpolationType.ToString(), Description = $"Time-series interpolation type. {EnumOptions<InterpolationType>()}"},
+                new Option {Key = nameof(context.Publish), Setter = value => context.Publish = bool.Parse(value), Getter = () => context.Publish.ToString(), Description = "Publish flag."},
+                new Option {Key = nameof(context.Description), Setter = value => context.Description = value, Getter = () => context.Description, Description = "Time-series description"},
+                new Option {Key = nameof(context.Comment), Setter = value => context.Comment = value, Getter = () => context.Comment, Description = "Time-series comment"},
+                new Option {Key = nameof(context.Method), Setter = value => context.Method = value, Getter = () => context.Method, Description = "Time-series monitoring method"},
+                new Option {Key = nameof(context.ComputationIdentifier), Setter = value => context.ComputationIdentifier = value, Getter = () => context.ComputationIdentifier, Description = "Time-series computation identifier"},
+                new Option {Key = nameof(context.ComputationPeriodIdentifier), Setter = value => context.ComputationPeriodIdentifier = value, Getter = () => context.ComputationPeriodIdentifier, Description = "Time-series computation period identifier"},
+                new Option {Key = nameof(context.SubLocationIdentifier), Setter = value => context.SubLocationIdentifier = value, Getter = () => context.SubLocationIdentifier, Description = "Time-series sub-location identifier"},
+                new Option {Key = nameof(context.ExtendedAttributeValues), Setter = value => ParseExtendedAttributeValue(context, value), Getter = () => string.Empty, Description = "Extended attribute values in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value syntax. Can be set multiple times."},
 
                 new Option(), new Option {Description = "Copy points from another time-series:"},
                 new Option {Key = nameof(context.SourceTimeSeries), Setter = value => {if (TimeSeriesIdentifier.TryParse(value, out var tsi)) context.SourceTimeSeries = tsi; }, Getter = () => context.SourceTimeSeries?.ToString(), Description = "Source time-series to copy. Prefix with [server2] or [server2:username2:password2] to copy from another server"},
@@ -349,6 +362,23 @@ private static Interval ParseInterval(string text)
                 components[1].FromJson<Instant>());
         }
 
+        private static void ParseExtendedAttributeValue(Context context, string text)
+        {
+            var components = text.Split(new[] {'='}, 2);
+
+            if (components.Length < 2)
+                throw new ExpectedException($"'{text}' is not in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value format.");
+
+            var columnIdentifier = components[0].Trim();
+            var value = components[1].Trim();
+
+            context.ExtendedAttributeValues.Add(new ExtendedAttributeValue
+            {
+                ColumnIdentifier = columnIdentifier,
+                Value = value
+            });
+        }
+
         private readonly Context _context;
 
         public Program(Context context)

TimeSeries/PublicApis/SdkExamples/PointZilla/Readme.md

@@ -227,9 +227,22 @@ Supported -option=value settings (/option=value works too):
   -Command                  Append operation to perform.  One of Auto, Append, OverwriteAppend, Reflected, DeleteAllPoints. [default: Auto]
   -GradeCode                Optional grade code for all appended points
   -Qualifiers               Optional qualifier list for all appended points
-  -CreateMode               Mode for creating missing time-series.  One of Never, Basic, Reflected. [default: Never]
-  -GapTolerance             Set the gap tolerance for newly-created time-series. [default: "MaxDuration"]
-  -UtcOffset                Set the UTC offset for any created location. [default: Use system timezone]
+
+  ========================= Time-series creation options:
+  -CreateMode               Mode for creating missing time-series. One of Never, Basic, Reflected. [default: Never]
+  -GapTolerance             Gap tolerance for newly-created time-series. [default: "MaxDuration"]
+  -UtcOffset                UTC offset for any created time-series or location. [default: Use system timezone]
+  -Unit                     Time-series unit
+  -InterpolationType        Time-series interpolation type. One of InstantaneousValues, PrecedingConstant, PrecedingTotals, InstantaneousTotals, DiscreteValues, SucceedingConstant.
+  -Publish                  Publish flag. [default: False]
+  -Description              Time-series description [default: Created by PointZilla]
+  -Comment                  Time-series comment
+  -Method                   Time-series monitoring method
+  -ComputationIdentifier    Time-series computation identifier
+  -ComputationPeriodIdentifier Time-series computation period identifier
+  -SubLocationIdentifier    Time-series sub-location identifier
+  -ExtendedAttributeValues  Extended attribute values in UPPERCASE_COLUMN_NAME@UPPERCASE_TABLE_NAME=value syntax. Can be set multiple times.
+
 
   ========================= Copy points from another time-series:
   -SourceTimeSeries         Source time-series to copy. Prefix with [server2] or [server2:username2:password2] to copy from another server