Merge pull request #290 from DougSchmidt-AI/feature/PF-1365-NtdnrRepo…

…rtingTool

PF-1365 - First working cut of ObservationReportExporter

Samples/DotNetSdk/ObservationReportExporter/Context.cs

@@ -11,7 +11,7 @@ public class Context
         public string TimeSeriesUsername { get; set; }
         public string TimeSeriesPassword { get; set; }
         public string ExportTemplateName { get; set; }
-        public string AttachmentFilename { get; set; } = "Report.xlsx";
+        public string AttachmentFilename { get; set; } = FilenameGenerator.DefaultAttachmentFilename;
         public bool DryRun { get; set; }
         public bool DeleteExistingAttachments { get; set; } = true;
 
@@ -21,6 +21,7 @@ public class Context
         public List<string> LocationGroupIds { get; } = new List<string>();
         public List<string> AnalyticalGroupIds { get; } = new List<string>();
         public List<string> ObservedPropertyIds { get; } = new List<string>();
+        public DateTimeOffset? ExportTime { get; set; }
         public DateTimeOffset? StartTime { get; set; }
         public DateTimeOffset? EndTime { get; set; }
     }

Samples/DotNetSdk/ObservationReportExporter/GetExchangeConfigurations.cs → Samples/DotNetSdk/ObservationReportExporter/ExtraApis/Samples/GetExchangeConfigurations.cs

@@ -2,7 +2,7 @@
 using Aquarius.Samples.Client.ServiceModel;
 using ServiceStack;
 
-namespace ObservationReportExporter
+namespace ObservationReportExporter.ExtraApis.Samples
 {
     [Route("/v1/exchangeconfigurations", HttpMethods.Get)]
     public class GetExchangeConfigurations : IReturn<SearchResultsExchangeConfigurations>

Samples/DotNetSdk/ObservationReportExporter/ExtraApis/TimeSeries/DeleteAttachmentById.cs

@@ -0,0 +1,10 @@
+using ServiceStack;
+
+namespace ObservationReportExporter.ExtraApis.TimeSeries
+{
+    [Route("/attachments/{Id}", HttpMethods.Delete)]
+    public class DeleteAttachmentById : IReturnVoid
+    {
+        public string Id { get; set; }
+    }
+}

Samples/DotNetSdk/ObservationReportExporter/FilenameGenerator.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+namespace ObservationReportExporter
+{
+    public class FilenameGenerator
+    {
+        public const string TemplatePattern = "Template";
+        public const string LocationPattern = "Location";
+        public const string TimePattern = "Time";
+
+        public const string DefaultAttachmentFilename = "{" + TemplatePattern + "}-{" + LocationPattern + "}.xlsx";
+
+        public static string GenerateAttachmentFilename(string attachmentFilename, string exportTemplateName, string locationIdentifier, DateTimeOffset exportTime)
+        {
+            var patterns = new Dictionary<string, Func<string, string>>(StringComparer.InvariantCultureIgnoreCase)
+            {
+                { TemplatePattern, _ => exportTemplateName },
+                { LocationPattern, _ => locationIdentifier },
+                { TimePattern, format => exportTime.ToString(string.IsNullOrWhiteSpace(format) ? "yyyy-MM-dd" : format) }
+            };
+
+            return PatternRegex.Replace(attachmentFilename, match =>
+            {
+                var pattern = match.Groups["pattern"].Value.Trim();
+                var format = match.Groups["format"].Value.Trim();
+
+                if (!patterns.TryGetValue(pattern, out var replacement))
+                    throw new ExpectedException($"'{pattern}' is not a known attachment filename substitution pattern. Try one of {{{string.Join("}, {", patterns.Keys)}}}.");
+
+                return replacement(format);
+            });
+        }
+
+        private static readonly Regex PatternRegex = new Regex(@"\{(?<pattern>[^:}]+)(:(?<format>[^}]+))?\}");
+    }
+}

Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj

@@ -238,11 +238,13 @@
     </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ExtraApis\TimeSeries\DeleteAttachmentById.cs" />
+    <Compile Include="FilenameGenerator.cs" />
     <Compile Include="Context.cs" />
     <Compile Include="ExeHelper.cs" />
     <Compile Include="ExpectedException.cs" />
     <Compile Include="Exporter.cs" />
-    <Compile Include="GetExchangeConfigurations.cs" />
+    <Compile Include="ExtraApis\Samples\GetExchangeConfigurations.cs" />
     <Compile Include="Option.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -256,6 +258,7 @@
   <ItemGroup>
     <Content Include="FodyWeavers.xml" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="..\packages\Fody.6.5.5\build\Fody.targets" Condition="Exists('..\packages\Fody.6.5.5\build\Fody.targets')" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">

Samples/DotNetSdk/ObservationReportExporter/Program.cs

@@ -136,6 +136,13 @@ private static Context ParseArgs(string[] args)
 
                 new Option(), new Option { Description = "Export options:" },
                 new Option
+                {
+                    Key = nameof(context.DryRun),
+                    Setter = value => context.DryRun = ParseBoolean(value),
+                    Getter = () => $"{context.DryRun}",
+                    Description = "When true, don't export and upload reports, just validate what would be done."
+                },
+                new Option
                 {
                     Key = nameof(context.ExportTemplateName),
                     Setter = value => context.ExportTemplateName = value,
@@ -147,7 +154,7 @@ private static Context ParseArgs(string[] args)
                     Key = nameof(context.AttachmentFilename),
                     Setter = value => context.AttachmentFilename = value,
                     Getter = () => context.AttachmentFilename,
-                    Description = "Name of the generated report. Existing attachments with the same name will be deleted."
+                    Description = $"Filename of the exported attachment."
                 },
                 new Option
                 {
@@ -165,12 +172,13 @@ private static Context ParseArgs(string[] args)
                 },
                 new Option
                 {
-                    Key = nameof(context.DryRun),
-                    Setter = value => context.DryRun = ParseBoolean(value),
-                    Getter = () => $"{context.DryRun}",
-                    Description = "When true, don't export and upload reports, just validate what would be done."
+                    Key = nameof(context.ExportTime),
+                    Setter = value => context.ExportTime = ParseDateTimeOffset(value),
+                    Getter = () => string.Empty,
+                    Description = $"The timestamp used for all {{{FilenameGenerator.TimePattern}}} pattern substitutions. [default: The current time]"
                 },
 
+
                 new Option(),
                 new Option
                 {
@@ -181,14 +189,14 @@ private static Context ParseArgs(string[] args)
                     Key = nameof(context.StartTime),
                     Setter = value => context.StartTime = ParseDateTimeOffset(value),
                     Getter = () => string.Empty,
-                    Description = "Include observations after this time."
+                    Description = "Include observations after this time. [default: Start of record]"
                 },
                 new Option
                 {
                     Key = nameof(context.EndTime),
                     Setter = value => context.EndTime = ParseDateTimeOffset(value),
                     Getter = () => string.Empty,
-                    Description = "Include observations before this time."
+                    Description = "Include observations before this time. [default: End of record]"
                 },
                 new Option
                 {
@@ -221,7 +229,7 @@ private static Context ParseArgs(string[] args)
             };
 
             var usageMessage
-                    = $"Export observations from AQUARIUS Samples using a spreadsheet template and into AQUARIUS Time-Series."
+                    = $"Export observations from AQUARIUS Samples using a spreadsheet template into AQUARIUS Time-Series locations as attachments."
                       + $"\n"
                       + $"\nusage: {ExeHelper.ExeName} [-option=value] [@optionsFile] ..."
                       + $"\n"

Samples/DotNetSdk/ObservationReportExporter/Readme.md

@@ -8,8 +8,8 @@ The intent is to run this tool on a schedule so that the uploaded location attac
 
 ## Features
 
-- With no filters specified, all observed values will be exported.
-- Results can be filtered by optional date range, location, project, property, or analytical group.
+- A `/DryRun=true` can be used to quickly validate all the configured options, without actually exporting the observations and uploading the exported file to AQUARIUS Time-Series.
+- Results can be filtered by optional date range, location, location group, property, or analytical group.
 - All filters are cumulative (ie. they are AND-ed together). The more filters you add, the fewer results will be exported.
 - An exit code of 0 indicates the export was successful. An exit code greater than 0 indicates an error. This allows for easy error checking when invoked from scripts.
 
@@ -33,18 +33,55 @@ Two options are required to tell the tool how to access your AQUARIUS Samples in
 - The `/SamplesServer=` option sets the URL to the server, usually something like `/SamplesServer=https://mydomain.aqsamples.com`.
 - The `/SamplesApiToken=` option sets the API token used to authenticate your account. Navigate to https://[your_account].aqsamples.com/api/ and follow the instructions to get the token.
 
-
 ## Authentication with AQUARIUS Time-Series
 
 Three options are required to tell the tool how to access your AQUARIUS Time-Series system.
 
 - The `/TimeSeriesServer=` option sets the name of the AQTS server, usually something like `/TimeSeriesServer=https://myappserver`.
-- The `/TimeSeriesUsername=` and `TimeSeriesPassword` options set the AQTS credentials to use for uploading the generated reports.
+- The `/TimeSeriesUsername=` and `/TimeSeriesPassword=` options set the AQTS credentials to use for uploading the generated reports.
+
+## Filtering the exported observations
+
+- You cannot mix `/LocationId=` and `/LocationGroupId=` options, but at least one must be specified.
+- You cannot mix `/AnalyticalGroupId=` and `/ObservedPropertyId=` options, but at least one must be specified.
+- All the other filter options are AND-ed together, to reduce the observations exported using the template.
+
+## Controlling the AQTS attachment filename
+
+The exported observations spreadsheet is downloaded from AQUARIUS Samples and uploaded to the equivalent AQUARIUS Time-Series
+location as a location attachment with the specified location tags applied.
+
+The `/AttachmentFilename=` option controls the name of the uploaded location attachment.
+
+Supported \{replacement patterns} include:
+
+| Pattern | Replaced with: |
+|---|---|
+| `{Template}` | The name of the AQUARIUS Samples Observation Export template. |
+| `{Location}` | The AQUARIUS Time-Series location identifier. |
+| `{Time`*:format*`}` | The `/ExportTime` value. <br/><br/> The optional *format* is a [.NET DateTimeOffset format string](https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings). <br/>If omitted, the default format of `yyyy-MM-dd` is used.|
+
+The default `/AttachmentFilename=` value is `{Template}-{Location}.xlsx`.
+
+The resulting attachment filename will also interact with the `/DeleteExistingAttachment=` option.
+By default, any existing attachments at the location with the same filename will be deleted before the new attachment is uploaded, to avoid cluttering the location's attachment list.
+
+## Easy integration with WebPortal security using the `/AttachmentTags=` option
+
+The `/AttachmentTags=` option can be set multiple times to apply as many tags as needed to all the uploaded attachments.
+
+- Tags are specified in key:value format, with a colon separating the two parts.
+- The tag must be configured with AppliedToAttachments = true.
+- If a tag has its ValueType of None, then no value portion is required.
+
+AQUARIUS WebPortal can be configured to display attachments with specific tag patterns to specific view groups.
+
+See your AQUARIUS WebPortal Admin Guide for configuration details.
 
 ### `/help` screen
 
 ```
-Export observations from AQUARIUS Samples using a spreadsheet template and into AQUARIUS Time-Series.
+Export observations from AQUARIUS Samples using a spreadsheet template into AQUARIUS Time-Series locations as attachments.
 
 usage: ObservationReportExporter [-option=value] [@optionsFile] ...
 
@@ -60,17 +97,18 @@ Supported -option=value settings (/option=value works too):
   -TimeSeriesPassword         AQTS password
 
   =========================== Export options:
+  -DryRun                     When true, don't export and upload reports, just validate what would be done. [default: False]
   -ExportTemplateName         The Observation Export Spreadsheet Template to use for all exports.
-  -AttachmentFilename         Name of the generated report. Existing attachments with the same name will be deleted. [default: Report.xlsx]
+  -AttachmentFilename         Filename of the exported attachment. [default: {Template}-{Location}.xlsx]
   -AttachmentTags             Uploaded attachments will have these tag values applies, in key:value format.
   -DeleteExistingAttachments  Delete any existing location attachments with the same name. [default: True]
-  -DryRun                     When true, don't export and upload reports, just validate what would be done. [default: False]
+  -ExportTime                 The timestamp used for all {Time} pattern substitutions. [default: The current time]
 
   =========================== Cumulative filter options: (ie. AND-ed together). Can be set multiple times.
-  -StartTime                  Include observations after this time.
-  -EndTime                    Include observations before this time.
+  -StartTime                  Include observations after this time. [default: Start of record]
+  -EndTime                    Include observations before this time. [default: End of record]
   -LocationId                 Observations matching these sampling locations.
-  -LocationGroupId            Observations matching these sample location groups.
+  -LocationGroupId            Observations matching these sampling location groups.
   -AnalyticalGroupId          Observations matching these analytical groups.
   -ObservedPropertyId         Observations matching these observed properties.