Skip to content

Commit

Permalink
Merge pull request #291 from DougSchmidt-AI/feature/PF-1365-NtdnRepor…
Browse files Browse the repository at this point in the history
…tingTweaks

PF-1365 - Minor tweaks
  • Loading branch information
Doug Schmidt authored Feb 9, 2022
2 parents dc77e81 + b245225 commit 8978eb3
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 22 deletions.
6 changes: 3 additions & 3 deletions Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
<Reference Include="NodaTime, Version=1.3.0.0, Culture=neutral, PublicKeyToken=4226afe0d9b296d1, processorArchitecture=MSIL">
<HintPath>..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll</HintPath>
</Reference>
<Reference Include="RestSharp, Version=106.11.7.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
<HintPath>..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll</HintPath>
<Reference Include="RestSharp, Version=106.15.0.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
<HintPath>..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Client, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll</HintPath>
Expand All @@ -73,7 +73,7 @@
<HintPath>..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Logging.Log4Net, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Text, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll</HintPath>
Expand Down
4 changes: 2 additions & 2 deletions Samples/DotNetSdk/LabFileImporter/packages.config
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
<package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
<package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net472" />
<package id="NodaTime" version="1.3.0" targetFramework="net472" />
<package id="RestSharp" version="106.12.0" targetFramework="net472" />
<package id="RestSharp" version="106.15.0" targetFramework="net472" />
<package id="ServiceStack.Client" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.HttpClient" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Interfaces" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Logging.Log4Net" version="5.9.0" targetFramework="net472" />
<package id="ServiceStack.Logging.Log4Net" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Text" version="5.10.4" targetFramework="net472" />
<package id="System.Buffers" version="4.5.1" targetFramework="net472" />
<package id="System.Memory" version="4.5.4" targetFramework="net472" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@
<Reference Include="NodaTime, Version=1.3.0.0, Culture=neutral, PublicKeyToken=4226afe0d9b296d1, processorArchitecture=MSIL">
<HintPath>..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll</HintPath>
</Reference>
<Reference Include="RestSharp, Version=106.11.7.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
<HintPath>..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll</HintPath>
<Reference Include="RestSharp, Version=106.15.0.0, Culture=neutral, PublicKeyToken=598062e77f915f75, processorArchitecture=MSIL">
<HintPath>..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Client, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll</HintPath>
Expand All @@ -71,7 +71,7 @@
<HintPath>..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Logging.Log4Net, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
<HintPath>..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll</HintPath>
</Reference>
<Reference Include="ServiceStack.Text, Version=5.0.0.0, Culture=neutral, PublicKeyToken=02c12cbda47e6587, processorArchitecture=MSIL">
<HintPath>..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll</HintPath>
Expand Down
4 changes: 2 additions & 2 deletions Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
<package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
<package id="Microsoft.Net.Http" version="2.2.29" targetFramework="net472" />
<package id="NodaTime" version="1.3.0" targetFramework="net472" />
<package id="RestSharp" version="106.12.0" targetFramework="net472" />
<package id="RestSharp" version="106.15.0" targetFramework="net472" />
<package id="ServiceStack.Client" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.HttpClient" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Interfaces" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Logging.Log4Net" version="5.9.0" targetFramework="net472" />
<package id="ServiceStack.Logging.Log4Net" version="5.10.4" targetFramework="net472" />
<package id="ServiceStack.Text" version="5.10.4" targetFramework="net472" />
<package id="System.Buffers" version="4.5.1" targetFramework="net472" />
<package id="System.Memory" version="4.5.4" targetFramework="net472" />
Expand Down
22 changes: 17 additions & 5 deletions Samples/DotNetSdk/ObservationReportExporter/Exporter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class Exporter
private List<ApplyTagRequest> AppliedTags { get; } = new List<ApplyTagRequest>();

private int ExportedLocations { get; set; }
private int ExportedObservations { get; set; }
private int SkippedLocations { get; set; }
private int Errors { get; set; }
private int DeletedAttachments { get; set; }
Expand All @@ -59,7 +60,7 @@ public void Run()
ExportAllLocations();
}

LogAction($"Exported observations to {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}.");
LogAction($"Exported {"observation".ToQuantity(ExportedObservations)} using '{ExportTemplate.CustomId}' template into {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}.");
}

private void LogAction(string message)
Expand Down Expand Up @@ -428,10 +429,6 @@ private void ExportLocation(SamplingLocation location)
DeleteExistingAttachment(locationData, existingAttachment);
}

LogAction($"Exporting observations from '{location.CustomId}' ...");

++ExportedLocations;

var exportRequest = new GetExportObservations
{
EndObservedTime = FromDateTimeOffset(Context.EndTime),
Expand All @@ -441,6 +438,21 @@ private void ExportLocation(SamplingLocation location)
AnalyticalGroupIds = AnalyticalGroupIds,
};

var exportedObservationCount = Samples.Get(new GetObservationsV2
{
EndObservedTime = exportRequest.EndObservedTime,
StartObservedTime = exportRequest.StartObservedTime,
SamplingLocationIds = exportRequest.SamplingLocationIds,
ObservedPropertyIds = exportRequest.ObservedPropertyIds,
AnalyticalGroupIds = exportRequest.AnalyticalGroupIds
}).TotalCount;

LogAction($"Exporting {"observation".ToQuantity(exportedObservationCount)} from '{location.CustomId}' ...");

++ExportedLocations;
ExportedObservations += exportedObservationCount;

// Need to hack the URL until WI-4928 is fixed
var url = $"{(Samples.Client as JsonServiceClient)?.BaseUri}{exportRequest.ToGetUrl()}&observationTemplateAttachmentId={ExportTemplate.Attachments.Single().Id}&format=xlsx";

if (Context.DryRun)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@
<HintPath>..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Net" />
<Reference Include="System.Net.Http, Version=4.1.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Net.Http.4.3.0\lib\net46\System.Net.Http.dll</HintPath>
<Reference Include="System.Net.Http, Version=4.1.1.3, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll</HintPath>
<Private>True</Private>
<Private>True</Private>
</Reference>
Expand Down Expand Up @@ -215,7 +215,7 @@
</Reference>
<Reference Include="System.ServiceModel" />
<Reference Include="System.Text.RegularExpressions, Version=4.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Text.RegularExpressions.4.3.0\lib\net463\System.Text.RegularExpressions.dll</HintPath>
<HintPath>..\packages\System.Text.RegularExpressions.4.3.1\lib\net463\System.Text.RegularExpressions.dll</HintPath>
<Private>True</Private>
<Private>True</Private>
</Reference>
Expand Down Expand Up @@ -248,6 +248,7 @@
<Compile Include="Option.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="SingleInstanceGuard.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
Expand Down
35 changes: 31 additions & 4 deletions Samples/DotNetSdk/ObservationReportExporter/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
using System.IO;
using System.Linq;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Aquarius.Samples.Client;
Expand Down Expand Up @@ -64,17 +66,15 @@ private static void ConfigureLogging()

log4net.Config.XmlConfigurator.Configure(xml.DocumentElement);

// ReSharper disable once PossibleNullReferenceException
_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
_log = LogManager.GetLogger(GetProgramType());

ServiceStack.Logging.LogManager.LogFactory = new Log4NetFactory();
}
}

private static byte[] LoadEmbeddedResource(string path)
{
// ReSharper disable once PossibleNullReferenceException
var resourceName = $"{MethodBase.GetCurrentMethod().DeclaringType.Namespace}.{path}";
var resourceName = $"{GetProgramType().Namespace}.{path}";

using (var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName))
{
Expand All @@ -85,6 +85,11 @@ private static byte[] LoadEmbeddedResource(string path)
}
}

private static Type GetProgramType()
{
return MethodBase.GetCurrentMethod()?.DeclaringType;
}

private static Context ParseArgs(string[] args)
{
var context = new Context();
Expand Down Expand Up @@ -349,11 +354,33 @@ public Program(Context context)

private void Run()
{
using (var guard = new SingleInstanceGuard(GetContextName()))
{
if (!guard.IsAnotherInstanceRunning())
{
new Exporter { Context = _context }
.Run();
}
else
{
_log.Warn($"Exiting while another instance of {guard.Name} is running.");
}
}
new Exporter
{
Context = _context
}
.Run();
}

private string GetContextName()
{
var jsonText = _context.ToJson();

using (var sha256 = new SHA256Managed())
{
return BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(jsonText)));
}
}
}
}
32 changes: 32 additions & 0 deletions Samples/DotNetSdk/ObservationReportExporter/Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,28 @@ The intent is to run this tool on a schedule so that the uploaded location attac
- Results can be filtered by optional date range, location, location group, property, or analytical group.
- All filters are cumulative (ie. they are AND-ed together). The more filters you add, the fewer results will be exported.
- An exit code of 0 indicates the export was successful. An exit code greater than 0 indicates an error. This allows for easy error checking when invoked from scripts.
- A log file named `ObservationReportExporter.log` will be created in the same folder as the EXE.

## Requirements

- The .NET 4.7 runtime is required, which is pre-installed on all Windows 10 and Windows Server 2016 systems, and on nearly all up-to-date Windows 7 and Windows Server 2008 systems.
- No installer is needed. It is just a single .EXE which can be run from any folder.

## Basic workflow

Each run of the tool will:
- Validate all the provided options and stop right away if any errors are detected.
- Determine the AQUARIUS Samples locations from which observations will be exported. This is either the list of `-LocationId=` options locations or locations belonging to the `-LocationGroupId=` options.
- For each exported location:
- Find the matching AQUARIUS Time Series location.
- If the Time Series location is not found:
- Log a warning and move on to the next export.
- Otherwise:
- Determine the exported attachment filename (see [controlling the attachment filename](#controlling-the-aqts-attachment-filename) for details).
- Delete any existing location attachments with the same filename.
- Export all the filtered observations from the AQUARIUS Samples location using the named template.
- Upload the exported spreadsheet of location-specific observations to the AQUARIUS Time-Series location.

### Command line option syntax

All command line options are case-insensitive, and support both common shell syntaxes: either `/Name=value` (for CMD.EXE) or `-Name=value` (for bash and PowerShell).
Expand All @@ -26,6 +42,22 @@ In addition, the [`@options.txt` syntax](https://github.com/AquaticInformatics/e

Try the `/help` option for a detailed list of options and their default values.

## Running the tool on a periodic schedule

The `ObservationReportExporter.exe` too can be run from Windows Task Scheduler, or other scheduling software.

Typical configuration involves:
- Storing all required command-line options in a single text file, in the same folder as the EXE.
- Specifying the executable as full path to the `ObservationReportExporter.exe` tool.
- Setting the working directory to the folder containing the EXE.
- Setting the arguments to the `@Options.txt` file containing all the required options.

No special Windows account is required to run the tool. All the required credentials are supplied as command-line options, so it is fine to run the tool using a built-in Windows account like LocalSystem (NT_AUTHORITY/SYSTEM).

The tool can be scheduled to run at whatever frequency you would like. The tool will quickly exit if it detects an identical export request already in progress. This allows for simple scheduling for normal loads.

If a complete export cycle normally takes 2 hours, but can sometimes take 4 hours to complete, you can still safely schedule the tool to run every 3 hours, and tool will detect when a previous cycle hasn't completed and won't go crazy.

## Authentication with AQUARIUS Samples

Two options are required to tell the tool how to access your AQUARIUS Samples instance.
Expand Down
73 changes: 73 additions & 0 deletions Samples/DotNetSdk/ObservationReportExporter/SingleInstanceGuard.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
using System;
using System.Reflection;
using System.Threading;
using log4net;

namespace ObservationReportExporter
{
public class SingleInstanceGuard : IDisposable
{
// ReSharper disable once PossibleNullReferenceException
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

private Mutex InstanceMutex { get; set; }
public string Name { get; }
private bool ShouldRelease { get; set; }

public SingleInstanceGuard(string name)
{
Name = $"{ExeHelper.ExeName}.{name}";
InstanceMutex = new Mutex(true, Name);
ShouldRelease = true;
}

public bool IsAnotherInstanceRunning()
{
try
{
var isAnotherInstanceRunning = !InstanceMutex.WaitOne(TimeSpan.Zero, true);

if (isAnotherInstanceRunning)
{
ShouldRelease = false;
}

return isAnotherInstanceRunning;
}
catch (AbandonedMutexException)
{
Log.Debug($"Previous run of the program did not clear the '{Name}' mutex cleanly.");

return false;
}
catch (Exception ex)
{
Log.Warn($"Error occurred while checking if the program is still running:'{ex.Message}'. Will continue.");
}

return false;
}

public void Dispose()
{
Release();

InstanceMutex?.Dispose();
}

private void Release()
{
if (!ShouldRelease)
return;

try
{
InstanceMutex?.ReleaseMutex();
}
catch (Exception e)
{
Log.Warn($"Can't release mutex '{Name}': {e.Message}");
}
}
}
}

0 comments on commit 8978eb3

Please sign in to comment.