diff --git a/Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj b/Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj index 6b517ae..1e038c2 100644 --- a/Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj +++ b/Samples/DotNetSdk/LabFileImporter/LabFileImporter.csproj @@ -60,8 +60,8 @@ ..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll - - ..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll + + ..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll ..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll @@ -73,7 +73,7 @@ ..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll - ..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll + ..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll ..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll diff --git a/Samples/DotNetSdk/LabFileImporter/packages.config b/Samples/DotNetSdk/LabFileImporter/packages.config index 3f36e7d..7d765c6 100644 --- a/Samples/DotNetSdk/LabFileImporter/packages.config +++ b/Samples/DotNetSdk/LabFileImporter/packages.config @@ -13,11 +13,11 @@ - + - + diff --git a/Samples/DotNetSdk/NWFWMD-LabFileImporter/NWFWMD-LabFileImporter.csproj b/Samples/DotNetSdk/NWFWMD-LabFileImporter/NWFWMD-LabFileImporter.csproj index ee84f73..990dc02 100644 --- a/Samples/DotNetSdk/NWFWMD-LabFileImporter/NWFWMD-LabFileImporter.csproj +++ b/Samples/DotNetSdk/NWFWMD-LabFileImporter/NWFWMD-LabFileImporter.csproj @@ -58,8 +58,8 @@ ..\packages\NodaTime.1.3.0\lib\net35-Client\NodaTime.dll - - ..\packages\RestSharp.106.11.7\lib\net452\RestSharp.dll + + ..\packages\RestSharp.106.15.0\lib\net452\RestSharp.dll ..\packages\ServiceStack.Client.5.10.4\lib\net45\ServiceStack.Client.dll @@ -71,7 +71,7 @@ ..\packages\ServiceStack.Interfaces.5.10.4\lib\net472\ServiceStack.Interfaces.dll - ..\packages\ServiceStack.Logging.Log4Net.5.9.0\lib\net45\ServiceStack.Logging.Log4Net.dll + ..\packages\ServiceStack.Logging.Log4Net.5.10.4\lib\net45\ServiceStack.Logging.Log4Net.dll ..\packages\ServiceStack.Text.5.10.4\lib\net45\ServiceStack.Text.dll diff --git a/Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config b/Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config index c6bad63..ed84b68 100644 --- a/Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config +++ b/Samples/DotNetSdk/NWFWMD-LabFileImporter/packages.config @@ -12,11 +12,11 @@ - + - + diff --git a/Samples/DotNetSdk/ObservationReportExporter/Exporter.cs b/Samples/DotNetSdk/ObservationReportExporter/Exporter.cs index 8ea5093..285e861 100644 --- a/Samples/DotNetSdk/ObservationReportExporter/Exporter.cs +++ b/Samples/DotNetSdk/ObservationReportExporter/Exporter.cs @@ -39,6 +39,7 @@ public class Exporter private List AppliedTags { get; } = new List(); private int ExportedLocations { get; set; } + private int ExportedObservations { get; set; } private int SkippedLocations { get; set; } private int Errors { get; set; } private int DeletedAttachments { get; set; } @@ -59,7 +60,7 @@ public void Run() ExportAllLocations(); } - LogAction($"Exported observations to {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}."); + LogAction($"Exported {"observation".ToQuantity(ExportedObservations)} using '{ExportTemplate.CustomId}' template into {"location".ToQuantity(ExportedLocations)}, skipping {"unknown location".ToQuantity(SkippedLocations)}, deleting {"existing attachment".ToQuantity(DeletedAttachments)}, with {"detected error".ToQuantity(Errors)} in {stopwatch.Elapsed.Humanize()}."); } private void LogAction(string message) @@ -428,10 +429,6 @@ private void ExportLocation(SamplingLocation location) DeleteExistingAttachment(locationData, existingAttachment); } - LogAction($"Exporting observations from '{location.CustomId}' ..."); - - ++ExportedLocations; - var exportRequest = new GetExportObservations { EndObservedTime = FromDateTimeOffset(Context.EndTime), @@ -441,6 +438,21 @@ private void ExportLocation(SamplingLocation location) AnalyticalGroupIds = AnalyticalGroupIds, }; + var exportedObservationCount = Samples.Get(new GetObservationsV2 + { + EndObservedTime = exportRequest.EndObservedTime, + StartObservedTime = exportRequest.StartObservedTime, + SamplingLocationIds = exportRequest.SamplingLocationIds, + ObservedPropertyIds = exportRequest.ObservedPropertyIds, + AnalyticalGroupIds = exportRequest.AnalyticalGroupIds + }).TotalCount; + + LogAction($"Exporting {"observation".ToQuantity(exportedObservationCount)} from '{location.CustomId}' ..."); + + ++ExportedLocations; + ExportedObservations += exportedObservationCount; + + // Need to hack the URL until WI-4928 is fixed var url = $"{(Samples.Client as JsonServiceClient)?.BaseUri}{exportRequest.ToGetUrl()}&observationTemplateAttachmentId={ExportTemplate.Attachments.Single().Id}&format=xlsx"; if (Context.DryRun) diff --git a/Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj b/Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj index 2d6f47b..cabab2d 100644 --- a/Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj +++ b/Samples/DotNetSdk/ObservationReportExporter/ObservationReportExporter.csproj @@ -144,8 +144,8 @@ ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll - - ..\packages\System.Net.Http.4.3.0\lib\net46\System.Net.Http.dll + + ..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll True True @@ -215,7 +215,7 @@ - ..\packages\System.Text.RegularExpressions.4.3.0\lib\net463\System.Text.RegularExpressions.dll + ..\packages\System.Text.RegularExpressions.4.3.1\lib\net463\System.Text.RegularExpressions.dll True True @@ -248,6 +248,7 @@ + diff --git a/Samples/DotNetSdk/ObservationReportExporter/Program.cs b/Samples/DotNetSdk/ObservationReportExporter/Program.cs index a4e9022..46fec49 100644 --- a/Samples/DotNetSdk/ObservationReportExporter/Program.cs +++ b/Samples/DotNetSdk/ObservationReportExporter/Program.cs @@ -3,6 +3,8 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using System.Xml; using Aquarius.Samples.Client; @@ -64,8 +66,7 @@ private static void ConfigureLogging() log4net.Config.XmlConfigurator.Configure(xml.DocumentElement); - // ReSharper disable once PossibleNullReferenceException - _log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + _log = LogManager.GetLogger(GetProgramType()); ServiceStack.Logging.LogManager.LogFactory = new Log4NetFactory(); } @@ -73,8 +74,7 @@ private static void ConfigureLogging() private static byte[] LoadEmbeddedResource(string path) { - // ReSharper disable once PossibleNullReferenceException - var resourceName = $"{MethodBase.GetCurrentMethod().DeclaringType.Namespace}.{path}"; + var resourceName = $"{GetProgramType().Namespace}.{path}"; using (var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName)) { @@ -85,6 +85,11 @@ private static byte[] LoadEmbeddedResource(string path) } } + private static Type GetProgramType() + { + return MethodBase.GetCurrentMethod()?.DeclaringType; + } + private static Context ParseArgs(string[] args) { var context = new Context(); @@ -349,11 +354,33 @@ public Program(Context context) private void Run() { + using (var guard = new SingleInstanceGuard(GetContextName())) + { + if (!guard.IsAnotherInstanceRunning()) + { + new Exporter { Context = _context } + .Run(); + } + else + { + _log.Warn($"Exiting while another instance of {guard.Name} is running."); + } + } new Exporter { Context = _context } .Run(); } + + private string GetContextName() + { + var jsonText = _context.ToJson(); + + using (var sha256 = new SHA256Managed()) + { + return BitConverter.ToString(sha256.ComputeHash(Encoding.UTF8.GetBytes(jsonText))); + } + } } } diff --git a/Samples/DotNetSdk/ObservationReportExporter/Readme.md b/Samples/DotNetSdk/ObservationReportExporter/Readme.md index 81327e2..cc3cfe0 100644 --- a/Samples/DotNetSdk/ObservationReportExporter/Readme.md +++ b/Samples/DotNetSdk/ObservationReportExporter/Readme.md @@ -12,12 +12,28 @@ The intent is to run this tool on a schedule so that the uploaded location attac - Results can be filtered by optional date range, location, location group, property, or analytical group. - All filters are cumulative (ie. they are AND-ed together). The more filters you add, the fewer results will be exported. - An exit code of 0 indicates the export was successful. An exit code greater than 0 indicates an error. This allows for easy error checking when invoked from scripts. +- A log file named `ObservationReportExporter.log` will be created in the same folder as the EXE. ## Requirements - The .NET 4.7 runtime is required, which is pre-installed on all Windows 10 and Windows Server 2016 systems, and on nearly all up-to-date Windows 7 and Windows Server 2008 systems. - No installer is needed. It is just a single .EXE which can be run from any folder. +## Basic workflow + +Each run of the tool will: +- Validate all the provided options and stop right away if any errors are detected. +- Determine the AQUARIUS Samples locations from which observations will be exported. This is either the list of `-LocationId=` options locations or locations belonging to the `-LocationGroupId=` options. +- For each exported location: + - Find the matching AQUARIUS Time Series location. + - If the Time Series location is not found: + - Log a warning and move on to the next export. + - Otherwise: + - Determine the exported attachment filename (see [controlling the attachment filename](#controlling-the-aqts-attachment-filename) for details). + - Delete any existing location attachments with the same filename. + - Export all the filtered observations from the AQUARIUS Samples location using the named template. + - Upload the exported spreadsheet of location-specific observations to the AQUARIUS Time-Series location. + ### Command line option syntax All command line options are case-insensitive, and support both common shell syntaxes: either `/Name=value` (for CMD.EXE) or `-Name=value` (for bash and PowerShell). @@ -26,6 +42,22 @@ In addition, the [`@options.txt` syntax](https://github.com/AquaticInformatics/e Try the `/help` option for a detailed list of options and their default values. +## Running the tool on a periodic schedule + +The `ObservationReportExporter.exe` too can be run from Windows Task Scheduler, or other scheduling software. + +Typical configuration involves: +- Storing all required command-line options in a single text file, in the same folder as the EXE. +- Specifying the executable as full path to the `ObservationReportExporter.exe` tool. +- Setting the working directory to the folder containing the EXE. +- Setting the arguments to the `@Options.txt` file containing all the required options. + +No special Windows account is required to run the tool. All the required credentials are supplied as command-line options, so it is fine to run the tool using a built-in Windows account like LocalSystem (NT_AUTHORITY/SYSTEM). + +The tool can be scheduled to run at whatever frequency you would like. The tool will quickly exit if it detects an identical export request already in progress. This allows for simple scheduling for normal loads. + +If a complete export cycle normally takes 2 hours, but can sometimes take 4 hours to complete, you can still safely schedule the tool to run every 3 hours, and tool will detect when a previous cycle hasn't completed and won't go crazy. + ## Authentication with AQUARIUS Samples Two options are required to tell the tool how to access your AQUARIUS Samples instance. diff --git a/Samples/DotNetSdk/ObservationReportExporter/SingleInstanceGuard.cs b/Samples/DotNetSdk/ObservationReportExporter/SingleInstanceGuard.cs new file mode 100644 index 0000000..b46efd1 --- /dev/null +++ b/Samples/DotNetSdk/ObservationReportExporter/SingleInstanceGuard.cs @@ -0,0 +1,73 @@ +using System; +using System.Reflection; +using System.Threading; +using log4net; + +namespace ObservationReportExporter +{ + public class SingleInstanceGuard : IDisposable + { + // ReSharper disable once PossibleNullReferenceException + private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + private Mutex InstanceMutex { get; set; } + public string Name { get; } + private bool ShouldRelease { get; set; } + + public SingleInstanceGuard(string name) + { + Name = $"{ExeHelper.ExeName}.{name}"; + InstanceMutex = new Mutex(true, Name); + ShouldRelease = true; + } + + public bool IsAnotherInstanceRunning() + { + try + { + var isAnotherInstanceRunning = !InstanceMutex.WaitOne(TimeSpan.Zero, true); + + if (isAnotherInstanceRunning) + { + ShouldRelease = false; + } + + return isAnotherInstanceRunning; + } + catch (AbandonedMutexException) + { + Log.Debug($"Previous run of the program did not clear the '{Name}' mutex cleanly."); + + return false; + } + catch (Exception ex) + { + Log.Warn($"Error occurred while checking if the program is still running:'{ex.Message}'. Will continue."); + } + + return false; + } + + public void Dispose() + { + Release(); + + InstanceMutex?.Dispose(); + } + + private void Release() + { + if (!ShouldRelease) + return; + + try + { + InstanceMutex?.ReleaseMutex(); + } + catch (Exception e) + { + Log.Warn($"Can't release mutex '{Name}': {e.Message}"); + } + } + } +}