From adc8a008f6c8bcc99731d51ae22ae0992c49d232 Mon Sep 17 00:00:00 2001 From: Doug Schmidt Date: Mon, 4 Feb 2019 14:14:42 -0800 Subject: [PATCH] Issue-115 - First cut at working WaterWatch sensor export By default, all new measurements for all sensors in the WaterWatch organisation will be exported to standard output. Filtering options are provided to exclude specific sensors by name or serial number patterns. --- .../WaterWatchPreProcessor/Context.cs | 17 ++- .../WaterWatchPreProcessor/Dtos/SavedState.cs | 11 ++ .../Dtos/WaterWatch/Alarm.cs | 8 ++ .../Dtos/WaterWatch/Config.cs | 16 +++ .../Dtos/WaterWatch/DisplayInfo.cs | 12 ++ .../Dtos/WaterWatch/GetMeasurementsRequest.cs | 16 +++ .../WaterWatch/GetMeasurementsResponse.cs | 10 ++ .../Dtos/WaterWatch/GetSensorsRequest.cs | 11 ++ .../Dtos/WaterWatch/LatestData.cs | 15 ++ .../Dtos/WaterWatch/Measurement.cs | 10 ++ .../Dtos/WaterWatch/ReferenceLine.cs | 9 ++ .../Dtos/WaterWatch/Sensor.cs | 19 +++ .../WaterWatchPreProcessor/Exporter.cs | 112 ++++++++++++++- .../WaterWatchPreProcessor/Filters/Filter.cs | 28 ++++ .../WaterWatchPreProcessor/Filters/IFilter.cs | 7 + .../Filters/RegexFilter.cs | 10 ++ .../WaterWatchPreProcessor/OutputMode.cs | 8 ++ .../WaterWatchPreProcessor/Program.cs | 131 ++++++++++++++++-- .../WaterWatchPreProcessor/Readme.md | 67 ++++++++- .../WaterWatchPreProcessor.csproj | 15 ++ 20 files changed, 513 insertions(+), 19 deletions(-) create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/SavedState.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Alarm.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Config.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/DisplayInfo.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsRequest.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsResponse.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetSensorsRequest.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/LatestData.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Measurement.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/ReferenceLine.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Sensor.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/Filter.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/IFilter.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/RegexFilter.cs create mode 100644 TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/OutputMode.cs diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Context.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Context.cs index 88427143..9ca28d0b 100644 --- a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Context.cs +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Context.cs @@ -1,10 +1,19 @@ -namespace WaterWatchPreProcessor +using System; +using System.Collections.Generic; +using WaterWatchPreProcessor.Filters; + +namespace WaterWatchPreProcessor { public class Context { - public string WaterWaterOrgId { get; set; } - public string WaterWaterApiKey { get; set; } - public string WaterWaterApiToken { get; set; } + public string WaterWatchOrgId { get; set; } + public string WaterWatchApiKey { get; set; } + public string WaterWatchApiToken { get; set; } public string SaveStatePath { get; set; } = "WaterWatchSaveState.json"; + public OutputMode OutputMode { get; set; } + public List SensorSerialFilters { get; set; } = new List(); + public List SensorNameFilters { get; set; } = new List(); + public DateTime? SyncFromUtc { get; set; } + public int NewSensorSyncDays { get; set; } = 5; } } diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/SavedState.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/SavedState.cs new file mode 100644 index 00000000..9c42d3d1 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/SavedState.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace WaterWatchPreProcessor.Dtos +{ + public class SavedState + { + public Dictionary LastSeenBySensorSerial { get; set; } = + new Dictionary(StringComparer.InvariantCultureIgnoreCase); + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Alarm.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Alarm.cs new file mode 100644 index 00000000..51bffb0a --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Alarm.cs @@ -0,0 +1,8 @@ +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class Alarm + { + public int Threshold { get; set; } + public int Type { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Config.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Config.cs new file mode 100644 index 00000000..57f6f3a6 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Config.cs @@ -0,0 +1,16 @@ +using System; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class Config + { + public bool DeltaCompressionEnabled { get; set; } + public DateTime? LastSchedulerOnTime { get; set; } + public int MeasurementInterval { get; set; } + public bool MeasurementTransmissionEnabled { get; set; } + public bool SchedulerEnabled { get; set; } + public bool ServerSideAlarm { get; set; } + public int TransmissionInterval { get; set; } + public Alarm Alarm { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/DisplayInfo.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/DisplayInfo.cs new file mode 100644 index 00000000..7faaa3a3 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/DisplayInfo.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class DisplayInfo + { + public double? MaxLevel { get; set; } + public double? MinLevel { get; set; } + public double? OffsetMeasurement { get; set; } + public IList ReferenceLines { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsRequest.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsRequest.cs new file mode 100644 index 00000000..6e1d8243 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsRequest.cs @@ -0,0 +1,16 @@ +using System; +using ServiceStack; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + [Route("/organisations/{OrganisationId}/sensors/{SensorSerial}/measurements", HttpMethods.Get)] + public class GetMeasurementsRequest : IReturn + { + public string OrganisationId { get; set; } + public string SensorSerial { get; set; } + public DateTime StartTime { get; set; } + public DateTime? EndTime { get; set; } + public string Order { get; set; } + public string Start { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsResponse.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsResponse.cs new file mode 100644 index 00000000..1ea4158c --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsResponse.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class GetMeasurementsResponse + { + public IList Measurements { get; set; } + public string Next { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetSensorsRequest.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetSensorsRequest.cs new file mode 100644 index 00000000..a14cea9a --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetSensorsRequest.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using ServiceStack; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + [Route("/organisations/{OrganisationId}/sensors", HttpMethods.Get)] + public class GetSensorsRequest : IReturn> + { + public string OrganisationId { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/LatestData.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/LatestData.cs new file mode 100644 index 00000000..a8b69960 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/LatestData.cs @@ -0,0 +1,15 @@ +using System; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class LatestData + { + public bool AlertAsserted { get; set; } + public bool OfflineAsserted { get; set; } + public DateTime LastSeen { get; set; } + public Measurement LastMeasurement { get; set; } + public int SignalLevel { get; set; } + public double Battery { get; set; } + public string LinkQuality { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Measurement.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Measurement.cs new file mode 100644 index 00000000..cd2b9045 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Measurement.cs @@ -0,0 +1,10 @@ +using System; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class Measurement + { + public DateTime Time { get; set; } + public double RawDistance { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/ReferenceLine.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/ReferenceLine.cs new file mode 100644 index 00000000..ed8c66e5 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/ReferenceLine.cs @@ -0,0 +1,9 @@ +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class ReferenceLine + { + public string Color { get; set; } + public int Value { get; set; } + public string Label { get; set; } + } +} \ No newline at end of file diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Sensor.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Sensor.cs new file mode 100644 index 00000000..e2997ce1 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Sensor.cs @@ -0,0 +1,19 @@ +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace WaterWatchPreProcessor.Dtos.WaterWatch +{ + public class Sensor + { + public string OrganisationId { get; set; } + public string Name { get; set; } + public string Serial { get; set; } + public string SensorType { get; set; } + public double? Longitude { get; set; } + public double? Latitude { get; set; } + public DisplayInfo DisplayInfo { get; set; } + public Config Config { get; set; } + public LatestData LatestData { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Exporter.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Exporter.cs index 736c9b86..521c9276 100644 --- a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Exporter.cs +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Exporter.cs @@ -1,11 +1,121 @@ -namespace WaterWatchPreProcessor +using System; +using System.Collections.Generic; +using System.IO; +using ServiceStack; +using ServiceStack.Text; +using WaterWatchPreProcessor.Dtos; +using WaterWatchPreProcessor.Dtos.WaterWatch; +using WaterWatchPreProcessor.Filters; + +namespace WaterWatchPreProcessor { public class Exporter { public Context Context { get; set; } + private JsonServiceClient Client { get; set; } + + private SavedState SavedState { get; set; } = new SavedState(); + public void Run() { + RestoreSavedState(); + + CreateConnectedClient(); + + var sensors = GetSensors(); + + var nameFilter = new Filter(Context.SensorNameFilters); + var serialFilter = new Filter(Context.SensorSerialFilters); + + foreach (var sensor in sensors) + { + if (nameFilter.IsFiltered(f => f.Regex.IsMatch(sensor.Name)) + || serialFilter.IsFiltered(f => f.Regex.IsMatch(sensor.Serial))) + continue; + + foreach (var measurement in GetSensorMeasurements(sensor)) + { + Console.WriteLine($"{measurement.Time:yyyy-MM-ddTHH:mm:ss.fffZ}, {sensor.SensorType}, {sensor.Serial}, {GetSensorValue(sensor, measurement.RawDistance)}"); + } + + SavedState.LastSeenBySensorSerial[sensor.Serial] = sensor.LatestData.LastSeen; + } + + PersistSavedState(); + } + + private void RestoreSavedState() + { + if (!File.Exists(Context.SaveStatePath)) + return; + + SavedState = File.ReadAllText(Context.SaveStatePath).FromJson(); + } + + private void PersistSavedState() + { + File.WriteAllText(Context.SaveStatePath, SavedState.ToJson().IndentJson()); + } + + private void CreateConnectedClient() + { + Client = new JsonServiceClient("https://api.waterwatch.io/v1") + { + AlwaysSendBasicAuthHeader = true, + UserName = Context.WaterWatchApiKey, + Password = Context.WaterWatchApiToken + }; + } + + private IList GetSensors() + { + return Client.Get(new GetSensorsRequest + { + OrganisationId = Context.WaterWatchOrgId + }); + } + + private IEnumerable GetSensorMeasurements(Sensor sensor) + { + if (!SavedState.LastSeenBySensorSerial.TryGetValue(sensor.Serial, out var lastSeenTime)) + { + lastSeenTime = DateTime.UtcNow.Date.AddDays(-Context.NewSensorSyncDays); + } + + if (Context.SyncFromUtc.HasValue) + lastSeenTime = Context.SyncFromUtc.Value; + + var request = new GetMeasurementsRequest + { + OrganisationId = sensor.OrganisationId, + SensorSerial = sensor.Serial, + StartTime = lastSeenTime, + Order = "asc" + }; + + do + { + var response = Client.Get(request); + + foreach (var measurement in response.Measurements) + { + yield return measurement; + } + + request.Start = response.Next; + + } while (!string.IsNullOrEmpty(request.Start)); + } + + private double GetSensorValue(Sensor sensor, double rawDistance) + { + if (Context.OutputMode == OutputMode.OffsetCorrected && (sensor.DisplayInfo?.OffsetMeasurement.HasValue ?? false)) + { + return sensor.DisplayInfo.OffsetMeasurement.Value - rawDistance; + } + + return rawDistance; } } } diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/Filter.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/Filter.cs new file mode 100644 index 00000000..cfc683e4 --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/Filter.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace WaterWatchPreProcessor.Filters +{ + public class Filter where TFilter : IFilter + { + private List IncludeFilters { get; } + private List ExcludeFilters { get; } + public int Count { get; private set; } + + public Filter(List filters) + { + IncludeFilters = filters.Where(filter => !filter.Exclude).ToList(); + ExcludeFilters = filters.Where(filter => filter.Exclude).ToList(); + } + + public bool IsFiltered(Func predicate) + { + if ((!IncludeFilters.Any() || IncludeFilters.Any(predicate)) && !ExcludeFilters.Any(predicate)) + return false; + + ++Count; + return true; + } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/IFilter.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/IFilter.cs new file mode 100644 index 00000000..8abff4ed --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/IFilter.cs @@ -0,0 +1,7 @@ +namespace WaterWatchPreProcessor.Filters +{ + public interface IFilter + { + bool Exclude { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/RegexFilter.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/RegexFilter.cs new file mode 100644 index 00000000..b3d7bf0f --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/RegexFilter.cs @@ -0,0 +1,10 @@ +using System.Text.RegularExpressions; + +namespace WaterWatchPreProcessor.Filters +{ + public class RegexFilter : IFilter + { + public bool Exclude { get; set; } + public Regex Regex { get; set; } + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/OutputMode.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/OutputMode.cs new file mode 100644 index 00000000..d620731d --- /dev/null +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/OutputMode.cs @@ -0,0 +1,8 @@ +namespace WaterWatchPreProcessor +{ + public enum OutputMode + { + OffsetCorrected, + RawDistance, + } +} diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Program.cs b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Program.cs index e9bf116e..e03a03c8 100644 --- a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Program.cs +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -8,6 +9,8 @@ using log4net; using ServiceStack; using ServiceStack.Logging.Log4Net; +using ServiceStack.Text; +using WaterWatchPreProcessor.Filters; namespace WaterWatchPreProcessor { @@ -47,10 +50,8 @@ public static void Main(string[] args) else logAction($"{exception.Message}\n{exception.StackTrace}"); } - } - private static void ConfigureLogging() { using (var stream = new MemoryStream(EmbeddedResource.LoadEmbeddedResource("log4net.config"))) @@ -69,6 +70,8 @@ private static void ConfigureLogging() private static void ConfigureJson() { + JsConfig.EmitCamelCaseNames = true; + JsConfig.DateHandler = DateHandler.UnixTimeMs; } private static string GetProgramName() @@ -86,10 +89,71 @@ private static Context ParseArgs(string[] args) var options = new[] { - new Option {Key = nameof(context.WaterWaterOrgId), Setter = value => context.WaterWaterOrgId = value, Getter = () => context.WaterWaterOrgId, Description = "WaterWatch.io organisation Id"}, - new Option {Key = nameof(context.WaterWaterApiKey), Setter = value => context.WaterWaterApiKey = value, Getter = () => context.WaterWaterApiKey, Description = "WaterWatch.io API key"}, - new Option {Key = nameof(context.WaterWaterApiToken), Setter = value => context.WaterWaterApiToken = value, Getter = () => context.WaterWaterApiToken, Description = "WaterWatch.io API token"}, - new Option {Key = nameof(context.SaveStatePath), Setter = value => context.SaveStatePath = value, Getter = () => context.SaveStatePath, Description = "Path to persisted state file"}, + new Option {Description = "https://waterwatch.io credentials"}, + new Option + { + Key = nameof(context.WaterWatchOrgId), + Description = "WaterWatch.io organisation Id", + Setter = value => context.WaterWatchOrgId = value, + Getter = () => context.WaterWatchOrgId, + }, + new Option + { + Key = nameof(context.WaterWatchApiKey), + Description = "WaterWatch.io API key", + Setter = value => context.WaterWatchApiKey = value, + Getter = () => context.WaterWatchApiKey, + }, + new Option + { + Key = nameof(context.WaterWatchApiToken), + Description = "WaterWatch.io API token", + Setter = value => context.WaterWatchApiToken = value, + Getter = () => context.WaterWatchApiToken, + }, + + new Option(), new Option {Description = "Configuration options"}, + new Option + { + Key = nameof(context.OutputMode), + Description = $"Measurement value output mode. One of {string.Join(", ", Enum.GetNames(typeof(OutputMode)))}.", + Setter = value => context.OutputMode = (OutputMode) Enum.Parse(typeof(OutputMode), value, true), + Getter = () => context.OutputMode.ToString(), + }, + new Option + { + Key = nameof(context.SaveStatePath), + Description = "Path to persisted state file", + Setter = value => context.SaveStatePath = value, + Getter = () => context.SaveStatePath, + }, + new Option + { + Key = nameof(context.SyncFromUtc), + Description = "Optional UTC sync time. [default: last known sensor time]", + Setter = value => context.SyncFromUtc = ParseDateTime(value), + }, + new Option + { + Key = nameof(context.NewSensorSyncDays), + Description = "Number of days to sync data when a new sensor is detected.", + Setter = value => context.NewSensorSyncDays = int.Parse(value), + Getter = () => context.NewSensorSyncDays.ToString(), + }, + + new Option(), new Option {Description = "Sensor filtering options"}, + new Option + { + Key = "SensorName", + Description = "Sensor name regular expression filter. Can be specified multiple times.", + Setter = value => context.SensorNameFilters.Add(ParseRegexFilter(value)) + }, + new Option + { + Key = "SensorSerial", + Description = "Sensor serial number regular expression filter. Can be specified multiple times.", + Setter = value => context.SensorSerialFilters.Add(ParseRegexFilter(value)) + }, }; var usageMessage @@ -99,6 +163,10 @@ var usageMessage + $"\n" + $"\nSupported -option=value settings (/option=value works too):\n\n {string.Join("\n ", options.Select(o => o.UsageText()))}" + $"\n" + + $"\nSupported /{nameof(context.SyncFromUtc)} date formats:" + + $"\n" + + $"\n {string.Join("\n ", SupportedDateFormats)}" + + $"\n" + $"\nUse the @optionsFile syntax to read more options from a file." + $"\n" + $"\n Each line in the file is treated as a command line option." @@ -132,9 +200,9 @@ var usageMessage option.Setter(value); } - if (string.IsNullOrWhiteSpace(context.WaterWaterOrgId) - || string.IsNullOrEmpty(context.WaterWaterApiKey) - || string.IsNullOrEmpty(context.WaterWaterApiToken)) + if (string.IsNullOrWhiteSpace(context.WaterWatchOrgId) + || string.IsNullOrEmpty(context.WaterWatchApiKey) + || string.IsNullOrEmpty(context.WaterWatchApiToken)) throw new ExpectedException($"Ensure your WaterWatch account credentials are set."); return context; @@ -165,5 +233,50 @@ private static IEnumerable ResolveOptionsFromFile(string arg) .Where(s => !s.StartsWith("#") && !s.StartsWith("//")); } + private static RegexFilter ParseRegexFilter(string value) + { + var filter = ParseExclusionFiler(value); + + return new RegexFilter + { + Exclude = filter.Exclude, + Regex = new Regex(filter.Text) + }; + } + + private static (bool Exclude, string Text) ParseExclusionFiler(string value) + { + var exclude = false; + var text = value; + + if (value.StartsWith("+")) + { + text = value.Substring(1); + } + else if (value.StartsWith("-")) + { + exclude = true; + text = value.Substring(1); + } + + return (exclude, text); + } + + private static DateTime ParseDateTime(string value) + { + return DateTime.ParseExact( + value, + SupportedDateFormats, + CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal); + } + + private static readonly string[] SupportedDateFormats = + { + "yyyy-MM-dd", + "yyyy-MM-ddTHH:mm", + "yyyy-MM-ddTHH:mm:ss", + "yyyy-MM-ddTHH:mm:ss.fff", + }; } } diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Readme.md b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Readme.md index 655e285b..dcea4715 100644 --- a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Readme.md +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Readme.md @@ -15,16 +15,56 @@ The tool supports the [common command line options](https://github.com/AquaticIn At a minimum, this tool needs three pieces of information from your WaterWatch account, which can be found in the [Settings/APIs & Integration](https://help.waterwatch.io/article/20-waterwatch-rest-api-documentation) menu of the WaterWatch web application: - `/WaterWatchOrgId=` - Organisation ID - `/WaterWatchApiKey=` - API Key -- `/WaterWatchApiKey=` - API Token +- `/WaterWatchApiToken=` - API Token + +## Forcing a resync from a specific time + +Use the `/SyncFromUtc=` option to force a resync of data from a specific time. + +When not specified, the tool will sync from the last known time for each sensor. + +## Filter sensors by regular expression + +You can specify [.NET regular expressions](http://regexstorm.net/tester?p=WW&i=WWSensor1%0aSensor2%0aWWSensor3) to filter the list of exported sensors by patterns in the sensor name or sensor serial text properties. + +Use the `/SensorName=regex` option to filter by the sensor name. + +Use the `/SensorSerial=regex` option to filter by the sensor serial number. + +The `regex` regular expression just needs to match a portion of the text property being filtered. So `/SensorName=Public` will match all sensors with a name containing the phrase "Public". + +- Regular expressions perform case-sensitve matching. +- More than one `/SensorName=regex` or `/SensorSerial=regex` filter option can be specified on the command line or in a configuration file. +- Each filter operates as either an **inclusion** or **exclusion** filter. +- A filter is assumed to be an **inclusion** filter, unless the `regex` value begins with a minus-sign. +- If no **inclusion** filters are specified, all sensors will be included by default. + +This pattern of inclusion and exclusion filtering can be used to precisely control which sensors are exported. + +Eg. Export all the sensors starting with "Public", except for serial number "ABC123": + +```cmd +WaterWatchPreProcessor /SerialName=^Public /SensorSerial="-^ABC123$" (other options ...) +``` + +## `/OutputMode=OffsetCorrected` is enabled by default + +The value reported for each sensor will be the [offset-corrected value](https://help.waterwatch.io/article/23-what-is-the-offset-and-how-do-i-set-it-up), if the sensor has been configured with an offset. + +`OffsetCorrectedValue = Offset - RawDistance;` + +Use `/OutputMode=RawDistance` to always show the sensor's rawDistance value. ## Output text stream format Each time the tool runs, it queries the WatchWatch API for any new measurements. +`Iso8601UtcTime, SensorType, SensorSerial, Value` + Each new measurement will be output to standard out as the following CSV stream: + ```csv -Iso8601UtcTime, SensorType, SensorSerial, Value 2018-12-31T14:35:00.000Z, LS1, 418892, 92.23456 2018-12-31T15:35:00.000Z, LS1, 418892, 89.47586 ``` @@ -38,10 +78,27 @@ usage: WaterWatchPreProcessor [-option=value] [@optionsFile] ... Supported -option=value settings (/option=value works too): - -WaterWaterOrgId WaterWatch.io organisation Id - -WaterWaterApiKey WaterWatch.io API key - -WaterWaterApiToken WaterWatch.io API token + ===================== https://waterwatch.io credentials + -WaterWatchOrgId WaterWatch.io organisation Id + -WaterWatchApiKey WaterWatch.io API key + -WaterWatchApiToken WaterWatch.io API token + + ===================== Configuration options + -OutputMode Measurement value output mode. One of OffsetCorrected, RawDistance. [default: OffsetCorrected] -SaveStatePath Path to persisted state file [default: WaterWatchSaveState.json] + -SyncFromUtc Optional UTC sync time. [default: last known sensor time] + -NewSensorSyncDays Number of days to sync data when a new sensor is detected. [default: 5] + + ===================== Sensor filtering options + -SensorName Sensor name regular expression filter. Can be specified multiple times. + -SensorSerial Sensor serial number regular expression filter. Can be specified multiple times. + +Supported /SyncFromUtc date formats: + + yyyy-MM-dd + yyyy-MM-ddTHH:mm + yyyy-MM-ddTHH:mm:ss + yyyy-MM-ddTHH:mm:ss.fff Use the @optionsFile syntax to read more options from a file. diff --git a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/WaterWatchPreProcessor.csproj b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/WaterWatchPreProcessor.csproj index 415432a9..952dea50 100644 --- a/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/WaterWatchPreProcessor.csproj +++ b/TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/WaterWatchPreProcessor.csproj @@ -65,10 +65,25 @@ + + + + + + + + + + + + + + +