Issue-115 - First cut at working WaterWatch sensor export

By default, all new measurements for all sensors in the WaterWatch organisation will be exported to standard output.

Filtering options are provided to exclude specific sensors by name or serial number patterns.

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Context.cs

@@ -1,10 +1,19 @@
-namespace WaterWatchPreProcessor
+using System;
+using System.Collections.Generic;
+using WaterWatchPreProcessor.Filters;
+
+namespace WaterWatchPreProcessor
 {
     public class Context
     {
-        public string WaterWaterOrgId { get; set; }
-        public string WaterWaterApiKey { get; set; }
-        public string WaterWaterApiToken { get; set; }
+        public string WaterWatchOrgId { get; set; }
+        public string WaterWatchApiKey { get; set; }
+        public string WaterWatchApiToken { get; set; }
         public string SaveStatePath { get; set; } = "WaterWatchSaveState.json";
+        public OutputMode OutputMode { get; set; }
+        public List<RegexFilter> SensorSerialFilters { get; set; } = new List<RegexFilter>();
+        public List<RegexFilter> SensorNameFilters { get; set; } = new List<RegexFilter>();
+        public DateTime? SyncFromUtc { get; set; }
+        public int NewSensorSyncDays { get; set; } = 5;
     }
 }

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/SavedState.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+
+namespace WaterWatchPreProcessor.Dtos
+{
+    public class SavedState
+    {
+        public Dictionary<string, DateTime> LastSeenBySensorSerial { get; set; } =
+            new Dictionary<string, DateTime>(StringComparer.InvariantCultureIgnoreCase);
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Alarm.cs

@@ -0,0 +1,8 @@
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class Alarm
+    {
+        public int Threshold { get; set; }
+        public int Type { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Config.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class Config
+    {
+        public bool DeltaCompressionEnabled { get; set; }
+        public DateTime? LastSchedulerOnTime { get; set; }
+        public int MeasurementInterval { get; set; }
+        public bool MeasurementTransmissionEnabled { get; set; }
+        public bool SchedulerEnabled { get; set; }
+        public bool ServerSideAlarm { get; set; }
+        public int TransmissionInterval { get; set; }
+        public Alarm Alarm { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/DisplayInfo.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class DisplayInfo
+    {
+        public double? MaxLevel { get; set; }
+        public double? MinLevel { get; set; }
+        public double? OffsetMeasurement { get; set; }
+        public IList<ReferenceLine> ReferenceLines { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsRequest.cs

@@ -0,0 +1,16 @@
+using System;
+using ServiceStack;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    [Route("/organisations/{OrganisationId}/sensors/{SensorSerial}/measurements", HttpMethods.Get)]
+    public class GetMeasurementsRequest : IReturn<GetMeasurementsResponse>
+    {
+        public string OrganisationId { get; set; }
+        public string SensorSerial { get; set; }
+        public DateTime StartTime { get; set; }
+        public DateTime? EndTime { get; set; }
+        public string Order { get; set; }
+        public string Start { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetMeasurementsResponse.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class GetMeasurementsResponse
+    {
+        public IList<Measurement> Measurements { get; set; }
+        public string Next { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/GetSensorsRequest.cs

@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+using ServiceStack;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    [Route("/organisations/{OrganisationId}/sensors", HttpMethods.Get)]
+    public class GetSensorsRequest : IReturn<IList<Sensor>>
+    {
+        public string OrganisationId { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/LatestData.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class LatestData
+    {
+        public bool AlertAsserted { get; set; }
+        public bool OfflineAsserted { get; set; }
+        public DateTime LastSeen { get; set; }
+        public Measurement LastMeasurement { get; set; }
+        public int SignalLevel { get; set; }
+        public double Battery { get; set; }
+        public string LinkQuality { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Measurement.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class Measurement
+    {
+        public DateTime Time { get; set; }
+        public double RawDistance { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/ReferenceLine.cs

@@ -0,0 +1,9 @@
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class ReferenceLine
+    {
+        public string Color { get; set; }
+        public int Value { get; set; }
+        public string Label { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Dtos/WaterWatch/Sensor.cs

@@ -0,0 +1,19 @@
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace WaterWatchPreProcessor.Dtos.WaterWatch
+{
+    public class Sensor
+    {
+        public string OrganisationId { get; set; }
+        public string Name { get; set; }
+        public string Serial { get; set; }
+        public string SensorType { get; set; }
+        public double? Longitude { get; set; }
+        public double? Latitude { get; set; }
+        public DisplayInfo DisplayInfo { get; set; }
+        public Config Config { get; set; }
+        public LatestData LatestData { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Exporter.cs

@@ -1,11 +1,121 @@
-namespace WaterWatchPreProcessor
+using System;
+using System.Collections.Generic;
+using System.IO;
+using ServiceStack;
+using ServiceStack.Text;
+using WaterWatchPreProcessor.Dtos;
+using WaterWatchPreProcessor.Dtos.WaterWatch;
+using WaterWatchPreProcessor.Filters;
+
+namespace WaterWatchPreProcessor
 {
     public class Exporter
     {
         public Context Context { get; set; }
 
+        private JsonServiceClient Client { get; set; }
+
+        private SavedState SavedState { get; set; } = new SavedState();
+
         public void Run()
         {
+            RestoreSavedState();
+
+            CreateConnectedClient();
+
+            var sensors = GetSensors();
+
+            var nameFilter = new Filter<RegexFilter>(Context.SensorNameFilters);
+            var serialFilter = new Filter<RegexFilter>(Context.SensorSerialFilters);
+
+            foreach (var sensor in sensors)
+            {
+                if (nameFilter.IsFiltered(f => f.Regex.IsMatch(sensor.Name))
+                    || serialFilter.IsFiltered(f => f.Regex.IsMatch(sensor.Serial)))
+                    continue;
+
+                foreach (var measurement in GetSensorMeasurements(sensor))
+                {
+                    Console.WriteLine($"{measurement.Time:yyyy-MM-ddTHH:mm:ss.fffZ}, {sensor.SensorType}, {sensor.Serial}, {GetSensorValue(sensor, measurement.RawDistance)}");
+                }
+
+                SavedState.LastSeenBySensorSerial[sensor.Serial] = sensor.LatestData.LastSeen;
+            }
+
+            PersistSavedState();
+        }
+
+        private void RestoreSavedState()
+        {
+            if (!File.Exists(Context.SaveStatePath))
+                return;
+
+            SavedState = File.ReadAllText(Context.SaveStatePath).FromJson<SavedState>();
+        }
+
+        private void PersistSavedState()
+        {
+            File.WriteAllText(Context.SaveStatePath, SavedState.ToJson().IndentJson());
+        }
+
+        private void CreateConnectedClient()
+        {
+            Client = new JsonServiceClient("https://api.waterwatch.io/v1")
+            {
+                AlwaysSendBasicAuthHeader = true,
+                UserName = Context.WaterWatchApiKey,
+                Password = Context.WaterWatchApiToken
+            };
+        }
+
+        private IList<Sensor> GetSensors()
+        {
+            return Client.Get(new GetSensorsRequest
+            {
+                OrganisationId = Context.WaterWatchOrgId
+            });
+        }
+
+        private IEnumerable<Measurement> GetSensorMeasurements(Sensor sensor)
+        {
+            if (!SavedState.LastSeenBySensorSerial.TryGetValue(sensor.Serial, out var lastSeenTime))
+            {
+                lastSeenTime = DateTime.UtcNow.Date.AddDays(-Context.NewSensorSyncDays);
+            }
+
+            if (Context.SyncFromUtc.HasValue)
+                lastSeenTime = Context.SyncFromUtc.Value;
+
+            var request = new GetMeasurementsRequest
+            {
+                OrganisationId = sensor.OrganisationId,
+                SensorSerial = sensor.Serial,
+                StartTime = lastSeenTime,
+                Order = "asc"
+            };
+
+            do
+            {
+                var response = Client.Get(request);
+
+                foreach (var measurement in response.Measurements)
+                {
+                    yield return measurement;
+                }
+
+                request.Start = response.Next;
+
+            } while (!string.IsNullOrEmpty(request.Start));
+        }
+
+        private double GetSensorValue(Sensor sensor, double rawDistance)
+        {
+            if (Context.OutputMode == OutputMode.OffsetCorrected && (sensor.DisplayInfo?.OffsetMeasurement.HasValue ?? false))
+            {
+                return sensor.DisplayInfo.OffsetMeasurement.Value - rawDistance;
+            }
+
+            return rawDistance;
         }
     }
 }

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/Filter.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace WaterWatchPreProcessor.Filters
+{
+    public class Filter<TFilter> where TFilter : IFilter
+    {
+        private List<TFilter> IncludeFilters { get; }
+        private List<TFilter> ExcludeFilters { get; }
+        public int Count { get; private set; }
+
+        public Filter(List<TFilter> filters)
+        {
+            IncludeFilters = filters.Where(filter => !filter.Exclude).ToList();
+            ExcludeFilters = filters.Where(filter => filter.Exclude).ToList();
+        }
+
+        public bool IsFiltered(Func<TFilter, bool> predicate)
+        {
+            if ((!IncludeFilters.Any() || IncludeFilters.Any(predicate)) && !ExcludeFilters.Any(predicate))
+                return false;
+
+            ++Count;
+            return true;
+        }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/IFilter.cs

@@ -0,0 +1,7 @@
+namespace WaterWatchPreProcessor.Filters
+{
+    public interface IFilter
+    {
+        bool Exclude { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/Filters/RegexFilter.cs

@@ -0,0 +1,10 @@
+using System.Text.RegularExpressions;
+
+namespace WaterWatchPreProcessor.Filters
+{
+    public class RegexFilter : IFilter
+    {
+        public bool Exclude { get; set; }
+        public Regex Regex { get; set; }
+    }
+}

TimeSeries/PublicApis/SdkExamples/WaterWatchPreProcessor/OutputMode.cs

@@ -0,0 +1,8 @@
+namespace WaterWatchPreProcessor
+{
+    public enum OutputMode
+    {
+        OffsetCorrected,
+        RawDistance,
+    }
+}