Fix #28. Encoding and line feed not being calculated correctly when f…

…ile is indexed

Documents/Fast Scrolling.md

@@ -32,4 +32,4 @@ I would love you tech people, testers or system adminstrators out there to trial
 
 But being as I only started this project 2 weeks ago I am very happy for now.
 
-For you techs out there feel free to build this yourself. For the non-techs or people without Visual Studio 2015 I will regularly be putting releases  @ [Tail Blazer Release Page](https://github.com/RolandPheasant/TailBlazer/releases) where all you have to do is extract the file and double click TrailBlazer.exe
+For you techs out there feel free to build this yourself. For the non-techs or people without Visual Studio 2015 I will regularly be putting releases   the  [Tail Blazer release page](https://github.com/RolandPheasant/TailBlazer/releases) where all you have to do is extract the file and double click TrailBlazer.exe

Source/TailBlazer.Domain/FileHandling/FileInfoEx.cs

@@ -29,8 +29,7 @@ public static IObservable<FileNotification> WatchFile(this FileInfo file, TimeSp
 
                 //TODO: create a cool-off period after a poll to account for over running jobs
                 Func<IObservable<FileNotification>> poller = () => Observable.Interval(refresh, scheduler)
-                      .StartWith(0)
-                    // .ObserveOn(scheduler)
+                     // .StartWith(0)
                     .Scan((FileNotification) null, (state, _) => state == null
                         ? new FileNotification(file)
                         : new FileNotification(state))
@@ -52,7 +51,7 @@ public static IObservable<FileNotification> WatchFile(this FileInfo file, IObser
             {
                 //TODO: create a cool-off period after a poll to account for over running jobs
                 Func<IObservable<FileNotification>> poller = () => pulse.StartWith(Unit.Default)
-                    // .ObserveOn(scheduler)
+
                     .Scan((FileNotification)null, (state, _) => state == null
                        ? new FileNotification(file)
                        : new FileNotification(state))
@@ -91,7 +90,7 @@ public static IObservable<LineIndicies> Index(this IObservable<FileNotification>
                              .Scan((LineIndicies)null, (state, notification) =>
                              {
                                  var lines = indexer.ReadToEnd().ToArray();
-                                 return new LineIndicies(lines, state);
+                                 return new LineIndicies(lines, indexer.Encoding, indexer.LineFeedSize, state);
                              })
                              .SubscribeSafe(observer);
 
@@ -144,19 +143,15 @@ public static IEnumerable<Line> ReadLines(this FileInfo source, int[] lines, Fun
             }
         }
 
-        public static IEnumerable<T> ReadLines<T>(this FileInfo source, IEnumerable<LineIndex> lines, Func<LineIndex,string, T>  selector, Encoding encoding=null)
+        public static IEnumerable<T> ReadLines<T>(this FileInfo source, IEnumerable<LineIndex> lines, Func<LineIndex,string, T>  selector, Encoding encoding=null, int lineFeedLength=2)
         {
-            encoding = encoding ?? Encoding.Default;
-
-            //TODO: This assumes a windows file whereas Unix / max have length=1.  
-            //This is accounted for on the LineIndexer but we need to pass the length and current encoding here ()
-            const int delimiterLength = 2;
+            encoding = encoding ?? Encoding.UTF8;
 
             using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
             {
                 foreach (var lineIndex in lines.OrderBy(l=>l.Index))
                 {
-                    var data = new byte[lineIndex.Size- delimiterLength];
+                    var data = new byte[lineIndex.Size- lineFeedLength];
                     stream.Seek(lineIndex.Start, SeekOrigin.Begin);
                     stream.Read(data, 0, data.Length);
 

Source/TailBlazer.Domain/FileHandling/FileTailer.cs

@@ -76,21 +76,21 @@ public FileTailer(FileInfo file,
                 .Select(result =>
                 {
                     var scroll = result.Scroll;
-                    var allLines = result.Incidies;
+                    var indicies = result.Incidies;
                     var matched = result.MatchedLines;
 
                     IEnumerable<LineIndex> indices;
                     if (result.MatchedLines.ChangedReason == LineMatchChangedReason.None)
                     {
                         indices = scroll.Mode == ScrollingMode.Tail
-                            ? allLines.GetTail(scroll)
-                            : allLines.GetFromIndex(scroll);
+                            ? indicies.GetTail(scroll)
+                            : indicies.GetFromIndex(scroll);
                     }
                     else
                     {
                         indices = scroll.Mode == ScrollingMode.Tail
-                            ? allLines.GetTail(scroll, matched)
-                            : allLines.GetFromIndex(scroll, matched);
+                            ? indicies.GetTail(scroll, matched)
+                            : indicies.GetFromIndex(scroll, matched);
                     }
 
                     var currentPage = indices.ToArray();
@@ -102,12 +102,10 @@ public FileTailer(FileInfo file,
                     //finally we can load the line from the file
                     var newLines =  file.ReadLines(added, (lineIndex, text) =>
                     {
-                        var isEndOfTail = allLines.ChangedReason != LinesChangedReason.Loaded && lineIndex.Line > allLines.TailStartsAt;
+                        var isEndOfTail = indicies.ChangedReason != LinesChangedReason.Loaded && lineIndex.Line > indicies.TailStartsAt;
 
                         return new Line(lineIndex, text, isEndOfTail ? DateTime.Now : (DateTime?) null);
-                    }).ToArray();
-
-
+                    }, indicies.Encoding).ToArray();
 
                     return new { NewLines = newLines, OldLines = removedLines };
                 })

Source/TailBlazer.Domain/FileHandling/LineIndexer.cs

@@ -9,44 +9,42 @@ public class LineIndexer : IDisposable
     {
         private readonly FileInfo _info;
         private readonly FileStream _stream;
-        private readonly StreamReader _reader;
+        private readonly StreamReaderWithPosition _reader;
 
         public  Encoding Encoding { get; }
+
+        public int LineFeedSize => _lineFeedSize;
         public int Positon => _postion;
         public int Lines => _index;
 
         private int _postion;
         private int _index = -1;
 
-        private int _delimiterSize =-1;
+        private int _lineFeedSize =-1;
 
-        public LineIndexer(FileInfo info, Encoding encoding = null)
+        public LineIndexer(FileInfo info)
         {
             _info = info;
-            Encoding = encoding ?? Encoding.Default;
+
+            Encoding = info.GetEncoding();
 
             _stream = File.Open(info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite);
-            _reader = new StreamReader(_stream, Encoding, true);
+            _reader = new StreamReaderWithPosition( _stream, Encoding,false, 10 * 1024);
         }
 
         public IEnumerable<int> ReadToEnd()
         {
-            string line;
-
-            if (_reader.EndOfStream)
-                yield break;
-
-            if (_delimiterSize==-1 )
+            if (_lineFeedSize==-1 )
             {
-                _delimiterSize = _info.FindDelimiter();
-                if (_delimiterSize==-1)
+                _lineFeedSize = _info.FindDelimiter();
+                if (_lineFeedSize==-1)
                     throw new FileLoadException("Cannot determine new line delimiter");
             }
 
-            while ((line = _reader.ReadLine()) != null)
+            while ((_reader.ReadLine()) != null)
             {
                 _index++;
-                _postion = _postion + line.Length + _delimiterSize;
+                _postion = _postion + _reader.LineLength;
                 yield return _postion;
             }
 

Source/TailBlazer.Domain/FileHandling/LineIndicies.cs

@@ -1,16 +1,23 @@
-namespace TailBlazer.Domain.FileHandling
+using System.Text;
+
+namespace TailBlazer.Domain.FileHandling
 {
     public class LineIndicies
     {
+        public Encoding Encoding { get;  }
+        public int LineFeedSize { get;  }
+
         public int[] Lines { get; }
         public int Count => Lines.Length;
         public int Diff { get; }
         public LinesChangedReason ChangedReason { get; }
         public int TailStartsAt { get; }
 
 
-        public LineIndicies(int[] lines, LineIndicies previous = null)
+        public LineIndicies(int[] lines, Encoding encoding, int lineFeedSize, LineIndicies previous = null)
         {
+            Encoding = encoding;
+            LineFeedSize = lineFeedSize;
             if (previous == null)
             {
                 Lines = lines;
@@ -32,6 +39,5 @@ public LineIndicies(int[] lines, LineIndicies previous = null)
                 TailStartsAt = previous.Count - 1;
             }
         }
-
     }
 }

Source/TailBlazer.Domain/FileHandling/LineReader.cs

@@ -72,5 +72,21 @@ public static int  FindDelimiter(this FileInfo source)
                 }
             }
         }
+
+        /// <summary>
+        /// Determines the encoding of a file
+        /// </summary>
+        /// <returns></returns>
+        public static Encoding GetEncoding(this FileInfo source)
+        {
+            using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
+            {
+                using (var reader = new StreamReader(stream, true))
+                {
+                    var something = reader.Peek();
+                    return reader.CurrentEncoding;
+                }
+            }
+        }
     }
 }