private static IEnumerable <T> ScanLines <T>(StreamReaderExtended source,
int compression,
Func <long, T> selector,
Func <string, long, bool> shouldBreak)
{
int i = 0;
if (source.EndOfStream)
{
yield break;
}
string line;
while ((line = source.ReadLine()) != null)
{
i++;
var position = source.AbsolutePosition();
if (shouldBreak(line, position))
{
yield break;
}
if (i == compression)
{
yield return(selector(position));
i = 0;
}
;
}
}
private FileReadResult ReadLines(string file, long firstPosition)
{
var list = new List <string>();
long endPosition;
using (var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream))
{
//go to starting point
if (firstPosition == -1)
{
stream.Seek(0, SeekOrigin.End);
}
else
{
stream.Seek(firstPosition, SeekOrigin.Begin);
}
string line;
while ((line = reader.ReadLine()) != null)
{
list.Add(line);
}
endPosition = reader.AbsolutePosition();
}
}
return(new FileReadResult(list, endPosition));
}
public static IEnumerable <T> SearchLines <T>(this StreamReaderExtended source,
Func <string, bool> predicate,
Func <long, T> selector,
Func <string, long, bool> shouldBreak)
{
if (source.EndOfStream)
{
yield break;
}
long previousPostion = source.AbsolutePosition();
string line;
while ((line = source.ReadLine()) != null)
{
long position = source.AbsolutePosition();
if (predicate(line))
{
yield return(selector(previousPostion));
}
if (shouldBreak(line, position))
{
yield break;
}
previousPostion = position;
}
}
private FileSegmentSearchResult Search(long start, long end)
{
long lastPosition = 0;
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
if (stream.Length < start)
{
start = 0;
end = stream.Length;
}
long[] lines;
using (var reader = new StreamReaderExtended(stream, true))
{
stream.Seek(start, SeekOrigin.Begin);
if (reader.EndOfStream)
{
return(new FileSegmentSearchResult(start, end));
}
lines = reader.SearchLines(_predicate, i => i, (line, position) =>
{
lastPosition = position;//this is end of line po
return(end != -1 && lastPosition > end);
}).ToArray();
}
return(new FileSegmentSearchResult(start, lastPosition, lines));
}
}
/// <summary>
/// Finds the delimiter by looking for the first line in the file and comparing chars
/// </summary>
/// <param name="source">The source.</param>
/// <returns></returns>
public static int FindDelimiter(this FileInfo source)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding.Default, true))
{
if (reader.EndOfStream)
{
return(-1);
}
do
{
var ch = (char)reader.Read();
// Note the following common line feed chars:
// \n - UNIX \r\n - DOS \r - Mac
switch (ch)
{
case '\r':
var next = (char)reader.Peek();
//with \n is WINDOWS delimiter. Otherwise mac
return(next == '\n' ? 2 : 1);
case '\n':
return(1);
}
} while (!reader.EndOfStream);
return(-1);
}
}
}
/// <summary>
/// Finds the delimiter by looking for the first line in the file and comparing chars
/// </summary>
/// <param name="source">The source.</param>
/// <returns></returns>
public static int FindDelimiter(this FileInfo source)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding.Default, true))
{
if (reader.EndOfStream)
return -1;
do
{
var ch = (char)reader.Read();
// Note the following common line feed chars:
// \n - UNIX \r\n - DOS \r - Mac
switch (ch)
{
case '\r':
var next = (char)reader.Peek();
//with \n is WINDOWS delimiter. Otherwise mac
return next == '\n' ? 2 : 1;
case '\n':
return 1;
}
} while (!reader.EndOfStream);
return -1;
}
}
}
/// <summary>
/// Determines the encoding of a file
/// </summary>
/// <returns></returns>
public static Encoding GetEncoding(this FileInfo source)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, true))
{
return reader.CurrentEncoding;
}
}
}
/// <summary>
/// Determines the encoding of a file
/// </summary>
/// <returns></returns>
public static Encoding GetEncoding(this FileInfo source)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, true))
{
return(reader.CurrentEncoding);
}
}
}
public static long FindNextEndOfLinePosition(this StreamReaderExtended source, long initialPosition,
SeekOrigin origin = SeekOrigin.Begin)
{
if (source.EndOfStream)
{
return(-1);
}
source.BaseStream.Seek(initialPosition, origin);
source.ReadLine();
return(source.AbsolutePosition());
}
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll)
{
var page = GetPage(scroll);
var relativeIndex = CalculateRelativeIndex(page.Start);
if (relativeIndex == null)
{
yield break;
}
var offset = relativeIndex.LinesOffset;
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
//go to starting point
stream.Seek(relativeIndex.Start, SeekOrigin.Begin);
if (offset > 0)
{
//skip number of lines offset
for (int i = 0; i < offset; i++)
{
reader.ReadLine();
}
}
//if estimate move to the next start of line
if (relativeIndex.IsEstimate && relativeIndex.Start != 0)
{
reader.ReadLine();
}
foreach (var i in Enumerable.Range(page.Start, page.Size))
{
var startPosition = reader.AbsolutePosition();
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(i + 1, i, startPosition, endPosition);
var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return(new Line(info, line, ontail));
}
}
}
}
public IEnumerable <FileSegment> LoadSegments()
{
using (var stream = File.Open(_info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
var fileLength = stream.Length;
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReaderExtended(stream, true))
{
if (reader.EndOfStream || fileLength == 0)
{
yield return(new FileSegment(0, 0, 0, FileSegmentType.Tail));
yield break;
}
if (fileLength < _initialTail)
{
yield return(new FileSegment(0, 0, fileLength, FileSegmentType.Tail));
yield break;
}
var headStartsAt = reader.FindNextEndOfLinePosition(fileLength - _initialTail);
long currentEnfOfPage = 0;
long previousEndOfPage = 0;
int index = 0;
do
{
var approximateEndOfPage = currentEnfOfPage + _segmentSize;
if (approximateEndOfPage >= headStartsAt)
{
yield return(new FileSegment(index, previousEndOfPage, headStartsAt, FileSegmentType.Head));
break;
}
currentEnfOfPage = reader.FindNextEndOfLinePosition(approximateEndOfPage);
yield return(new FileSegment(index, previousEndOfPage, currentEnfOfPage, FileSegmentType.Head));
index++;
previousEndOfPage = currentEnfOfPage;
} while (true);
index++;
yield return(new FileSegment(index, headStartsAt, fileLength, FileSegmentType.Tail));
}
}
}
public IEnumerable <Line> ReadLines(ScrollRequest scroll)
{
var page = GetPage(scroll);
if (page.Size == 0)
{
yield break;
}
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
if (page.Size == 0)
{
yield break;
}
foreach (var i in Enumerable.Range(page.Start, page.Size))
{
if (i > Count - 1)
{
continue;
}
var start = Matches[i];
var startPosition = reader.AbsolutePosition();
if (startPosition != start)
{
reader.DiscardBufferedData();
reader.BaseStream.Seek(start, SeekOrigin.Begin);
}
startPosition = reader.AbsolutePosition();
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(i + 1, i, startPosition, endPosition);
var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return(new Line(info, line, ontail));
}
}
}
}
public IEnumerable<FileSegment> LoadSegments()
{
using (var stream = File.Open(_info.FullName, FileMode.Open, FileAccess.Read,FileShare.Delete | FileShare.ReadWrite))
{
var fileLength = stream.Length;
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReaderExtended(stream, true))
{
if (reader.EndOfStream || fileLength == 0)
{
yield return new FileSegment(0, 0, 0, FileSegmentType.Tail);
yield break;
}
if (fileLength < _initialTail)
{
yield return new FileSegment(0, 0, fileLength, FileSegmentType.Tail);
yield break;
}
var headStartsAt = reader.FindNextEndOfLinePosition(fileLength - _initialTail);
long currentEnfOfPage = 0;
long previousEndOfPage = 0;
int index = 0;
do
{
var approximateEndOfPage = currentEnfOfPage + _segmentSize;
if (approximateEndOfPage >= headStartsAt)
{
yield return new FileSegment(index, previousEndOfPage, headStartsAt, FileSegmentType.Head);
break;
}
currentEnfOfPage = reader.FindNextEndOfLinePosition(approximateEndOfPage);
yield return new FileSegment(index, previousEndOfPage, currentEnfOfPage, FileSegmentType.Head);
index++;
previousEndOfPage = currentEnfOfPage;
} while (true);
index ++;
yield return new FileSegment(index, headStartsAt, fileLength, FileSegmentType.Tail);
}
}
}
private Index Scan(long start, long end, int compression)
{
var count = 0;
long lastPosition = 0;
using (
var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite))
{
long[] lines;
using (var reader = new StreamReaderExtended(stream, Info.GetEncoding(), false))
{
var currentPosition = reader.AbsolutePosition();
if (currentPosition != start)
{
stream.Seek(start, SeekOrigin.Begin);
}
if (reader.EndOfStream)
{
return(null);
}
lines = ScanLines(reader, compression, i => i, (line, position) =>
{
var shouldBreak = end != -1 && lastPosition >= end;
if (!shouldBreak)
{
//do not count the last line as this will take us one line over
lastPosition = position;
count++;
}
return(shouldBreak);
}).ToArray();
}
if (end != -1 && lastPosition > end)
{
count--;
lastPosition = end;
lines = lines.Take(lines.Length - 1).ToArray();
}
return(new Index(start, lastPosition, lines, compression, count,
end == -1 ? IndexType.Tail : IndexType.Page));
}
}
public static IEnumerable <Line> ReadLinesByPosition(this FileInfo source, long[] positions, Func <int, bool> isEndOfTail = null)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding.Default, true))
{
foreach (var position in positions)
{
if (reader.AbsolutePosition() != position)
{
reader.DiscardBufferedData();
stream.Seek(position, SeekOrigin.Begin);
}
var line = reader.ReadLine();
yield return(new Line((int)position, line, null));
}
}
}
}
private IEnumerable<Line> ReadLinesByIndex(ScrollRequest scroll)
{
var page = GetPage(scroll);
var relativeIndex = CalculateRelativeIndex(page.Start);
if (relativeIndex == null) yield break;
var offset = relativeIndex.LinesOffset;
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
//go to starting point
stream.Seek(relativeIndex.Start, SeekOrigin.Begin);
if (offset > 0)
{
//skip number of lines offset
for (int i = 0; i < offset; i++)
reader.ReadLine();
}
//if estimate move to the next start of line
if (relativeIndex.IsEstimate && relativeIndex.Start != 0)
reader.ReadLine();
foreach (var i in Enumerable.Range(page.Start, page.Size))
{
var startPosition = reader.AbsolutePosition();
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(i + 1, i, startPosition, endPosition);
var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return new Line(info, line, ontail);
}
}
}
}
private IEnumerable <Line> ReadLinesByPosition(ScrollRequest scroll)
{
//TODO: Calculate initial index of first item.
//scroll from specified position
using (
var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite))
{
var taken = 0;
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
var startPosition = scroll.Position;
var first = (int)CalculateIndexByPositon(startPosition);
reader.BaseStream.Seek(startPosition, SeekOrigin.Begin);
do
{
var line = reader.ReadLine();
if (line == null)
{
yield break;
}
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(first + taken + 1, first + taken, startPosition, endPosition);
var ontail = endPosition >= TailInfo.TailStartsAt &&
DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return(new Line(info, line, ontail));
startPosition = endPosition;
taken++;
} while (taken < scroll.PageSize);
}
}
}
public IEnumerable <Line> ReadLines(ScrollRequest scroll)
{
var current = ReverseLinking(this);
var page = GetPage(scroll, this);
var lastValueWrapper = new LastValueWrapper();
if (page.Size == 0)
{
yield break;
}
while (current != null)
{
using (
var stream = File.Open(current.Info.FullName, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, current.Encoding, false))
{
if (page.Size == 0)
{
yield break;
}
if (lastValueWrapper.LastPageIndex == page.Start + page.Size)
{
yield break;
}
var counter = 0;
long lastEndPos = 0;
foreach (
var i in
Enumerable.Range(
lastValueWrapper.LastPageIndex > 0 ? lastValueWrapper.LastPageIndex : page.Start,
page.Size - counter))
{
if (i == page.Start + page.Size)
{
yield break;
}
if (i - lastValueWrapper.LastMatchesSize > current.Count - 1)
{
lastValueWrapper.LastEndPosition += lastEndPos + 1;
lastValueWrapper.LastMatchesSize += current.Count;
break;
}
var start = current.Matches[i - lastValueWrapper.LastMatchesSize];
var startPosition = reader.AbsolutePosition();
if (startPosition != start)
{
reader.DiscardBufferedData();
reader.BaseStream.Seek(start, SeekOrigin.Begin);
}
startPosition = reader.AbsolutePosition();
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(i + 1, i, startPosition + lastValueWrapper.LastEndPosition,
endPosition + lastValueWrapper.LastEndPosition);
var ontail = endPosition >= TailInfo.TailStartsAt &&
DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return(new Line(info, line, ontail));
lastValueWrapper.LastPageIndex = i + 1;
counter++;
lastEndPos = endPosition;
if (reader.EndOfStream)
{
lastValueWrapper.LastEndPosition += endPosition + 1;
lastValueWrapper.LastMatchesSize += current.Count;
break;
}
}
}
}
current = current.Next as FileSearchResult;
}
}
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll)
{
var current = this;
var lastValueWrapper = new LastValueWrapper();
var iterationCounter = 0;
var page = GetPage(scroll, current);
var relativeIndex = CalculateRelativeIndex(page.Start, ref current, lastValueWrapper);
while (relativeIndex != null && current != null)
{
if (current.Indicies.Length > 0 && current.Indicies.Any(t => t.Indicies.Count == 0))
{
iterationCounter++;
current = current.Previous as IndexCollection;
continue;
}
if (lastValueWrapper.LastPageIndex == page.Start + page.Size)
{
yield break;
}
var offset = relativeIndex.LinesOffset;
using (
var stream = File.Open(current.Info.FullName, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, current.Encoding, false))
{
//go to starting point
stream.Seek((iterationCounter > 0) ? 0 : relativeIndex.Start, SeekOrigin.Begin);
if (iterationCounter == 0 && offset > 0)
{
//skip number of lines offset
for (var i = 0; i < offset; i++)
{
reader.ReadLine();
}
}
//if estimate move to the next start of line
if (iterationCounter == 0 && relativeIndex.IsEstimate && relativeIndex.Start != 0)
{
reader.ReadLine();
}
foreach (
var i in
Enumerable.Range((iterationCounter > 0) ? lastValueWrapper.LastPageIndex : page.Start,
page.Size))
{
if (i == page.Start + page.Size)
{
yield break;
}
var startPosition = reader.AbsolutePosition() + lastValueWrapper.LastEndPosition;
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition() + lastValueWrapper.LastEndPosition;
var info = new LineInfo(i + 1, i, startPosition, endPosition);
var ontail = startPosition >= current.TailInfo.TailStartsAt &&
DateTime.Now.Subtract(current.TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return(new Line(info, line, ontail));
lastValueWrapper.LastPageIndex = i + 1;
if (reader.EndOfStream)
{
lastValueWrapper.LastEndPosition += endPosition + 1;
break;
}
}
}
}
iterationCounter++;
current = current.Previous as IndexCollection;
}
}
private IEnumerable<Line> ReadLinesByPosition(ScrollRequest scroll)
{
//TODO: Calculate initial index of first item.
//scroll from specified position
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read,FileShare.Delete | FileShare.ReadWrite))
{
int taken = 0;
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
var startPosition = scroll.Position;
var first = (int)CalculateIndexByPositon(startPosition);
reader.BaseStream.Seek(startPosition, SeekOrigin.Begin);
do
{
var line = reader.ReadLine();
if (line==null) yield break;
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(first + taken + 1, first + taken, startPosition, endPosition);
var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.Now
: (DateTime?)null;
yield return new Line(info, line, ontail);
startPosition = endPosition;
taken++;
} while (taken < scroll.PageSize);
}
}
}
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll)
{
var page = GetPage(scroll);
var relativeIndex = CalculateRelativeIndex(page.Start);
if (relativeIndex == null)
{
yield break;
}
var offset = relativeIndex.LinesOffset;
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding, false))
{
//go to starting point
stream.Seek(relativeIndex.Start, SeekOrigin.Begin);
if (offset > 0)
{
//skip number of lines offset
for (int i = 0; i < offset; i++)
{
reader.ReadLine();
}
}
//if estimate move to the next start of line
if (relativeIndex.IsEstimate && relativeIndex.Start != 0)
{
reader.ReadLine();
}
foreach (var i in Enumerable.Range(page.Start, page.Size))
{
var startPosition = reader.AbsolutePosition();
var line = reader.ReadLine();
var endPosition = reader.AbsolutePosition();
var info = new LineInfo(i + 1, i, startPosition, endPosition);
var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.UtcNow.Subtract(TailInfo.LastTail).TotalSeconds < 1
? DateTime.UtcNow
: (DateTime?)null;
if (!line.StartsWith("{"))
{
yield return(new Line(info, line, ontail));
}
else
{
var data = JsonSerializer.Deserialize <Dictionary <string, object> >(line);
var timestamp = DateTime.Parse(data["@t"].ToString());
string message = data["@mt"].ToString();
var timestampPrefix = timestamp.ToString("[dd/MM/yyyy hh:mm:ss.fff tt] ");
string formattedMessage = SubstituteRegex.Replace(message, match =>
{
var name = match.Groups["exp"].Value;
return(data[name].ToString());
});
if (data.TryGetValue("@x", out var exception) && !string.IsNullOrEmpty(exception.ToString()))
{
yield return(new Line(info, timestampPrefix + formattedMessage + " (" + exception.ToString() + ")", ontail));
}
else
{
yield return(new Line(info, timestampPrefix + formattedMessage, ontail));
}
}
}
}
}
}
private FileSegmentSearchResult Search(long start, long end)
{
long lastPosition = 0;
using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
if (stream.Length < start)
{
start = 0;
end = stream.Length;
}
long[] lines;
using (var reader = new StreamReaderExtended(stream, true))
{
stream.Seek(start, SeekOrigin.Begin);
if (reader.EndOfStream)
return new FileSegmentSearchResult(start,end);
lines = reader.SearchLines(_predicate, i => i, (line, position) =>
{
lastPosition = position;//this is end of line po
return end != -1 && lastPosition > end;
}).ToArray();
}
return new FileSegmentSearchResult(start, lastPosition, lines);
}
}
public static IEnumerable<Line> ReadLinesByPosition(this FileInfo source, long[] positions, Func<int, bool> isEndOfTail = null)
{
using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream, Encoding.Default, true))
{
foreach (var position in positions)
{
if (reader.AbsolutePosition() != position)
{
reader.DiscardBufferedData();
stream.Seek(position, SeekOrigin.Begin);
}
var line = reader.ReadLine();
yield return new Line((int)position, line,null);
}
}
}
}
private FileReadResult ReadLines(string file, long firstPosition)
{
var list = new List<string>();
long endPosition;
using (var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
{
using (var reader = new StreamReaderExtended(stream))
{
//go to starting point
if (firstPosition == -1)
{
stream.Seek(0, SeekOrigin.End);
}
else
{
stream.Seek(firstPosition, SeekOrigin.Begin);
}
string line;
while ((line = reader.ReadLine()) != null)
{
list.Add(line);
}
endPosition = reader.AbsolutePosition();
}
}
return new FileReadResult(list, endPosition);
}
