private static IEnumerable <T> ScanLines <T>(StreamReaderExtended source, int compression, Func <long, T> selector, Func <string, long, bool> shouldBreak) { int i = 0; if (source.EndOfStream) { yield break; } string line; while ((line = source.ReadLine()) != null) { i++; var position = source.AbsolutePosition(); if (shouldBreak(line, position)) { yield break; } if (i == compression) { yield return(selector(position)); i = 0; } ; } }
private FileReadResult ReadLines(string file, long firstPosition) { var list = new List <string>(); long endPosition; using (var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream)) { //go to starting point if (firstPosition == -1) { stream.Seek(0, SeekOrigin.End); } else { stream.Seek(firstPosition, SeekOrigin.Begin); } string line; while ((line = reader.ReadLine()) != null) { list.Add(line); } endPosition = reader.AbsolutePosition(); } } return(new FileReadResult(list, endPosition)); }
public static IEnumerable <T> SearchLines <T>(this StreamReaderExtended source, Func <string, bool> predicate, Func <long, T> selector, Func <string, long, bool> shouldBreak) { if (source.EndOfStream) { yield break; } long previousPostion = source.AbsolutePosition(); string line; while ((line = source.ReadLine()) != null) { long position = source.AbsolutePosition(); if (predicate(line)) { yield return(selector(previousPostion)); } if (shouldBreak(line, position)) { yield break; } previousPostion = position; } }
private FileSegmentSearchResult Search(long start, long end) { long lastPosition = 0; using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { if (stream.Length < start) { start = 0; end = stream.Length; } long[] lines; using (var reader = new StreamReaderExtended(stream, true)) { stream.Seek(start, SeekOrigin.Begin); if (reader.EndOfStream) { return(new FileSegmentSearchResult(start, end)); } lines = reader.SearchLines(_predicate, i => i, (line, position) => { lastPosition = position;//this is end of line po return(end != -1 && lastPosition > end); }).ToArray(); } return(new FileSegmentSearchResult(start, lastPosition, lines)); } }
/// <summary> /// Finds the delimiter by looking for the first line in the file and comparing chars /// </summary> /// <param name="source">The source.</param> /// <returns></returns> public static int FindDelimiter(this FileInfo source) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding.Default, true)) { if (reader.EndOfStream) { return(-1); } do { var ch = (char)reader.Read(); // Note the following common line feed chars: // \n - UNIX \r\n - DOS \r - Mac switch (ch) { case '\r': var next = (char)reader.Peek(); //with \n is WINDOWS delimiter. Otherwise mac return(next == '\n' ? 2 : 1); case '\n': return(1); } } while (!reader.EndOfStream); return(-1); } } }
/// <summary> /// Finds the delimiter by looking for the first line in the file and comparing chars /// </summary> /// <param name="source">The source.</param> /// <returns></returns> public static int FindDelimiter(this FileInfo source) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding.Default, true)) { if (reader.EndOfStream) return -1; do { var ch = (char)reader.Read(); // Note the following common line feed chars: // \n - UNIX \r\n - DOS \r - Mac switch (ch) { case '\r': var next = (char)reader.Peek(); //with \n is WINDOWS delimiter. Otherwise mac return next == '\n' ? 2 : 1; case '\n': return 1; } } while (!reader.EndOfStream); return -1; } } }
/// <summary> /// Determines the encoding of a file /// </summary> /// <returns></returns> public static Encoding GetEncoding(this FileInfo source) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, true)) { return reader.CurrentEncoding; } } }
/// <summary> /// Determines the encoding of a file /// </summary> /// <returns></returns> public static Encoding GetEncoding(this FileInfo source) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, true)) { return(reader.CurrentEncoding); } } }
public static long FindNextEndOfLinePosition(this StreamReaderExtended source, long initialPosition, SeekOrigin origin = SeekOrigin.Begin) { if (source.EndOfStream) { return(-1); } source.BaseStream.Seek(initialPosition, origin); source.ReadLine(); return(source.AbsolutePosition()); }
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll) { var page = GetPage(scroll); var relativeIndex = CalculateRelativeIndex(page.Start); if (relativeIndex == null) { yield break; } var offset = relativeIndex.LinesOffset; using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding, false)) { //go to starting point stream.Seek(relativeIndex.Start, SeekOrigin.Begin); if (offset > 0) { //skip number of lines offset for (int i = 0; i < offset; i++) { reader.ReadLine(); } } //if estimate move to the next start of line if (relativeIndex.IsEstimate && relativeIndex.Start != 0) { reader.ReadLine(); } foreach (var i in Enumerable.Range(page.Start, page.Size)) { var startPosition = reader.AbsolutePosition(); var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition(); var info = new LineInfo(i + 1, i, startPosition, endPosition); var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return(new Line(info, line, ontail)); } } } }
public IEnumerable <FileSegment> LoadSegments() { using (var stream = File.Open(_info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { var fileLength = stream.Length; stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReaderExtended(stream, true)) { if (reader.EndOfStream || fileLength == 0) { yield return(new FileSegment(0, 0, 0, FileSegmentType.Tail)); yield break; } if (fileLength < _initialTail) { yield return(new FileSegment(0, 0, fileLength, FileSegmentType.Tail)); yield break; } var headStartsAt = reader.FindNextEndOfLinePosition(fileLength - _initialTail); long currentEnfOfPage = 0; long previousEndOfPage = 0; int index = 0; do { var approximateEndOfPage = currentEnfOfPage + _segmentSize; if (approximateEndOfPage >= headStartsAt) { yield return(new FileSegment(index, previousEndOfPage, headStartsAt, FileSegmentType.Head)); break; } currentEnfOfPage = reader.FindNextEndOfLinePosition(approximateEndOfPage); yield return(new FileSegment(index, previousEndOfPage, currentEnfOfPage, FileSegmentType.Head)); index++; previousEndOfPage = currentEnfOfPage; } while (true); index++; yield return(new FileSegment(index, headStartsAt, fileLength, FileSegmentType.Tail)); } } }
public IEnumerable <Line> ReadLines(ScrollRequest scroll) { var page = GetPage(scroll); if (page.Size == 0) { yield break; } using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding, false)) { if (page.Size == 0) { yield break; } foreach (var i in Enumerable.Range(page.Start, page.Size)) { if (i > Count - 1) { continue; } var start = Matches[i]; var startPosition = reader.AbsolutePosition(); if (startPosition != start) { reader.DiscardBufferedData(); reader.BaseStream.Seek(start, SeekOrigin.Begin); } startPosition = reader.AbsolutePosition(); var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition(); var info = new LineInfo(i + 1, i, startPosition, endPosition); var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return(new Line(info, line, ontail)); } } } }
public IEnumerable<FileSegment> LoadSegments() { using (var stream = File.Open(_info.FullName, FileMode.Open, FileAccess.Read,FileShare.Delete | FileShare.ReadWrite)) { var fileLength = stream.Length; stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReaderExtended(stream, true)) { if (reader.EndOfStream || fileLength == 0) { yield return new FileSegment(0, 0, 0, FileSegmentType.Tail); yield break; } if (fileLength < _initialTail) { yield return new FileSegment(0, 0, fileLength, FileSegmentType.Tail); yield break; } var headStartsAt = reader.FindNextEndOfLinePosition(fileLength - _initialTail); long currentEnfOfPage = 0; long previousEndOfPage = 0; int index = 0; do { var approximateEndOfPage = currentEnfOfPage + _segmentSize; if (approximateEndOfPage >= headStartsAt) { yield return new FileSegment(index, previousEndOfPage, headStartsAt, FileSegmentType.Head); break; } currentEnfOfPage = reader.FindNextEndOfLinePosition(approximateEndOfPage); yield return new FileSegment(index, previousEndOfPage, currentEnfOfPage, FileSegmentType.Head); index++; previousEndOfPage = currentEnfOfPage; } while (true); index ++; yield return new FileSegment(index, headStartsAt, fileLength, FileSegmentType.Tail); } } }
private Index Scan(long start, long end, int compression) { var count = 0; long lastPosition = 0; using ( var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { long[] lines; using (var reader = new StreamReaderExtended(stream, Info.GetEncoding(), false)) { var currentPosition = reader.AbsolutePosition(); if (currentPosition != start) { stream.Seek(start, SeekOrigin.Begin); } if (reader.EndOfStream) { return(null); } lines = ScanLines(reader, compression, i => i, (line, position) => { var shouldBreak = end != -1 && lastPosition >= end; if (!shouldBreak) { //do not count the last line as this will take us one line over lastPosition = position; count++; } return(shouldBreak); }).ToArray(); } if (end != -1 && lastPosition > end) { count--; lastPosition = end; lines = lines.Take(lines.Length - 1).ToArray(); } return(new Index(start, lastPosition, lines, compression, count, end == -1 ? IndexType.Tail : IndexType.Page)); } }
public static IEnumerable <Line> ReadLinesByPosition(this FileInfo source, long[] positions, Func <int, bool> isEndOfTail = null) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding.Default, true)) { foreach (var position in positions) { if (reader.AbsolutePosition() != position) { reader.DiscardBufferedData(); stream.Seek(position, SeekOrigin.Begin); } var line = reader.ReadLine(); yield return(new Line((int)position, line, null)); } } } }
private IEnumerable<Line> ReadLinesByIndex(ScrollRequest scroll) { var page = GetPage(scroll); var relativeIndex = CalculateRelativeIndex(page.Start); if (relativeIndex == null) yield break; var offset = relativeIndex.LinesOffset; using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding, false)) { //go to starting point stream.Seek(relativeIndex.Start, SeekOrigin.Begin); if (offset > 0) { //skip number of lines offset for (int i = 0; i < offset; i++) reader.ReadLine(); } //if estimate move to the next start of line if (relativeIndex.IsEstimate && relativeIndex.Start != 0) reader.ReadLine(); foreach (var i in Enumerable.Range(page.Start, page.Size)) { var startPosition = reader.AbsolutePosition(); var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition(); var info = new LineInfo(i + 1, i, startPosition, endPosition); var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return new Line(info, line, ontail); } } } }
private IEnumerable <Line> ReadLinesByPosition(ScrollRequest scroll) { //TODO: Calculate initial index of first item. //scroll from specified position using ( var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { var taken = 0; using (var reader = new StreamReaderExtended(stream, Encoding, false)) { var startPosition = scroll.Position; var first = (int)CalculateIndexByPositon(startPosition); reader.BaseStream.Seek(startPosition, SeekOrigin.Begin); do { var line = reader.ReadLine(); if (line == null) { yield break; } var endPosition = reader.AbsolutePosition(); var info = new LineInfo(first + taken + 1, first + taken, startPosition, endPosition); var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return(new Line(info, line, ontail)); startPosition = endPosition; taken++; } while (taken < scroll.PageSize); } } }
public IEnumerable <Line> ReadLines(ScrollRequest scroll) { var current = ReverseLinking(this); var page = GetPage(scroll, this); var lastValueWrapper = new LastValueWrapper(); if (page.Size == 0) { yield break; } while (current != null) { using ( var stream = File.Open(current.Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, current.Encoding, false)) { if (page.Size == 0) { yield break; } if (lastValueWrapper.LastPageIndex == page.Start + page.Size) { yield break; } var counter = 0; long lastEndPos = 0; foreach ( var i in Enumerable.Range( lastValueWrapper.LastPageIndex > 0 ? lastValueWrapper.LastPageIndex : page.Start, page.Size - counter)) { if (i == page.Start + page.Size) { yield break; } if (i - lastValueWrapper.LastMatchesSize > current.Count - 1) { lastValueWrapper.LastEndPosition += lastEndPos + 1; lastValueWrapper.LastMatchesSize += current.Count; break; } var start = current.Matches[i - lastValueWrapper.LastMatchesSize]; var startPosition = reader.AbsolutePosition(); if (startPosition != start) { reader.DiscardBufferedData(); reader.BaseStream.Seek(start, SeekOrigin.Begin); } startPosition = reader.AbsolutePosition(); var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition(); var info = new LineInfo(i + 1, i, startPosition + lastValueWrapper.LastEndPosition, endPosition + lastValueWrapper.LastEndPosition); var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return(new Line(info, line, ontail)); lastValueWrapper.LastPageIndex = i + 1; counter++; lastEndPos = endPosition; if (reader.EndOfStream) { lastValueWrapper.LastEndPosition += endPosition + 1; lastValueWrapper.LastMatchesSize += current.Count; break; } } } } current = current.Next as FileSearchResult; } }
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll) { var current = this; var lastValueWrapper = new LastValueWrapper(); var iterationCounter = 0; var page = GetPage(scroll, current); var relativeIndex = CalculateRelativeIndex(page.Start, ref current, lastValueWrapper); while (relativeIndex != null && current != null) { if (current.Indicies.Length > 0 && current.Indicies.Any(t => t.Indicies.Count == 0)) { iterationCounter++; current = current.Previous as IndexCollection; continue; } if (lastValueWrapper.LastPageIndex == page.Start + page.Size) { yield break; } var offset = relativeIndex.LinesOffset; using ( var stream = File.Open(current.Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, current.Encoding, false)) { //go to starting point stream.Seek((iterationCounter > 0) ? 0 : relativeIndex.Start, SeekOrigin.Begin); if (iterationCounter == 0 && offset > 0) { //skip number of lines offset for (var i = 0; i < offset; i++) { reader.ReadLine(); } } //if estimate move to the next start of line if (iterationCounter == 0 && relativeIndex.IsEstimate && relativeIndex.Start != 0) { reader.ReadLine(); } foreach ( var i in Enumerable.Range((iterationCounter > 0) ? lastValueWrapper.LastPageIndex : page.Start, page.Size)) { if (i == page.Start + page.Size) { yield break; } var startPosition = reader.AbsolutePosition() + lastValueWrapper.LastEndPosition; var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition() + lastValueWrapper.LastEndPosition; var info = new LineInfo(i + 1, i, startPosition, endPosition); var ontail = startPosition >= current.TailInfo.TailStartsAt && DateTime.Now.Subtract(current.TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return(new Line(info, line, ontail)); lastValueWrapper.LastPageIndex = i + 1; if (reader.EndOfStream) { lastValueWrapper.LastEndPosition += endPosition + 1; break; } } } } iterationCounter++; current = current.Previous as IndexCollection; } }
private IEnumerable<Line> ReadLinesByPosition(ScrollRequest scroll) { //TODO: Calculate initial index of first item. //scroll from specified position using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read,FileShare.Delete | FileShare.ReadWrite)) { int taken = 0; using (var reader = new StreamReaderExtended(stream, Encoding, false)) { var startPosition = scroll.Position; var first = (int)CalculateIndexByPositon(startPosition); reader.BaseStream.Seek(startPosition, SeekOrigin.Begin); do { var line = reader.ReadLine(); if (line==null) yield break; var endPosition = reader.AbsolutePosition(); var info = new LineInfo(first + taken + 1, first + taken, startPosition, endPosition); var ontail = endPosition >= TailInfo.TailStartsAt && DateTime.Now.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.Now : (DateTime?)null; yield return new Line(info, line, ontail); startPosition = endPosition; taken++; } while (taken < scroll.PageSize); } } }
private IEnumerable <Line> ReadLinesByIndex(ScrollRequest scroll) { var page = GetPage(scroll); var relativeIndex = CalculateRelativeIndex(page.Start); if (relativeIndex == null) { yield break; } var offset = relativeIndex.LinesOffset; using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding, false)) { //go to starting point stream.Seek(relativeIndex.Start, SeekOrigin.Begin); if (offset > 0) { //skip number of lines offset for (int i = 0; i < offset; i++) { reader.ReadLine(); } } //if estimate move to the next start of line if (relativeIndex.IsEstimate && relativeIndex.Start != 0) { reader.ReadLine(); } foreach (var i in Enumerable.Range(page.Start, page.Size)) { var startPosition = reader.AbsolutePosition(); var line = reader.ReadLine(); var endPosition = reader.AbsolutePosition(); var info = new LineInfo(i + 1, i, startPosition, endPosition); var ontail = startPosition >= TailInfo.TailStartsAt && DateTime.UtcNow.Subtract(TailInfo.LastTail).TotalSeconds < 1 ? DateTime.UtcNow : (DateTime?)null; if (!line.StartsWith("{")) { yield return(new Line(info, line, ontail)); } else { var data = JsonSerializer.Deserialize <Dictionary <string, object> >(line); var timestamp = DateTime.Parse(data["@t"].ToString()); string message = data["@mt"].ToString(); var timestampPrefix = timestamp.ToString("[dd/MM/yyyy hh:mm:ss.fff tt] "); string formattedMessage = SubstituteRegex.Replace(message, match => { var name = match.Groups["exp"].Value; return(data[name].ToString()); }); if (data.TryGetValue("@x", out var exception) && !string.IsNullOrEmpty(exception.ToString())) { yield return(new Line(info, timestampPrefix + formattedMessage + " (" + exception.ToString() + ")", ontail)); } else { yield return(new Line(info, timestampPrefix + formattedMessage, ontail)); } } } } } }
private FileSegmentSearchResult Search(long start, long end) { long lastPosition = 0; using (var stream = File.Open(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { if (stream.Length < start) { start = 0; end = stream.Length; } long[] lines; using (var reader = new StreamReaderExtended(stream, true)) { stream.Seek(start, SeekOrigin.Begin); if (reader.EndOfStream) return new FileSegmentSearchResult(start,end); lines = reader.SearchLines(_predicate, i => i, (line, position) => { lastPosition = position;//this is end of line po return end != -1 && lastPosition > end; }).ToArray(); } return new FileSegmentSearchResult(start, lastPosition, lines); } }
public static IEnumerable<Line> ReadLinesByPosition(this FileInfo source, long[] positions, Func<int, bool> isEndOfTail = null) { using (var stream = File.Open(source.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream, Encoding.Default, true)) { foreach (var position in positions) { if (reader.AbsolutePosition() != position) { reader.DiscardBufferedData(); stream.Seek(position, SeekOrigin.Begin); } var line = reader.ReadLine(); yield return new Line((int)position, line,null); } } } }
private FileReadResult ReadLines(string file, long firstPosition) { var list = new List<string>(); long endPosition; using (var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) { using (var reader = new StreamReaderExtended(stream)) { //go to starting point if (firstPosition == -1) { stream.Seek(0, SeekOrigin.End); } else { stream.Seek(firstPosition, SeekOrigin.Begin); } string line; while ((line = reader.ReadLine()) != null) { list.Add(line); } endPosition = reader.AbsolutePosition(); } } return new FileReadResult(list, endPosition); }