/// <summary> /// Read the log using the given index entries /// </summary> /// <param name="index">Enumerable returning the index entries to read</param> /// <param name="progress">Action to report reading progress based on bytes consumed</param> /// <param name="cancellation">CancellationToken for canceling the read operation</param> /// <returns>Enumerable returning the log entries</returns> internal IEnumerable <LogItem> Read(IEnumerable <IndexItem> index, Action <long, long> progress, CancellationToken cancellation) { var linesRead = 0; ZipArchive archive = null; Stream stream = null; var sw = Stopwatch.StartNew(); try { var indexEnumerator = index.GetEnumerator(); var readerFile = string.Empty; var readerMember = string.Empty; var readerPosition = 0L; var readerLine = 0; var readerBytes = 0; CountingReader reader = null; while (true) { // no more index entries to read if (!indexEnumerator.MoveNext()) { break; } // the file of the current index entry changed if (readerFile != indexEnumerator.Current.File) { if (stream != null) { stream.Dispose(); reader = null; } if (archive != null) { archive.Dispose(); } try { stream = new FileStream(indexEnumerator.Current.File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); readerFile = indexEnumerator.Current.File; readerMember = string.Empty; readerPosition = 0; readerLine = 0; } catch (Exception ex) { _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}: {ex.Message}"); break; } if (!string.IsNullOrEmpty(indexEnumerator.Current.Member)) { try { archive = new ZipArchive(stream); stream = archive.GetEntry(indexEnumerator.Current.Member).Open(); readerMember = indexEnumerator.Current.Member; } catch (Exception ex) { _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}"); break; } } } // the member of the current index entry changed if (readerMember != indexEnumerator.Current.Member && !string.IsNullOrEmpty(indexEnumerator.Current.Member)) { if (stream != null) { stream.Dispose(); reader = null; } try { stream = archive.GetEntry(indexEnumerator.Current.Member).Open(); readerMember = indexEnumerator.Current.Member; readerPosition = 0; readerLine = 0; } catch (Exception ex) { _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}"); break; } } // create the reader if not yet created if (reader == null) { try { reader = new CountingReader(stream, p => progress?.Invoke(p, _progressMaximum), cancellation); } catch (Exception ex) { _logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}"); break; } } // the current index entry points to a position further into the file if (indexEnumerator.Current.Position > readerPosition) { try { readerPosition = reader.Seek(indexEnumerator.Current.Position, SeekOrigin.Begin); readerLine = indexEnumerator.Current.Line; } catch (Exception ex) { _logger.Error($"Log::Read(): Error while seeking in {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}"); break; } } // read the line string data; try { data = reader.ReadLine(out readerBytes); } catch (Exception ex) { _logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}"); break; } // return the log item yield return(new LogItem(data, readerFile, readerMember, readerPosition, readerLine)); readerPosition += readerBytes; readerLine += 1; linesRead += 1; } } finally { if (stream != null) { stream.Dispose(); } if (archive != null) { archive.Dispose(); } sw.Stop(); _logger.Info($"Log::Read(): Reading completed in {sw.ElapsedMilliseconds}ms ({linesRead} lines returned)"); cancellation.ThrowIfCancellationRequested(); } }
/// <summary> /// Updates the index for the log /// </summary> /// <param name="progress">Action to report indexing progress</param> /// <param name="cancellationToken">CancellationToken for cancelling the index update</param> public bool Update(Action <double> progress, CancellationToken cancellationToken) { var sw = Stopwatch.StartNew(); var updated = false; IsUpdating = true; // find all files of the log on disk var files = FindFiles(); // remove indexed file which are no longer present foreach (var file in _index.Files.Where(f => !files.Any(p => p.Item1 == f.Item1 && p.Item2 == f.Item2))) { _index.Remove(file.Item1, file.Item2); updated = true; } // find the files which were changed since the last indexing or which are not present in the index var indexFiles = _index.Files; var indexTimestamp = indexFiles.Select(f => f.Item3).Concat(new[] { DateTime.MinValue }).Max(); var changedFiles = files .GroupJoin(indexFiles, f => f.Item1 + f.Item2, f => f.Item1 + f.Item2, (file, index) => new { file, index = index.FirstOrDefault() }) .Where(a => a.index == null || a.file.Item3 > a.index.Item3) .Select(a => a.file) .ToList(); // when the tailing log file has changed just process the new lines, otherwise completely index all changed files if (changedFiles.Count > 0) { // update the maximum progress value _progressMaximum = changedFiles.Sum(f => f.Item4); // update the index with the changed files var progressCount = 0L; var stateCollection = new ConcurrentStack <object[]>(); Parallel.ForEach(changedFiles, element => { // create the indexer state for the file var state = _indexers.Select(i => i.Initialize(element.Item5, element.Item1, element.Item2, element.Item4, element.Item3, false)).ToArray(); // create a stream for the file or archive member var length = 0L; var stream = OpenFileStream(element.Item1, element.Item2, out length); // read and tokenize the file CountingReader streamreader = null; TokenReader tokenreader = null; var sw2 = Stopwatch.StartNew(); try { var buffer = new Token[1024]; streamreader = new CountingReader(stream, p => progress?.Invoke(Interlocked.Add(ref progressCount, p) * 100.0 / _progressMaximum), cancellationToken); tokenreader = new TokenReader(streamreader, element.Item1, element.Item2, 0, streamreader.CurrentEncoding); var count = 0; while ((count = tokenreader.Read(buffer, 0, buffer.Length)) > 0) { for (var i = 0; i < _indexers.Length; i++) { _indexers[i].Update(state[i], buffer, count); } } } finally { if (tokenreader != null) { tokenreader.Dispose(); } else if (streamreader != null) { streamreader.Dispose(); } sw2.Stop(); _logger.Info($"Log::Read(): Reading {element.Item1}:{element.Item2} completed in {sw2.ElapsedMilliseconds}ms"); } // complete the files on the indexers for (var i = 0; i < _indexers.Length; i++) { _indexers[i].Complete(state[i]); } }); // complete the indexers for (var i = 0; i < _indexers.Length; i++) { _indexers[i].Complete(); } updated = true; } // send the collection changed event if (updated) { CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } sw.Stop(); // signal the end of the update IsUpdating = false; // raise property changes if (updated) { _logger.Info($"Log::Update(): Updating index completed in {sw.ElapsedMilliseconds}ms"); PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Files))); PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Count))); PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Tokens))); } else { _logger.Info("Log::Update(): Index is up to date"); } return(updated); }