/// <summary>
/// Read the log using the given index entries
/// </summary>
/// <param name="index">Enumerable returning the index entries to read</param>
/// <param name="progress">Action to report reading progress based on bytes consumed</param>
/// <param name="cancellation">CancellationToken for canceling the read operation</param>
/// <returns>Enumerable returning the log entries</returns>
internal IEnumerable <LogItem> Read(IEnumerable <IndexItem> index, Action <long, long> progress, CancellationToken cancellation)
{
var linesRead = 0;
ZipArchive archive = null;
Stream stream = null;
var sw = Stopwatch.StartNew();
try
{
var indexEnumerator = index.GetEnumerator();
var readerFile = string.Empty;
var readerMember = string.Empty;
var readerPosition = 0L;
var readerLine = 0;
var readerBytes = 0;
CountingReader reader = null;
while (true)
{
// no more index entries to read
if (!indexEnumerator.MoveNext())
{
break;
}
// the file of the current index entry changed
if (readerFile != indexEnumerator.Current.File)
{
if (stream != null)
{
stream.Dispose();
reader = null;
}
if (archive != null)
{
archive.Dispose();
}
try
{
stream = new FileStream(indexEnumerator.Current.File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
readerFile = indexEnumerator.Current.File;
readerMember = string.Empty;
readerPosition = 0;
readerLine = 0;
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}: {ex.Message}");
break;
}
if (!string.IsNullOrEmpty(indexEnumerator.Current.Member))
{
try
{
archive = new ZipArchive(stream);
stream = archive.GetEntry(indexEnumerator.Current.Member).Open();
readerMember = indexEnumerator.Current.Member;
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
break;
}
}
}
// the member of the current index entry changed
if (readerMember != indexEnumerator.Current.Member && !string.IsNullOrEmpty(indexEnumerator.Current.Member))
{
if (stream != null)
{
stream.Dispose();
reader = null;
}
try
{
stream = archive.GetEntry(indexEnumerator.Current.Member).Open();
readerMember = indexEnumerator.Current.Member;
readerPosition = 0;
readerLine = 0;
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
break;
}
}
// create the reader if not yet created
if (reader == null)
{
try
{
reader = new CountingReader(stream, p => progress?.Invoke(p, _progressMaximum), cancellation);
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
break;
}
}
// the current index entry points to a position further into the file
if (indexEnumerator.Current.Position > readerPosition)
{
try
{
readerPosition = reader.Seek(indexEnumerator.Current.Position, SeekOrigin.Begin);
readerLine = indexEnumerator.Current.Line;
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while seeking in {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
break;
}
}
// read the line
string data;
try
{
data = reader.ReadLine(out readerBytes);
}
catch (Exception ex)
{
_logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
break;
}
// return the log item
yield return(new LogItem(data, readerFile, readerMember, readerPosition, readerLine));
readerPosition += readerBytes;
readerLine += 1;
linesRead += 1;
}
}
finally
{
if (stream != null)
{
stream.Dispose();
}
if (archive != null)
{
archive.Dispose();
}
sw.Stop();
_logger.Info($"Log::Read(): Reading completed in {sw.ElapsedMilliseconds}ms ({linesRead} lines returned)");
cancellation.ThrowIfCancellationRequested();
}
}
/// <summary>
/// Updates the index for the log
/// </summary>
/// <param name="progress">Action to report indexing progress</param>
/// <param name="cancellationToken">CancellationToken for cancelling the index update</param>
public bool Update(Action <double> progress, CancellationToken cancellationToken)
{
var sw = Stopwatch.StartNew();
var updated = false;
IsUpdating = true;
// find all files of the log on disk
var files = FindFiles();
// remove indexed file which are no longer present
foreach (var file in _index.Files.Where(f => !files.Any(p => p.Item1 == f.Item1 && p.Item2 == f.Item2)))
{
_index.Remove(file.Item1, file.Item2);
updated = true;
}
// find the files which were changed since the last indexing or which are not present in the index
var indexFiles = _index.Files;
var indexTimestamp = indexFiles.Select(f => f.Item3).Concat(new[] { DateTime.MinValue }).Max();
var changedFiles = files
.GroupJoin(indexFiles, f => f.Item1 + f.Item2, f => f.Item1 + f.Item2, (file, index) => new { file, index = index.FirstOrDefault() })
.Where(a => a.index == null || a.file.Item3 > a.index.Item3)
.Select(a => a.file)
.ToList();
// when the tailing log file has changed just process the new lines, otherwise completely index all changed files
if (changedFiles.Count > 0)
{
// update the maximum progress value
_progressMaximum = changedFiles.Sum(f => f.Item4);
// update the index with the changed files
var progressCount = 0L;
var stateCollection = new ConcurrentStack <object[]>();
Parallel.ForEach(changedFiles,
element =>
{
// create the indexer state for the file
var state = _indexers.Select(i => i.Initialize(element.Item5, element.Item1, element.Item2, element.Item4, element.Item3, false)).ToArray();
// create a stream for the file or archive member
var length = 0L;
var stream = OpenFileStream(element.Item1, element.Item2, out length);
// read and tokenize the file
CountingReader streamreader = null;
TokenReader tokenreader = null;
var sw2 = Stopwatch.StartNew();
try
{
var buffer = new Token[1024];
streamreader = new CountingReader(stream, p => progress?.Invoke(Interlocked.Add(ref progressCount, p) * 100.0 / _progressMaximum), cancellationToken);
tokenreader = new TokenReader(streamreader, element.Item1, element.Item2, 0, streamreader.CurrentEncoding);
var count = 0;
while ((count = tokenreader.Read(buffer, 0, buffer.Length)) > 0)
{
for (var i = 0; i < _indexers.Length; i++)
{
_indexers[i].Update(state[i], buffer, count);
}
}
}
finally
{
if (tokenreader != null)
{
tokenreader.Dispose();
}
else if (streamreader != null)
{
streamreader.Dispose();
}
sw2.Stop();
_logger.Info($"Log::Read(): Reading {element.Item1}:{element.Item2} completed in {sw2.ElapsedMilliseconds}ms");
}
// complete the files on the indexers
for (var i = 0; i < _indexers.Length; i++)
{
_indexers[i].Complete(state[i]);
}
});
// complete the indexers
for (var i = 0; i < _indexers.Length; i++)
{
_indexers[i].Complete();
}
updated = true;
}
// send the collection changed event
if (updated)
{
CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
}
sw.Stop();
// signal the end of the update
IsUpdating = false;
// raise property changes
if (updated)
{
_logger.Info($"Log::Update(): Updating index completed in {sw.ElapsedMilliseconds}ms");
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Files)));
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Count)));
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Tokens)));
}
else
{
_logger.Info("Log::Update(): Index is up to date");
}
return(updated);
}
