Пример #1
0
        /// <summary>
        /// Read the log using the given index entries
        /// </summary>
        /// <param name="index">Enumerable returning the index entries to read</param>
        /// <param name="progress">Action to report reading progress based on bytes consumed</param>
        /// <param name="cancellation">CancellationToken for canceling the read operation</param>
        /// <returns>Enumerable returning the log entries</returns>
        internal IEnumerable <LogItem> Read(IEnumerable <IndexItem> index, Action <long, long> progress, CancellationToken cancellation)
        {
            var        linesRead = 0;
            ZipArchive archive   = null;
            Stream     stream    = null;
            var        sw        = Stopwatch.StartNew();

            try
            {
                var            indexEnumerator = index.GetEnumerator();
                var            readerFile      = string.Empty;
                var            readerMember    = string.Empty;
                var            readerPosition  = 0L;
                var            readerLine      = 0;
                var            readerBytes     = 0;
                CountingReader reader          = null;
                while (true)
                {
                    // no more index entries to read
                    if (!indexEnumerator.MoveNext())
                    {
                        break;
                    }

                    // the file of the current index entry changed
                    if (readerFile != indexEnumerator.Current.File)
                    {
                        if (stream != null)
                        {
                            stream.Dispose();
                            reader = null;
                        }

                        if (archive != null)
                        {
                            archive.Dispose();
                        }

                        try
                        {
                            stream         = new FileStream(indexEnumerator.Current.File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                            readerFile     = indexEnumerator.Current.File;
                            readerMember   = string.Empty;
                            readerPosition = 0;
                            readerLine     = 0;
                        }
                        catch (Exception ex)
                        {
                            _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}: {ex.Message}");
                            break;
                        }

                        if (!string.IsNullOrEmpty(indexEnumerator.Current.Member))
                        {
                            try
                            {
                                archive      = new ZipArchive(stream);
                                stream       = archive.GetEntry(indexEnumerator.Current.Member).Open();
                                readerMember = indexEnumerator.Current.Member;
                            }
                            catch (Exception ex)
                            {
                                _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
                                break;
                            }
                        }
                    }

                    // the member of the current index entry changed
                    if (readerMember != indexEnumerator.Current.Member && !string.IsNullOrEmpty(indexEnumerator.Current.Member))
                    {
                        if (stream != null)
                        {
                            stream.Dispose();
                            reader = null;
                        }

                        try
                        {
                            stream         = archive.GetEntry(indexEnumerator.Current.Member).Open();
                            readerMember   = indexEnumerator.Current.Member;
                            readerPosition = 0;
                            readerLine     = 0;
                        }
                        catch (Exception ex)
                        {
                            _logger.Error($"Log::Read(): Error while opening {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
                            break;
                        }
                    }

                    // create the reader if not yet created
                    if (reader == null)
                    {
                        try
                        {
                            reader = new CountingReader(stream, p => progress?.Invoke(p, _progressMaximum), cancellation);
                        }
                        catch (Exception ex)
                        {
                            _logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
                            break;
                        }
                    }

                    // the current index entry points to a position further into the file
                    if (indexEnumerator.Current.Position > readerPosition)
                    {
                        try
                        {
                            readerPosition = reader.Seek(indexEnumerator.Current.Position, SeekOrigin.Begin);
                            readerLine     = indexEnumerator.Current.Line;
                        }
                        catch (Exception ex)
                        {
                            _logger.Error($"Log::Read(): Error while seeking in {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
                            break;
                        }
                    }

                    // read the line
                    string data;
                    try
                    {
                        data = reader.ReadLine(out readerBytes);
                    }
                    catch (Exception ex)
                    {
                        _logger.Error($"Log::Read(): Error while creating reader for {indexEnumerator.Current.File}::{indexEnumerator.Current.Member}: {ex.Message}");
                        break;
                    }

                    // return the log item
                    yield return(new LogItem(data, readerFile, readerMember, readerPosition, readerLine));

                    readerPosition += readerBytes;
                    readerLine     += 1;
                    linesRead      += 1;
                }
            }
            finally
            {
                if (stream != null)
                {
                    stream.Dispose();
                }

                if (archive != null)
                {
                    archive.Dispose();
                }

                sw.Stop();
                _logger.Info($"Log::Read(): Reading completed in {sw.ElapsedMilliseconds}ms ({linesRead} lines returned)");

                cancellation.ThrowIfCancellationRequested();
            }
        }
Пример #2
0
        /// <summary>
        /// Updates the index for the log
        /// </summary>
        /// <param name="progress">Action to report indexing progress</param>
        /// <param name="cancellationToken">CancellationToken for cancelling the index update</param>
        public bool Update(Action <double> progress, CancellationToken cancellationToken)
        {
            var sw = Stopwatch.StartNew();

            var updated = false;

            IsUpdating = true;

            // find all files of the log on disk
            var files = FindFiles();

            // remove indexed file which are no longer present
            foreach (var file in _index.Files.Where(f => !files.Any(p => p.Item1 == f.Item1 && p.Item2 == f.Item2)))
            {
                _index.Remove(file.Item1, file.Item2);
                updated = true;
            }

            // find the files which were changed since the last indexing or which are not present in the index
            var indexFiles     = _index.Files;
            var indexTimestamp = indexFiles.Select(f => f.Item3).Concat(new[] { DateTime.MinValue }).Max();
            var changedFiles   = files
                                 .GroupJoin(indexFiles, f => f.Item1 + f.Item2, f => f.Item1 + f.Item2, (file, index) => new { file, index = index.FirstOrDefault() })
                                 .Where(a => a.index == null || a.file.Item3 > a.index.Item3)
                                 .Select(a => a.file)
                                 .ToList();

            // when the tailing log file has changed just process the new lines, otherwise completely index all changed files
            if (changedFiles.Count > 0)
            {
                // update the maximum progress value
                _progressMaximum = changedFiles.Sum(f => f.Item4);

                // update the index with the changed files
                var progressCount   = 0L;
                var stateCollection = new ConcurrentStack <object[]>();
                Parallel.ForEach(changedFiles,
                                 element =>
                {
                    // create the indexer state for the file
                    var state = _indexers.Select(i => i.Initialize(element.Item5, element.Item1, element.Item2, element.Item4, element.Item3, false)).ToArray();

                    // create a stream for the file or archive member
                    var length = 0L;
                    var stream = OpenFileStream(element.Item1, element.Item2, out length);

                    // read and tokenize the file
                    CountingReader streamreader = null;
                    TokenReader tokenreader     = null;

                    var sw2 = Stopwatch.StartNew();
                    try
                    {
                        var buffer   = new Token[1024];
                        streamreader = new CountingReader(stream, p => progress?.Invoke(Interlocked.Add(ref progressCount, p) * 100.0 / _progressMaximum), cancellationToken);
                        tokenreader  = new TokenReader(streamreader, element.Item1, element.Item2, 0, streamreader.CurrentEncoding);

                        var count = 0;
                        while ((count = tokenreader.Read(buffer, 0, buffer.Length)) > 0)
                        {
                            for (var i = 0; i < _indexers.Length; i++)
                            {
                                _indexers[i].Update(state[i], buffer, count);
                            }
                        }
                    }
                    finally
                    {
                        if (tokenreader != null)
                        {
                            tokenreader.Dispose();
                        }
                        else if (streamreader != null)
                        {
                            streamreader.Dispose();
                        }
                        sw2.Stop();
                        _logger.Info($"Log::Read(): Reading {element.Item1}:{element.Item2} completed in {sw2.ElapsedMilliseconds}ms");
                    }

                    // complete the files on the indexers
                    for (var i = 0; i < _indexers.Length; i++)
                    {
                        _indexers[i].Complete(state[i]);
                    }
                });

                // complete the indexers
                for (var i = 0; i < _indexers.Length; i++)
                {
                    _indexers[i].Complete();
                }


                updated = true;
            }

            // send the collection changed event
            if (updated)
            {
                CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset));
            }

            sw.Stop();

            // signal the end of the update
            IsUpdating = false;

            // raise property changes
            if (updated)
            {
                _logger.Info($"Log::Update(): Updating index completed in {sw.ElapsedMilliseconds}ms");
                PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Files)));
                PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Count)));
                PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Tokens)));
            }
            else
            {
                _logger.Info("Log::Update(): Index is up to date");
            }

            return(updated);
        }