/// <inheritdoc /> public bool SetValue(IReadOnlyPropertyDescriptor property, object value) { lock (_syncRoot) { return(_storage.SetValue(property, value)); } }
/// <summary> /// Initializes this text log file. /// </summary> /// <param name="taskScheduler"></param> /// <param name="fileName"></param> /// <param name="format"></param> /// <param name="encoding"></param> internal StreamingTextLogSource(ITaskScheduler taskScheduler, string fileName, ILogFileFormat format, Encoding encoding) { _taskScheduler = taskScheduler; _encoding = encoding; _listeners = new LogSourceListenerCollection(this); _sourceDoesNotExist = new SourceDoesNotExist(fileName); _sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName); _fileName = fileName ?? throw new ArgumentNullException(nameof(fileName)); _index = new LogBufferList(StreamingTextLogSource.LineOffsetInBytes); _propertiesBuffer = new PropertiesBufferList(); _propertiesBuffer.SetValue(Core.Properties.Name, _fileName); _propertiesBuffer.SetValue(Core.Properties.Format, format); _propertiesBuffer.SetValue(TextProperties.RequiresBuffer, true); _propertiesBuffer.SetValue(TextProperties.LineCount, 0); _properties = new ConcurrentPropertiesList(Core.Properties.Minimum); SynchronizeProperties(); _cancellationTokenSource = new CancellationTokenSource(); _columns = new IColumnDescriptor[] { Core.Columns.Index, StreamingTextLogSource.LineOffsetInBytes, Core.Columns.RawContent }; _pendingReadRequests = new ConcurrentQueue <IReadRequest>(); _fileScanTask = _taskScheduler.StartPeriodic(() => RunFileScan(_cancellationTokenSource.Token)); _fileReadTask = _taskScheduler.StartPeriodic(() => RunFileRead(_cancellationTokenSource.Token)); }
private void UpdateProperties() { // First we want to retrieve all properties from the source _source.GetAllProperties(_propertiesBuffer); // Then we'll add / overwrite properties _propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, ComputePercentageProcessed()); _propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _indices.Count); // And last but not least we'll update our own properties to the new values // It's important we do this in one go so clients can retrieve all those properties // in a consistent state _properties.CopyFrom(_propertiesBuffer); }
private void UpdateProperties() { Size? size = null; DateTime? lastModified = null; DateTime? startTimestamp = null; DateTime? endTimestamp = null; int maxCharactersPerLine = 0; Percentage processed = Percentage.HundredPercent; for (int n = 0; n < _sources.Count; ++n) { var source = _sources[n]; source.GetAllProperties(_propertiesBuffer); var sourceSize = _propertiesBuffer.GetValue(Core.Properties.Size); if (size == null) { size = sourceSize; } else if (sourceSize != null) { size += sourceSize; } var last = _propertiesBuffer.GetValue(Core.Properties.LastModified); if (last != null && (last > lastModified || lastModified == null)) { lastModified = last; } var start = _propertiesBuffer.GetValue(Core.Properties.StartTimestamp); if (start != null && (start < startTimestamp || startTimestamp == null)) { startTimestamp = start; } var end = _propertiesBuffer.GetValue(Core.Properties.EndTimestamp); if (end != null && (end > endTimestamp || endTimestamp == null)) { endTimestamp = end; } maxCharactersPerLine = Math.Max(maxCharactersPerLine, _propertiesBuffer.GetValue(TextProperties.MaxCharactersInLine)); var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed); processed *= sourceProcessed; } _propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _index.Count); _propertiesBuffer.SetValue(TextProperties.MaxCharactersInLine, maxCharactersPerLine); _propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, processed); _propertiesBuffer.SetValue(Core.Properties.LastModified, lastModified); _propertiesBuffer.SetValue(Core.Properties.Size, size); _propertiesBuffer.SetValue(Core.Properties.StartTimestamp, startTimestamp); _propertiesBuffer.SetValue(Core.Properties.EndTimestamp, endTimestamp); _propertiesBuffer.SetValue(Core.Properties.Duration, endTimestamp - startTimestamp); // We want to ensure that we modify all properties at once so that users of this log file don't // see an inconsistent state of properties when they retrieve them. _properties.CopyFrom(_propertiesBuffer); }
public FileLogSource(IServiceContainer services, string fileName, TimeSpan maximumWaitTime) : base(services.Retrieve <ITaskScheduler>()) { _syncRoot = new object(); _filesystem = services.Retrieve <IFilesystem>(); _services = services; _fullFilename = Path.IsPathRooted(fileName) ? fileName : Path.Combine(Directory.GetCurrentDirectory(), fileName); _maximumWaitTime = maximumWaitTime; _sourceDoesNotExist = new SourceDoesNotExist(fileName); _sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName); var formatMatcher = services.Retrieve <ILogFileFormatMatcher>(); _encodingDetector = new EncodingDetector(); _formatDetector = new FileFormatDetector(formatMatcher); _buffer = new LogBufferArray(MaximumLineCount, Core.Columns.RawContent); _pendingSections = new ConcurrentQueue <KeyValuePair <ILogSource, LogSourceModification> >(); _propertiesBuffer = new PropertiesBufferList(); _propertiesBuffer.SetValue(Core.Properties.Name, _fullFilename); _properties = new ConcurrentPropertiesList(); StartTask(); }
/// <summary> /// Initializes this object. /// </summary> /// <remarks> /// Plugin authors are deliberately prevented from calling this constructor directly because it's signature may change /// over time. In order to create an instance of this type, simply call <see cref="ILogSourceFactory.CreateMultiLineLogFile"/>. /// </remarks> /// <param name="taskScheduler"></param> /// <param name="source"></param> /// <param name="maximumWaitTime"></param> public MultiLineLogSource(ITaskScheduler taskScheduler, ILogSource source, TimeSpan maximumWaitTime) : base(taskScheduler) { if (source == null) { throw new ArgumentNullException(nameof(source)); } _maximumWaitTime = maximumWaitTime; _pendingModifications = new ConcurrentQueue <LogSourceModification>(); _syncRoot = new object(); _specialColumns = new HashSet <IColumnDescriptor> { Core.Columns.LogEntryIndex, Core.Columns.Timestamp, Core.Columns.LogLevel }; _indices = new List <LogEntryInfo>(); // The log file we were given might offer even more properties than the minimum set and we // want to expose those as well. _propertiesBuffer = new PropertiesBufferList(Core.Properties.CombineWithMinimum(source.Properties)); _propertiesBuffer.SetValue(Core.Properties.EmptyReason, null); _properties = new ConcurrentPropertiesList(Core.Properties.CombineWithMinimum(source.Properties)); _properties.CopyFrom(_propertiesBuffer); _currentLogEntry = new LogEntryInfo(-1, 0); _source = source; _source.AddListener(this, maximumWaitTime, MaximumBatchSize); StartTask(); }
private void SynchronizeProperties() { _source.GetAllProperties(_propertiesBufferView); var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed); var sourceCount = _propertiesBuffer.GetValue(Core.Properties.LogEntryCount); var ownProgress = sourceCount > 0 ? Percentage.Of(_count, sourceCount).Clamped() : Percentage.HundredPercent; var totalProgress = (sourceProcessed * ownProgress).Clamped(); _propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, totalProgress); _propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _count); GetOverwrittenProperties(_propertiesBuffer); _properties.CopyFrom(_propertiesBuffer); }
/// <summary> /// Initializes this text log file. /// </summary> /// <param name="filesystem"></param> /// <param name="taskScheduler"></param> /// <param name="fileName"></param> /// <param name="format"></param> /// <param name="encoding"></param> internal TextLogSource(IFilesystem filesystem, ITaskScheduler taskScheduler, string fileName, ILogFileFormat format, Encoding encoding) : base(taskScheduler) { _filesystem = filesystem; _fileName = fileName ?? throw new ArgumentNullException(nameof(fileName)); _encoding = encoding ?? throw new ArgumentNullException(nameof(encoding)); _entries = new LogBufferList(Core.Columns.RawContent); _columns = new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.OriginalIndex, Core.Columns.LogEntryIndex, Core.Columns.LineNumber, Core.Columns.OriginalLineNumber, Core.Columns.OriginalDataSourceName, Core.Columns.RawContent, PageBufferedLogSource.RetrievalState }; _sourceDoesNotExist = new SourceDoesNotExist(fileName); _sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName); _localProperties = new PropertiesBufferList(Core.Properties.Minimum); _localProperties.SetValue(Core.Properties.Name, _fileName); _localProperties.Add(TextProperties.LineCount); _localProperties.SetValue(Core.Properties.Format, format); _localProperties.SetValue(TextProperties.LineCount, 0); _localProperties.SetValue(TextProperties.RequiresBuffer, false); _properties = new ConcurrentPropertiesList(Core.Properties.Minimum); SynchronizePropertiesWithUser(); _syncRoot = new object(); _properties.SetValue(TextProperties.AutoDetectedEncoding, encoding); Log.DebugFormat("Log File '{0}' is interpreted using {1}", _fileName, _encoding.EncodingName); StartTask(); }
/// <summary> /// Initializes this object. /// </summary> /// <param name="columns"></param> /// <param name="properties"></param> public InMemoryLogSource(IEnumerable <IColumnDescriptor> columns, IReadOnlyDictionary <IReadOnlyPropertyDescriptor, object> properties) { if (columns == null) { throw new ArgumentNullException(nameof(columns)); } _syncRoot = new object(); _logBuffer = new LogBufferList(Core.Columns.CombineWithMinimum(columns)); _listeners = new LogSourceListenerCollection(this); _properties = new PropertiesBufferList(Core.Properties.Minimum); _properties.SetValue(Core.Properties.Size, Size.Zero); _properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent); foreach (var pair in properties) { _properties.SetValue(pair.Key, pair.Value); } }
/// <inheritdoc /> protected override TimeSpan RunOnce(CancellationToken token) { bool read = false; try { if (!File.Exists(_fileName)) { SetDoesNotExist(); } else { var info = new FileInfo(_fileName); var fileSize = info.Length; _localProperties.SetValue(Core.Properties.LastModified, info.LastWriteTimeUtc); _localProperties.SetValue(Core.Properties.Created, info.CreationTimeUtc); _localProperties.SetValue(Core.Properties.Size, Size.FromBytes(fileSize)); UpdatePercentageProcessed(_lastStreamPosition, fileSize, allow100Percent: true); SynchronizePropertiesWithUser(); using (var stream = new FileStream(_fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { using (var reader = new StreamReaderEx(stream, _encoding)) { // We change the error flag explicitly AFTER opening // the stream because that operation might throw if we're // not allowed to access the file (in which case a different // error must be set). _localProperties.SetValue(Core.Properties.EmptyReason, null); if (stream.Length >= _lastStreamPosition) { stream.Position = _lastStreamPosition; } else { OnReset(stream, out _numberOfLinesRead, out _lastStreamPosition); } int numProcessed = 0; string currentLine; while ((currentLine = reader.ReadLine()) != null) { token.ThrowIfCancellationRequested(); bool lastLineHadNewline = _lastLineHadNewline; var trimmedLine = currentLine.TrimNewlineEnd(out _lastLineHadNewline); var entryCount = _entries.Count; if (entryCount > 0 && !lastLineHadNewline) { // We need to remove the last line and create a new line // that consists of the entire content. RemoveLast(); trimmedLine = _untrimmedLastLine + trimmedLine; _untrimmedLastLine = _untrimmedLastLine + currentLine; } else { _untrimmedLastLine = currentLine; ++_numberOfLinesRead; read = true; } Add(trimmedLine, _numberOfLinesRead); if (++numProcessed % 1000 == 0) { // Here's the deal: Since we're processing the file in chunks, we advance the underlying // stream faster than we're actually consuming lines. This means that it's quite likely // that at the end of the file, we have moved the stream to the end, but have not quite // yet processed the underlying buffer from StreamReaderEx. The percentage processed // should be accurate enough so that if it is at 100%, then no more log entries are added. // We can only guarantee that when we have processed all lines and therefore we reserve // setting the percentage to 100% ONLY when we can read no more lines // (See the SetEndOfSourceReached() call below, outside the loop). UpdatePercentageProcessed(stream.Position, fileSize, allow100Percent: false); SynchronizePropertiesWithUser(); } } _lastStreamPosition = stream.Position; _localProperties.SetValue(TextProperties.LineCount, _entries.Count); _localProperties.SetValue(Core.Properties.LogEntryCount, _entries.Count); } } Listeners.OnRead(_numberOfLinesRead); SetEndOfSourceReached(); } } catch (FileNotFoundException e) { SetError(_sourceDoesNotExist); Log.Debug(e); } catch (DirectoryNotFoundException e) { SetError(_sourceDoesNotExist); Log.Debug(e); } catch (OperationCanceledException e) { Log.Debug(e); } catch (UnauthorizedAccessException e) { SetError(_sourceCannotBeAccessed); Log.Debug(e); } catch (IOException e) { SetError(_sourceCannotBeAccessed); Log.Debug(e); } catch (Exception e) { Log.Debug(e); } if (read) { return(TimeSpan.Zero); } return(TimeSpan.FromMilliseconds(100)); }
private void Process(LogSourceSection section) { DateTime?startTime = _propertiesBuffer.GetValue(Core.Properties.StartTimestamp); DateTime?endTime = _propertiesBuffer.GetValue(Core.Properties.EndTimestamp); int traceCount = _propertiesBuffer.GetValue(Core.Properties.TraceLogEntryCount); int debugCount = _propertiesBuffer.GetValue(Core.Properties.DebugLogEntryCount); int infoCount = _propertiesBuffer.GetValue(Core.Properties.InfoLogEntryCount); int warningCount = _propertiesBuffer.GetValue(Core.Properties.WarningLogEntryCount); int errorCount = _propertiesBuffer.GetValue(Core.Properties.ErrorLogEntryCount); int fatalCount = _propertiesBuffer.GetValue(Core.Properties.FatalLogEntryCount); int otherCount = _propertiesBuffer.GetValue(Core.Properties.OtherLogEntryCount); _source.GetEntries(section, _buffer, 0, LogSourceQueryOptions.Default); bool evaluateTimestamp = _requiredColumns.Contains(Core.Columns.Timestamp); bool evaluateLevel = _requiredColumns.Contains(Core.Columns.LogLevel); foreach (var entry in _buffer) { if (!entry.Index.IsValid) { break; } if (evaluateTimestamp) { var timestamp = entry.Timestamp; if (timestamp != null) { if (startTime == null) { startTime = timestamp; } else if (timestamp < startTime) { startTime = timestamp; } if (endTime == null) { endTime = timestamp; } else if (timestamp > endTime) { endTime = timestamp; } } } if (evaluateLevel) { var level = entry.LogLevel; switch (level) { case LevelFlags.Fatal: ++fatalCount; break; case LevelFlags.Error: ++errorCount; break; case LevelFlags.Warning: ++warningCount; break; case LevelFlags.Info: ++infoCount; break; case LevelFlags.Debug: ++debugCount; break; case LevelFlags.Trace: ++traceCount; break; case LevelFlags.Other: ++otherCount; break; } } } if (evaluateTimestamp) { _propertiesBuffer.SetValue(Core.Properties.StartTimestamp, startTime); _propertiesBuffer.SetValue(Core.Properties.EndTimestamp, endTime); _propertiesBuffer.SetValue(Core.Properties.Duration, endTime - startTime); } if (evaluateLevel) { _propertiesBuffer.SetValue(Core.Properties.TraceLogEntryCount, traceCount); _propertiesBuffer.SetValue(Core.Properties.DebugLogEntryCount, debugCount); _propertiesBuffer.SetValue(Core.Properties.InfoLogEntryCount, infoCount); _propertiesBuffer.SetValue(Core.Properties.WarningLogEntryCount, warningCount); _propertiesBuffer.SetValue(Core.Properties.ErrorLogEntryCount, errorCount); _propertiesBuffer.SetValue(Core.Properties.FatalLogEntryCount, fatalCount); _propertiesBuffer.SetValue(Core.Properties.OtherLogEntryCount, otherCount); } }
private void UpdateFormat() { try { if (DetectFileChange(out var overwrittenEncoding)) { if (!_filesystem.FileExists(_fullFilename)) { SetError(_sourceDoesNotExist); } else { using (var stream = TryOpenRead(_fullFilename, out var error)) { if (stream == null) { SetError(error); } else { var autoDetectedEncoding = _encodingDetector.TryFindEncoding(stream); var defaultEncoding = _services.TryRetrieve <Encoding>() ?? Encoding.Default; var format = _formatDetector.TryDetermineFormat(_fullFilename, stream, overwrittenEncoding ?? autoDetectedEncoding ?? defaultEncoding); var encoding = PickEncoding(overwrittenEncoding, format, autoDetectedEncoding, defaultEncoding); var formatChanged = _propertiesBuffer.SetValue(Core.Properties.Format, format); _propertiesBuffer.SetValue(TextProperties.AutoDetectedEncoding, autoDetectedEncoding); _propertiesBuffer.SetValue(TextProperties.ByteOrderMark, autoDetectedEncoding != null); _propertiesBuffer.SetValue(Core.Properties.EmptyReason, error); var encodingChanged = _propertiesBuffer.SetValue(TextProperties.Encoding, encoding); var currentLogSources = _logSources; if (currentLogSources == null || currentLogSources.Count == 0 || formatChanged || encodingChanged) { Dispose(currentLogSources); // Depending on the log file we're actually opening, the plugins we have installed, the final log source // that we expose here could be anything. It could be the raw log source which reads everything line by line, // but it also could be a plugin's ILogSource implementation which does all kinds of magic. var newLogSources = CreateAllLogSources(_services, _fullFilename, format, encoding, _maximumWaitTime); if (!StartListenTo(newLogSources)) { Dispose(newLogSources); } } } } } } } catch (IOException e) { Log.DebugFormat("Caught exception while reading '{0}': {1}", _fullFilename, e); } catch (Exception e) { Log.DebugFormat("Caught unexpected exception while reading '{0}': {1}", _fullFilename, e); } }
public EmptyLogSource() { _properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent); _properties.SetValue(Core.Properties.Size, Size.Zero); }
/// <inheritdoc /> public void SetProperty(IPropertyDescriptor property, object value) { _properties.SetValue(property, value); }