/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal StreamingTextLogSource(ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
{
_taskScheduler = taskScheduler;
_encoding = encoding;
_listeners = new LogSourceListenerCollection(this);
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_index = new LogBufferList(StreamingTextLogSource.LineOffsetInBytes);
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fileName);
_propertiesBuffer.SetValue(Core.Properties.Format, format);
_propertiesBuffer.SetValue(TextProperties.RequiresBuffer, true);
_propertiesBuffer.SetValue(TextProperties.LineCount, 0);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizeProperties();
_cancellationTokenSource = new CancellationTokenSource();
_columns = new IColumnDescriptor[] { Core.Columns.Index, StreamingTextLogSource.LineOffsetInBytes, Core.Columns.RawContent };
_pendingReadRequests = new ConcurrentQueue <IReadRequest>();
_fileScanTask = _taskScheduler.StartPeriodic(() => RunFileScan(_cancellationTokenSource.Token));
_fileReadTask = _taskScheduler.StartPeriodic(() => RunFileRead(_cancellationTokenSource.Token));
}
/// <summary>
/// Returns all properties from this log file.
/// </summary>
/// <returns></returns>
public static IPropertiesBuffer GetAllProperties(this ILogSource that)
{
var destination = new PropertiesBufferList();
that.GetAllProperties(destination);
return(destination);
}
public FileLogSource(IServiceContainer services, string fileName, TimeSpan maximumWaitTime)
: base(services.Retrieve <ITaskScheduler>())
{
_syncRoot = new object();
_filesystem = services.Retrieve <IFilesystem>();
_services = services;
_fullFilename = Path.IsPathRooted(fileName)
? fileName
: Path.Combine(Directory.GetCurrentDirectory(), fileName);
_maximumWaitTime = maximumWaitTime;
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
var formatMatcher = services.Retrieve <ILogFileFormatMatcher>();
_encodingDetector = new EncodingDetector();
_formatDetector = new FileFormatDetector(formatMatcher);
_buffer = new LogBufferArray(MaximumLineCount, Core.Columns.RawContent);
_pendingSections = new ConcurrentQueue <KeyValuePair <ILogSource, LogSourceModification> >();
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fullFilename);
_properties = new ConcurrentPropertiesList();
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <remarks>
/// Plugin authors are deliberately prevented from calling this constructor directly because it's signature may change
/// over time. In order to create an instance of this type, simply call <see cref="ILogSourceFactory.CreateMultiLineLogFile"/>.
/// </remarks>
/// <param name="taskScheduler"></param>
/// <param name="source"></param>
/// <param name="maximumWaitTime"></param>
public MultiLineLogSource(ITaskScheduler taskScheduler, ILogSource source, TimeSpan maximumWaitTime)
: base(taskScheduler)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
_maximumWaitTime = maximumWaitTime;
_pendingModifications = new ConcurrentQueue <LogSourceModification>();
_syncRoot = new object();
_specialColumns = new HashSet <IColumnDescriptor> {
Core.Columns.LogEntryIndex, Core.Columns.Timestamp, Core.Columns.LogLevel
};
_indices = new List <LogEntryInfo>();
// The log file we were given might offer even more properties than the minimum set and we
// want to expose those as well.
_propertiesBuffer = new PropertiesBufferList(Core.Properties.CombineWithMinimum(source.Properties));
_propertiesBuffer.SetValue(Core.Properties.EmptyReason, null);
_properties = new ConcurrentPropertiesList(Core.Properties.CombineWithMinimum(source.Properties));
_properties.CopyFrom(_propertiesBuffer);
_currentLogEntry = new LogEntryInfo(-1, 0);
_source = source;
_source.AddListener(this, maximumWaitTime, MaximumBatchSize);
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <remarks>
/// Plugin authors are deliberately prevented from calling this constructor directly because it's signature may change
/// over time. In order to create an instance of this type, simply call <see cref="ILogSourceFactory.CreateMergedLogFile"/>.
/// </remarks>
/// <param name="scheduler"></param>
/// <param name="maximumWaitTime"></param>
/// <param name="sources"></param>
internal MergedLogSource(ITaskScheduler scheduler, TimeSpan maximumWaitTime, params ILogSource[] sources)
: base(scheduler)
{
if (sources == null)
{
throw new ArgumentNullException(nameof(sources));
}
if (sources.Any(x => x == null))
{
throw new ArgumentException("sources.Any(x => x == null)", nameof(sources));
}
if (sources.Length > LogEntrySourceId.MaxSources)
{
throw new ArgumentException(string.Format("Only up to {0} sources are supported ({1} were given)", LogEntrySourceId.MaxSources, sources.Length));
}
_sources = sources;
_index = new MergedLogSourceIndex(sources);
_pendingModifications = new ConcurrentQueue <MergedLogSourcePendingModification>();
_maximumWaitTime = maximumWaitTime;
_columns = sources.SelectMany(x => x.Columns).Concat(new[] { Core.Columns.SourceId }).Distinct().ToList();
_propertiesBuffer = new PropertiesBufferList(Core.Properties.Minimum);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
foreach (var logFile in _sources)
{
logFile.AddListener(this, maximumWaitTime, MaximumBatchSizePerSource);
}
StartTask();
}
/// <summary>
/// </summary>
/// <param name="properties"></param>
public ConcurrentPropertiesList(IEnumerable <IReadOnlyPropertyDescriptor> properties)
{
if (properties == null)
{
throw new ArgumentNullException(nameof(properties));
}
_syncRoot = new object();
_storage = new PropertiesBufferList(properties);
}
public LogSourcePropertyAdorner(ITaskScheduler scheduler, ILogSource source, TimeSpan maximumWaitTime, IReadOnlyList <IReadOnlyPropertyDescriptor> adornedProperties)
: base(scheduler)
{
_source = source;
_maximumWaitTime = maximumWaitTime;
_adornedProperties = adornedProperties;
_propertiesBuffer = new PropertiesBufferList(_adornedProperties);
_properties = new ConcurrentPropertiesList(_adornedProperties);
_pendingSections = new ConcurrentQueue <LogSourceModification>();
_requiredColumns = GetColumnsRequiredFor(_adornedProperties);
const int bufferSize = 1000;
_buffer = new LogBufferArray(bufferSize, _requiredColumns);
_source.AddListener(this, maximumWaitTime, bufferSize);
StartTask();
}
/// <summary>
///
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="source"></param>
/// <param name="columnsToFetch"></param>
/// <param name="overwrittenProperties"></param>
/// <param name="maximumWaitTime"></param>
/// <param name="maxEntryCount"></param>
protected AbstractProcessingLogSource(ITaskScheduler taskScheduler,
ILogSource source,
IReadOnlyList <IColumnDescriptor> columnsToFetch,
IReadOnlyList <IReadOnlyPropertyDescriptor> overwrittenProperties,
TimeSpan maximumWaitTime,
int maxEntryCount = 1000)
: base(taskScheduler)
{
_fetchBuffer = new LogBufferArray(maxEntryCount, columnsToFetch);
_pendingSections = new ConcurrentQueue <LogSourceModification>();
_source = source;
_maximumWaitTime = maximumWaitTime;
_maxEntryCount = maxEntryCount;
_propertiesBuffer = new PropertiesBufferList();
_propertiesBufferView = new PropertiesBufferHidingView(_propertiesBuffer, overwrittenProperties);
_properties = new ConcurrentPropertiesList();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="maximumWaitTime"></param>
/// <param name="maxEntryCount"></param>
public LogSourceProxy(ITaskScheduler taskScheduler, TimeSpan maximumWaitTime, int maxEntryCount = DefaultMaxEntryCount)
{
if (taskScheduler == null)
{
throw new ArgumentNullException(nameof(taskScheduler));
}
_taskScheduler = taskScheduler;
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
_properties.SetValue(Core.Properties.EmptyReason, null);
_sourceProperties = new PropertiesBufferList();
_pendingSections = new ConcurrentQueue <KeyValuePair <ILogSource, LogSourceModification> >();
_listeners = new LogSourceListenerCollection(this);
_task = _taskScheduler.StartPeriodic(RunOnce, "Log File Proxy");
_maximumWaitTime = maximumWaitTime;
_maxEntryCount = maxEntryCount;
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="filesystem"></param>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal TextLogSource(IFilesystem filesystem,
ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
: base(taskScheduler)
{
_filesystem = filesystem;
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_encoding = encoding ?? throw new ArgumentNullException(nameof(encoding));
_entries = new LogBufferList(Core.Columns.RawContent);
_columns = new IColumnDescriptor[]
{
Core.Columns.Index,
Core.Columns.OriginalIndex,
Core.Columns.LogEntryIndex,
Core.Columns.LineNumber,
Core.Columns.OriginalLineNumber,
Core.Columns.OriginalDataSourceName,
Core.Columns.RawContent,
PageBufferedLogSource.RetrievalState
};
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_localProperties = new PropertiesBufferList(Core.Properties.Minimum);
_localProperties.SetValue(Core.Properties.Name, _fileName);
_localProperties.Add(TextProperties.LineCount);
_localProperties.SetValue(Core.Properties.Format, format);
_localProperties.SetValue(TextProperties.LineCount, 0);
_localProperties.SetValue(TextProperties.RequiresBuffer, false);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizePropertiesWithUser();
_syncRoot = new object();
_properties.SetValue(TextProperties.AutoDetectedEncoding, encoding);
Log.DebugFormat("Log File '{0}' is interpreted using {1}", _fileName, _encoding.EncodingName);
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="columns"></param>
/// <param name="properties"></param>
public InMemoryLogSource(IEnumerable <IColumnDescriptor> columns, IReadOnlyDictionary <IReadOnlyPropertyDescriptor, object> properties)
{
if (columns == null)
{
throw new ArgumentNullException(nameof(columns));
}
_syncRoot = new object();
_logBuffer = new LogBufferList(Core.Columns.CombineWithMinimum(columns));
_listeners = new LogSourceListenerCollection(this);
_properties = new PropertiesBufferList(Core.Properties.Minimum);
_properties.SetValue(Core.Properties.Size, Size.Zero);
_properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent);
foreach (var pair in properties)
{
_properties.SetValue(pair.Key, pair.Value);
}
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="scheduler"></param>
/// <param name="maximumWaitTime"></param>
/// <param name="source"></param>
/// <param name="logLineFilter"></param>
/// <param name="logEntryFilter"></param>
public FilteredLogSource(ITaskScheduler scheduler,
TimeSpan maximumWaitTime,
ILogSource source,
ILogLineFilter logLineFilter,
ILogEntryFilter logEntryFilter)
: base(scheduler)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
_properties = new ConcurrentPropertiesList(source.Properties);
_propertiesBuffer = new PropertiesBufferList(); //< Will be used as temporary storage to hold the properties from the source
_logLineFilter = logLineFilter ?? new NoFilter();
_logEntryFilter = logEntryFilter ?? new NoFilter();
_pendingModifications = new ConcurrentQueue <LogSourceModification>();
_indices = new List <int>();
_logEntryIndices = new Dictionary <int, int>();
_array = new LogBufferArray(BatchSize, Core.Columns.Minimum);
_lastLogBuffer = new LogBufferList(Core.Columns.Minimum);
_maximumWaitTime = maximumWaitTime;
_source.AddListener(this, maximumWaitTime, BatchSize);
StartTask();
}
protected override void GetOverwrittenProperties(PropertiesBufferList destination)
{
}
/// <summary> /// This method is called when properties are synchronized with the clients of this class. /// If the implementation of this class has some overwritten properties, then now is the time to write them /// into the <paramref name="destination" /> buffer. /// </summary> /// <param name="destination"></param> protected abstract void GetOverwrittenProperties(PropertiesBufferList destination);