/// <summary>
/// Initializes a new instance of the <see cref="BaseMultiFormatIssueProviderSettings{TIssueProvider, TSettings}"/> class
/// for a log file content in memoy.
/// </summary>
/// <param name="logFileContent">Content of the log file.
/// The log file needs to be in the format as defined by the <paramref name="format"/> parameter.</param>
/// <param name="format">Format of the provided log file.</param>
public BaseMultiFormatIssueProviderSettings(byte[] logFileContent, ILogFileFormat <TIssueProvider, TSettings> format)
: base(logFileContent)
{
format.NotNull(nameof(format));
this.Format = format;
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseMultiFormatIssueProviderSettings{TIssueProvider, TSettings}"/> class
/// for reading a log file on disk.
/// </summary>
/// <param name="logFilePath">Path to the log file.
/// The log file needs to be in the format as defined by the <paramref name="format"/> parameter.</param>
/// <param name="format">Format of the provided log file.</param>
public BaseMultiFormatIssueProviderSettings(FilePath logFilePath, ILogFileFormat <TIssueProvider, TSettings> format)
: base(logFilePath)
{
format.NotNull(nameof(format));
this.Format = format;
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal StreamingTextLogSource(ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
{
_taskScheduler = taskScheduler;
_encoding = encoding;
_listeners = new LogSourceListenerCollection(this);
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_index = new LogBufferList(StreamingTextLogSource.LineOffsetInBytes);
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fileName);
_propertiesBuffer.SetValue(Core.Properties.Format, format);
_propertiesBuffer.SetValue(TextProperties.RequiresBuffer, true);
_propertiesBuffer.SetValue(TextProperties.LineCount, 0);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizeProperties();
_cancellationTokenSource = new CancellationTokenSource();
_columns = new IColumnDescriptor[] { Core.Columns.Index, StreamingTextLogSource.LineOffsetInBytes, Core.Columns.RawContent };
_pendingReadRequests = new ConcurrentQueue <IReadRequest>();
_fileScanTask = _taskScheduler.StartPeriodic(() => RunFileScan(_cancellationTokenSource.Token));
_fileReadTask = _taskScheduler.StartPeriodic(() => RunFileRead(_cancellationTokenSource.Token));
}
private void ReplaceCustomFormat(CustomLogFileFormat newCustomFormat, ILogFileFormat newLogFileFormat)
{
_logFileFormatRegistry.Replace(_customFormat, newCustomFormat, newLogFileFormat);
_settings.CustomFormats.Remove(_customFormat);
_settings.CustomFormats.Add(newCustomFormat);
_customFormat = newCustomFormat;
}
public bool TryMatchFormat(string fileName,
byte[] header,
Encoding encoding,
out ILogFileFormat format,
out Certainty certainty)
{
try
{
if (_inner == null)
{
format = null;
certainty = Certainty.Sure;
return(false);
}
return(_inner.TryMatchFormat(fileName, header, encoding, out format, out certainty));
}
catch (Exception e)
{
Log.ErrorFormat("Caught unexpected exception: {0}", e);
format = null;
certainty = Certainty.Uncertain;
return(false);
}
}
static MyLogFileFormatMatcherPlugin()
{
MyCustomFormat = new MyCustomLogFileFormat
{
Name = "Mylog v2"
};
}
public void Add(CustomLogFileFormat customFormat, ILogFileFormat format)
{
lock (_syncRoot)
{
_formats.Add(format);
}
}
public bool TryMatchFormat(string fileName,
byte[] header,
Encoding encoding,
out ILogFileFormat format,
out Certainty certainty)
{
var formats = _repository.Formats.OfType <SerilogFileFormat>();
foreach (var serilogFormat in formats)
{
var parser = serilogFormat.Parser;
using (var memoryStream = new MemoryStream(header))
using (var reader = new StreamReader(memoryStream, serilogFormat.Encoding ?? encoding))
{
if (TryParseFormat(reader, parser))
{
format = serilogFormat;
certainty = Certainty.Sure;
return(true);
}
}
}
certainty = header.Length >= 512
? Certainty.Sure
: Certainty.Uncertain;
format = null;
return(false);
}
/// <summary>
/// Initializes a new instance of the <see cref="MsBuildIssuesSettings"/> class.
/// </summary>
/// <param name="logFileContent">Content of the the MsBuild log file.
/// The log file needs to be in the format as defined by the <paramref name="format"/> parameter.</param>
/// <param name="format">Format of the provided MsBuild log file.</param>
protected MsBuildIssuesSettings(string logFileContent, ILogFileFormat format)
{
logFileContent.NotNullOrWhiteSpace(nameof(logFileContent));
format.NotNull(nameof(format));
this.LogFileContent = logFileContent;
this.Format = format;
}
private ILogSource CreateSource(ILogFileFormat format)
{
var source = new Mock <ILogSource>();
source.Setup(x => x.GetProperty(Properties.Format)).Returns(format);
source.Setup(x => x.Columns).Returns(new IColumnDescriptor[0]);
return(source.Object);
}
private IDataSource CreateDataSource(ILogFileFormat format)
{
var dataSource = new Mock <IDataSource>();
var logFile = new Mock <ILogSource>();
logFile.Setup(x => x.GetProperty(Properties.Format)).Returns(format);
dataSource.Setup(x => x.UnfilteredLogSource).Returns(logFile.Object);
return(dataSource.Object);
}
public static ICodeAnalysisProvider MsBuildIssuesFromFilePath(
this ICakeContext context,
FilePath logFilePath,
ILogFileFormat format)
{
context.NotNull(nameof(context));
logFilePath.NotNull(nameof(logFilePath));
format.NotNull(nameof(format));
return(context.MsBuildIssues(MsBuildIssuesSettings.FromFilePath(logFilePath, format)));
}
public static ICodeAnalysisProvider MsBuildIssuesFromContent(
this ICakeContext context,
string logFileContent,
ILogFileFormat format)
{
context.NotNull(nameof(context));
logFileContent.NotNullOrWhiteSpace(nameof(logFileContent));
format.NotNull(nameof(format));
return(context.MsBuildIssues(MsBuildIssuesSettings.FromContent(logFileContent, format)));
}
public ILogEntryParser CreateParser(IServiceContainer services, ILogFileFormat format)
{
var serilogFormat = format as SerilogFileFormat;
if (serilogFormat == null)
{
return(null);
}
return(serilogFormat.Parser);
}
static LogFileFormats()
{
GenericText = new TextLogFileFormat("Generic Text");
Csv = new TextLogFileFormat("CSV");
CommonLogFormat = new TextLogFileFormat("Common Log Format",
"The Common Log Format, also known as the NCSA Common log format is a standardized text file format used by web servers when generating server log files.",
Encoding.ASCII);
ExtendedLogFormat = new TextLogFileFormat("Extended Log Format",
"Extended Log Format (ELF) is a standardised text file format, like Common Log Format (CLF), that is used by web servers when generating log files, but ELF files provide more information and flexibility.",
Encoding.ASCII);
}
/// <inheritdoc />
public ILogSource OpenRead(string fileName, ILogFileFormat format, Encoding encoding)
{
if (format.IsText)
{
return(new TextLogSource(_filesystem, _taskScheduler, fileName, format, encoding));
}
else
{
Log.WarnFormat("Log file {0} has been determined to be a binary file ({1}) - processing binary files is not implemented", fileName, format);
return(null);
}
}
private bool Matches(ILogFileOutlinePlugin plugin, ILogFileFormat format)
{
try
{
return(plugin.SupportedFormats.Any(x => Equals(x, format)));
}
catch (Exception e)
{
Log.ErrorFormat("Caught unexpected exception: {0}", e);
return(false);
}
}
public ILogEntryParser CreateParser(IServiceContainer services, ILogFileFormat format)
{
foreach (var plugin in _pluginLoader.LoadAllOfType <ILogEntryParserPlugin>())
{
if (TryCreateParser(plugin, services, format, out var parser))
{
return(parser);
}
}
return(CreateDefaultParser(services));
}
public bool TryMatchFormat(string fileName, byte[] initialContent, out ILogFileFormat format)
{
foreach (var matcher in _matchers)
{
if (matcher.TryMatchFormat(fileName, initialContent, out format))
{
return(true);
}
}
format = null;
return(false);
}
public bool TryMatchFormat(string fileName, byte[] initialContent, out ILogFileFormat format)
{
try
{
return(_inner.TryMatchFormat(fileName, initialContent, out format));
}
catch (Exception e)
{
Log.ErrorFormat("Caught unexpected exception: {0}", e);
format = null;
return(false);
}
}
public bool TryMatchFormat(string fileName,
byte[] header,
Encoding encoding,
out ILogFileFormat format,
out Certainty certainty)
{
Header = header;
Encoding = encoding;
NumInvocations++;
format = Format;
certainty = Certainty;
return(format != null);
}
public ILogEntryParser CreateParser(IServiceContainer services, ILogFileFormat format)
{
// We want to make sure that we only influence those log files we actually
// know the structure of. Therefore we check that the format of the log file
// we're supposed to parse now IS the format our matcher has detected.
if (format == MyLogFileFormatMatcherPlugin.MyCustomFormat)
{
return(new MyCustomLogLevelParser());
}
// If it is a different format, then we don't do anything and instead return null
// so that a different plugin OR Tailviewer itself may parse its contents.
return(null);
}
/// <summary>
/// Initializes a new instance of the <see cref="MsBuildIssuesSettings"/> class.
/// </summary>
/// <param name="logFilePath">Path to the the MsBuild log file.
/// The log file needs to be in the format as defined by the <paramref name="format"/> parameter.</param>
/// <param name="format">Format of the provided MsBuild log file.</param>
protected MsBuildIssuesSettings(FilePath logFilePath, ILogFileFormat format)
{
logFilePath.NotNull(nameof(logFilePath));
format.NotNull(nameof(format));
this.Format = format;
using (var stream = new FileStream(logFilePath.FullPath, FileMode.Open, FileAccess.Read))
{
using (var sr = new StreamReader(stream))
{
this.LogFileContent = sr.ReadToEnd();
}
}
}
public bool TryCreate(IServiceContainer serviceContainer,
ICustomLogFileFormat format,
out ILogFileFormat logFileFormat)
{
try
{
logFileFormat = new SerilogFileFormat(format.Name, format.Format, format.Encoding);
return(true);
}
catch (Exception e)
{
Log.WarnFormat("Unable to create serilog format from '{0}': {1}", format, e);
logFileFormat = null;
return(false);
}
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="filesystem"></param>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal TextLogSource(IFilesystem filesystem,
ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
: base(taskScheduler)
{
_filesystem = filesystem;
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_encoding = encoding ?? throw new ArgumentNullException(nameof(encoding));
_entries = new LogBufferList(Core.Columns.RawContent);
_columns = new IColumnDescriptor[]
{
Core.Columns.Index,
Core.Columns.OriginalIndex,
Core.Columns.LogEntryIndex,
Core.Columns.LineNumber,
Core.Columns.OriginalLineNumber,
Core.Columns.OriginalDataSourceName,
Core.Columns.RawContent,
PageBufferedLogSource.RetrievalState
};
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_localProperties = new PropertiesBufferList(Core.Properties.Minimum);
_localProperties.SetValue(Core.Properties.Name, _fileName);
_localProperties.Add(TextProperties.LineCount);
_localProperties.SetValue(Core.Properties.Format, format);
_localProperties.SetValue(TextProperties.LineCount, 0);
_localProperties.SetValue(TextProperties.RequiresBuffer, false);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizePropertiesWithUser();
_syncRoot = new object();
_properties.SetValue(TextProperties.AutoDetectedEncoding, encoding);
Log.DebugFormat("Log File '{0}' is interpreted using {1}", _fileName, _encoding.EncodingName);
StartTask();
}
private ILogFileFormat TryFindFormat(FileStream stream)
{
var pos = stream.Position;
const int maxHeaderLength = 512;
var length = Math.Min(maxHeaderLength, stream.Length - pos);
var header = new byte[length];
stream.Read(header, 0, header.Length);
ILogFileFormat format = null;
_formatMatcher?.TryMatchFormat(_fullFilename, header, out format);
if (format == null && length == maxHeaderLength)
{
return(LogFileFormats.GenericText);
}
return(format);
}
private Encoding PickEncoding(Encoding overwrittenEncoding, ILogFileFormat format, Encoding detectedEncoding, Encoding defaultEncoding)
{
var formatEncoding = format?.Encoding;
if (overwrittenEncoding != null)
{
if (formatEncoding != null)
{
Log.DebugFormat("File {0} format specifies Encoding {1} but the user forced the encoding to be {2} instead",
_fullFilename,
formatEncoding.WebName,
overwrittenEncoding.WebName);
}
return(overwrittenEncoding);
}
if (formatEncoding != null)
{
if (detectedEncoding != null)
{
Log.WarnFormat("File {0} has been detected to be encoded with {1}, but its format ({2}) says it's encoded with {3}, choosing the latter....",
_fullFilename,
detectedEncoding.WebName,
format,
formatEncoding.WebName);
}
return(formatEncoding);
}
if (detectedEncoding != null)
{
Log.DebugFormat("File {0}: Encoding was auto detected to be {1}", _fullFilename, detectedEncoding);
return(detectedEncoding);
}
Log.DebugFormat("File {0}: No encoding could be determined, falling back to {1}", _fullFilename, defaultEncoding);
return(defaultEncoding);
}
/// <summary>
/// </summary>
/// <param name="fileName"></param>
/// <param name="fileStream"></param>
/// <param name="encoding"></param>
/// <returns></returns>
public ILogFileFormat TryDetermineFormat(string fileName,
Stream fileStream,
Encoding encoding)
{
try
{
if (_format == null || _detectionCertainty != Certainty.Sure)
{
_format = TryFindFormatOf(fileName, fileStream, encoding, out _detectionCertainty);
}
}
catch (IOException e)
{
Log.DebugFormat("Caught exception: {0}", e);
}
catch (Exception e)
{
Log.WarnFormat("Caught unexpected exception: {0}", e);
}
return(_format);
}
public bool TryMatchFormat(string fileName,
byte[] data,
Encoding encoding,
out ILogFileFormat format,
out Certainty certainty)
{
// In this example we are able to detect our log format by the custom extension
// 'mylog' as well as by a file-name pattern "log_*.logv2". If the filename
// matches either of these, then we assume that we're dealing with the desired
// format and forward its descriptor to Tailviewer.
// Other plugins may then compare log file format descriptors and thereby choose
// which formats they want to be used for (and which ones they do not).
var extension = Path.GetExtension(fileName);
if (extension == "mylog")
{
format = MyCustomFormat;
certainty = Certainty.Sure;
return(true);
}
var name = Path.GetFileName(fileName);
if (Regex.Match(name, "log_*\\.logv2").Success)
{
format = MyCustomFormat;
certainty = Certainty.Sure;
return(true);
}
// Since the filename doesn't match our expectations, we can say with 100% certainty
// that we don't know what type of log file this is.
// This causes Tailviewer to no longer invoke this matcher for that file anymore, even
// if the file changes / is appended to / etc...
format = null;
certainty = Certainty.Sure;
return(false);
}
public bool TryMatchFormat(string fileName,
byte[] header,
Encoding encoding,
out ILogFileFormat format,
out Certainty certainty)
{
var pluginLoader = _services.Retrieve <IPluginLoader>();
var matchers = pluginLoader.LoadAllOfType <ILogFileFormatMatcherPlugin>()
.Select(x => new NoThrowLogFileFormatMatcher(x, _services))
.ToList();
foreach (var matcher in matchers)
{
if (matcher.TryMatchFormat(fileName, header, encoding, out format, out certainty))
{
return(true);
}
}
format = null;
certainty = Certainty.None;
return(false);
}
