/// <summary>
/// Finds the first and the last available messages in the reader.
/// </summary>
/// <param name="reader">Messages reader to read from</param>
/// <param name="cachedFirstMessage">When the caller passes non-null value
/// the function doesn't search for the first message in the reader and return the
/// value precalculated by the client instead. That can be user for optimization:
/// if the client is sure that the first message didn't change then it can
/// pass the value calculated before. If <paramref name="firstMessage"/> is <value>null</value>
/// the function will search for the first message in the reader.</param>
/// <param name="firstMessage">When the function returns <paramref name="firstMessage"/> receives
/// the message with the smallest available position.</param>
/// <param name="lastMessage">When the function returns
/// <paramref name="lastMessage"/> receives the message with the largest available position.</param>
public static async Task <(IMessage firstMessage, IMessage lastMessage)> GetBoundaryMessages(
IPositionedMessagesReader reader,
IMessage cachedFirstMessage)
{
IMessage firstMessage, lastMessage;
if (cachedFirstMessage == null)
{
firstMessage = await ReadNearestMessage(reader, reader.BeginPosition);
}
else
{
firstMessage = cachedFirstMessage;
}
lastMessage = firstMessage;
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.EndPosition,
null, MessagesParserFlag.Default, MessagesParserDirection.Backward)), async parser =>
{
IMessage tmp = await parser.ReadNext();
if (tmp != null)
{
lastMessage = tmp;
}
});
return(firstMessage, lastMessage);
}
public static long?FindNextMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
// Validate the input.
if (originalMessagePos < reader.BeginPosition)
{
return(null);
}
if (originalMessagePos >= reader.EndPosition)
{
return(null);
}
using (IPositionedMessagesParser parser = reader.CreateParser(new CreateParserParams(originalMessagePos,
null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageTimeIsNotNeeded,
MessagesParserDirection.Forward, null)))
{
if (parser.ReadNext() == null)
{
return(null);
}
IMessage p = parser.ReadNext();
if (p == null)
{
return(null);
}
return(p.Position);
}
}
public static async Task <long?> FindNextMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
// Validate the input.
if (originalMessagePos < reader.BeginPosition)
{
return(null);
}
if (originalMessagePos >= reader.EndPosition)
{
return(null);
}
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos,
null, MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageTimeIsNotNeeded,
MessagesParserDirection.Forward, null)), async parser =>
{
if (parser.ReadNext() == null)
{
return (long?)null;
}
IMessage p = await parser.ReadNext();
if (p == null)
{
return null;
}
return p.Position;
}));
}
public static async Task <long?> FindPrevMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
long nextMessagePos = 0;
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(originalMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Forward)), async p =>
{
var msgAtOriginalPos = await p.ReadNext();
if (msgAtOriginalPos != null)
{
nextMessagePos = msgAtOriginalPos.Position;
}
else
{
nextMessagePos = reader.EndPosition;
}
});
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(nextMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Backward)), async p =>
{
IMessage msg = await p.ReadNext();
if (msg != null)
{
return msg.Position;
}
return (long?)null;
}));
}
/// <summary>
/// Finds the first and the last available messages in the reader.
/// </summary>
/// <param name="reader">Messages reader to read from</param>
/// <param name="cachedFirstMessage">When the caller passes non-null value
/// the function doesn't search for the first message in the reader and return the
/// value precalculated by the client instead. That can be user for optimization:
/// if the client is sure that the first message didn't change then it can
/// pass the value calculated before. If <paramref name="firstMessage"/> is <value>null</value>
/// the function will search for the first message in the reader.</param>
/// <param name="firstMessage">When the function returns <paramref name="firstMessage"/> receives
/// the message with the smallest available position.</param>
/// <param name="lastMessage">When the function returns
/// <paramref name="lastMessage"/> receives the message with the largest available position.</param>
public static void GetBoundaryMessages(
IPositionedMessagesReader reader,
IMessage cachedFirstMessage,
out IMessage firstMessage, out IMessage lastMessage)
{
if (cachedFirstMessage == null)
{
firstMessage = ReadNearestMessage(reader, reader.BeginPosition);
}
else
{
firstMessage = cachedFirstMessage;
}
lastMessage = firstMessage;
using (IPositionedMessagesParser parser = reader.CreateParser(new CreateParserParams(reader.EndPosition,
null, MessagesParserFlag.Default, MessagesParserDirection.Backward)))
{
IMessage tmp = parser.ReadNext();
if (tmp != null)
{
lastMessage = tmp;
}
}
}
public static async Task Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media)))
{
await reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition)), async parser =>
{
for (; ;)
{
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
});
expectation.StartVerification();
for (int i = 0; i < msgs.Count; ++i)
{
expectation.Verify(i, msgs[i]);
}
expectation.FinishVerification();
}
}
public static void Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media, tempFilesManager, new TraceSourceFactory())))
{
reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
using (var parser = reader.CreateParser(new CreateParserParams(reader.BeginPosition)))
{
for (; ;)
{
var msg = parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
}
expectation.StartVerification();
for (int i = 0; i < msgs.Count; ++i)
{
expectation.Verify(i, msgs[i]);
}
expectation.FinishVerification();
}
}
public void TextWriterTraceListener_FindPrevMessagePositionTest()
{
var testLog =
@"SampleApp Information: 0 : No free data file found. Going sleep.
ProcessId=4756
ThreadId=7
DateTime=2011-07-12T12:10:00.0000000Z
SampleApp Information: 0 : Searching for data files
ProcessId=4756
ThreadId=7
DateTime=2011-07-12T12:12:00.0000000Z
SampleApp Information: 0 : No free data file found. Going sleep.
ProcessId=4756
ThreadId=7
DateTime=2011-07-12T12:14:00.0000000Z
";
using (StringStreamMedia media = new StringStreamMedia(testLog, Encoding.ASCII))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = CreateFactory().CreateMessagesReader(new MediaBasedReaderParams(threads, media, new TempFilesManager(), new TraceSourceFactory())))
{
reader.UpdateAvailableBounds(false);
long?prevMessagePos = PositionedMessagesUtils.FindPrevMessagePosition(reader, 0x0000004A);
Assert.IsTrue(prevMessagePos.HasValue);
Assert.AreEqual(0, prevMessagePos.Value);
}
}
public static long?FindPrevMessagePosition(IPositionedMessagesReader reader,
long originalMessagePos)
{
long nextMessagePos;
using (IPositionedMessagesParser p = reader.CreateParser(new CreateParserParams(originalMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Forward)))
{
var msgAtOriginalPos = p.ReadNext();
if (msgAtOriginalPos != null)
{
nextMessagePos = msgAtOriginalPos.Position;
}
else
{
nextMessagePos = reader.EndPosition;
}
}
using (IPositionedMessagesParser p = reader.CreateParser(new CreateParserParams(nextMessagePos, null,
MessagesParserFlag.HintMessageContentIsNotNeeed | MessagesParserFlag.HintMessageContentIsNotNeeed,
MessagesParserDirection.Backward)))
{
IMessage msg = p.ReadNext();
if (msg != null)
{
return(msg.Position);
}
return(null);
}
}
public bool Update()
{
using (owner.trace.NewNamedFrame("Updating {0}", this.fileName))
{
CheckDisposed();
try
{
if (simpleMedia == null)
{
owner.trace.Info("SimpleMedia object not created yet. Creating");
simpleMedia = new SimpleFileMedia(
owner.fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(Path.Combine(owner.baseDirectory, FileName))
);
}
owner.trace.Info("Updating simple media");
simpleMedia.Update();
if (!simpleMedia.IsAvailable)
{
owner.trace.Info("File is not avaliable (i.e. has been deleted)");
return(false);
}
if (firstMessageTime == null)
{
owner.trace.Info("First message time is unknown. Calcalating it");
using (IPositionedMessagesReader reader = (IPositionedMessagesReader)Activator.CreateInstance(
owner.logReaderType, new MediaBasedReaderParams(owner.tempThreads, SimpleMedia, owner.tempFilesManager), owner.logFormatInfo))
{
owner.trace.Info("Reader created");
reader.UpdateAvailableBounds(false);
owner.trace.Info("Bounds found");
IMessage first = PositionedMessagesUtils.ReadNearestMessage(reader, reader.BeginPosition);
if (first == null)
{
owner.trace.Warning("No messages found");
return(false);
}
owner.trace.Info("First message: {0} '{1}'", first.Time, first.Text);
firstMessageTime = first.Time;
}
}
owner.trace.Info("Part updated OK");
return(true);
}
catch (Exception e)
{
owner.trace.Error(e, "Failure during part update");
return(false);
}
}
}
static public async Task <IMessage> ReadNearestMessage(IPositionedMessagesReader reader, long position, MessagesParserFlag flags)
{
return(await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(position, null, flags, MessagesParserDirection.Forward)), async parser =>
{
IMessage ret = await parser.ReadNext();
return ret;
}));
}
static public IMessage ReadNearestMessage(IPositionedMessagesReader reader, long position, MessagesParserFlag flags)
{
using (IPositionedMessagesParser parser = reader.CreateParser(new CreateParserParams(position, null, flags, MessagesParserDirection.Forward)))
{
IMessage ret = parser.ReadNext();
return(ret);
}
}
public DelegatingMessagesReader(IPositionedMessagesReader underliyingReader)
{
if (underliyingReader == null)
{
throw new ArgumentNullException("underliyingReader");
}
this.underliyingReader = underliyingReader;
}
protected void StartAsyncReader(string threadName, IPositionedMessagesReader reader)
{
Debug.Assert(this.thread == null);
Debug.Assert(reader != null);
this.reader = reader;
this.thread = Task.Run(() => Run());
}
public static MessageTimestamp?ReadNearestMessageTimestamp(IPositionedMessagesReader reader, long position)
{
IMessage m = ReadNearestMessage(reader, position, MessagesParserFlag.HintMessageContentIsNotNeeed);
if (m != null)
{
return(m.Time);
}
return(null);
}
public void EnsureRangeIsSet(IPositionedMessagesReader reader)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (Range == null)
{
Range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
}
}
public static async Task <long> LocateDateBound(IPositionedMessagesReader reader, DateTime date, ValueBound bound, CancellationToken cancellation)
{
var d = new MessageTimestamp(date);
long begin = reader.BeginPosition;
long end = reader.EndPosition;
long pos = begin;
long count = end - begin;
for (; 0 < count;)
{
long count2 = count / 2;
MessageTimestamp d2 = (await ReadNearestMessageTimestamp(reader, pos + count2)).GetValueOrDefault(MessageTimestamp.MaxValue);
bool moveRight = false;
switch (bound)
{
case ValueBound.Lower:
case ValueBound.UpperReversed:
moveRight = d2 < d;
break;
case ValueBound.Upper:
case ValueBound.LowerReversed:
moveRight = d2 <= d;
break;
}
if (moveRight)
{
pos += count2 + 1;
count -= count2 + 1;
}
else
{
count = count2;
}
cancellation.ThrowIfCancellationRequested();
}
if (bound == ValueBound.LowerReversed || bound == ValueBound.UpperReversed)
{
long?tmp = await FindPrevMessagePosition(reader, pos);
if (tmp == null)
{
return(begin - 1);
}
pos = tmp.Value;
}
return(pos);
}
public static async Task <StreamParser> Create(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache)
{
var parser = new StreamParser(owner, p, textStreamPositioningParams, globalSettings, strategiesCache);
await parser.Strategy.ParserCreated(parser.InitialParams);
return(parser);
}
public MessagesReaderExtensions(IPositionedMessagesReader owner, XmlInitializationParams initializationData = null)
{
this.owner = owner;
if (initializationData != null)
{
try
{
initializationData.InitializeInstance(this);
}
catch
{
Dispose();
throw;
}
}
}
public static long NormalizeMessagePosition(IPositionedMessagesReader reader,
long position)
{
if (position == reader.BeginPosition)
{
return(position);
}
IMessage m = ReadNearestMessage(reader, position,
MessagesParserFlag.HintMessageTimeIsNotNeeded | MessagesParserFlag.HintMessageContentIsNotNeeed);
if (m != null)
{
return(m.Position);
}
return(reader.EndPosition);
}
public SearchingParser(
IPositionedMessagesReader owner,
CreateSearchingParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
DejitteringParams?dejitteringParams,
Stream rawStream,
Encoding streamEncoding,
bool allowPlainTextSearchOptimization,
LoadedRegex headerRe,
ILogSourceThreads threads,
ITraceSourceFactory traceSourceFactory,
RegularExpressions.IRegexFactory regexFactory
)
{
this.owner = owner;
this.parserParams = p;
this.plainTextSearchOptimizationAllowed = allowPlainTextSearchOptimization && ((p.Flags & MessagesParserFlag.DisablePlainTextSearchOptimization) == 0);
this.threads = threads;
this.requestedRange = p.Range;
this.textStreamPositioningParams = textStreamPositioningParams;
this.dejitteringParams = dejitteringParams;
this.rawStream = rawStream;
this.streamEncoding = streamEncoding;
this.regexFactory = regexFactory;
this.trace = traceSourceFactory.CreateTraceSource("LogSource", "srchp." + GetHashCode().ToString("x"));
this.dummyFilter = new Filter(FilterAction.Include, "", true, new Search.Options(), null, regexFactory);
var continuationToken = p.ContinuationToken as ContinuationToken;
if (continuationToken != null)
{
this.requestedRange = new FileRange.Range(continuationToken.NextPosition, requestedRange.End);
}
this.aligmentTextAccess = new StreamTextAccess(rawStream, streamEncoding, textStreamPositioningParams);
this.aligmentSplitter = new MessagesSplitter(aligmentTextAccess, headerRe.Clone().Regex, headerRe.GetHeaderReSplitterFlags());
this.aligmentCapture = new TextMessageCapture();
this.progressAndCancellation = new ProgressAndCancellation()
{
progressHandler = p.ProgressHandler,
cancellationToken = p.Cancellation,
continuationToken = new ContinuationToken()
{
NextPosition = requestedRange.Begin
}
};
this.impl = Enum();
}
void UpdateLoadedTimeStats(IPositionedMessagesReader reader)
{
MessagesContainers.RangesManagingCollection tmp = buffer;
tracer.Info("Current messages: {0}", tmp);
long bytesCount = 0;
foreach (MessagesContainers.MessagesRange lr in tmp.Ranges)
{
bytesCount += reader.PositionRangeToBytes(lr.LoadedRange);
}
DateRange loadedTime;
int c = tmp.Count;
if (c != 0)
{
DateTime begin = new DateTime();
DateTime end = new DateTime();
foreach (IndexedMessage l in tmp.Forward(0, 1))
{
tracer.Info("First message: {0}, {1}", l.Message.Time, l.Message.Text);
begin = l.Message.Time.ToLocalDateTime();
}
foreach (IndexedMessage l in tmp.Reverse(c - 1, c - 2))
{
tracer.Info("Last message: {0}, {1}", l.Message.Time, l.Message.Text);
end = l.Message.Time.ToLocalDateTime();
}
loadedTime = DateRange.MakeFromBoundaryValues(begin, end);
}
else
{
loadedTime = DateRange.MakeEmpty();
}
tracer.Info("Calculated statistics: LoadedTime={0}, BytesLoaded={1}", loadedTime, bytesCount);
owner.StatsTransaction(stats =>
{
stats.LoadedTime = loadedTime;
stats.LoadedBytes = bytesCount;
return(LogProviderStatsFlag.CachedTime | LogProviderStatsFlag.BytesCount);
});
}
public StreamLogProvider(
ILogProviderHost host,
ILogProviderFactory factory,
IConnectionParams connectParams,
StreamBasedFormatInfo formatInfo,
Type readerType
) :
base(host, factory, connectParams)
{
using (tracer.NewFrame)
{
tracer.Info("readerType={0}", readerType);
if (connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam] != null)
{
media = new RollingFilesMedia(
LogMedia.FileSystemImpl.Instance,
readerType,
formatInfo,
tracer,
new GenericRollingMediaStrategy(connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam]),
host.TempFilesManager,
host.TraceSourceFactory
);
}
else
{
media = new SimpleFileMedia(connectParams);
}
reader = (IPositionedMessagesReader)Activator.CreateInstance(
readerType, new MediaBasedReaderParams(this.threads, media, host.TempFilesManager, host.TraceSourceFactory,
settingsAccessor: host.GlobalSettings, parentLoggingPrefix: tracer.Prefix), formatInfo);
ITimeOffsets initialTimeOffset;
if (LogJoint.TimeOffsets.TryParse(
connectionParams[ConnectionParamsKeys.TimeOffsetConnectionParam] ?? "", out initialTimeOffset))
{
reader.TimeOffsets = initialTimeOffset;
}
StartAsyncReader("Reader thread: " + connectParams.ToString(), reader);
InitPathDependentMembers(connectParams);
}
}
private StreamParser(
IPositionedMessagesReader owner,
CreateParserParams p,
TextStreamPositioningParams textStreamPositioningParams,
IGlobalSettingsAccessor globalSettings,
StrategiesCache strategiesCache
)
{
p.EnsureRangeIsSet(owner);
this.InitialParams = p;
this.isSequentialReadingParser = (p.Flags & MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading) != 0;
this.multithreadingDisabled = (p.Flags & MessagesParserFlag.DisableMultithreading) != 0 ||
globalSettings.MultithreadedParsingDisabled;
CreateParsingStrategy(p, textStreamPositioningParams, strategiesCache, out this.Strategy);
}
public StreamLogProvider(
ILogProviderHost host,
ILogProviderFactory factory,
IConnectionParams connectParams,
Func <MediaBasedReaderParams, IPositionedMessagesReader> readerCreator
) :
base(host, factory, connectParams)
{
using (tracer.NewFrame)
{
StartAsyncReader(async() => {
if (connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam] != null)
{
media = new RollingFilesMedia(
host.FileSystem,
readerCreator,
tracer,
new GenericRollingMediaStrategy(
connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam],
ConnectionParamsUtils.GetRotatedLogPatterns(connectParams)
)
);
}
else
{
media = await SimpleFileMedia.Create(host.FileSystem, connectParams);
}
reader = readerCreator(new MediaBasedReaderParams(this.threads, media,
settingsAccessor: host.GlobalSettings, parentLoggingPrefix: tracer.Prefix));
ITimeOffsets initialTimeOffset;
if (LogJoint.TimeOffsets.TryParse(
connectionParams[ConnectionParamsKeys.TimeOffsetConnectionParam] ?? "", out initialTimeOffset))
{
reader.TimeOffsets = initialTimeOffset;
}
return(reader);
});
InitPathDependentMembers(connectParams);
}
}
public static void Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreads threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media, tempFilesManager)))
{
reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
using (var parser = reader.CreateParser(new CreateParserParams(reader.BeginPosition)))
{
for (; ;)
{
var msg = parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
}
expectation.StartVerification();
int frameLevel = 0;
for (int i = 0; i < msgs.Count; ++i)
{
switch (msgs[i].Flags & MessageFlag.TypeMask)
{
case MessageFlag.StartFrame:
++frameLevel;
break;
case MessageFlag.EndFrame:
--frameLevel;
break;
}
expectation.Verify(i, msgs[i], frameLevel);
}
expectation.FinishVerification();
}
}
protected virtual long CalcTotalBytesStats(IPositionedMessagesReader reader)
{
return(reader.SizeInBytes);
}
protected override long CalcTotalBytesStats(IPositionedMessagesReader reader)
{
return(sizeInBytesStat);
}
public static long LocateDateBound(IPositionedMessagesReader reader, DateTime date, ValueBound bound)
{
return(LocateDateBound(reader, date, bound, CancellationToken.None));
}
static public IMessage ReadNearestMessage(IPositionedMessagesReader reader, long position)
{
return(ReadNearestMessage(reader, position, MessagesParserFlag.Default));
}
