public override void ParserCreated(CreateParserParams p) { tracer.Info("Parser created"); attachedToParser = true; currentParams = p; base.ParserCreated(p); }
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Backward(CancellationToken cancellationToken) { Stream stream = owner.media.DataStream; CreateParserParams parserParams = owner.currentParams; FileRange.Range range = parserParams.Range.Value; TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams); long beginStreamPos = new TextStreamPosition(range.Begin, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize; long endStreamPos = startPosition.StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize; if (beginStreamPos != 0 && !owner.encoding.IsSingleByte) { int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1); beginStreamPos -= maxBytesPerCharacter; } PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); { firstPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos); if (firstPieceOfWork.streamData.IsEmpty) { yield break; } firstPieceOfWork.startTextPosition = startPosition.Value; firstPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread; firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); endStreamPos -= owner.BytesToParsePerThread; } PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork; for (; ;) { cancellationToken.ThrowIfCancellationRequested(); PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); nextPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos); nextPieceOfWork.nextStreamData = pieceOfWorkToYieldNextTime.streamData; nextPieceOfWork.startTextPosition = endStreamPos; nextPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread; nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); pieceOfWorkToYieldNextTime.prevStreamData = nextPieceOfWork.streamData; yield return(pieceOfWorkToYieldNextTime); if (endStreamPos < beginStreamPos) { break; } if (nextPieceOfWork.streamData.IsEmpty) { break; } pieceOfWorkToYieldNextTime = nextPieceOfWork; endStreamPos -= owner.BytesToParsePerThread; } }
public override Task ParserCreated(CreateParserParams p) { tracer.Info("Parser created"); attachedToParser = true; currentParams = p; return(base.ParserCreated(p)); }
public ParserImpl(LogEntry[] logContent, CreateParserParams parserParams) { this.logContent = logContent; if (parserParams.Range.HasValue) { effectiveRange = parserParams.Range.Value; if (effectiveRange.Begin < 0) { effectiveRange = new Range(0, effectiveRange.End); } if (effectiveRange.End > logContent.Length) { effectiveRange = new Range(effectiveRange.Begin, logContent.Length); } } else { effectiveRange = new Range(0, logContent.Length); } reverse = parserParams.Direction == MessagesParserDirection.Backward; if (!reverse) { pos = Math.Max(parserParams.StartPosition, effectiveRange.Begin); } else { pos = Math.Min(parserParams.StartPosition - 1, effectiveRange.End); } }
public override void ParserCreated(CreateParserParams p) { postprocessor = p.PostprocessorsFactory?.Invoke(); textSplitter.BeginSplittingSession(p.Range.Value, p.StartPosition, p.Direction); // todo //if (textSplitter.CurrentMessageIsEmpty) //{ // if (direction == MessagesParserDirection.Forward) // { // if ((initParams.startPosition == reader.BeginPosition) // || ((reader.EndPosition - initParams.startPosition) >= StreamTextAccess.MaxTextBufferSize)) // { // throw new InvalidFormatException(); // } // } // else // { // // todo // } //} }
void DoTest(LogEntry[] logContent, CreateParserParams originalParams, int jitterBufferSize, LogEntry[] expectedParsedMessages) { if (originalParams.Range == null) { originalParams.Range = new Range(0, logContent.Length); } CreateParserParams validatedParams = originalParams; validatedParams.EnsureStartPositionIsInRange(); using (DejitteringMessagesParser jitter = new DejitteringMessagesParser(p => new ParserImpl(logContent, p), originalParams, jitterBufferSize)) { int messageIdx; int idxStep; if (originalParams.Direction == MessagesParserDirection.Forward) { messageIdx = 0; idxStep = 1; } else { messageIdx = -1; idxStep = -1; } foreach (LogEntry expectedMessage in expectedParsedMessages) { IMessage actualMessage = jitter.ReadNext(); Assert.IsNotNull(actualMessage); Assert.AreEqual((long)expectedMessage.Time, actualMessage.Time.ToLocalDateTime().Ticks); Assert.AreEqual(expectedMessage.Msg, actualMessage.Text.Value); Assert.AreEqual(validatedParams.StartPosition + messageIdx, actualMessage.Position); messageIdx += idxStep; } IMessage lastMessage = jitter.ReadNext(); Assert.IsNull(lastMessage); } }
public override Task ParserCreated(CreateParserParams p) { return(base.ParserCreated(p)); }
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Forward(CancellationToken cancellationToken) { Stream stream = owner.media.DataStream; CreateParserParams parserParams = owner.currentParams; FileRange.Range range = parserParams.Range.Value; TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams); long beginStreamPos = startPosition.StreamPositionAlignedToBlockSize; long endStreamPos = new TextStreamPosition(range.End, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize; PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); if (beginStreamPos != 0 && !owner.encoding.IsSingleByte) { int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1); firstPieceOfWork.prevStreamData = new StreamData( beginStreamPos - maxBytesPerCharacter, new byte[maxBytesPerCharacter]); stream.Position = beginStreamPos - maxBytesPerCharacter; stream.Read(firstPieceOfWork.prevStreamData.Bytes, 0, maxBytesPerCharacter); } else { stream.Position = beginStreamPos; } { firstPieceOfWork.streamData = AllocateAndReadStreamData(stream); if (firstPieceOfWork.streamData.IsEmpty) { yield break; } firstPieceOfWork.startTextPosition = startPosition.Value; firstPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread; firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); beginStreamPos += owner.BytesToParsePerThread; } PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork; for (; ;) { cancellationToken.ThrowIfCancellationRequested(); PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); nextPieceOfWork.streamData = AllocateAndReadStreamData(stream); nextPieceOfWork.prevStreamData = pieceOfWorkToYieldNextTime.streamData; nextPieceOfWork.startTextPosition = beginStreamPos; nextPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread; nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); pieceOfWorkToYieldNextTime.nextStreamData = nextPieceOfWork.streamData; owner.tracer.Info("Start processing new peice of work. Currently being processed: {0}", Interlocked.Increment(ref owner.peicesOfWorkBeingProgressed)); yield return(pieceOfWorkToYieldNextTime); if (beginStreamPos > endStreamPos) { break; } if (nextPieceOfWork.streamData.IsEmpty) { break; } pieceOfWorkToYieldNextTime = nextPieceOfWork; beginStreamPos += owner.BytesToParsePerThread; } }
public IPositionedMessagesParser CreateParser(CreateParserParams p) { return(new Parser(media)); }
public virtual void ParserCreated(CreateParserParams p) { }
public virtual Task ParserCreated(CreateParserParams p) { return(Task.CompletedTask); }
public override void ParserCreated(CreateParserParams p) { base.ParserCreated(p); flags = ParserFlagsToMakeMessageFlags(p.Flags); }
void DoTest(string logContent, CreateParserParams originalParams, int jitterBufferSize, string expectedParsedMessages) { DoTest(ParseTestLog(logContent), originalParams, jitterBufferSize, ParseTestLog(expectedParsedMessages)); }
public override async Task ParserCreated(CreateParserParams p) { await base.ParserCreated(p); flags = ParserFlagsToMakeMessageFlags(p.Flags); }
public async Task <IPositionedMessagesParser> CreateParser(CreateParserParams p) { CheckDisposed(); return(new Parser(this, p.StartPosition, p.Range, p.Direction)); }
public override void ParserCreated(CreateParserParams p) { base.ParserCreated(p); }
public async Task <IPositionedMessagesParser> CreateParser(CreateParserParams p) { return(new Parser(media)); }