public override void ParserCreated(CreateParserParams p)
{
tracer.Info("Parser created");
attachedToParser = true;
currentParams = p;
base.ParserCreated(p);
}
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Backward(CancellationToken cancellationToken)
{
Stream stream = owner.media.DataStream;
CreateParserParams parserParams = owner.currentParams;
FileRange.Range range = parserParams.Range.Value;
TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);
long beginStreamPos = new TextStreamPosition(range.Begin, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize;
long endStreamPos = startPosition.StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;
if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
{
int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
beginStreamPos -= maxBytesPerCharacter;
}
PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
{
firstPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos);
if (firstPieceOfWork.streamData.IsEmpty)
{
yield break;
}
firstPieceOfWork.startTextPosition = startPosition.Value;
firstPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread;
firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
endStreamPos -= owner.BytesToParsePerThread;
}
PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
nextPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos);
nextPieceOfWork.nextStreamData = pieceOfWorkToYieldNextTime.streamData;
nextPieceOfWork.startTextPosition = endStreamPos;
nextPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread;
nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
pieceOfWorkToYieldNextTime.prevStreamData = nextPieceOfWork.streamData;
yield return(pieceOfWorkToYieldNextTime);
if (endStreamPos < beginStreamPos)
{
break;
}
if (nextPieceOfWork.streamData.IsEmpty)
{
break;
}
pieceOfWorkToYieldNextTime = nextPieceOfWork;
endStreamPos -= owner.BytesToParsePerThread;
}
}
public override Task ParserCreated(CreateParserParams p)
{
tracer.Info("Parser created");
attachedToParser = true;
currentParams = p;
return(base.ParserCreated(p));
}
public ParserImpl(LogEntry[] logContent, CreateParserParams parserParams)
{
this.logContent = logContent;
if (parserParams.Range.HasValue)
{
effectiveRange = parserParams.Range.Value;
if (effectiveRange.Begin < 0)
{
effectiveRange = new Range(0, effectiveRange.End);
}
if (effectiveRange.End > logContent.Length)
{
effectiveRange = new Range(effectiveRange.Begin, logContent.Length);
}
}
else
{
effectiveRange = new Range(0, logContent.Length);
}
reverse = parserParams.Direction == MessagesParserDirection.Backward;
if (!reverse)
{
pos = Math.Max(parserParams.StartPosition, effectiveRange.Begin);
}
else
{
pos = Math.Min(parserParams.StartPosition - 1, effectiveRange.End);
}
}
public override void ParserCreated(CreateParserParams p)
{
postprocessor = p.PostprocessorsFactory?.Invoke();
textSplitter.BeginSplittingSession(p.Range.Value, p.StartPosition, p.Direction);
// todo
//if (textSplitter.CurrentMessageIsEmpty)
//{
// if (direction == MessagesParserDirection.Forward)
// {
// if ((initParams.startPosition == reader.BeginPosition)
// || ((reader.EndPosition - initParams.startPosition) >= StreamTextAccess.MaxTextBufferSize))
// {
// throw new InvalidFormatException();
// }
// }
// else
// {
// // todo
// }
//}
}
void DoTest(LogEntry[] logContent, CreateParserParams originalParams, int jitterBufferSize, LogEntry[] expectedParsedMessages)
{
if (originalParams.Range == null)
{
originalParams.Range = new Range(0, logContent.Length);
}
CreateParserParams validatedParams = originalParams;
validatedParams.EnsureStartPositionIsInRange();
using (DejitteringMessagesParser jitter = new DejitteringMessagesParser(p => new ParserImpl(logContent, p), originalParams, jitterBufferSize))
{
int messageIdx;
int idxStep;
if (originalParams.Direction == MessagesParserDirection.Forward)
{
messageIdx = 0;
idxStep = 1;
}
else
{
messageIdx = -1;
idxStep = -1;
}
foreach (LogEntry expectedMessage in expectedParsedMessages)
{
IMessage actualMessage = jitter.ReadNext();
Assert.IsNotNull(actualMessage);
Assert.AreEqual((long)expectedMessage.Time, actualMessage.Time.ToLocalDateTime().Ticks);
Assert.AreEqual(expectedMessage.Msg, actualMessage.Text.Value);
Assert.AreEqual(validatedParams.StartPosition + messageIdx, actualMessage.Position);
messageIdx += idxStep;
}
IMessage lastMessage = jitter.ReadNext();
Assert.IsNull(lastMessage);
}
}
public override Task ParserCreated(CreateParserParams p)
{
return(base.ParserCreated(p));
}
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Forward(CancellationToken cancellationToken)
{
Stream stream = owner.media.DataStream;
CreateParserParams parserParams = owner.currentParams;
FileRange.Range range = parserParams.Range.Value;
TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);
long beginStreamPos = startPosition.StreamPositionAlignedToBlockSize;
long endStreamPos = new TextStreamPosition(range.End, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;
PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
{
int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
firstPieceOfWork.prevStreamData = new StreamData(
beginStreamPos - maxBytesPerCharacter, new byte[maxBytesPerCharacter]);
stream.Position = beginStreamPos - maxBytesPerCharacter;
stream.Read(firstPieceOfWork.prevStreamData.Bytes, 0, maxBytesPerCharacter);
}
else
{
stream.Position = beginStreamPos;
}
{
firstPieceOfWork.streamData = AllocateAndReadStreamData(stream);
if (firstPieceOfWork.streamData.IsEmpty)
{
yield break;
}
firstPieceOfWork.startTextPosition = startPosition.Value;
firstPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread;
firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
beginStreamPos += owner.BytesToParsePerThread;
}
PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
nextPieceOfWork.streamData = AllocateAndReadStreamData(stream);
nextPieceOfWork.prevStreamData = pieceOfWorkToYieldNextTime.streamData;
nextPieceOfWork.startTextPosition = beginStreamPos;
nextPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread;
nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
pieceOfWorkToYieldNextTime.nextStreamData = nextPieceOfWork.streamData;
owner.tracer.Info("Start processing new peice of work. Currently being processed: {0}", Interlocked.Increment(ref owner.peicesOfWorkBeingProgressed));
yield return(pieceOfWorkToYieldNextTime);
if (beginStreamPos > endStreamPos)
{
break;
}
if (nextPieceOfWork.streamData.IsEmpty)
{
break;
}
pieceOfWorkToYieldNextTime = nextPieceOfWork;
beginStreamPos += owner.BytesToParsePerThread;
}
}
public IPositionedMessagesParser CreateParser(CreateParserParams p)
{
return(new Parser(media));
}
public virtual void ParserCreated(CreateParserParams p)
{
}
public virtual Task ParserCreated(CreateParserParams p)
{
return(Task.CompletedTask);
}
public override void ParserCreated(CreateParserParams p)
{
base.ParserCreated(p);
flags = ParserFlagsToMakeMessageFlags(p.Flags);
}
void DoTest(string logContent, CreateParserParams originalParams, int jitterBufferSize, string expectedParsedMessages)
{
DoTest(ParseTestLog(logContent), originalParams, jitterBufferSize, ParseTestLog(expectedParsedMessages));
}
public override async Task ParserCreated(CreateParserParams p)
{
await base.ParserCreated(p);
flags = ParserFlagsToMakeMessageFlags(p.Flags);
}
public async Task <IPositionedMessagesParser> CreateParser(CreateParserParams p)
{
CheckDisposed();
return(new Parser(this, p.StartPosition, p.Range, p.Direction));
}
public override void ParserCreated(CreateParserParams p)
{
base.ParserCreated(p);
}
public async Task <IPositionedMessagesParser> CreateParser(CreateParserParams p)
{
return(new Parser(media));
}
