IEnumerable <PieceOfWork> ReadRawDataFromMedia_Backward(CancellationToken cancellationToken)
{
Stream stream = owner.media.DataStream;
CreateParserParams parserParams = owner.currentParams;
FileRange.Range range = parserParams.Range.Value;
TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);
long beginStreamPos = new TextStreamPosition(range.Begin, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize;
long endStreamPos = startPosition.StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;
if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
{
int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
beginStreamPos -= maxBytesPerCharacter;
}
PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
{
firstPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos);
if (firstPieceOfWork.streamData.IsEmpty)
{
yield break;
}
firstPieceOfWork.startTextPosition = startPosition.Value;
firstPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread;
firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
endStreamPos -= owner.BytesToParsePerThread;
}
PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
nextPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos);
nextPieceOfWork.nextStreamData = pieceOfWorkToYieldNextTime.streamData;
nextPieceOfWork.startTextPosition = endStreamPos;
nextPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread;
nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
pieceOfWorkToYieldNextTime.prevStreamData = nextPieceOfWork.streamData;
yield return(pieceOfWorkToYieldNextTime);
if (endStreamPos < beginStreamPos)
{
break;
}
if (nextPieceOfWork.streamData.IsEmpty)
{
break;
}
pieceOfWorkToYieldNextTime = nextPieceOfWork;
endStreamPos -= owner.BytesToParsePerThread;
}
}
public void TextStreamPosition_ConstructorTest()
{
long streamPosition = 64 * 1024 * 12;
int textPositionInsideBuffer = 1234;
TextStreamPosition pos = new TextStreamPosition(streamPosition, textPositionInsideBuffer);
Assert.AreEqual(pos.Value, streamPosition + textPositionInsideBuffer);
}
public async Task StreamPositionToCharIndexTest_InvalidBigTextStreamPositionIsMappedToPastTheEndCharIndex()
{
StreamTextAccess buf = new StreamTextAccess(
S().Add('a', blockSz * 3).ToStream(Encoding.Unicode),
Encoding.Unicode
);
await buf.BeginReading(0, TextAccessDirection.Forward);
// valid Unicode text stream positions are from 0 to blockSz/2. below is invalid position.
var invalidTextStreamPosition = new TextStreamPosition(blockSz - 10);
Assert.AreEqual(blockSz / 2, buf.PositionToCharIndex(invalidTextStreamPosition));
}
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Forward(CancellationToken cancellationToken)
{
Stream stream = owner.media.DataStream;
CreateParserParams parserParams = owner.currentParams;
FileRange.Range range = parserParams.Range.Value;
TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);
long beginStreamPos = startPosition.StreamPositionAlignedToBlockSize;
long endStreamPos = new TextStreamPosition(range.End, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;
PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
{
int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
firstPieceOfWork.prevStreamData = new StreamData(
beginStreamPos - maxBytesPerCharacter, new byte[maxBytesPerCharacter]);
stream.Position = beginStreamPos - maxBytesPerCharacter;
stream.Read(firstPieceOfWork.prevStreamData.Bytes, 0, maxBytesPerCharacter);
}
else
{
stream.Position = beginStreamPos;
}
{
firstPieceOfWork.streamData = AllocateAndReadStreamData(stream);
if (firstPieceOfWork.streamData.IsEmpty)
{
yield break;
}
firstPieceOfWork.startTextPosition = startPosition.Value;
firstPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread;
firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
beginStreamPos += owner.BytesToParsePerThread;
}
PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
nextPieceOfWork.streamData = AllocateAndReadStreamData(stream);
nextPieceOfWork.prevStreamData = pieceOfWorkToYieldNextTime.streamData;
nextPieceOfWork.startTextPosition = beginStreamPos;
nextPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread;
nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer();
pieceOfWorkToYieldNextTime.nextStreamData = nextPieceOfWork.streamData;
owner.tracer.Info("Start processing new peice of work. Currently being processed: {0}", Interlocked.Increment(ref owner.peicesOfWorkBeingProgressed));
yield return(pieceOfWorkToYieldNextTime);
if (beginStreamPos > endStreamPos)
{
break;
}
if (nextPieceOfWork.streamData.IsEmpty)
{
break;
}
pieceOfWorkToYieldNextTime = nextPieceOfWork;
beginStreamPos += owner.BytesToParsePerThread;
}
}
static void TestCharPosMapping(StreamTextAccess sut, TextStreamPosition pos, int charIdx)
{
Assert.AreEqual(pos.Value, sut.CharIndexToPosition(charIdx).Value);
Assert.AreEqual(charIdx, sut.PositionToCharIndex(pos));
}