IEnumerable <PieceOfWork> ReadRawDataFromMedia_Backward(CancellationToken cancellationToken) { Stream stream = owner.media.DataStream; CreateParserParams parserParams = owner.currentParams; FileRange.Range range = parserParams.Range.Value; TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams); long beginStreamPos = new TextStreamPosition(range.Begin, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize; long endStreamPos = startPosition.StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize; if (beginStreamPos != 0 && !owner.encoding.IsSingleByte) { int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1); beginStreamPos -= maxBytesPerCharacter; } PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); { firstPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos); if (firstPieceOfWork.streamData.IsEmpty) { yield break; } firstPieceOfWork.startTextPosition = startPosition.Value; firstPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread; firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); endStreamPos -= owner.BytesToParsePerThread; } PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork; for (; ;) { cancellationToken.ThrowIfCancellationRequested(); PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); nextPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos); nextPieceOfWork.nextStreamData = pieceOfWorkToYieldNextTime.streamData; nextPieceOfWork.startTextPosition = endStreamPos; nextPieceOfWork.stopTextPosition = endStreamPos - owner.BytesToParsePerThread; nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); pieceOfWorkToYieldNextTime.prevStreamData = nextPieceOfWork.streamData; yield return(pieceOfWorkToYieldNextTime); if (endStreamPos < beginStreamPos) { break; } if (nextPieceOfWork.streamData.IsEmpty) { break; } pieceOfWorkToYieldNextTime = nextPieceOfWork; endStreamPos -= owner.BytesToParsePerThread; } }
public void TextStreamPosition_ConstructorTest() { long streamPosition = 64 * 1024 * 12; int textPositionInsideBuffer = 1234; TextStreamPosition pos = new TextStreamPosition(streamPosition, textPositionInsideBuffer); Assert.AreEqual(pos.Value, streamPosition + textPositionInsideBuffer); }
public async Task StreamPositionToCharIndexTest_InvalidBigTextStreamPositionIsMappedToPastTheEndCharIndex() { StreamTextAccess buf = new StreamTextAccess( S().Add('a', blockSz * 3).ToStream(Encoding.Unicode), Encoding.Unicode ); await buf.BeginReading(0, TextAccessDirection.Forward); // valid Unicode text stream positions are from 0 to blockSz/2. below is invalid position. var invalidTextStreamPosition = new TextStreamPosition(blockSz - 10); Assert.AreEqual(blockSz / 2, buf.PositionToCharIndex(invalidTextStreamPosition)); }
IEnumerable <PieceOfWork> ReadRawDataFromMedia_Forward(CancellationToken cancellationToken) { Stream stream = owner.media.DataStream; CreateParserParams parserParams = owner.currentParams; FileRange.Range range = parserParams.Range.Value; TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams); long beginStreamPos = startPosition.StreamPositionAlignedToBlockSize; long endStreamPos = new TextStreamPosition(range.End, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize; PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); if (beginStreamPos != 0 && !owner.encoding.IsSingleByte) { int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1); firstPieceOfWork.prevStreamData = new StreamData( beginStreamPos - maxBytesPerCharacter, new byte[maxBytesPerCharacter]); stream.Position = beginStreamPos - maxBytesPerCharacter; stream.Read(firstPieceOfWork.prevStreamData.Bytes, 0, maxBytesPerCharacter); } else { stream.Position = beginStreamPos; } { firstPieceOfWork.streamData = AllocateAndReadStreamData(stream); if (firstPieceOfWork.streamData.IsEmpty) { yield break; } firstPieceOfWork.startTextPosition = startPosition.Value; firstPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread; firstPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); beginStreamPos += owner.BytesToParsePerThread; } PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork; for (; ;) { cancellationToken.ThrowIfCancellationRequested(); PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer); nextPieceOfWork.streamData = AllocateAndReadStreamData(stream); nextPieceOfWork.prevStreamData = pieceOfWorkToYieldNextTime.streamData; nextPieceOfWork.startTextPosition = beginStreamPos; nextPieceOfWork.stopTextPosition = beginStreamPos + owner.BytesToParsePerThread; nextPieceOfWork.outputBuffer = owner.AllocateOutputBuffer(); pieceOfWorkToYieldNextTime.nextStreamData = nextPieceOfWork.streamData; owner.tracer.Info("Start processing new peice of work. Currently being processed: {0}", Interlocked.Increment(ref owner.peicesOfWorkBeingProgressed)); yield return(pieceOfWorkToYieldNextTime); if (beginStreamPos > endStreamPos) { break; } if (nextPieceOfWork.streamData.IsEmpty) { break; } pieceOfWorkToYieldNextTime = nextPieceOfWork; beginStreamPos += owner.BytesToParsePerThread; } }
static void TestCharPosMapping(StreamTextAccess sut, TextStreamPosition pos, int charIdx) { Assert.AreEqual(pos.Value, sut.CharIndexToPosition(charIdx).Value); Assert.AreEqual(charIdx, sut.PositionToCharIndex(pos)); }