Exemplo n.º 1
0
            IEnumerable <PieceOfWork> ReadRawDataFromMedia_Backward(CancellationToken cancellationToken)
            {
                Stream             stream       = owner.media.DataStream;
                CreateParserParams parserParams = owner.currentParams;

                FileRange.Range    range         = parserParams.Range.Value;
                TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);

                long beginStreamPos = new TextStreamPosition(range.Begin, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize;
                long endStreamPos   = startPosition.StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;

                if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
                {
                    int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
                    beginStreamPos -= maxBytesPerCharacter;
                }

                PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);

                {
                    firstPieceOfWork.streamData = AllocateAndReadStreamData_Backward(stream, endStreamPos);
                    if (firstPieceOfWork.streamData.IsEmpty)
                    {
                        yield break;
                    }
                    firstPieceOfWork.startTextPosition = startPosition.Value;
                    firstPieceOfWork.stopTextPosition  = endStreamPos - owner.BytesToParsePerThread;
                    firstPieceOfWork.outputBuffer      = owner.AllocateOutputBuffer();
                    endStreamPos -= owner.BytesToParsePerThread;
                }

                PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;

                for (; ;)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
                    nextPieceOfWork.streamData        = AllocateAndReadStreamData_Backward(stream, endStreamPos);
                    nextPieceOfWork.nextStreamData    = pieceOfWorkToYieldNextTime.streamData;
                    nextPieceOfWork.startTextPosition = endStreamPos;
                    nextPieceOfWork.stopTextPosition  = endStreamPos - owner.BytesToParsePerThread;
                    nextPieceOfWork.outputBuffer      = owner.AllocateOutputBuffer();

                    pieceOfWorkToYieldNextTime.prevStreamData = nextPieceOfWork.streamData;

                    yield return(pieceOfWorkToYieldNextTime);

                    if (endStreamPos < beginStreamPos)
                    {
                        break;
                    }
                    if (nextPieceOfWork.streamData.IsEmpty)
                    {
                        break;
                    }

                    pieceOfWorkToYieldNextTime = nextPieceOfWork;
                    endStreamPos -= owner.BytesToParsePerThread;
                }
            }
Exemplo n.º 2
0
        public void TextStreamPosition_ConstructorTest()
        {
            long streamPosition           = 64 * 1024 * 12;
            int  textPositionInsideBuffer = 1234;
            TextStreamPosition pos        = new TextStreamPosition(streamPosition, textPositionInsideBuffer);

            Assert.AreEqual(pos.Value, streamPosition + textPositionInsideBuffer);
        }
Exemplo n.º 3
0
        public async Task StreamPositionToCharIndexTest_InvalidBigTextStreamPositionIsMappedToPastTheEndCharIndex()
        {
            StreamTextAccess buf = new StreamTextAccess(
                S().Add('a', blockSz * 3).ToStream(Encoding.Unicode),
                Encoding.Unicode
                );
            await buf.BeginReading(0, TextAccessDirection.Forward);

            // valid Unicode text stream positions are from 0 to blockSz/2. below is invalid position.
            var invalidTextStreamPosition = new TextStreamPosition(blockSz - 10);

            Assert.AreEqual(blockSz / 2, buf.PositionToCharIndex(invalidTextStreamPosition));
        }
Exemplo n.º 4
0
            IEnumerable <PieceOfWork> ReadRawDataFromMedia_Forward(CancellationToken cancellationToken)
            {
                Stream             stream       = owner.media.DataStream;
                CreateParserParams parserParams = owner.currentParams;

                FileRange.Range    range         = parserParams.Range.Value;
                TextStreamPosition startPosition = new TextStreamPosition(parserParams.StartPosition, owner.textStreamPositioningParams);

                long beginStreamPos = startPosition.StreamPositionAlignedToBlockSize;
                long endStreamPos   = new TextStreamPosition(range.End, owner.textStreamPositioningParams).StreamPositionAlignedToBlockSize + owner.textStreamPositioningParams.AlignmentBlockSize;

                PieceOfWork firstPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);

                if (beginStreamPos != 0 && !owner.encoding.IsSingleByte)
                {
                    int maxBytesPerCharacter = owner.encoding.GetMaxByteCount(1);
                    firstPieceOfWork.prevStreamData = new StreamData(
                        beginStreamPos - maxBytesPerCharacter, new byte[maxBytesPerCharacter]);
                    stream.Position = beginStreamPos - maxBytesPerCharacter;
                    stream.Read(firstPieceOfWork.prevStreamData.Bytes, 0, maxBytesPerCharacter);
                }
                else
                {
                    stream.Position = beginStreamPos;
                }

                {
                    firstPieceOfWork.streamData = AllocateAndReadStreamData(stream);
                    if (firstPieceOfWork.streamData.IsEmpty)
                    {
                        yield break;
                    }
                    firstPieceOfWork.startTextPosition = startPosition.Value;
                    firstPieceOfWork.stopTextPosition  = beginStreamPos + owner.BytesToParsePerThread;
                    firstPieceOfWork.outputBuffer      = owner.AllocateOutputBuffer();
                    beginStreamPos += owner.BytesToParsePerThread;
                }

                PieceOfWork pieceOfWorkToYieldNextTime = firstPieceOfWork;

                for (; ;)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    PieceOfWork nextPieceOfWork = new PieceOfWork(Interlocked.Increment(ref owner.nextPieceOfWorkId), owner.tracer);
                    nextPieceOfWork.streamData        = AllocateAndReadStreamData(stream);
                    nextPieceOfWork.prevStreamData    = pieceOfWorkToYieldNextTime.streamData;
                    nextPieceOfWork.startTextPosition = beginStreamPos;
                    nextPieceOfWork.stopTextPosition  = beginStreamPos + owner.BytesToParsePerThread;
                    nextPieceOfWork.outputBuffer      = owner.AllocateOutputBuffer();

                    pieceOfWorkToYieldNextTime.nextStreamData = nextPieceOfWork.streamData;

                    owner.tracer.Info("Start processing new peice of work. Currently being processed: {0}", Interlocked.Increment(ref owner.peicesOfWorkBeingProgressed));
                    yield return(pieceOfWorkToYieldNextTime);

                    if (beginStreamPos > endStreamPos)
                    {
                        break;
                    }
                    if (nextPieceOfWork.streamData.IsEmpty)
                    {
                        break;
                    }

                    pieceOfWorkToYieldNextTime = nextPieceOfWork;
                    beginStreamPos            += owner.BytesToParsePerThread;
                }
            }
Exemplo n.º 5
0
 static void TestCharPosMapping(StreamTextAccess sut, TextStreamPosition pos, int charIdx)
 {
     Assert.AreEqual(pos.Value, sut.CharIndexToPosition(charIdx).Value);
     Assert.AreEqual(charIdx, sut.PositionToCharIndex(pos));
 }