public void Append(ReadIterator begin, ReadIterator end)
{
var clonedBegin = MemoryBlockSegment.Clone(begin, end);
var clonedEnd = clonedBegin;
while (clonedEnd.Next != null)
{
clonedEnd = clonedEnd.Next;
}
if (_tail == null)
{
_head = clonedBegin;
_headIndex = clonedBegin.Start;
}
else
{
Debug.Assert(_tail.Block != null);
Debug.Assert(_tail.Next == null);
Debug.Assert(_tail.End == _tailIndex);
_tail.Next = clonedBegin;
}
_tail = clonedEnd;
_tailIndex = clonedEnd.End;
}
internal void Append(WritableBuffer buffer)
{
lock (_sync)
{
if (Interlocked.CompareExchange(ref _producingState, 0, 1) != 1)
{
throw new InvalidOperationException("No ongoing producing operation to complete.");
}
if (buffer.IsDefault)
{
// REVIEW: Should we signal the completion?
return;
}
if (_head == null)
{
// Update the head to point to the head of the buffer. This
// happens if we called alloc(0) then write
_head = buffer.Head;
_head.Start = buffer.HeadIndex;
}
// If buffer.Head == tail it means we appended data to the tail
else if (_tail != null && buffer.Head != _tail)
{
// If we have a tail point next to the head of the buffer
Volatile.Write(ref _tail.Next, buffer.Head);
}
// Always update tail to the buffer's tail
_tail = buffer.Tail;
_tail.End = buffer.TailIndex;
}
}
public static MemoryBlockSegment Clone(ReadIterator beginBuffer, ReadIterator endBuffer)
{
var beginOrig = beginBuffer.Segment;
var endOrig = endBuffer.Segment;
if (beginOrig == endOrig)
{
return(new MemoryBlockSegment(beginOrig.Block, beginBuffer.Index, endBuffer.Index));
}
var beginClone = new MemoryBlockSegment(beginOrig.Block, beginBuffer.Index, beginOrig.End);
var endClone = beginClone;
beginOrig = beginOrig.Next;
while (beginOrig != endOrig)
{
endClone.Next = new MemoryBlockSegment(beginOrig.Block, beginOrig.Start, beginOrig.End);
endClone = endClone.Next;
beginOrig = beginOrig.Next;
}
endClone.Next = new MemoryBlockSegment(endOrig.Block, endOrig.Start, endBuffer.Index);
return(beginClone);
}
public static MemoryBlockSegment Clone(ReadCursor beginBuffer, ReadCursor endBuffer, out MemoryBlockSegment lastBlockSegment)
{
var beginOrig = beginBuffer.Segment;
var endOrig = endBuffer.Segment;
if (beginOrig == endOrig)
{
lastBlockSegment = new MemoryBlockSegment(beginOrig.Block, beginBuffer.Index, endBuffer.Index);
return(lastBlockSegment);
}
var beginClone = new MemoryBlockSegment(beginOrig.Block, beginBuffer.Index, beginOrig.End);
var endClone = beginClone;
beginOrig = beginOrig.Next;
while (beginOrig != endOrig)
{
endClone.Next = new MemoryBlockSegment(beginOrig.Block, beginOrig.Start, beginOrig.End);
endClone = endClone.Next;
beginOrig = beginOrig.Next;
}
lastBlockSegment = new MemoryBlockSegment(endOrig.Block, endOrig.Start, endBuffer.Index);
endClone.Next = lastBlockSegment;
return(beginClone);
}
/// <summary>
/// Save the data at the current location then move to the next available space.
/// </summary>
/// <param name="data">The byte to be saved.</param>
/// <returns>true if the operation successes. false if can't find available space.</returns>
internal bool Put(byte data)
{
if (_segment == null)
{
return(false);
}
var segment = _segment;
var index = _index;
while (true)
{
var wasLastBlock = segment.Next == null;
if (index < segment.End)
{
_segment = segment;
_index = index + 1;
segment.Block.Array[index] = data;
return(true);
}
else if (wasLastBlock)
{
return(false);
}
else
{
segment = segment.Next;
index = segment.Start;
}
}
}
internal int CopyTo(byte[] array, int offset, int count)
{
if (IsDefault)
{
return(0);
}
var actual = 0;
var segment = _segment;
var index = _index;
var remaining = count;
while (true)
{
// Determine if we might attempt to copy data from segment.Next before
// calculating "following" so we don't risk skipping data that could
// be added after segment.End when we decide to copy from segment.Next.
// segment.End will always be advanced before segment.Next is set.
var wasLastBlock = segment.Next == null;
var following = segment.End - index;
if (remaining <= following)
{
actual = count;
if (array != null)
{
Buffer.BlockCopy(segment.Block.Array, index, array, offset, remaining);
}
_segment = segment;
_index = index + remaining;
return(actual);
}
else if (wasLastBlock)
{
actual = count - remaining + following;
if (array != null)
{
Buffer.BlockCopy(segment.Block.Array, index, array, offset, following);
}
_segment = segment;
_index = following;
return(actual);
}
else
{
if (array != null)
{
Buffer.BlockCopy(segment.Block.Array, index, array, offset, following);
}
offset += following;
remaining -= following;
segment = segment.Next;
index = segment.Start;
}
}
}
internal WritableBuffer(MemoryPool pool, MemoryBlockSegment segment)
{
_pool = pool;
_tail = segment;
_tailIndex = segment?.End ?? 0;
_head = segment;
_headIndex = _tailIndex;
}
internal bool TryGetBuffer(ReadCursor end, out BufferSpan span)
{
span = default(BufferSpan);
if (IsDefault)
{
return(false);
}
var segment = _segment;
var index = _index;
// Determine if we might attempt to copy data from segment.Next before
// calculating "following" so we don't risk skipping data that could
// be added after segment.End when we decide to copy from segment.Next.
// segment.End will always be advanced before segment.Next is set.
int following = 0;
while (true)
{
var wasLastBlock = segment.Next == null || end.Segment == segment;
if (end.Segment == segment)
{
following = end.Index - index;
}
else
{
following = segment.End - index;
}
if (following > 0)
{
break;
}
if (wasLastBlock)
{
return(false);
}
else
{
segment = segment.Next;
index = segment.Start;
}
}
span = new BufferSpan(segment, index, following);
_segment = segment;
_index = index + following;
return(true);
}
public void Write(byte[] data, int offset, int count)
{
if (_tail == null)
{
_tail = new MemoryBlockSegment(_pool.Lease());
_tailIndex = _tail.End;
_head = _tail;
_headIndex = _tail.Start;
}
Debug.Assert(_tail.Block != null);
Debug.Assert(_tail.Next == null);
Debug.Assert(_tail.End == _tailIndex);
var segment = _tail;
var block = _tail.Block;
var blockIndex = _tailIndex;
var bufferIndex = offset;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
while (remaining > 0)
{
// Try the block empty if the segment is reaodnly
if (bytesLeftInBlock == 0 || segment.ReadOnly)
{
var nextBlock = _pool.Lease();
var nextSegment = new MemoryBlockSegment(nextBlock);
segment.End = blockIndex;
Volatile.Write(ref segment.Next, nextSegment);
segment = nextSegment;
block = nextBlock;
blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
blockIndex += bytesToCopy;
bufferIndex += bytesToCopy;
remaining -= bytesToCopy;
bytesLeftInBlock -= bytesToCopy;
}
segment.End = blockIndex;
segment.Block = block;
_tail = segment;
_tailIndex = blockIndex;
}
internal WritableBuffer(Channel channel, MemoryPool pool, MemoryBlockSegment segment)
{
_channel = channel;
_pool = pool;
_tail = segment;
_tailIndex = segment?.End ?? 0;
_head = segment;
_headIndex = _tailIndex;
_comitted = false;
}
internal int Seek(int bytes)
{
if (_segment == null || IsEnd)
{
return(0);
}
var wasLastBlock = _segment.Next == null;
var following = _segment.End - _index;
if (following >= bytes)
{
_index += bytes;
return(bytes);
}
var segment = _segment;
var index = _index;
while (true)
{
if (wasLastBlock)
{
_segment = segment;
_index = index + following;
return(following);
}
else
{
bytes -= following;
segment = segment.Next;
index = segment.Start;
}
wasLastBlock = segment.Next == null;
following = segment.End - index;
if (following >= bytes)
{
_segment = segment;
_index = index + bytes;
return(bytes);
}
}
}
public ReadableBuffer Clone()
{
var begin = _start;
var end = _end;
var segmentHead = MemoryBlockSegment.Clone(begin, end);
var segmentTail = segmentHead;
while (segmentTail.Next != null)
{
segmentTail = segmentTail.Next;
}
begin = new ReadIterator(segmentHead);
end = new ReadIterator(segmentTail, segmentTail.End);
return(new ReadableBuffer(begin, end, isOwner: true));
}
internal void Skip(int bytesToSkip)
{
if (_segment == null)
{
return;
}
var wasLastBlock = _segment.Next == null;
var following = _segment.End - _index;
if (following >= bytesToSkip)
{
_index += bytesToSkip;
return;
}
var segment = _segment;
var index = _index;
while (true)
{
if (wasLastBlock)
{
throw new InvalidOperationException("Attempted to skip more bytes than available.");
}
else
{
bytesToSkip -= following;
segment = segment.Next;
index = segment.Start;
}
wasLastBlock = segment.Next == null;
following = segment.End - index;
if (following >= bytesToSkip)
{
_segment = segment;
_index = index + bytesToSkip;
return;
}
}
}
public WritableBuffer Alloc(int minimumSize = 0)
{
if (Interlocked.CompareExchange(ref _producingState, 1, 0) != 0)
{
throw new InvalidOperationException("Already producing.");
}
MemoryBlockSegment segment = null;
if (_tail != null && !_tail.ReadOnly)
{
// Try to return the tail so the calling code can append to it
int remaining = _tail.Block.Data.Offset + _tail.Block.Data.Count - _tail.End;
if (minimumSize <= remaining)
{
segment = _tail;
}
}
if (segment == null && minimumSize > 0)
{
// We're out of tail space so lease a new segment only if the requested size > 0
segment = new MemoryBlockSegment(_pool.Lease());
}
lock (_sync)
{
if (_head == null)
{
_head = segment;
}
else if (segment != null && segment != _tail)
{
// Append the segment to the tail if it's non-null
Volatile.Write(ref _tail.Next, segment);
_tail = segment;
}
return(new WritableBuffer(this, _pool, segment));
}
}
private ReadableBuffer(ref ReadableBuffer buffer)
{
_channel = buffer._channel;
var begin = buffer._start;
var end = buffer._end;
MemoryBlockSegment segmentTail;
var segmentHead = MemoryBlockSegment.Clone(begin, end, out segmentTail);
begin = new ReadCursor(segmentHead);
end = new ReadCursor(segmentTail, segmentTail.End);
_start = begin;
_end = end;
_isOwner = true;
_span = buffer._span;
_length = buffer._length;
}
public void EndRead(
ReadCursor consumed,
ReadCursor examined)
{
MemoryBlockSegment returnStart = null;
MemoryBlockSegment returnEnd = null;
lock (_sync)
{
if (!consumed.IsDefault)
{
returnStart = _head;
returnEnd = consumed.Segment;
_head = consumed.Segment;
_head.Start = consumed.Index;
}
if (!examined.IsDefault &&
examined.IsEnd &&
Completion.Status == TaskStatus.WaitingForActivation)
{
Interlocked.CompareExchange(
ref _awaitableState,
_awaitableIsNotCompleted,
_awaitableIsCompleted);
}
}
while (returnStart != returnEnd)
{
var returnSegment = returnStart;
returnStart = returnStart.Next;
returnSegment.Dispose();
}
if (Interlocked.CompareExchange(ref _consumingState, 0, 1) != 1)
{
throw new InvalidOperationException("No ongoing consuming operation to complete.");
}
}
internal int Take()
{
var segment = _segment;
if (segment == null)
{
return(-1);
}
var index = _index;
var wasLastBlock = segment.Next == null;
if (index < segment.End)
{
_index = index + 1;
return(segment.Block.Array[index]);
}
do
{
if (wasLastBlock)
{
return(-1);
}
else
{
segment = segment.Next;
index = segment.Start;
}
wasLastBlock = segment.Next == null;
if (index < segment.End)
{
_segment = segment;
_index = index + 1;
return(segment.Block.Array[index]);
}
} while (true);
}
public void Append(ref ReadableBuffer buffer)
{
MemoryBlockSegment clonedEnd;
var clonedBegin = MemoryBlockSegment.Clone(buffer.Start, buffer.End, out clonedEnd);
if (_tail == null)
{
_head = clonedBegin;
_headIndex = clonedBegin.Start;
}
else
{
Debug.Assert(_tail.Block != null);
Debug.Assert(_tail.Next == null);
Debug.Assert(_tail.End == _tailIndex);
_tail.Next = clonedBegin;
}
_tail = clonedEnd;
_tailIndex = clonedEnd.End;
}
public void Ensure(int count)
{
if (_tail == null)
{
_tail = new MemoryBlockSegment(_pool.Lease());
_tailIndex = _tail.End;
_head = _tail;
_headIndex = _tail.Start;
}
Debug.Assert(_tail.Block != null);
Debug.Assert(_tail.Next == null);
Debug.Assert(_tail.End == _tailIndex);
var segment = _tail;
var block = _tail.Block;
var blockIndex = _tailIndex;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
// If inadequate bytes left or if the segment is readonly
if (bytesLeftInBlock < count || segment.ReadOnly)
{
var nextBlock = _pool.Lease();
var nextSegment = new MemoryBlockSegment(nextBlock);
segment.End = blockIndex;
Volatile.Write(ref segment.Next, nextSegment);
segment = nextSegment;
block = nextBlock;
blockIndex = block.Data.Offset;
segment.End = blockIndex;
segment.Block = block;
_tail = segment;
_tailIndex = blockIndex;
}
}
public void Dispose()
{
Debug.Assert(_completedWriting, "Not completed writing");
Debug.Assert(_completedReading, "Not completed reading");
lock (_sync)
{
// Return all segments
var segment = _head;
while (segment != null)
{
var returnSegment = segment;
segment = segment.Next;
returnSegment.Dispose();
}
_head = null;
_tail = null;
Interlocked.Exchange(ref _disposeCallback, null)?.Invoke();
}
}
private void Dispose()
{
Debug.Assert(_completedWriting, "Not completed writing");
Debug.Assert(_completedReading, "Not completed reading");
lock (_sync)
{
// Return all segments
var segment = _head;
while (segment != null)
{
var returnSegment = segment;
segment = segment.Next;
returnSegment.Dispose();
}
_head = null;
_tail = null;
_disposedTcs.TrySetResult(null);
}
}
internal BufferSpan(MemoryBlockSegment segment, int offset, int count)
{
_segment = segment;
_length = count;
_offset = offset;
}
internal ReadCursor(MemoryBlockSegment segment)
{
_segment = segment;
_index = segment?.Start ?? 0;
}
internal ReadCursor(MemoryBlockSegment segment, int index)
{
_segment = segment;
_index = index;
}
internal unsafe int Seek(ref Vector <byte> byte0Vector, ref Vector <byte> byte1Vector, ref Vector <byte> byte2Vector)
{
if (IsDefault)
{
return(-1);
}
var segment = _segment;
var index = _index;
var wasLastBlock = segment.Next == null;
var following = segment.End - index;
byte[] array;
int byte0Index = int.MaxValue;
int byte1Index = int.MaxValue;
int byte2Index = int.MaxValue;
var byte0 = byte0Vector[0];
var byte1 = byte1Vector[0];
var byte2 = byte2Vector[0];
while (true)
{
while (following == 0)
{
if (wasLastBlock)
{
_segment = segment;
_index = index;
return(-1);
}
segment = segment.Next;
index = segment.Start;
wasLastBlock = segment.Next == null;
following = segment.End - index;
}
array = segment.Block.Array;
while (following > 0)
{
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
{
var data = new Vector <byte>(array, index);
var byte0Equals = Vector.Equals(data, byte0Vector);
var byte1Equals = Vector.Equals(data, byte1Vector);
var byte2Equals = Vector.Equals(data, byte2Vector);
if (!byte0Equals.Equals(Vector <byte> .Zero))
{
byte0Index = FindFirstEqualByte(ref byte0Equals);
}
if (!byte1Equals.Equals(Vector <byte> .Zero))
{
byte1Index = FindFirstEqualByte(ref byte1Equals);
}
if (!byte2Equals.Equals(Vector <byte> .Zero))
{
byte2Index = FindFirstEqualByte(ref byte2Equals);
}
if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue)
{
following -= _vectorSpan;
index += _vectorSpan;
continue;
}
_segment = segment;
int toReturn, toMove;
if (byte0Index < byte1Index)
{
if (byte0Index < byte2Index)
{
toReturn = byte0;
toMove = byte0Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
else
{
if (byte1Index < byte2Index)
{
toReturn = byte1;
toMove = byte1Index;
}
else
{
toReturn = byte2;
toMove = byte2Index;
}
}
_index = index + toMove;
return(toReturn);
}
// Need unit tests to test Vector path
#if !DEBUG
}
#endif
var pCurrent = (segment.Block.DataFixedPtr + index);
var pEnd = pCurrent + following;
do
{
if (*pCurrent == byte0)
{
_segment = segment;
_index = index;
return(byte0);
}
if (*pCurrent == byte1)
{
_segment = segment;
_index = index;
return(byte1);
}
if (*pCurrent == byte2)
{
_segment = segment;
_index = index;
return(byte2);
}
pCurrent++;
index++;
} while (pCurrent != pEnd);
following = 0;
break;
}
}
}
