public SpanProcessor(ISpanWriter spanStorage, ILogger <SpanProcessor> logger)
{
_spanStorage = spanStorage;
_logger = logger;
var channel = Channel.CreateUnbounded <IEnumerable <Span> >();
_channelWriter = channel.Writer;
_channelReader = channel.Reader;
var taskFactory = new TaskFactory();
for (int i = 0; i < Environment.ProcessorCount; i++)
{
taskFactory.StartNew(async() =>
{
while (true)
{
try
{
await ConsumeAsync();
}
catch (Exception e)
{
_logger.LogError(e, "Failed to consume span");
}
}
}, TaskCreationOptions.LongRunning);
}
}
public SpanWriter(ISpanWriter controller)
{
Controller = controller;
Buf = new Span <byte>();
InitialBuffer = new Span <byte>();
HeapBuffer = null;
Controller.Init(ref this);
}
void MoveValuesContent(ISpanWriter writer, uint wastefulFileId, uint pvlFileId)
{
const uint blockSize = 256 * 1024;
var wastefulStream = _keyValueDB.FileCollection.GetFile(wastefulFileId);
var totalSize = wastefulStream !.GetSize();
var blocks = (int)((totalSize + blockSize - 1) / blockSize);
var wasteInMemory = new byte[blocks][];
var pos = 0UL;
var readLimiter = new BytesPerSecondLimiter(_keyValueDB.CompactorReadBytesPerSecondLimit);
for (var i = 0; i < blocks; i++)
{
_cancellation.ThrowIfCancellationRequested();
wasteInMemory[i] = new byte[blockSize];
var readSize = totalSize - pos;
if (readSize > blockSize)
{
readSize = blockSize;
}
wastefulStream.RandomRead(wasteInMemory[i].AsSpan(0, (int)readSize), pos, true);
pos += readSize;
readLimiter.Limit(pos);
}
_keyValueDB.IterateRoot(_root, (valueFileId, valueOfs, valueSize) =>
{
if (valueFileId != wastefulFileId)
{
return;
}
var size = (uint)Math.Abs(valueSize);
_newPositionMap.Add(((ulong)wastefulFileId << 32) | valueOfs,
((ulong)pvlFileId << 32) + (ulong)writer.GetCurrentPositionWithoutWriter());
pos = valueOfs;
while (size > 0)
{
_cancellation.ThrowIfCancellationRequested();
var blockId = pos / blockSize;
var blockStart = pos % blockSize;
var writeSize = (uint)(blockSize - blockStart);
if (writeSize > size)
{
writeSize = size;
}
writer.WriteBlockWithoutWriter(
ref MemoryMarshal.GetReference(wasteInMemory[blockId].AsSpan((int)blockStart, (int)writeSize)),
writeSize);
size -= writeSize;
pos += writeSize;
_writerBytesPerSecondLimiter.Limit((ulong)writer.GetCurrentPositionWithoutWriter());
}
});
}
public int Write(ISpanWriter writer, Span <byte> destination, T item)
{
if (object.ReferenceEquals(item, null))
{
throw new InvalidDataException("The root table may not be null.");
}
#if DEBUG
int expectedMaxSize = this.GetMaxSize(item);
#endif
var serializationContext = SerializationContext.ThreadLocalContext.Value;
serializationContext.Reset(destination.Length);
var sharedStringWriter = this.sharedStringWriter?.Value;
serializationContext.SharedStringWriter = sharedStringWriter;
serializationContext.Offset = 4; // first 4 bytes are reserved for uoffset to the first table.
try
{
sharedStringWriter?.PrepareWrite();
writer.InvokeWrite(this.innerSerializer, destination, item, 0, serializationContext);
if (sharedStringWriter != null)
{
writer.FlushSharedStrings(sharedStringWriter, destination, serializationContext);
}
serializationContext.InvokePostSerializeActions(destination);
}
catch (BufferTooSmallException ex)
{
ex.SizeNeeded = this.GetMaxSize(item);
throw;
}
#if DEBUG
Trace.WriteLine($"Serialized Size: {serializationContext.Offset + 1}, Max Size: {expectedMaxSize}");
Debug.Assert(serializationContext.Offset <= expectedMaxSize - 1);
#endif
return(serializationContext.Offset);
}
private void TableSerializationTest(
ISpanWriter writer,
Func <FlatBufferSerializer, byte[], PrimitiveTypesTable> parse)
{
PrimitiveTypesTable original = new()
{
A = true,
B = 1,
C = 2,
D = 3,
E = 4,
F = 5,
G = 6,
H = 7,
I = 8,
J = 9.1f,
K = 10.2,
L = "foobar",
M = "foobar2",
};
byte[] data = new byte[1024];
FlatBufferSerializer.Default.Serialize(original, data, writer);
PrimitiveTypesTable parsed = parse(FlatBufferSerializer.Default, data);
Assert.Equal(original.A, parsed.A);
Assert.Equal(original.B, parsed.B);
Assert.Equal(original.C, parsed.C);
Assert.Equal(original.D, parsed.D);
Assert.Equal(original.E, parsed.E);
Assert.Equal(original.F, parsed.F);
Assert.Equal(original.G, parsed.G);
Assert.Equal(original.H, parsed.H);
Assert.Equal(original.I, parsed.I);
Assert.Equal(original.J, parsed.J);
Assert.Equal(original.K, parsed.K);
Assert.Equal(original.L, parsed.L);
Assert.Equal(original.M, parsed.M);
}
