public SpanProcessor(ISpanWriter spanStorage, ILogger <SpanProcessor> logger) { _spanStorage = spanStorage; _logger = logger; var channel = Channel.CreateUnbounded <IEnumerable <Span> >(); _channelWriter = channel.Writer; _channelReader = channel.Reader; var taskFactory = new TaskFactory(); for (int i = 0; i < Environment.ProcessorCount; i++) { taskFactory.StartNew(async() => { while (true) { try { await ConsumeAsync(); } catch (Exception e) { _logger.LogError(e, "Failed to consume span"); } } }, TaskCreationOptions.LongRunning); } }
public SpanWriter(ISpanWriter controller) { Controller = controller; Buf = new Span <byte>(); InitialBuffer = new Span <byte>(); HeapBuffer = null; Controller.Init(ref this); }
void MoveValuesContent(ISpanWriter writer, uint wastefulFileId, uint pvlFileId) { const uint blockSize = 256 * 1024; var wastefulStream = _keyValueDB.FileCollection.GetFile(wastefulFileId); var totalSize = wastefulStream !.GetSize(); var blocks = (int)((totalSize + blockSize - 1) / blockSize); var wasteInMemory = new byte[blocks][]; var pos = 0UL; var readLimiter = new BytesPerSecondLimiter(_keyValueDB.CompactorReadBytesPerSecondLimit); for (var i = 0; i < blocks; i++) { _cancellation.ThrowIfCancellationRequested(); wasteInMemory[i] = new byte[blockSize]; var readSize = totalSize - pos; if (readSize > blockSize) { readSize = blockSize; } wastefulStream.RandomRead(wasteInMemory[i].AsSpan(0, (int)readSize), pos, true); pos += readSize; readLimiter.Limit(pos); } _keyValueDB.IterateRoot(_root, (valueFileId, valueOfs, valueSize) => { if (valueFileId != wastefulFileId) { return; } var size = (uint)Math.Abs(valueSize); _newPositionMap.Add(((ulong)wastefulFileId << 32) | valueOfs, ((ulong)pvlFileId << 32) + (ulong)writer.GetCurrentPositionWithoutWriter()); pos = valueOfs; while (size > 0) { _cancellation.ThrowIfCancellationRequested(); var blockId = pos / blockSize; var blockStart = pos % blockSize; var writeSize = (uint)(blockSize - blockStart); if (writeSize > size) { writeSize = size; } writer.WriteBlockWithoutWriter( ref MemoryMarshal.GetReference(wasteInMemory[blockId].AsSpan((int)blockStart, (int)writeSize)), writeSize); size -= writeSize; pos += writeSize; _writerBytesPerSecondLimiter.Limit((ulong)writer.GetCurrentPositionWithoutWriter()); } }); }
public int Write(ISpanWriter writer, Span <byte> destination, T item) { if (object.ReferenceEquals(item, null)) { throw new InvalidDataException("The root table may not be null."); } #if DEBUG int expectedMaxSize = this.GetMaxSize(item); #endif var serializationContext = SerializationContext.ThreadLocalContext.Value; serializationContext.Reset(destination.Length); var sharedStringWriter = this.sharedStringWriter?.Value; serializationContext.SharedStringWriter = sharedStringWriter; serializationContext.Offset = 4; // first 4 bytes are reserved for uoffset to the first table. try { sharedStringWriter?.PrepareWrite(); writer.InvokeWrite(this.innerSerializer, destination, item, 0, serializationContext); if (sharedStringWriter != null) { writer.FlushSharedStrings(sharedStringWriter, destination, serializationContext); } serializationContext.InvokePostSerializeActions(destination); } catch (BufferTooSmallException ex) { ex.SizeNeeded = this.GetMaxSize(item); throw; } #if DEBUG Trace.WriteLine($"Serialized Size: {serializationContext.Offset + 1}, Max Size: {expectedMaxSize}"); Debug.Assert(serializationContext.Offset <= expectedMaxSize - 1); #endif return(serializationContext.Offset); }
private void TableSerializationTest( ISpanWriter writer, Func <FlatBufferSerializer, byte[], PrimitiveTypesTable> parse) { PrimitiveTypesTable original = new() { A = true, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, I = 8, J = 9.1f, K = 10.2, L = "foobar", M = "foobar2", }; byte[] data = new byte[1024]; FlatBufferSerializer.Default.Serialize(original, data, writer); PrimitiveTypesTable parsed = parse(FlatBufferSerializer.Default, data); Assert.Equal(original.A, parsed.A); Assert.Equal(original.B, parsed.B); Assert.Equal(original.C, parsed.C); Assert.Equal(original.D, parsed.D); Assert.Equal(original.E, parsed.E); Assert.Equal(original.F, parsed.F); Assert.Equal(original.G, parsed.G); Assert.Equal(original.H, parsed.H); Assert.Equal(original.I, parsed.I); Assert.Equal(original.J, parsed.J); Assert.Equal(original.K, parsed.K); Assert.Equal(original.L, parsed.L); Assert.Equal(original.M, parsed.M); }