public void CouldAddNextChunkAfterCompleted()
{
Settings.DoAdditionalCorrectnessChecks = false;
var path = TestUtils.GetPath();
var processConfig = new ProcessConfig(path);
StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync;
StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; // LMDBEnvironmentFlags.MapAsync | LMDBEnvironmentFlags.NoMetaSync | LMDBEnvironmentFlags.WriteMap;
var slm = new StreamLogManager(processConfig, "CouldAddNextChunk", null, 512, true, true);
var bufferPool = slm.BufferPool;
var blockIndex = slm.BlockIndex;
var streamId = (StreamLogId)42;
short valueSize = 8;
var state = slm.StateStorage.GetState(streamId);
state.CheckInit(streamId, 8, SerializationFormat.Binary);
var sl = new StreamLog(slm, state, textId: "test_stream");
var count = 100;
SharedMemory currentBuffer = null;
using (Benchmark.Run("AddNext (KOPS)", count * 1000))
{
var version = 1UL;
var block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version, default);
Assert.AreEqual(2, block.SharedMemory.ReferenceCount);
// var firstBufferRef = block.SharedMemory.BufferRef;
for (int i = 0; i < count; i++)
{
version = (ulong)i + 1;
if (!block.Claim(version, 8).IsValid)
{
Assert.Fail("!slc.Claim(version, 8).IsValid");
}
block.Commit();
block.Complete();
var bufferRef = block.SharedMemory.BufferRef;
block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
var bufferRef1 = block.SharedMemory.BufferRef;
if (bufferRef1 == bufferRef)
{
Assert.Fail("bufferRef1 == bufferRef");
}
// As if someone has created the next buffer before the second call, forget the first call
var prev = block;
// block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
prev.DisposeFree();
var bufferRef2 = block.SharedMemory.BufferRef;
if (block.SharedMemory.ReferenceCount != 2)
{
Assert.Fail($"block.SharedMemory.RefCount {block.SharedMemory.ReferenceCount} != 1");
}
if (bufferRef1 != bufferRef2)
{
Assert.Fail($"bufferRef1 {bufferRef1} != bufferRef2 {bufferRef2}");
}
var nextVersion = block.NextVersion;
block.DisposeFree();
// block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, nextVersion);
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, nextVersion, default);
}
// SharedMemory.Free(buffer);
block.DisposeFree();
block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default);
block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, version + 1);
// SharedMemory.Free(buffer);
block.DisposeFree();
}
bufferPool.PrintBuffers();
bufferPool.PrintBuffersAfterPoolDispose = true;
Benchmark.Dump();
Console.WriteLine("Finished");
sl.Dispose();
slm.Dispose();
}
public void CouldWriteManyFinalizedChunksToTable()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
Settings.DoDetectBufferLeaks = true;
#pragma warning restore 618
ProcessConfig.InitDefault();
var path = TestUtils.GetPath();
ulong count = 100_000;
var rng = new Random(42);
var values = new SmallDecimal[count + 1];
for (ulong i = 1; i <= count; i++)
{
values[(int)i] = new SmallDecimal(10 + Math.Round(rng.NextDouble() * 2, 4), 4);
}
var storage = new SQLiteStorage($@"Filename={Path.Combine(path, "blockstorage.db")}");
var totalPayload = 0;
var totalCapacity = 0L;
//Settings.ZstdCompressionLevel = 1;
//Settings.LZ4CompressionLevel = 1;
//Settings.ZlibCompressionLevel = 5;
using (Benchmark.Run("Chunks", (int)(count)))
{
var chunkSize = 4032;
var rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, 1);
var block = new StreamBlock(rm, (StreamLogId)1, 8, 1);
var chunkCount = 1;
for (ulong i = 1; i <= count; i++)
{
var claim = block.Claim(i, 8);
if (!claim.IsValid)
{
Assert.IsTrue(block.IsCompleted);
var(inserted, rowid) = storage.InsertBlock(block);
//var couldRead = file.TryReadChunk(chunk.FirstVersion, out var readChunk);
//Assert.IsTrue(couldRead);
//Assert.AreEqual(chunk.WriteEnd, readChunk.WriteEnd);
//var fv = readChunk.FirstVersion;
//for (int j = 0; j < readChunk.Count; j++)
//{
// var ii = fv + (ulong)j;
// var readValue = readChunk[j].Read<SmallDecimal>(0);
// if (readValue != values[ii])
// {
// Assert.Fail();
// }
//}
//readChunk.Dispose();
block.DisposeFree();
chunkCount++;
if (chunkCount % 100 == 0)
{
storage.Checkpoint(true);
}
//if (chunkCount % 50 == 0)
//{
// file.Complete();
// totalPayload += file.PayloadSize;
// totalCapacity += file.FileCapacity;
// fileCount++;
// file.Dispose();
// filePath = Path.Combine(path, $"{fileCount}.slc");
// file = new StreamLogChunkFile(filePath, true);
//}
rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, i);
block = new StreamBlock(rm, (StreamLogId)1, 8, i);
claim = block.Claim(i, 8);
}
claim.Write(0, values[i]);
block.Commit();
}
var(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
var blockR = storage.TryGetStreamBlock(block.StreamLogIdLong, block.FirstVersion);
var lastVersion = blockR.CurrentVersion;
Assert.AreEqual(block.CurrentVersion, lastVersion);
blockR.DisposeFree();
block.Complete();
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
Assert.AreEqual(block.CurrentVersion, lastVersion);
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsFalse(inserted1);
//var couldRead1 = file.TryReadChunk(chunk.FirstVersion, out var readChunk1);
//Assert.IsTrue(couldRead1);
//Assert.AreEqual(chunk.WriteStart, readChunk1.WriteStart, "WriteStart");
//Assert.AreEqual(chunk.WriteEnd, readChunk1.WriteEnd, "WriteEnd");
//readChunk1.Dispose();
block.DisposeFree();
//totalPayload += file.PayloadSize;
//totalCapacity += file.FileCapacity;
Console.WriteLine("Chunk count: " + chunkCount);
//Console.WriteLine("Payload: " + totalPayload);
//Console.WriteLine("File size: " + totalCapacity);
Console.WriteLine("Useful size: " + count * 8);
//Console.WriteLine("Effective Compression: " + (totalPayload / (1.0 * count * 8)));
storage.Dispose();
}
}
