public unsafe void CountAndChecksum()
{
var path = TestUtils.GetPath();
var rounds = 10;
var count = 100_000;
var bufferSize = count * 16 + StreamBlock.DataOffset;
var pool = new BlockMemoryPool(path, 64, LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync, (Wpid)123,
maxBufferLength: BitUtil.FindNextPositivePowerOfTwo(bufferSize));
// TODO step on what happens when buffer is too large
var buffer = pool.RentMemory(bufferSize);
var rm = buffer.RetainBlockMemory(false);
var rm2 = rm.Clone();
var seed = 2u;
StreamBlock.TryInitialize(new DirectBuffer(rm2), (StreamLogId)1, default, 16, 1, seed, default);
public unsafe void StreamBlockHeaderLayout()
{
var ty = typeof(StreamBlock.StreamBlockHeader);
var rm = BufferPool.Retain(4096, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1L, 1, 1);
var sb = new StreamBlock(rm, (StreamLogId)1L);
Assert.AreEqual(0, StreamBlock.StreamBlockHeader.InitTimestampOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.InitTimestampOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.InitTimestamp)));
Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).InitTimestamp.GetType());
Assert.AreEqual(64, StreamBlock.StreamBlockHeader.VersionAndFlagsOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.VersionAndFlagsOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.VersionAndFlags)));
Assert.AreEqual(typeof(VersionAndFlags), default(StreamBlock.StreamBlockHeader).VersionAndFlags.GetType());
Assert.AreEqual(65, StreamBlock.StreamBlockHeader.AdditionalFlagsOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.AdditionalFlagsOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.AdditionalFlags)));
Assert.AreEqual(typeof(byte), default(StreamBlock.StreamBlockHeader).AdditionalFlags.GetType());
Assert.AreEqual(66, StreamBlock.StreamBlockHeader.ItemFixedSizeOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemFixedSizeOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemFixedSize)));
Assert.AreEqual(typeof(short), default(StreamBlock.StreamBlockHeader).ItemFixedSize.GetType());
Assert.AreEqual(68, StreamBlock.StreamBlockHeader.PayloadLengthOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.PayloadLengthOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PayloadLength)));
Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).PayloadLength.GetType());
Assert.AreEqual(72, StreamBlock.StreamBlockHeader.StreamLogIdOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.StreamLogIdOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.StreamLogId)));
Assert.AreEqual(typeof(StreamLogId), default(StreamBlock.StreamBlockHeader).StreamLogId.GetType());
Assert.AreEqual(typeof(StreamLogId), sb.StreamLogId.GetType());
Assert.AreEqual(typeof(long), sb.StreamLogIdLong.GetType());
Assert.AreEqual(80, StreamBlock.StreamBlockHeader.FirstVersionOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.FirstVersionOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.FirstVersion)));
Assert.AreEqual(typeof(ulong), default(StreamBlock.StreamBlockHeader).FirstVersion.GetType());
Assert.AreEqual(typeof(ulong), sb.FirstVersion.GetType());
Assert.AreEqual(1, sb.FirstVersion);
Assert.AreEqual(88, StreamBlock.StreamBlockHeader.CountOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.CountOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Count)));
Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).Count.GetType());
Assert.AreEqual(typeof(int), sb.CountVolatile.GetType());
Assert.AreEqual(92, StreamBlock.StreamBlockHeader.ChecksumOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ChecksumOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Checksum)));
Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).Checksum.GetType());
Assert.AreEqual(typeof(uint), sb.Checksum.GetType());
Assert.AreEqual(96, StreamBlock.StreamBlockHeader.WriteEndTimestampOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.WriteEndTimestampOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.WriteEnd)));
Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).WriteEnd.GetType());
Assert.AreEqual(typeof(Timestamp), sb.WriteEnd.GetType());
Assert.AreEqual(104, StreamBlock.StreamBlockHeader.HashOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.HashOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Hash)));
Assert.AreEqual(typeof(Hash16), default(StreamBlock.StreamBlockHeader).Hash.GetType());
Assert.AreEqual(typeof(Hash16), sb.Hash.GetType());
Assert.AreEqual(120, StreamBlock.StreamBlockHeader.PkContextIdOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.PkContextIdOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PkContextId)));
Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).PkContextId.GetType());
Assert.AreEqual(typeof(uint), sb.PkContextId.GetType());
Assert.AreEqual(124, StreamBlock.StreamBlockHeader.ItemDthOffset);
Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemDthOffset,
(int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemDth)));
Assert.AreEqual(typeof(DataTypeHeader), default(StreamBlock.StreamBlockHeader).ItemDth.GetType());
Assert.AreEqual(typeof(DataTypeHeader), sb.ItemDth.GetType());
rm.Dispose();
}
public void CouldWriteManyFinalizedChunksToTable()
{
#pragma warning disable 618
Settings.DoAdditionalCorrectnessChecks = false;
Settings.DoDetectBufferLeaks = true;
#pragma warning restore 618
ProcessConfig.InitDefault();
var path = TestUtils.GetPath();
ulong count = 100_000;
var rng = new Random(42);
var values = new SmallDecimal[count + 1];
for (ulong i = 1; i <= count; i++)
{
values[(int)i] = new SmallDecimal(10 + Math.Round(rng.NextDouble() * 2, 4), 4);
}
var storage = new SQLiteStorage($@"Filename={Path.Combine(path, "blockstorage.db")}");
var totalPayload = 0;
var totalCapacity = 0L;
//Settings.ZstdCompressionLevel = 1;
//Settings.LZ4CompressionLevel = 1;
//Settings.ZlibCompressionLevel = 5;
using (Benchmark.Run("Chunks", (int)(count)))
{
var chunkSize = 4032;
var rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, 1);
var block = new StreamBlock(rm, (StreamLogId)1, 8, 1);
var chunkCount = 1;
for (ulong i = 1; i <= count; i++)
{
var claim = block.Claim(i, 8);
if (!claim.IsValid)
{
Assert.IsTrue(block.IsCompleted);
var(inserted, rowid) = storage.InsertBlock(block);
//var couldRead = file.TryReadChunk(chunk.FirstVersion, out var readChunk);
//Assert.IsTrue(couldRead);
//Assert.AreEqual(chunk.WriteEnd, readChunk.WriteEnd);
//var fv = readChunk.FirstVersion;
//for (int j = 0; j < readChunk.Count; j++)
//{
// var ii = fv + (ulong)j;
// var readValue = readChunk[j].Read<SmallDecimal>(0);
// if (readValue != values[ii])
// {
// Assert.Fail();
// }
//}
//readChunk.Dispose();
block.DisposeFree();
chunkCount++;
if (chunkCount % 100 == 0)
{
storage.Checkpoint(true);
}
//if (chunkCount % 50 == 0)
//{
// file.Complete();
// totalPayload += file.PayloadSize;
// totalCapacity += file.FileCapacity;
// fileCount++;
// file.Dispose();
// filePath = Path.Combine(path, $"{fileCount}.slc");
// file = new StreamLogChunkFile(filePath, true);
//}
rm = BufferPool.Retain(chunkSize, true);
rm.Span.Clear();
StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, i);
block = new StreamBlock(rm, (StreamLogId)1, 8, i);
claim = block.Claim(i, 8);
}
claim.Write(0, values[i]);
block.Commit();
}
var(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
var blockR = storage.TryGetStreamBlock(block.StreamLogIdLong, block.FirstVersion);
var lastVersion = blockR.CurrentVersion;
Assert.AreEqual(block.CurrentVersion, lastVersion);
blockR.DisposeFree();
block.Complete();
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsTrue(inserted1);
Assert.AreEqual(block.CurrentVersion, lastVersion);
(inserted1, rowid1) = storage.InsertBlock(block);
Assert.IsFalse(inserted1);
//var couldRead1 = file.TryReadChunk(chunk.FirstVersion, out var readChunk1);
//Assert.IsTrue(couldRead1);
//Assert.AreEqual(chunk.WriteStart, readChunk1.WriteStart, "WriteStart");
//Assert.AreEqual(chunk.WriteEnd, readChunk1.WriteEnd, "WriteEnd");
//readChunk1.Dispose();
block.DisposeFree();
//totalPayload += file.PayloadSize;
//totalCapacity += file.FileCapacity;
Console.WriteLine("Chunk count: " + chunkCount);
//Console.WriteLine("Payload: " + totalPayload);
//Console.WriteLine("File size: " + totalCapacity);
Console.WriteLine("Useful size: " + count * 8);
//Console.WriteLine("Effective Compression: " + (totalPayload / (1.0 * count * 8)));
storage.Dispose();
}
}