public unsafe void CountAndChecksum() { var path = TestUtils.GetPath(); var rounds = 10; var count = 100_000; var bufferSize = count * 16 + StreamBlock.DataOffset; var pool = new BlockMemoryPool(path, 64, LMDBEnvironmentFlags.WriteMap | LMDBEnvironmentFlags.NoSync, (Wpid)123, maxBufferLength: BitUtil.FindNextPositivePowerOfTwo(bufferSize)); // TODO step on what happens when buffer is too large var buffer = pool.RentMemory(bufferSize); var rm = buffer.RetainBlockMemory(false); var rm2 = rm.Clone(); var seed = 2u; StreamBlock.TryInitialize(new DirectBuffer(rm2), (StreamLogId)1, default, 16, 1, seed, default);
public unsafe void StreamBlockHeaderLayout() { var ty = typeof(StreamBlock.StreamBlockHeader); var rm = BufferPool.Retain(4096, true); rm.Span.Clear(); StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1L, 1, 1); var sb = new StreamBlock(rm, (StreamLogId)1L); Assert.AreEqual(0, StreamBlock.StreamBlockHeader.InitTimestampOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.InitTimestampOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.InitTimestamp))); Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).InitTimestamp.GetType()); Assert.AreEqual(64, StreamBlock.StreamBlockHeader.VersionAndFlagsOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.VersionAndFlagsOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.VersionAndFlags))); Assert.AreEqual(typeof(VersionAndFlags), default(StreamBlock.StreamBlockHeader).VersionAndFlags.GetType()); Assert.AreEqual(65, StreamBlock.StreamBlockHeader.AdditionalFlagsOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.AdditionalFlagsOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.AdditionalFlags))); Assert.AreEqual(typeof(byte), default(StreamBlock.StreamBlockHeader).AdditionalFlags.GetType()); Assert.AreEqual(66, StreamBlock.StreamBlockHeader.ItemFixedSizeOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemFixedSizeOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemFixedSize))); Assert.AreEqual(typeof(short), default(StreamBlock.StreamBlockHeader).ItemFixedSize.GetType()); Assert.AreEqual(68, StreamBlock.StreamBlockHeader.PayloadLengthOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.PayloadLengthOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PayloadLength))); Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).PayloadLength.GetType()); Assert.AreEqual(72, StreamBlock.StreamBlockHeader.StreamLogIdOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.StreamLogIdOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.StreamLogId))); Assert.AreEqual(typeof(StreamLogId), default(StreamBlock.StreamBlockHeader).StreamLogId.GetType()); Assert.AreEqual(typeof(StreamLogId), sb.StreamLogId.GetType()); Assert.AreEqual(typeof(long), sb.StreamLogIdLong.GetType()); Assert.AreEqual(80, StreamBlock.StreamBlockHeader.FirstVersionOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.FirstVersionOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.FirstVersion))); Assert.AreEqual(typeof(ulong), default(StreamBlock.StreamBlockHeader).FirstVersion.GetType()); Assert.AreEqual(typeof(ulong), sb.FirstVersion.GetType()); Assert.AreEqual(1, sb.FirstVersion); Assert.AreEqual(88, StreamBlock.StreamBlockHeader.CountOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.CountOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Count))); Assert.AreEqual(typeof(int), default(StreamBlock.StreamBlockHeader).Count.GetType()); Assert.AreEqual(typeof(int), sb.CountVolatile.GetType()); Assert.AreEqual(92, StreamBlock.StreamBlockHeader.ChecksumOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.ChecksumOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Checksum))); Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).Checksum.GetType()); Assert.AreEqual(typeof(uint), sb.Checksum.GetType()); Assert.AreEqual(96, StreamBlock.StreamBlockHeader.WriteEndTimestampOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.WriteEndTimestampOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.WriteEnd))); Assert.AreEqual(typeof(Timestamp), default(StreamBlock.StreamBlockHeader).WriteEnd.GetType()); Assert.AreEqual(typeof(Timestamp), sb.WriteEnd.GetType()); Assert.AreEqual(104, StreamBlock.StreamBlockHeader.HashOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.HashOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.Hash))); Assert.AreEqual(typeof(Hash16), default(StreamBlock.StreamBlockHeader).Hash.GetType()); Assert.AreEqual(typeof(Hash16), sb.Hash.GetType()); Assert.AreEqual(120, StreamBlock.StreamBlockHeader.PkContextIdOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.PkContextIdOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.PkContextId))); Assert.AreEqual(typeof(uint), default(StreamBlock.StreamBlockHeader).PkContextId.GetType()); Assert.AreEqual(typeof(uint), sb.PkContextId.GetType()); Assert.AreEqual(124, StreamBlock.StreamBlockHeader.ItemDthOffset); Assert.AreEqual(StreamBlock.StreamBlockHeader.ItemDthOffset, (int)Marshal.OffsetOf(ty, nameof(StreamBlock.StreamBlockHeader.ItemDth))); Assert.AreEqual(typeof(DataTypeHeader), default(StreamBlock.StreamBlockHeader).ItemDth.GetType()); Assert.AreEqual(typeof(DataTypeHeader), sb.ItemDth.GetType()); rm.Dispose(); }
public void CouldWriteManyFinalizedChunksToTable() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; Settings.DoDetectBufferLeaks = true; #pragma warning restore 618 ProcessConfig.InitDefault(); var path = TestUtils.GetPath(); ulong count = 100_000; var rng = new Random(42); var values = new SmallDecimal[count + 1]; for (ulong i = 1; i <= count; i++) { values[(int)i] = new SmallDecimal(10 + Math.Round(rng.NextDouble() * 2, 4), 4); } var storage = new SQLiteStorage($@"Filename={Path.Combine(path, "blockstorage.db")}"); var totalPayload = 0; var totalCapacity = 0L; //Settings.ZstdCompressionLevel = 1; //Settings.LZ4CompressionLevel = 1; //Settings.ZlibCompressionLevel = 5; using (Benchmark.Run("Chunks", (int)(count))) { var chunkSize = 4032; var rm = BufferPool.Retain(chunkSize, true); rm.Span.Clear(); StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, 1); var block = new StreamBlock(rm, (StreamLogId)1, 8, 1); var chunkCount = 1; for (ulong i = 1; i <= count; i++) { var claim = block.Claim(i, 8); if (!claim.IsValid) { Assert.IsTrue(block.IsCompleted); var(inserted, rowid) = storage.InsertBlock(block); //var couldRead = file.TryReadChunk(chunk.FirstVersion, out var readChunk); //Assert.IsTrue(couldRead); //Assert.AreEqual(chunk.WriteEnd, readChunk.WriteEnd); //var fv = readChunk.FirstVersion; //for (int j = 0; j < readChunk.Count; j++) //{ // var ii = fv + (ulong)j; // var readValue = readChunk[j].Read<SmallDecimal>(0); // if (readValue != values[ii]) // { // Assert.Fail(); // } //} //readChunk.Dispose(); block.DisposeFree(); chunkCount++; if (chunkCount % 100 == 0) { storage.Checkpoint(true); } //if (chunkCount % 50 == 0) //{ // file.Complete(); // totalPayload += file.PayloadSize; // totalCapacity += file.FileCapacity; // fileCount++; // file.Dispose(); // filePath = Path.Combine(path, $"{fileCount}.slc"); // file = new StreamLogChunkFile(filePath, true); //} rm = BufferPool.Retain(chunkSize, true); rm.Span.Clear(); StreamBlock.TryInitialize(new DirectBuffer(rm), (StreamLogId)1, 8, i); block = new StreamBlock(rm, (StreamLogId)1, 8, i); claim = block.Claim(i, 8); } claim.Write(0, values[i]); block.Commit(); } var(inserted1, rowid1) = storage.InsertBlock(block); Assert.IsTrue(inserted1); var blockR = storage.TryGetStreamBlock(block.StreamLogIdLong, block.FirstVersion); var lastVersion = blockR.CurrentVersion; Assert.AreEqual(block.CurrentVersion, lastVersion); blockR.DisposeFree(); block.Complete(); (inserted1, rowid1) = storage.InsertBlock(block); Assert.IsTrue(inserted1); Assert.AreEqual(block.CurrentVersion, lastVersion); (inserted1, rowid1) = storage.InsertBlock(block); Assert.IsFalse(inserted1); //var couldRead1 = file.TryReadChunk(chunk.FirstVersion, out var readChunk1); //Assert.IsTrue(couldRead1); //Assert.AreEqual(chunk.WriteStart, readChunk1.WriteStart, "WriteStart"); //Assert.AreEqual(chunk.WriteEnd, readChunk1.WriteEnd, "WriteEnd"); //readChunk1.Dispose(); block.DisposeFree(); //totalPayload += file.PayloadSize; //totalCapacity += file.FileCapacity; Console.WriteLine("Chunk count: " + chunkCount); //Console.WriteLine("Payload: " + totalPayload); //Console.WriteLine("File size: " + totalCapacity); Console.WriteLine("Useful size: " + count * 8); //Console.WriteLine("Effective Compression: " + (totalPayload / (1.0 * count * 8))); storage.Dispose(); } }