public void CouldOpenCloseSlm() { var path = TestUtils.GetPath(); var pc = new ProcessConfig(path); var slm = new StreamLogManager(pc, "test"); var slid = new StreamLogId(-1, 1); var inited = slm.InitStreamLog(slid, 0, StreamLogFlags.IsBinary); Assert.IsTrue(inited); var sl = slm.OpenStreamLog(slid); var mem = sl.ClaimRestOfBlockMemory(1); var freeSpace = mem.Length; mem.Span[0] = 123; sl.Commit(1); var mem1 = sl.ClaimRestOfBlockMemory(1); Assert.AreEqual(mem1.Length, freeSpace - 1 - 4); sl.Commit(1); var db = sl.DangerousGetUnpacked(1); Assert.AreEqual(123, db.ReadByte(0)); sl.Dispose(); slm.Dispose(); }
public void CouldRotateNotificationLog() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 3_000_000; using (Benchmark.Run("Log0.Append", count)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); } } Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
private void Dispose(bool disposing) { MetadataManager.Dispose(); StreamLogManager.Dispose(); _slmHandle.Free(); _mmHandle.Free(); // TODO other members }
public void CouldWriteReadLog0MoveGT() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 3_000_000; using (Benchmark.Run("Log0.Append", count)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); } } using (Benchmark.Run("Log0 MoveGT", count)) { using (var reader = new NotificationLog.Reader(log0, CancellationToken.None, false)) { for (long i = 0; i < count; i++) { if (reader.MoveGT((ulong)i)) { if (reader.CurrentVersion != (ulong)(i + 1)) { Assert.Fail($"reader.CurrentVersion {reader.CurrentVersion} != [(ulong) (i + 1)] {i + 1}"); } } } } } // readerTask.Wait(); Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
public void CouldInitNotificationLog() { var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, false, true); slm.Dispose(); }
public void CouldGetVersionFromNewllyCreatedSl() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); Assert.IsFalse(sl.IsCompleted); Assert.AreEqual(0, sl.CurrentVersion); Assert.AreEqual(sl.Count, sl.CurrentVersion); sl.Dispose(); slm.Dispose(); }
public void CouldAddNextChunkBeforeCompleted() { var path = TestUtils.GetPath(); var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, "CouldAddNextChunk", null, 512, true, true); var bufferPool = slm.BufferPool; var blockIndex = slm.BlockIndex; var streamId = (StreamLogId)42; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, 8, SerializationFormat.Binary); var sl = new StreamLog(slm, state, textId: "test_stream"); var count = 100; SharedMemory currentBuffer = null; using (Benchmark.Run("AddNext (KOPS)", count * 1000)) { var version = 1UL; var block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version, default); // var slc = new StreamBlock(buffer.RetainChunkMemory(false), streamId, valueSize, 1); var firstBufferRef = block.SharedMemory.BufferRef; for (int i = 0; i < count; i++) { version = (ulong)i + 1; if (!block.Claim(version, 8).IsValid) { Assert.Fail("!slc.Claim(version, 8).IsValid"); } block.Commit(); var bufferRef = block.SharedMemory.BufferRef; blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); block.Complete(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default); var bufferRef1 = block.SharedMemory.BufferRef; // As if someone has created the next buffer before the second call, forget the first call block.DisposeFree(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default); var bufferRef2 = block.SharedMemory.BufferRef; if (block.SharedMemory.ReferenceCount != 2) { Assert.Fail($"buffer.RefCount {block.SharedMemory.ReferenceCount} != 1"); } if (bufferRef1 != bufferRef2) { Assert.Fail($"bufferRef1 {bufferRef1} != bufferRef2 {bufferRef2}"); } block.Complete(); var nextVersion = block.NextVersion; block.DisposeFree(); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); // block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, nextVersion); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, nextVersion, default); } if (!block.Claim(version + 1, 8).IsValid) { Assert.Fail("!slc.Claim(version, 8).IsValid"); } block.Commit(); block.Complete(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, block.NextVersion, default); Assert.AreEqual(2, block.SharedMemory.ReferenceCount); block.DisposeFree(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 2, default); Assert.IsFalse(block.IsCompleted); // TODO test replace //// replace existing //var toReplaceVersion = slc.FirstVersion; //var toReplaceRef = buffer.BufferRef; //slc.Complete(); //// SharedMemory.Free(buffer); //// slc.DisposeFree(); //buffer = chunkIndex.GetOrCreateNextWritableChunkBuffer(sl, toReplaceVersion, toReplaceRef, 1); //slc = new StreamLogChunk(buffer.RetainChunkMemory(false), streamId, valueSize, version + 2); //Assert.AreNotEqual(toReplaceRef, buffer.BufferRef); // SharedMemory.Free(buffer); block.DisposeFree(); } //bufferPool.PrintBuffers(); //bufferPool.PrintBuffersAfterPoolDispose = true; Benchmark.Dump(); Console.WriteLine("Finished"); sl.Dispose(); slm.Dispose(); }
public void CouldReadWriteZeroLogViaThreadPool() { Console.WriteLine("Starting test"); Console.Out.Flush(); #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: true); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 20 * 1024, false, true); var count = TestUtils.GetBenchCount(200 * 1024 * 1024L, 1000); var tp = new SpreadsThreadPool(new ThreadPoolSettings(6, ThreadType.Foreground, "Log0Pool", ApartmentState.Unknown, null, 0, ThreadPriority.AboveNormal)); var writer = new Thread(() => { for (int i = 1; i <= count; i++) { var x = i; ThreadPool.UnsafeQueueUserWorkItem(o => { (o as NotificationLog).Append((StreamLogNotification)(ulong)x); } , slm.Log0); } }); writer.Priority = ThreadPriority.Highest; writer.Start(); var reader = new Thread(() => { try { var c = 0L; using (Benchmark.Run("ZL Read", count, true)) { var zlr = new NotificationLog.Reader(slm.Log0, CancellationToken.None); var sw = new SpinWait(); while (zlr.MoveNext()) { // if (zlr.MoveNext()) { // Thread.SpinWait(1); c++; if (c % 10_000_000 == 0) { Console.WriteLine($"{c:N0} | MQL {zlr.MissedQueueLength}"); } if (c >= count - 1000) { if (zlr.MissedQueueLength > 0) { Console.WriteLine("Missed queue non empty"); continue; } Console.WriteLine( $"Finished at CurrentVersion: " + zlr.CurrentVersion.ToString("N")); break; } } //else //{ // sw.SpinOnce(); // if (sw.NextSpinWillYield) // { // sw.Reset(); // if (zlr.CurrentVersion > slm.Log0.State.GetZeroLogVersion()) // { // Console.WriteLine("Reached the end"); // // break; // } // // Console.WriteLine($"Spinning in ZLMN: " + zlr.CurrentVersion.ToString("N")); // } //} } if (zlr.MaxStall > MaxStall) { MaxStall = zlr.MaxStall; } Console.WriteLine("STALLS: " + zlr.StallCount); Console.WriteLine("-------------------------"); Console.WriteLine("MAX STALL: " + zlr.MaxStall.ToString("N")); Console.WriteLine("GLOBAL MAX STALL: " + MaxStall.ToString("N")); Console.WriteLine("-------------------------"); zlr.Dispose(); } Console.WriteLine("Read count: " + c.ToString("N")); } catch (Exception ex) { Console.WriteLine("Reader exception: " + ex); } }); reader.Priority = ThreadPriority.Highest; reader.Start(); reader.Join(); tp.Dispose(); tp.WaitForThreadsExit(); Benchmark.Dump(); Thread.Sleep(100); Console.WriteLine("ROTATE CNT: " + (slm.Log0.RotateCount)); Console.WriteLine("STALE VERSION CNT: " + (slm.Log0.StaleVersionCount)); //slm.Dispose(); //slm2.Dispose(); GC.KeepAlive(slm); GC.KeepAlive(slm); slm.Dispose(); Thread.Sleep(1000); }
public void CouldReadWriteZeroLog() { Console.WriteLine("Starting test"); Console.Out.Flush(); #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: true); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); var processConfig2 = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 20 * 1024, disableNotificationLog: false, true); var slm2 = new StreamLogManager(processConfig2, repoName, null, 20 * 1024, disableNotificationLog: false, true); slm.Log0.ActiveBlock.SharedMemory.Increment(); slm2.Log0.ActiveBlock.SharedMemory.Increment(); if (slm.Log0.State.StreamLogId != slm2.Log0.State.StreamLogId) { Assert.Fail("slm.Log0.State.StreamLogId != slm2.Log0.State.StreamLogId"); } var count = TestUtils.GetBenchCount(200 * 1024 * 1024L, 1000); // must be pow2 because we divide count by it var taskPerProcessCount = 6; //var tasks1 = new List<Task>(); //var tasks2 = new List<Task>(); var threads1 = new List <Thread>(); var threads2 = new List <Thread>(); for (int t = 0; t < taskPerProcessCount; t++) { var x = t; threads1.Add(new Thread(() => { Thread.CurrentThread.Name = "W1_" + x; using (Benchmark.Run("ZL Write", count / (2 * taskPerProcessCount), false)) { for (long i = 1; i <= count / (2 * taskPerProcessCount); i++) { try { { if (slm.Log0.Append((StreamLogNotification)(ulong)i) == 0) { Assert.Fail("Cannot append"); } // Thread.SpinWait(5); //if (i % 50 == 0) //{ // if (!Thread.Yield()) // { // Thread.Sleep(0); // } //} } } catch (Exception ex) { Console.WriteLine(ex); } } Console.WriteLine("W1 done"); } })); } for (int t = 0; t < taskPerProcessCount; t++) { var x = t; threads2.Add(new Thread(() => { try { Thread.CurrentThread.Name = "W2_" + x; using (Benchmark.Run("ZL Write", count / (2 * taskPerProcessCount), false)) { for (long i = 1; i <= count / (2 * taskPerProcessCount); i++) { try { { if (slm2.Log0.Append((StreamLogNotification)(ulong)i) == 0) { Assert.Fail("Cannot Append: " + i); } ; // Thread.SpinWait(5); //if (i % 50 == 0) //{ // if (!Thread.Yield()) // { // Thread.Sleep(0); // } //} } } catch (Exception ex) { Console.WriteLine(ex); } } Console.WriteLine("W2 done"); //slm2.Log0.Dispose(); } } catch (Exception ex) { Console.WriteLine(ex); } })); } foreach (var thread in threads1) { thread.Priority = ThreadPriority.Normal; thread.Start(); } foreach (var thread in threads2) { thread.Priority = ThreadPriority.Normal; thread.Start(); } var reader = new Thread(() => { try { var c = 0L; using (Benchmark.Run("ZL Read", count, true)) { var zlr = new NotificationLog.Reader(slm.Log0, CancellationToken.None); var sw = new SpinWait(); while (zlr.MoveNext()) { // if (zlr.MoveNext()) { // Thread.SpinWait(1); c++; if (c % 10_000_000 == 0) { Console.WriteLine($"{c:N0} | MQL {zlr.MissedQueueLength}"); } if (c >= count - 2 * taskPerProcessCount - 1000) { if (zlr.MissedQueueLength > 0) { Console.WriteLine("Missed queue non empty"); continue; } Console.WriteLine( $"Finished at CurrentVersion: " + zlr.CurrentVersion.ToString("N")); break; } } //else //{ // sw.SpinOnce(); // if (sw.NextSpinWillYield) // { // sw.Reset(); // if (zlr.CurrentVersion > slm.Log0.State.GetZeroLogVersion()) // { // Console.WriteLine("Reached the end"); // // break; // } // // Console.WriteLine($"Spinning in ZLMN: " + zlr.CurrentVersion.ToString("N")); // } //} } if (zlr.MaxStall > MaxStall) { MaxStall = zlr.MaxStall; } Console.WriteLine("STALLS: " + zlr.StallCount); Console.WriteLine("-------------------------"); Console.WriteLine("MAX STALL: " + zlr.MaxStall.ToString("N")); Console.WriteLine("GLOBAL MAX STALL: " + MaxStall.ToString("N")); Console.WriteLine("-------------------------"); zlr.Dispose(); } Console.WriteLine("Read count: " + c.ToString("N")); } catch (Exception ex) { Console.WriteLine("Reader exception: " + ex); } }); reader.Priority = ThreadPriority.Normal; reader.Start(); foreach (var thread in threads1) { thread.Join(); } foreach (var thread in threads2) { thread.Join(); } reader.Join(); //using (Benchmark.Run("Log0 MoveNext", count)) //{ // var readCount = 0; // try // { // var chunkCursor = // slm.Log0.State.StreamLogManager.LogChunkIndex.GetChunkRecordCursor(StreamLogId.Log0Id); // using (var txn = slm.Log0.State.StreamLogManager.LogChunkIndex._env.BeginReadOnlyTransaction()) // { // var chunks = slm.Log0.State.StreamLogManager.LogChunkIndex._chunksDb // .AsEnumerable<long, StreamLogChunkRecord>(txn, -1).ToArray(); // var previous = 0UL; // foreach (var r in chunks) // { // var chunk = slm.Log0.RentChunkFromRecord(r); // if (chunk.IsValid) // { // Console.WriteLine( // $"SLCR: version {r.Version} - chunk first version: {chunk.FirstVersion}"); // if (r.Version != chunk.FirstVersion && // r.Version != StreamLogChunkIndex.ReadyChunkVersion) // { // Assert.Fail("Bad versions"); // } // chunk.DisposeFree(); // if (previous != 0 && previous == r.Version) // { // Assert.Fail("Duplicate SLCR"); // } // previous = r.Version; // } // } // } // slm.BufferPool.PrintBuffers(); // Console.WriteLine("Move next"); // while (chunkCursor.MoveNext()) // { // var record = chunkCursor.Current.Value; // Console.WriteLine("Record version: " + record.Version); // if ((long)record.Version >= count) // { // Console.WriteLine("Break"); // break; // } // var chunk = slm.Log0.RentChunkFromRecord(record); // Console.WriteLine("Chunk first version: " + chunk.FirstVersion); // for (ulong i = 0; i < 1024 * 1024; i++) // { // var position = NotificationLog.Log0ItemPosition(chunk._pointer, // (long)(chunk.FirstVersion + i), out _); // var value = *(ulong*)position; // if (value == 0) // { // Assert.Fail("Zero value at: " + (chunk.FirstVersion + i)); // // Console.WriteLine("Zero value at: " + (chunk.FirstVersion + i)); // } // // Console.WriteLine("OK"); // readCount++; // } // chunk.DisposeFree(); // } // //var c = log0.GetContainerCursor(false); // //while (c.MoveNext()) // //{ // // if (c.CurrentValue.ReadUInt64(0) == 0) // // { // // // Console.WriteLine($"c.CurrentValue == 0 at {c.CurrentKey}"); // // Assert.Fail($"c.CurrentValue == 0 at {c.CurrentKey}"); // // } // // readCount++; // //} // //c.Dispose(); // // Assert.AreEqual(count, readCount); // } // catch (Exception ex) // { // Console.WriteLine("EX: " + ex); // } // Console.WriteLine("READ COUNT M: " + readCount * 1.0 / (1024 * 1024)); //} //Console.ReadLine(); //slm.Log0.Dispose(); //slm2.Log0.Dispose(); //slm.Dispose(); //slm2.Dispose(); Benchmark.Dump(); Thread.Sleep(100); Console.WriteLine("ROTATE CNT: " + (slm.Log0.RotateCount + slm2.Log0.RotateCount)); Console.WriteLine("STALE VERSION CNT: " + (slm.Log0.StaleVersionCount + slm2.Log0.StaleVersionCount)); slm.Dispose(); slm2.Dispose(); GC.KeepAlive(slm); GC.KeepAlive(slm2); }
public void CouldWriteReadLog0(bool delete) { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; Settings.DoDetectBufferLeaks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: delete); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, disableNotificationLog: false, disablePacker: false); // var log0 = new NotificationLog(slm); // will disable packing: log0._activeBuffer.Increment(); var count = TestUtils.GetBenchCount(100_000_000, 1000); var readerTask = Task.Run(() => { try { var step = count / 10; var stat = Benchmark.Run("Read", step); { using (var reader = new NotificationLog.Reader(slm.Log0, CancellationToken.None, false)) { var previous = 0UL; while (reader.MoveNext()) { var current = (ulong)reader.Current; //if (current != previous + 1) //{ // Assert.Fail($"current {current} != [previous + 1] {previous + 1}"); //} previous = current; if (_couldWriteReadLog0Finished) { break; } if ((reader.CurrentVersion % (ulong)(step)) == 0) { stat.Dispose(); stat = Benchmark.Run("Read", step); Console.WriteLine($"Running total: {reader.CurrentVersion:N0}"); } } Console.WriteLine("Reader done"); //if (previous != (ulong)count) //{ // Assert.Fail($"previous {previous} != count {count}"); //} } } stat.Dispose(); } catch (Exception ex) { Console.WriteLine("READER EX: " + ex); } }); var writerTask = Task.Run(() => { try { var stat = Benchmark.Run("Write to Log0", count); { for (long i = 1; i <= count; i++) { if (_couldWriteReadLog0Finished) { break; } slm.Log0.Append((StreamLogNotification)(ulong)i); //if (i % 10_000_000 == 0) //{ // stat.Dispose(); // stat = Benchmark.Run("Write to Log0", 10_000_000); //} } Console.WriteLine("W1 Done"); _couldWriteReadLog0Finished = true; } stat.Dispose(); } catch (Exception ex) { Console.WriteLine("EX: " + ex); } }); var writerTask2 = Task.Run(() => { try { using (Benchmark.Run("Write to Log0", count)) { for (long i = 1; i <= count; i++) { if (_couldWriteReadLog0Finished) { break; } slm.Log0.Append((StreamLogNotification)(ulong)i); } Console.WriteLine("W2 Done"); _couldWriteReadLog0Finished = true; } } catch (Exception ex) { Console.WriteLine("EX: " + ex); } }); writerTask2.Wait(); readerTask.Wait(); writerTask.Wait(); Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; // log0.Dispose(); slm.Dispose(); }
public void CouldWriteInParallelAllValuesNonZero() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 2 * 1024 * 1024; List <Task> tasks = new List <Task>(); var taskCount = 4; for (int j = 0; j < taskCount; j++) { tasks.Add(Task.Run(() => { try { using (Benchmark.Run("Log0.Append", count * taskCount)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); // Thread.Yield(); } } } catch (Exception ex) { Console.WriteLine("EX: " + ex); } })); } Task.WhenAll(tasks).Wait(); Benchmark.Dump(); using (Benchmark.Run("Log0 MoveNext", count * taskCount)) { var chunkCursor = log0.BlockIndex.GetBlockRecordCursor(StreamLogId.Log0Id); var readCount = 0; while (chunkCursor.MoveNext()) { var record = chunkCursor.Current.Value; if ((long)record.Version >= count * taskCount) { break; } if (record.IsPacked) { break; } var chunk = log0.RentChunkFromRecord(record); for (ulong i = 0; i < 1024 * 1024; i++) { var position = NotificationLog.Log0ItemPosition(chunk.Pointer, (long)(chunk.FirstVersion + i), out _); var value = *(ulong *)position; if (value == 0) { Assert.Fail("Zero value at: " + (chunk.FirstVersion + i)); } readCount++; } chunk.DisposeFree(); } //var c = log0.GetContainerCursor(false); //while (c.MoveNext()) //{ // if (c.CurrentValue.ReadUInt64(0) == 0) // { // // Console.WriteLine($"c.CurrentValue == 0 at {c.CurrentKey}"); // Assert.Fail($"c.CurrentValue == 0 at {c.CurrentKey}"); // } // readCount++; //} //c.Dispose(); // Assert.AreEqual(count * taskCount, readCount); Console.WriteLine("READ COUNT M: " + readCount * 1.0 / (1024 * 1024)); } // readerTask.Wait(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
public void StreamLogCursorWorks() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); var c = sl.GetCursor(); Assert.IsFalse(c.MoveFirst()); Assert.IsFalse(c.MoveLast()); Assert.IsFalse(c.MoveNext()); Assert.IsFalse(c.MovePrevious()); Assert.IsFalse(c.MoveAt(0, Lookup.LT)); Assert.IsFalse(c.MoveAt(0, Lookup.LE)); Assert.IsFalse(c.MoveAt(0, Lookup.EQ)); Assert.IsFalse(c.MoveAt(0, Lookup.GE)); Assert.IsFalse(c.MoveAt(0, Lookup.GT)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.LT)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.LE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.EQ)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GT)); Assert.IsFalse(c.TryGetValue(123, out _)); AddToSl(); Assert.IsTrue(c.MoveFirst()); Assert.IsTrue(c.MoveLast()); Assert.IsFalse(c.MoveNext()); Assert.IsFalse(c.MovePrevious()); Assert.IsFalse(c.MoveAt(0, Lookup.LT)); Assert.IsFalse(c.MoveAt(0, Lookup.LE)); Assert.IsFalse(c.MoveAt(0, Lookup.EQ)); Assert.IsTrue(c.MoveAt(0, Lookup.GE)); Assert.IsTrue(c.MoveAt(0, Lookup.GT)); Assert.IsTrue(c.MoveAt(ulong.MaxValue, Lookup.LT)); Assert.IsTrue(c.MoveAt(ulong.MaxValue, Lookup.LE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.EQ)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GT)); Assert.IsFalse(c.TryGetValue(123, out _)); void AddToSl() { var claim = sl.Claim(0, 8); if (claim.Length != 8) { Assert.Fail(); } claim.Write <long>(0, 123456); sl.Commit(); } sl.Dispose(); // Benchmark.Dump(); slm.Dispose(); }
public void CouldMoveAtAfterWriting() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, disablePacker: true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, StreamLogFlags.IsBinary | StreamLogFlags.NoPacking); // sl.DisablePacking = true; var sl = new StreamLog(slm, state, 1_000, "test_stream"); ulong count = (ulong)TestUtils.GetBenchCount(500_000, 500); var cnt = 0L; var c = sl.GetCursor(); using (Benchmark.Run("Write-MoveAt", (long)count)) { for (ulong i = 0; i < count; i++) { var claim = sl.Claim(0, valueSize); if (claim.Length != valueSize) { Assert.Fail("claim.Length != 8"); } claim.Write <long>(0, (long)i); var version = sl.Commit(); if (version != (ulong)i + 1) { Assert.Fail("version != (ulong) i + 1"); } if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; var position = c.CurrentKey; if (i > 1) { if (!c.MoveAt(i, Lookup.LT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.LT)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } if (!c.MoveAt(i - 1, Lookup.LE)) { Assert.Fail($"!c.MoveAt(i - 1 {i - 1}, Lookup.LE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } } if (i > 2 && i < count - 1) { if (!c.MoveAt(i - 1, Lookup.GT)) { Assert.Fail($"X: !c.MoveAt(i {i}, Lookup.GT)"); } if (c.CurrentKey != i) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } //if (i == 481) //{ // Console.WriteLine("fix me"); //} if (!c.MoveAt(i - 1, Lookup.GE)) { Assert.Fail($"!c.MoveAt(i - 1 {i}, Lookup.GE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } } if (!c.MoveAt(position, Lookup.EQ)) { Assert.Fail("Cannot move to position: !c.MoveAt(position, Lookup.EQ)"); } } else { Assert.Fail("Cannot move next after write: " + i); } } } using (Benchmark.Run("MoveAt", (long)count)) { for (ulong i = 2; i < count - 1; i++) { var position = i; if (i > 1) { if (!c.MoveAt(i, Lookup.LT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.LT)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } if (!c.MoveAt(i - 1, Lookup.LE)) { Assert.Fail($"!c.MoveAt(i - 1 {i - 1}, Lookup.LE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } } if (i < count - 1) { if (!c.MoveAt(i, Lookup.GT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.GT)"); } if (c.CurrentKey != i + 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } if (!c.MoveAt(i + 1, Lookup.GE)) { Assert.Fail($"!c.MoveAt(i + 1 {i}, Lookup.GE)"); } if (c.CurrentKey != i + 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } } if (!c.MoveAt(position, Lookup.EQ)) { Assert.Fail("Cannot move to position: !c.MoveAt(position, Lookup.EQ)"); } } } Benchmark.Dump(); c.Dispose(); sl.Dispose(); slm.Dispose(); }
public void CouldReadAfterWriting() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, StreamLogFlags.IsBinary | StreamLogFlags.NoPacking); // sl.DisablePacking = true; var sl = new StreamLog(slm, state, 100_000, "test_stream"); var count = TestUtils.GetBenchCount(1_000_000, 1000); var cnt = 0L; var c = sl.GetCursor(); for (int i = 0; i < count; i++) { var claim = sl.Claim(0, valueSize); if (claim.Length != valueSize) { Assert.Fail("claim.Length != 8"); } claim.Write <long>(0, i); var version = sl.Commit(); if (version != (ulong)i + 1) { Assert.Fail("version != (ulong) i + 1"); } if (i == 16360) { Console.WriteLine("stop"); } if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; } else { Assert.Fail("Cannot move next after write: " + i); } } Console.WriteLine("Writer finished"); c.Dispose(); sl.Dispose(); slm.Dispose(); }
public void CouldReadWhileWriting() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); var count = 1_000_000; var rt = Task.Run(() => { var cnt = 0L; var c = sl.GetCursor(); while (cnt < count) { if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; } else { // Console.WriteLine("false"); } } Console.WriteLine("Reader finished: " + cnt); c.Dispose(); }); for (int i = 0; i < count; i++) { var claim = sl.Claim(0, 8); if (claim.Length != 8) { Assert.Fail(); } claim.Write <long>(0, i); sl.Commit(); } Console.WriteLine("Writer finished"); rt.Wait(); sl.Dispose(); slm.Dispose(); }