public void CouldRotateNotificationLog() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 3_000_000; using (Benchmark.Run("Log0.Append", count)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); } } Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
public void CouldOpenCloseSlm() { var path = TestUtils.GetPath(); var pc = new ProcessConfig(path); var slm = new StreamLogManager(pc, "test"); var slid = new StreamLogId(-1, 1); var inited = slm.InitStreamLog(slid, 0, StreamLogFlags.IsBinary); Assert.IsTrue(inited); var sl = slm.OpenStreamLog(slid); var mem = sl.ClaimRestOfBlockMemory(1); var freeSpace = mem.Length; mem.Span[0] = 123; sl.Commit(1); var mem1 = sl.ClaimRestOfBlockMemory(1); Assert.AreEqual(mem1.Length, freeSpace - 1 - 4); sl.Commit(1); var db = sl.DangerousGetUnpacked(1); Assert.AreEqual(123, db.ReadByte(0)); sl.Dispose(); slm.Dispose(); }
public void CouldWriteReadLog0MoveGT() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 3_000_000; using (Benchmark.Run("Log0.Append", count)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); } } using (Benchmark.Run("Log0 MoveGT", count)) { using (var reader = new NotificationLog.Reader(log0, CancellationToken.None, false)) { for (long i = 0; i < count; i++) { if (reader.MoveGT((ulong)i)) { if (reader.CurrentVersion != (ulong)(i + 1)) { Assert.Fail($"reader.CurrentVersion {reader.CurrentVersion} != [(ulong) (i + 1)] {i + 1}"); } } } } } // readerTask.Wait(); Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
public void CouldInitNotificationLog() { var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, false, true); slm.Dispose(); }
/// <summary> /// If <paramref name="disablePacking"/> is set to true Packer only releases /// unused memory pages back to OS and does not perform compression. /// TODO This setting per stream /// </summary> /// <param name="streamLogManager"></param> /// <param name="stateStorage"></param> /// <param name="disablePacking"></param> public Packer(StreamLogManager streamLogManager, StreamLogStateStorage stateStorage, bool disablePacking = false) { _streamLogManager = streamLogManager; _stateStorage = stateStorage; _disablePacking = disablePacking; Pool = new SpreadsThreadPool( new ThreadPoolSettings(1 + Math.Max(Environment.ProcessorCount / 2 - 1, 0), ThreadType.Background, "Packer_pool", ApartmentState.Unknown, ex => { ThrowHelper.FailFast("Unhandled exception in Packer thread pool: \n" + ex); }, 0, ThreadPriority.BelowNormal)); }
public void CouldGetVersionFromNewllyCreatedSl() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); Assert.IsFalse(sl.IsCompleted); Assert.AreEqual(0, sl.CurrentVersion); Assert.AreEqual(sl.Count, sl.CurrentVersion); sl.Dispose(); slm.Dispose(); }
public void CouldReadWriteZeroLogViaThreadPool() { Console.WriteLine("Starting test"); Console.Out.Flush(); #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: true); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 20 * 1024, false, true); var count = TestUtils.GetBenchCount(200 * 1024 * 1024L, 1000); var tp = new SpreadsThreadPool(new ThreadPoolSettings(6, ThreadType.Foreground, "Log0Pool", ApartmentState.Unknown, null, 0, ThreadPriority.AboveNormal)); var writer = new Thread(() => { for (int i = 1; i <= count; i++) { var x = i; ThreadPool.UnsafeQueueUserWorkItem(o => { (o as NotificationLog).Append((StreamLogNotification)(ulong)x); } , slm.Log0); } }); writer.Priority = ThreadPriority.Highest; writer.Start(); var reader = new Thread(() => { try { var c = 0L; using (Benchmark.Run("ZL Read", count, true)) { var zlr = new NotificationLog.Reader(slm.Log0, CancellationToken.None); var sw = new SpinWait(); while (zlr.MoveNext()) { // if (zlr.MoveNext()) { // Thread.SpinWait(1); c++; if (c % 10_000_000 == 0) { Console.WriteLine($"{c:N0} | MQL {zlr.MissedQueueLength}"); } if (c >= count - 1000) { if (zlr.MissedQueueLength > 0) { Console.WriteLine("Missed queue non empty"); continue; } Console.WriteLine( $"Finished at CurrentVersion: " + zlr.CurrentVersion.ToString("N")); break; } } //else //{ // sw.SpinOnce(); // if (sw.NextSpinWillYield) // { // sw.Reset(); // if (zlr.CurrentVersion > slm.Log0.State.GetZeroLogVersion()) // { // Console.WriteLine("Reached the end"); // // break; // } // // Console.WriteLine($"Spinning in ZLMN: " + zlr.CurrentVersion.ToString("N")); // } //} } if (zlr.MaxStall > MaxStall) { MaxStall = zlr.MaxStall; } Console.WriteLine("STALLS: " + zlr.StallCount); Console.WriteLine("-------------------------"); Console.WriteLine("MAX STALL: " + zlr.MaxStall.ToString("N")); Console.WriteLine("GLOBAL MAX STALL: " + MaxStall.ToString("N")); Console.WriteLine("-------------------------"); zlr.Dispose(); } Console.WriteLine("Read count: " + c.ToString("N")); } catch (Exception ex) { Console.WriteLine("Reader exception: " + ex); } }); reader.Priority = ThreadPriority.Highest; reader.Start(); reader.Join(); tp.Dispose(); tp.WaitForThreadsExit(); Benchmark.Dump(); Thread.Sleep(100); Console.WriteLine("ROTATE CNT: " + (slm.Log0.RotateCount)); Console.WriteLine("STALE VERSION CNT: " + (slm.Log0.StaleVersionCount)); //slm.Dispose(); //slm2.Dispose(); GC.KeepAlive(slm); GC.KeepAlive(slm); slm.Dispose(); Thread.Sleep(1000); }
public void CouldAddNextChunkBeforeCompleted() { var path = TestUtils.GetPath(); var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, "CouldAddNextChunk", null, 512, true, true); var bufferPool = slm.BufferPool; var blockIndex = slm.BlockIndex; var streamId = (StreamLogId)42; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, 8, SerializationFormat.Binary); var sl = new StreamLog(slm, state, textId: "test_stream"); var count = 100; SharedMemory currentBuffer = null; using (Benchmark.Run("AddNext (KOPS)", count * 1000)) { var version = 1UL; var block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version, default); // var slc = new StreamBlock(buffer.RetainChunkMemory(false), streamId, valueSize, 1); var firstBufferRef = block.SharedMemory.BufferRef; for (int i = 0; i < count; i++) { version = (ulong)i + 1; if (!block.Claim(version, 8).IsValid) { Assert.Fail("!slc.Claim(version, 8).IsValid"); } block.Commit(); var bufferRef = block.SharedMemory.BufferRef; blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); block.Complete(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default); var bufferRef1 = block.SharedMemory.BufferRef; // As if someone has created the next buffer before the second call, forget the first call block.DisposeFree(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 1, default); var bufferRef2 = block.SharedMemory.BufferRef; if (block.SharedMemory.ReferenceCount != 2) { Assert.Fail($"buffer.RefCount {block.SharedMemory.ReferenceCount} != 1"); } if (bufferRef1 != bufferRef2) { Assert.Fail($"bufferRef1 {bufferRef1} != bufferRef2 {bufferRef2}"); } block.Complete(); var nextVersion = block.NextVersion; block.DisposeFree(); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); blockIndex.PrepareNextWritableStreamBlock(sl, length: 1); // block = new StreamBlock(block.SharedMemory.RetainBlockMemory(false), streamId, valueSize, nextVersion); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, nextVersion, default); } if (!block.Claim(version + 1, 8).IsValid) { Assert.Fail("!slc.Claim(version, 8).IsValid"); } block.Commit(); block.Complete(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, block.NextVersion, default); Assert.AreEqual(2, block.SharedMemory.ReferenceCount); block.DisposeFree(); block = blockIndex.RentNextWritableBlock(sl, minimumLength: 1, version + 2, default); Assert.IsFalse(block.IsCompleted); // TODO test replace //// replace existing //var toReplaceVersion = slc.FirstVersion; //var toReplaceRef = buffer.BufferRef; //slc.Complete(); //// SharedMemory.Free(buffer); //// slc.DisposeFree(); //buffer = chunkIndex.GetOrCreateNextWritableChunkBuffer(sl, toReplaceVersion, toReplaceRef, 1); //slc = new StreamLogChunk(buffer.RetainChunkMemory(false), streamId, valueSize, version + 2); //Assert.AreNotEqual(toReplaceRef, buffer.BufferRef); // SharedMemory.Free(buffer); block.DisposeFree(); } //bufferPool.PrintBuffers(); //bufferPool.PrintBuffersAfterPoolDispose = true; Benchmark.Dump(); Console.WriteLine("Finished"); sl.Dispose(); slm.Dispose(); }
public void CouldMoveAtAfterWriting() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, disablePacker: true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, StreamLogFlags.IsBinary | StreamLogFlags.NoPacking); // sl.DisablePacking = true; var sl = new StreamLog(slm, state, 1_000, "test_stream"); ulong count = (ulong)TestUtils.GetBenchCount(500_000, 500); var cnt = 0L; var c = sl.GetCursor(); using (Benchmark.Run("Write-MoveAt", (long)count)) { for (ulong i = 0; i < count; i++) { var claim = sl.Claim(0, valueSize); if (claim.Length != valueSize) { Assert.Fail("claim.Length != 8"); } claim.Write <long>(0, (long)i); var version = sl.Commit(); if (version != (ulong)i + 1) { Assert.Fail("version != (ulong) i + 1"); } if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; var position = c.CurrentKey; if (i > 1) { if (!c.MoveAt(i, Lookup.LT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.LT)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } if (!c.MoveAt(i - 1, Lookup.LE)) { Assert.Fail($"!c.MoveAt(i - 1 {i - 1}, Lookup.LE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } } if (i > 2 && i < count - 1) { if (!c.MoveAt(i - 1, Lookup.GT)) { Assert.Fail($"X: !c.MoveAt(i {i}, Lookup.GT)"); } if (c.CurrentKey != i) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } //if (i == 481) //{ // Console.WriteLine("fix me"); //} if (!c.MoveAt(i - 1, Lookup.GE)) { Assert.Fail($"!c.MoveAt(i - 1 {i}, Lookup.GE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } } if (!c.MoveAt(position, Lookup.EQ)) { Assert.Fail("Cannot move to position: !c.MoveAt(position, Lookup.EQ)"); } } else { Assert.Fail("Cannot move next after write: " + i); } } } using (Benchmark.Run("MoveAt", (long)count)) { for (ulong i = 2; i < count - 1; i++) { var position = i; if (i > 1) { if (!c.MoveAt(i, Lookup.LT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.LT)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } if (!c.MoveAt(i - 1, Lookup.LE)) { Assert.Fail($"!c.MoveAt(i - 1 {i - 1}, Lookup.LE)"); } if (c.CurrentKey != i - 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i - 1 {i - 1}"); } } if (i < count - 1) { if (!c.MoveAt(i, Lookup.GT)) { Assert.Fail($"!c.MoveAt(i {i}, Lookup.GT)"); } if (c.CurrentKey != i + 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } if (!c.MoveAt(i + 1, Lookup.GE)) { Assert.Fail($"!c.MoveAt(i + 1 {i}, Lookup.GE)"); } if (c.CurrentKey != i + 1) { Assert.Fail($"c.CurrentKey {c.CurrentKey} != i + 1 {i + 1}"); } } if (!c.MoveAt(position, Lookup.EQ)) { Assert.Fail("Cannot move to position: !c.MoveAt(position, Lookup.EQ)"); } } } Benchmark.Dump(); c.Dispose(); sl.Dispose(); slm.Dispose(); }
public void CouldReadAfterWriting() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, StreamLogFlags.IsBinary | StreamLogFlags.NoPacking); // sl.DisablePacking = true; var sl = new StreamLog(slm, state, 100_000, "test_stream"); var count = TestUtils.GetBenchCount(1_000_000, 1000); var cnt = 0L; var c = sl.GetCursor(); for (int i = 0; i < count; i++) { var claim = sl.Claim(0, valueSize); if (claim.Length != valueSize) { Assert.Fail("claim.Length != 8"); } claim.Write <long>(0, i); var version = sl.Commit(); if (version != (ulong)i + 1) { Assert.Fail("version != (ulong) i + 1"); } if (i == 16360) { Console.WriteLine("stop"); } if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; } else { Assert.Fail("Cannot move next after write: " + i); } } Console.WriteLine("Writer finished"); c.Dispose(); sl.Dispose(); slm.Dispose(); }
public void CouldWriteReadLog0(bool delete) { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; Settings.DoDetectBufferLeaks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: delete); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, disableNotificationLog: false, disablePacker: false); // var log0 = new NotificationLog(slm); // will disable packing: log0._activeBuffer.Increment(); var count = TestUtils.GetBenchCount(100_000_000, 1000); var readerTask = Task.Run(() => { try { var step = count / 10; var stat = Benchmark.Run("Read", step); { using (var reader = new NotificationLog.Reader(slm.Log0, CancellationToken.None, false)) { var previous = 0UL; while (reader.MoveNext()) { var current = (ulong)reader.Current; //if (current != previous + 1) //{ // Assert.Fail($"current {current} != [previous + 1] {previous + 1}"); //} previous = current; if (_couldWriteReadLog0Finished) { break; } if ((reader.CurrentVersion % (ulong)(step)) == 0) { stat.Dispose(); stat = Benchmark.Run("Read", step); Console.WriteLine($"Running total: {reader.CurrentVersion:N0}"); } } Console.WriteLine("Reader done"); //if (previous != (ulong)count) //{ // Assert.Fail($"previous {previous} != count {count}"); //} } } stat.Dispose(); } catch (Exception ex) { Console.WriteLine("READER EX: " + ex); } }); var writerTask = Task.Run(() => { try { var stat = Benchmark.Run("Write to Log0", count); { for (long i = 1; i <= count; i++) { if (_couldWriteReadLog0Finished) { break; } slm.Log0.Append((StreamLogNotification)(ulong)i); //if (i % 10_000_000 == 0) //{ // stat.Dispose(); // stat = Benchmark.Run("Write to Log0", 10_000_000); //} } Console.WriteLine("W1 Done"); _couldWriteReadLog0Finished = true; } stat.Dispose(); } catch (Exception ex) { Console.WriteLine("EX: " + ex); } }); var writerTask2 = Task.Run(() => { try { using (Benchmark.Run("Write to Log0", count)) { for (long i = 1; i <= count; i++) { if (_couldWriteReadLog0Finished) { break; } slm.Log0.Append((StreamLogNotification)(ulong)i); } Console.WriteLine("W2 Done"); _couldWriteReadLog0Finished = true; } } catch (Exception ex) { Console.WriteLine("EX: " + ex); } }); writerTask2.Wait(); readerTask.Wait(); writerTask.Wait(); Benchmark.Dump(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; // log0.Dispose(); slm.Dispose(); }
public void CouldReadWriteZeroLog() { Console.WriteLine("Starting test"); Console.Out.Flush(); #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(clear: true); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); var processConfig2 = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 20 * 1024, disableNotificationLog: false, true); var slm2 = new StreamLogManager(processConfig2, repoName, null, 20 * 1024, disableNotificationLog: false, true); slm.Log0.ActiveBlock.SharedMemory.Increment(); slm2.Log0.ActiveBlock.SharedMemory.Increment(); if (slm.Log0.State.StreamLogId != slm2.Log0.State.StreamLogId) { Assert.Fail("slm.Log0.State.StreamLogId != slm2.Log0.State.StreamLogId"); } var count = TestUtils.GetBenchCount(200 * 1024 * 1024L, 1000); // must be pow2 because we divide count by it var taskPerProcessCount = 6; //var tasks1 = new List<Task>(); //var tasks2 = new List<Task>(); var threads1 = new List <Thread>(); var threads2 = new List <Thread>(); for (int t = 0; t < taskPerProcessCount; t++) { var x = t; threads1.Add(new Thread(() => { Thread.CurrentThread.Name = "W1_" + x; using (Benchmark.Run("ZL Write", count / (2 * taskPerProcessCount), false)) { for (long i = 1; i <= count / (2 * taskPerProcessCount); i++) { try { { if (slm.Log0.Append((StreamLogNotification)(ulong)i) == 0) { Assert.Fail("Cannot append"); } // Thread.SpinWait(5); //if (i % 50 == 0) //{ // if (!Thread.Yield()) // { // Thread.Sleep(0); // } //} } } catch (Exception ex) { Console.WriteLine(ex); } } Console.WriteLine("W1 done"); } })); } for (int t = 0; t < taskPerProcessCount; t++) { var x = t; threads2.Add(new Thread(() => { try { Thread.CurrentThread.Name = "W2_" + x; using (Benchmark.Run("ZL Write", count / (2 * taskPerProcessCount), false)) { for (long i = 1; i <= count / (2 * taskPerProcessCount); i++) { try { { if (slm2.Log0.Append((StreamLogNotification)(ulong)i) == 0) { Assert.Fail("Cannot Append: " + i); } ; // Thread.SpinWait(5); //if (i % 50 == 0) //{ // if (!Thread.Yield()) // { // Thread.Sleep(0); // } //} } } catch (Exception ex) { Console.WriteLine(ex); } } Console.WriteLine("W2 done"); //slm2.Log0.Dispose(); } } catch (Exception ex) { Console.WriteLine(ex); } })); } foreach (var thread in threads1) { thread.Priority = ThreadPriority.Normal; thread.Start(); } foreach (var thread in threads2) { thread.Priority = ThreadPriority.Normal; thread.Start(); } var reader = new Thread(() => { try { var c = 0L; using (Benchmark.Run("ZL Read", count, true)) { var zlr = new NotificationLog.Reader(slm.Log0, CancellationToken.None); var sw = new SpinWait(); while (zlr.MoveNext()) { // if (zlr.MoveNext()) { // Thread.SpinWait(1); c++; if (c % 10_000_000 == 0) { Console.WriteLine($"{c:N0} | MQL {zlr.MissedQueueLength}"); } if (c >= count - 2 * taskPerProcessCount - 1000) { if (zlr.MissedQueueLength > 0) { Console.WriteLine("Missed queue non empty"); continue; } Console.WriteLine( $"Finished at CurrentVersion: " + zlr.CurrentVersion.ToString("N")); break; } } //else //{ // sw.SpinOnce(); // if (sw.NextSpinWillYield) // { // sw.Reset(); // if (zlr.CurrentVersion > slm.Log0.State.GetZeroLogVersion()) // { // Console.WriteLine("Reached the end"); // // break; // } // // Console.WriteLine($"Spinning in ZLMN: " + zlr.CurrentVersion.ToString("N")); // } //} } if (zlr.MaxStall > MaxStall) { MaxStall = zlr.MaxStall; } Console.WriteLine("STALLS: " + zlr.StallCount); Console.WriteLine("-------------------------"); Console.WriteLine("MAX STALL: " + zlr.MaxStall.ToString("N")); Console.WriteLine("GLOBAL MAX STALL: " + MaxStall.ToString("N")); Console.WriteLine("-------------------------"); zlr.Dispose(); } Console.WriteLine("Read count: " + c.ToString("N")); } catch (Exception ex) { Console.WriteLine("Reader exception: " + ex); } }); reader.Priority = ThreadPriority.Normal; reader.Start(); foreach (var thread in threads1) { thread.Join(); } foreach (var thread in threads2) { thread.Join(); } reader.Join(); //using (Benchmark.Run("Log0 MoveNext", count)) //{ // var readCount = 0; // try // { // var chunkCursor = // slm.Log0.State.StreamLogManager.LogChunkIndex.GetChunkRecordCursor(StreamLogId.Log0Id); // using (var txn = slm.Log0.State.StreamLogManager.LogChunkIndex._env.BeginReadOnlyTransaction()) // { // var chunks = slm.Log0.State.StreamLogManager.LogChunkIndex._chunksDb // .AsEnumerable<long, StreamLogChunkRecord>(txn, -1).ToArray(); // var previous = 0UL; // foreach (var r in chunks) // { // var chunk = slm.Log0.RentChunkFromRecord(r); // if (chunk.IsValid) // { // Console.WriteLine( // $"SLCR: version {r.Version} - chunk first version: {chunk.FirstVersion}"); // if (r.Version != chunk.FirstVersion && // r.Version != StreamLogChunkIndex.ReadyChunkVersion) // { // Assert.Fail("Bad versions"); // } // chunk.DisposeFree(); // if (previous != 0 && previous == r.Version) // { // Assert.Fail("Duplicate SLCR"); // } // previous = r.Version; // } // } // } // slm.BufferPool.PrintBuffers(); // Console.WriteLine("Move next"); // while (chunkCursor.MoveNext()) // { // var record = chunkCursor.Current.Value; // Console.WriteLine("Record version: " + record.Version); // if ((long)record.Version >= count) // { // Console.WriteLine("Break"); // break; // } // var chunk = slm.Log0.RentChunkFromRecord(record); // Console.WriteLine("Chunk first version: " + chunk.FirstVersion); // for (ulong i = 0; i < 1024 * 1024; i++) // { // var position = NotificationLog.Log0ItemPosition(chunk._pointer, // (long)(chunk.FirstVersion + i), out _); // var value = *(ulong*)position; // if (value == 0) // { // Assert.Fail("Zero value at: " + (chunk.FirstVersion + i)); // // Console.WriteLine("Zero value at: " + (chunk.FirstVersion + i)); // } // // Console.WriteLine("OK"); // readCount++; // } // chunk.DisposeFree(); // } // //var c = log0.GetContainerCursor(false); // //while (c.MoveNext()) // //{ // // if (c.CurrentValue.ReadUInt64(0) == 0) // // { // // // Console.WriteLine($"c.CurrentValue == 0 at {c.CurrentKey}"); // // Assert.Fail($"c.CurrentValue == 0 at {c.CurrentKey}"); // // } // // readCount++; // //} // //c.Dispose(); // // Assert.AreEqual(count, readCount); // } // catch (Exception ex) // { // Console.WriteLine("EX: " + ex); // } // Console.WriteLine("READ COUNT M: " + readCount * 1.0 / (1024 * 1024)); //} //Console.ReadLine(); //slm.Log0.Dispose(); //slm2.Log0.Dispose(); //slm.Dispose(); //slm2.Dispose(); Benchmark.Dump(); Thread.Sleep(100); Console.WriteLine("ROTATE CNT: " + (slm.Log0.RotateCount + slm2.Log0.RotateCount)); Console.WriteLine("STALE VERSION CNT: " + (slm.Log0.StaleVersionCount + slm2.Log0.StaleVersionCount)); slm.Dispose(); slm2.Dispose(); GC.KeepAlive(slm); GC.KeepAlive(slm2); }
public void StreamLogCursorWorks() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); var c = sl.GetCursor(); Assert.IsFalse(c.MoveFirst()); Assert.IsFalse(c.MoveLast()); Assert.IsFalse(c.MoveNext()); Assert.IsFalse(c.MovePrevious()); Assert.IsFalse(c.MoveAt(0, Lookup.LT)); Assert.IsFalse(c.MoveAt(0, Lookup.LE)); Assert.IsFalse(c.MoveAt(0, Lookup.EQ)); Assert.IsFalse(c.MoveAt(0, Lookup.GE)); Assert.IsFalse(c.MoveAt(0, Lookup.GT)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.LT)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.LE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.EQ)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GT)); Assert.IsFalse(c.TryGetValue(123, out _)); AddToSl(); Assert.IsTrue(c.MoveFirst()); Assert.IsTrue(c.MoveLast()); Assert.IsFalse(c.MoveNext()); Assert.IsFalse(c.MovePrevious()); Assert.IsFalse(c.MoveAt(0, Lookup.LT)); Assert.IsFalse(c.MoveAt(0, Lookup.LE)); Assert.IsFalse(c.MoveAt(0, Lookup.EQ)); Assert.IsTrue(c.MoveAt(0, Lookup.GE)); Assert.IsTrue(c.MoveAt(0, Lookup.GT)); Assert.IsTrue(c.MoveAt(ulong.MaxValue, Lookup.LT)); Assert.IsTrue(c.MoveAt(ulong.MaxValue, Lookup.LE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.EQ)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GE)); Assert.IsFalse(c.MoveAt(ulong.MaxValue, Lookup.GT)); Assert.IsFalse(c.TryGetValue(123, out _)); void AddToSl() { var claim = sl.Claim(0, 8); if (claim.Length != 8) { Assert.Fail(); } claim.Write <long>(0, 123456); sl.Commit(); } sl.Dispose(); // Benchmark.Dump(); slm.Dispose(); }
public void CouldWriteInParallelAllValuesNonZero() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog0"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 10 * 1024, true, true); var log0 = new NotificationLog(slm); // will disable packing log0.ActiveBuffer.Increment(); var count = 2 * 1024 * 1024; List <Task> tasks = new List <Task>(); var taskCount = 4; for (int j = 0; j < taskCount; j++) { tasks.Add(Task.Run(() => { try { using (Benchmark.Run("Log0.Append", count * taskCount)) { for (long i = 1; i <= count; i++) { log0.Append((StreamLogNotification)(ulong)i); // Thread.Yield(); } } } catch (Exception ex) { Console.WriteLine("EX: " + ex); } })); } Task.WhenAll(tasks).Wait(); Benchmark.Dump(); using (Benchmark.Run("Log0 MoveNext", count * taskCount)) { var chunkCursor = log0.BlockIndex.GetBlockRecordCursor(StreamLogId.Log0Id); var readCount = 0; while (chunkCursor.MoveNext()) { var record = chunkCursor.Current.Value; if ((long)record.Version >= count * taskCount) { break; } if (record.IsPacked) { break; } var chunk = log0.RentChunkFromRecord(record); for (ulong i = 0; i < 1024 * 1024; i++) { var position = NotificationLog.Log0ItemPosition(chunk.Pointer, (long)(chunk.FirstVersion + i), out _); var value = *(ulong *)position; if (value == 0) { Assert.Fail("Zero value at: " + (chunk.FirstVersion + i)); } readCount++; } chunk.DisposeFree(); } //var c = log0.GetContainerCursor(false); //while (c.MoveNext()) //{ // if (c.CurrentValue.ReadUInt64(0) == 0) // { // // Console.WriteLine($"c.CurrentValue == 0 at {c.CurrentKey}"); // Assert.Fail($"c.CurrentValue == 0 at {c.CurrentKey}"); // } // readCount++; //} //c.Dispose(); // Assert.AreEqual(count * taskCount, readCount); Console.WriteLine("READ COUNT M: " + readCount * 1.0 / (1024 * 1024)); } // readerTask.Wait(); slm.BufferPool.PrintBuffersAfterPoolDispose = true; log0.Dispose(); slm.Dispose(); }
public void CouldReadWhileWriting() { var path = TestUtils.GetPath(); var repoName = "CouldWriteToWAlFromStream"; var processConfig = new ProcessConfig(path); var slm = new StreamLogManager(processConfig, repoName, null, 1024, true, true); var streamId = (StreamLogId)42L; short valueSize = 8; var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, SerializationFormat.Binary); var sl = new StreamLog(slm, state, 10_000_000, "test_stream"); var count = 1_000_000; var rt = Task.Run(() => { var cnt = 0L; var c = sl.GetCursor(); while (cnt < count) { if (c.MoveNext()) { if (c.CurrentValue.Read <long>(0) != cnt) { Assert.Fail($"c.CurrentValue.Read<long>(0) {c.CurrentValue.Read<long>(0)} != cnt {cnt}"); } cnt++; } else { // Console.WriteLine("false"); } } Console.WriteLine("Reader finished: " + cnt); c.Dispose(); }); for (int i = 0; i < count; i++) { var claim = sl.Claim(0, 8); if (claim.Length != 8) { Assert.Fail(); } claim.Write <long>(0, i); sl.Commit(); } Console.WriteLine("Writer finished"); rt.Wait(); sl.Dispose(); slm.Dispose(); }
internal DataStore(ProcessConfig processConfig, uint id, string dataStoreName, string dataStoreDirectory = null, string walDirectory = null, string archiveDirectory = null, int maxLogSizeMb = 10 * 1024, Opt <bool> disableNotificationLog = default, Opt <bool> disablePacker = default) { var processConfig1 = processConfig; Id = id; Name = dataStoreName; dataStoreDirectory = Path.Combine(dataStoreDirectory ?? processConfig.DataRootPath, dataStoreName); Directory.CreateDirectory(dataStoreDirectory); DataStoreDirectory = dataStoreDirectory; if (walDirectory == null) { walDirectory = Path.Combine(DataStoreDirectory, "wal"); } else { walDirectory = Path.Combine(walDirectory, "wal_" + dataStoreName); } WALDirectory = walDirectory; Directory.CreateDirectory(WALDirectory); if (archiveDirectory != null) { ArchiveDirectory = Path.Combine(archiveDirectory, dataStoreName); Directory.CreateDirectory(ArchiveDirectory); } if (!processConfig1.IsLiteClient) { if (maxLogSizeMb < StartupConfig.MinimumLogSizeMb) { maxLogSizeMb = StartupConfig.MinimumLogSizeMb; } if (maxLogSizeMb > StartupConfig.StreamBlockTableMaxSizeMb) { maxLogSizeMb = StartupConfig.StreamBlockTableMaxSizeMb; } var disableNotificationLog1 = disableNotificationLog.IsPresent && disableNotificationLog.Present; var disablePacker1 = disablePacker.IsPresent && disablePacker.Present; var dataStoragePath = Path.Combine(DataStoreDirectory, "storage"); Directory.CreateDirectory(dataStoragePath); var path = Path.GetFullPath(Path.Combine(dataStoragePath, "data.db")); var uri = new Uri(path); var absoluteUri = uri.AbsoluteUri; var sqLiteStorage = new SQLiteStorage($@"Data Source={absoluteUri}?cache=shared"); StreamLogManager = new StreamLogManager( processConfig1, Name, dataStoreDirectory, (uint)maxLogSizeMb, disableNotificationLog1, disablePacker1, sqLiteStorage); _slmHandle = GCHandle.Alloc(StreamLogManager, GCHandleType.Normal); MetadataManager = new MetadataManager(DataStoreDirectory, sqLiteStorage, StreamLogManager.StateStorage); _mmHandle = GCHandle.Alloc(MetadataManager, GCHandleType.Normal); // TODO Connection manager } else { throw new NotImplementedException("Lite client is not implemented."); } }
public async Task CouldWriteUsingDswAsync() { #pragma warning disable 618 Settings.DoAdditionalCorrectnessChecks = false; #pragma warning restore 618 var path = TestUtils.GetPath(); var repoName = "CouldWriteAndReadLog"; var processConfig = new ProcessConfig(path); StartupConfig.StreamLogBufferPoolFlags = LMDBEnvironmentFlags.NoSync; StartupConfig.StreamBlockIndexFlags = LMDBEnvironmentFlags.NoSync; var slm = new StreamLogManager(processConfig, repoName, null, 20 * 1024, disableNotificationLog: false, disablePacker: false); var streamId = (StreamLogId)42L; var count = TestUtils.GetBenchCount(50_000_000, 1_000); var countSlice = TestUtils.GetBenchCount(10_000_000, 100); short valueSize = (short)(Timestamp.Size + Unsafe.SizeOf <long>()); var state = slm.StateStorage.GetState(streamId); state.CheckInit(streamId, valueSize, StreamLogFlags.IsBinary); var sl = new StreamLog(slm, state, 8_000_000, "test_stream"); Marshal.WriteInt64(sl.State.StatePointer + StreamLogState.StreamLogStateRecord.LockerOffset, 0); slm.OpenStreams.TryAdd(42, sl); var writer = new DataStreamWriter <long>(sl, KeySorting.NotEnforced, WriteMode.BatchWrite); try { var bench = Benchmark.Run("Write", countSlice); for (long i = 0; i < count; i++) { var addedVersion = await writer.TryAppend(100000L + i); if (addedVersion == 0) { Assert.Fail(); } if (i > 0 && i % countSlice == 0) { bench.Dispose(); bench = Benchmark.Run("Write", countSlice); } } bench.Dispose(); } catch (Exception ex) { Console.WriteLine($"EXCEPTION DURING WRITE: " + ex); throw; } for (int r = 0; r < 10; r++) { using (Benchmark.Run("Read", count)) { var slc = sl.GetCursor(); var dsc = new DataStreamCursor <long>(slc); var ds = dsc.Source; var c = ds.GetCursor(); var cnt = 0L; while (c.MoveNext()) { var cv = c.CurrentValue; var ts = cv.Timestamp; if (ts == default) { Assert.Fail("ts == default"); } var value = cv.Value; if (value != 100000L + cnt) { Assert.Fail($"value {value} != cnt {100000L + cnt}"); } cnt++; } c.Dispose(); if (cnt != count) { Assert.Fail($"cnt {cnt } != count {count}"); } } } slm.BufferPool.PrintBuffersAfterPoolDispose = true; Thread.Sleep(500); // TODO (!) Fix disposal // sl.Dispose(); // slm.Dispose(); Benchmark.Dump(); }