public void FasterLogTest6([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager }); byte[] data1 = new byte[1000]; for (int i = 0; i < 100; i++) { data1[i] = (byte)i; } for (int i = 0; i < 100; i++) { log.Enqueue(data1); } log.RefreshUncommitted(); Assert.IsTrue(log.SafeTailAddress == log.TailAddress); Assert.IsTrue(log.CommittedUntilAddress < log.SafeTailAddress); using (var iter = log.Scan(0, long.MaxValue, scanUncommitted: true)) { while (iter.GetNext(out _, out _, out _)) { log.TruncateUntil(iter.NextAddress); } Assert.IsTrue(iter.NextAddress == log.SafeTailAddress); log.Enqueue(data1); Assert.IsFalse(iter.GetNext(out _, out _, out _)); log.RefreshUncommitted(); Assert.IsTrue(iter.GetNext(out _, out _, out _)); } log.Dispose(); }
private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager) { log = new FasterLog(new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { int count = 0; while (iter.GetNext(out byte[] result, out int length, out long currentAddress)) { count++; Assert.IsTrue(result.SequenceEqual(entry)); if (count % 100 == 0) { log.TruncateUntil(iter.NextAddress); } } Assert.IsTrue(count == numEntries); } log.Dispose(); }
public static void Main() { commitPath = "FasterLogStress/"; // Clean up log files from previous test runs in case they weren't cleaned up // We loop to ensure clean-up as deleteOnClose does not always work for MLSD while (Directory.Exists(commitPath)) { Directory.Delete(commitPath, true); } // Create devices \ log for test device = new ManagedLocalStorageDevice(commitPath + "ManagedLocalStore.log", deleteOnClose: true); log = new FasterLog(new FasterLogSettings { LogDevice = device, PageSizeBits = 12, MemorySizeBits = 14 }); ManagedLocalStoreBasicTest(); log.Dispose(); device.Dispose(); // Clean up log files if (Directory.Exists(commitPath)) { Directory.Delete(commitPath, true); } }
static async Task Main() { var path = Path.GetTempPath() + "FasterLogPubSub\\"; var device = Devices.CreateLogDevice(path + "mylog"); var log = new FasterLog(new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9 }); using var cts = new CancellationTokenSource(); var producer = ProducerAsync(log, cts.Token); var commiter = CommiterAsync(log, cts.Token); var consumer = ConsumerAsync(log, cts.Token); Console.CancelKeyPress += (o, eventArgs) => { Console.WriteLine("Cancelling program..."); eventArgs.Cancel = true; cts.Cancel(); }; await producer; await consumer; await commiter; Console.WriteLine("Finished."); log.Dispose(); try { new DirectoryInfo(path).Delete(true); } catch { } }
public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.DeviceType deviceType) { string path = TestUtils.MethodTestDir + "/"; string filename = path + "TestDisposeRelease" + deviceType.ToString() + ".log"; DirectoryInfo di = Directory.CreateDirectory(path); IDevice device = TestUtils.CreateTestDevice(deviceType, filename); FasterLog fasterLog = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path, LogChecksum = LogChecksumType.PerEntry }); Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out _)); fasterLog.Commit(spinWait: true); fasterLog.Dispose(); device.Dispose(); while (true) { try { di.Delete(recursive: true); break; } catch { } } }
static async Task Main() { var device = Devices.CreateLogDevice(path + "mylog"); var log = new FasterLog(new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9 }); using var cts = new CancellationTokenSource(); var producer = ProducerAsync(log, cts.Token); var commiter = CommitterAsync(log, cts.Token); // Consumer on SAME FasterLog instance var consumer = ConsumerAsync(log, true, cts.Token); // Uncomment below to run consumer on SEPARATE read-only FasterLog instance // var consumer = SeparateConsumerAsync(cts.Token); Console.CancelKeyPress += (o, eventArgs) => { Console.WriteLine("Cancelling program..."); eventArgs.Cancel = true; cts.Cancel(); }; await producer; await consumer; await commiter; Console.WriteLine("Finished."); log.Dispose(); try { new DirectoryInfo(path).Delete(true); } catch { } }
public void FasterLogTest1([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum }); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { int count = 0; while (iter.GetNext(out byte[] result, out int length)) { count++; Assert.IsTrue(result.SequenceEqual(entry)); if (count % 100 == 0) { log.TruncateUntil(iter.CurrentAddress); } } Assert.IsTrue(count == numEntries); } log.Dispose(); }
public void TearDown() { log.Dispose(); device.Dispose(); // Clean up log files try { new DirectoryInfo(path).Delete(true); } catch { } }
public void TearDown() { log?.Dispose(); log = null; device?.Dispose(); device = null; // Clean up log files TestUtils.DeleteDirectory(path, wait: true); }
public void TearDown() { log?.Dispose(); log = null; device?.Dispose(); device = null; // Clean up log files TestUtils.DeleteDirectory(TestUtils.MethodTestDir); }
static async Task Main() { // This is two samples in one, enumerating over the same FasterLog instance that does commits, or over a separate // FasterLog that opens the log file read-only and continuously catches up with the first intance's commits. const bool sameInstance = true; #pragma warning disable CS0162 // Unreachable code detected if (!sameInstance) { // Because the SAME-instance iterator illustrates truncating the log, the SEPARATE-instance may encounter EOF // issues if it is run after that truncation without cleaning up the directory first. // In all other cases, the sample should run without needing to clean up the directory. //if (Directory.Exists(path)) Directory.Delete(path, true); } var device = Devices.CreateLogDevice(path + "mylog"); var log = new FasterLog(new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9 }); using var cts = new CancellationTokenSource(); var producer = ProducerAsync(log, cts.Token); var committer = CommitterAsync(log, cts.Token); Task consumer; if (sameInstance) { // Consumer on SAME FasterLog instance consumer = ConsumerAsync(log, true, cts.Token); } else { // Consumer on SEPARATE read-only FasterLog instance consumer = SeparateConsumerAsync(cts.Token); } #pragma warning restore CS0162 // Unreachable code detected Console.CancelKeyPress += (o, eventArgs) => { Console.WriteLine("Cancelling program..."); eventArgs.Cancel = true; cts.Cancel(); }; await producer; await consumer; await committer; Console.WriteLine("Finished."); log.Dispose(); device.Dispose(); try { new DirectoryInfo(path).Delete(true); } catch { } }
public async Task FasterLogTest2([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager }); byte[] data1 = new byte[10000]; for (int i = 0; i < 10000; i++) { data1[i] = (byte)i; } using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering)) { int i = 0; while (i++ < 500) { var waitingReader = iter.WaitAsync(); Assert.IsTrue(!waitingReader.IsCompleted); while (!log.TryEnqueue(data1, out _)) { ; } // We might have auto-committed at page boundary // Ensure we don't find new entry in iterator while (waitingReader.IsCompleted) { var _next = iter.GetNext(out _, out _, out _); Assert.IsFalse(_next); waitingReader = iter.WaitAsync(); } Assert.IsFalse(waitingReader.IsCompleted); await log.CommitAsync(); while (!waitingReader.IsCompleted) { ; } Assert.IsTrue(waitingReader.IsCompleted); var curr = iter.GetNext(out byte[] result, out _, out _); Assert.IsTrue(curr); Assert.IsTrue(result.SequenceEqual(data1)); var next = iter.GetNext(out _, out _, out _); Assert.IsFalse(next); } } log.Dispose(); }
public void TearDown() { log.Dispose(); device.Dispose(); logFullParams.Dispose(); deviceFullParams.Dispose(); // Clean up log files if (Directory.Exists(commitPath)) { Directory.Delete(commitPath, true); } }
public void TearDown() { log?.Dispose(); log = null; device?.Dispose(); device = null; logFullParams?.Dispose(); logFullParams = null; deviceFullParams?.Dispose(); deviceFullParams = null; // Clean up log TestUtils.DeleteDirectory(path); }
public void TearDown() { log?.Dispose(); log = null; device?.Dispose(); device = null; deviceUnCommitted?.Dispose(); deviceUnCommitted = null; logUncommitted?.Dispose(); logUncommitted = null; // Clean up log files TestUtils.DeleteDirectory(path); }
public void TearDown() { log?.Dispose(); log = null; device?.Dispose(); device = null; logReadOnly?.Dispose(); logReadOnly = null; deviceReadOnly?.Dispose(); deviceReadOnly = null; // Clean up log files TestUtils.DeleteDirectory(path); }
public void TearDown() { log.Dispose(); device.Dispose(); // Clean up log files try { if (Directory.Exists(path)) { Directory.Delete(path, true); } } catch { } }
public async Task FasterLogTest3([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum }); byte[] data1 = new byte[10000]; for (int i = 0; i < 10000; i++) { data1[i] = (byte)i; } using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering)) { var appendResult = log.TryEnqueue(data1, out _); Assert.IsTrue(appendResult); await log.CommitAsync(); await iter.WaitAsync(); var iterResult = iter.GetNext(out byte[] entry, out _, out _); Assert.IsTrue(iterResult); appendResult = log.TryEnqueue(data1, out _); Assert.IsFalse(appendResult); await iter.WaitAsync(); // Should read the "hole" and return false iterResult = iter.GetNext(out entry, out _, out _); Assert.IsFalse(iterResult); // Should wait for next item var task = iter.WaitAsync(); Assert.IsFalse(task.IsCompleted); appendResult = log.TryEnqueue(data1, out _); Assert.IsTrue(appendResult); await log.CommitAsync(); await task; iterResult = iter.GetNext(out entry, out _, out _); Assert.IsTrue(iterResult); } log.Dispose(); }
public async Task FasterLogTest4([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum }); byte[] data1 = new byte[100]; for (int i = 0; i < 100; i++) { data1[i] = (byte)i; } for (int i = 0; i < 100; i++) { log.Enqueue(data1); } Assert.IsTrue(log.CommittedUntilAddress == log.BeginAddress); await log.CommitAsync(); Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress); Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress); using (var iter = log.Scan(0, long.MaxValue)) { // Should read the "hole" and return false var iterResult = iter.GetNext(out byte[] entry, out _, out _); log.TruncateUntil(iter.NextAddress); Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress); Assert.IsTrue(log.CommittedBeginAddress < log.BeginAddress); Assert.IsTrue(iter.NextAddress == log.BeginAddress); await log.CommitAsync(); Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress); Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress); } log.Dispose(); }
public void TestDisposeReleasesFileLocksWithInprogressCommit() { string commitPath = TestContext.CurrentContext.TestDirectory + "/" + TestContext.CurrentContext.Test.Name + "/"; DirectoryInfo di = Directory.CreateDirectory(commitPath); IDevice device = Devices.CreateLogDevice(commitPath + "testDisposeReleasesFileLocksWithInprogressCommit.log", preallocateFile: true, deleteOnClose: false); FasterLog fasterLog = new FasterLog(new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.PerEntry }); Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out long beginAddress)); fasterLog.Commit(spinWait: false); fasterLog.Dispose(); device.Dispose(); while (true) { try { di.Delete(recursive: true); break; } catch { } } }
public async Task FasterLogTest5([Values] LogChecksumType logChecksum) { log = new FasterLog(new FasterLogSettings { LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum }); int headerSize = logChecksum == LogChecksumType.None ? 4 : 12; bool _disposed = false; var commit = new Thread(() => { while (!_disposed) { log.Commit(true); Thread.Sleep(1); } }); commit.Start(); // 65536=page size|headerSize|64=log header await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize - 64]); // 65536=page size|headerSize await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]); // 65536=page size|headerSize await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]); // 65536=page size|headerSize await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]); // 65536=page size|headerSize await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]); _disposed = true; commit.Join(); log.Dispose(); }
public void FasterLogSimpleFastCommitTest([Values] TestUtils.DeviceType deviceType) { var cookie = new byte[100]; new Random().NextBytes(cookie); var filename = path + "fastCommit" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true); var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true, TryRecoverLatest = false }; log = new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie1 = new byte[100]; new Random().NextBytes(cookie1); var commitSuccessful = log.CommitStrongly(out var commit1Addr, out _, true, cookie1, 1); Assert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie2 = new byte[100]; new Random().NextBytes(cookie2); commitSuccessful = log.CommitStrongly(out var commit2Addr, out _, true, cookie2, 2); Assert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie6 = new byte[100]; new Random().NextBytes(cookie6); commitSuccessful = log.CommitStrongly(out var commit6Addr, out _, true, cookie6, 6); Assert.IsTrue(commitSuccessful); // Wait for all metadata writes to be complete to avoid a concurrent access exception log.Dispose(); log = null; // be a deviant and remove commit metadata files manager.RemoveAllCommits(); // Recovery should still work var recoveredLog = new FasterLog(logSettings); recoveredLog.Recover(1); Assert.AreEqual(cookie1, recoveredLog.RecoveredCookie); Assert.AreEqual(commit1Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); recoveredLog = new FasterLog(logSettings); recoveredLog.Recover(2); Assert.AreEqual(cookie2, recoveredLog.RecoveredCookie); Assert.AreEqual(commit2Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); // Default argument should recover to most recent, if TryRecoverLatest is set logSettings.TryRecoverLatest = true; recoveredLog = new FasterLog(logSettings); Assert.AreEqual(cookie6, recoveredLog.RecoveredCookie); Assert.AreEqual(commit6Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); }
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType) { var logSettings = new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }; log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { var counter = new FasterLogTests.Counter(log); switch (iteratorType) { case FasterLogTests.IteratorType.AsyncByteVector: await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable()) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.AsyncMemoryOwner: await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared)) { Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.Sync: while (iter.GetNext(out byte[] result, out _, out _)) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(iter.NextAddress); } break; default: Assert.Fail("Unknown IteratorType"); break; } Assert.IsTrue(counter.count == numEntries); } log.Dispose(); }
public async ValueTask FlakyLogTestTolerateFailure([Values] IteratorType iteratorType) { var errorOptions = new ErrorSimulationOptions { readTransientErrorRate = 0, readPermanentErrorRate = 0, writeTransientErrorRate = 0, writePermanentErrorRate = 0.1, }; device = new SimulatedFlakyDevice(Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true), errorOptions); var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, TolerateDeviceFailure = true }; log = new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } // Ensure we write enough to trigger errors for (int i = 0; i < 1000; i++) { log.Enqueue(entry); try { if (IsAsync(iteratorType)) { await log.CommitAsync(); } else { log.Commit(); } } catch (CommitFailureException) { // Ignore failure } } // For surviving entries, scan should still work best-effort // If endAddress > log.TailAddress then GetAsyncEnumerable() will wait until more entries are added. var endAddress = IsAsync(iteratorType) ? log.CommittedUntilAddress : long.MaxValue; var recoveredLog = new FasterLog(logSettings); using var iter = recoveredLog.Scan(0, endAddress); switch (iteratorType) { case IteratorType.AsyncByteVector: await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable()) { Assert.IsTrue(result.SequenceEqual(entry)); } break; case IteratorType.AsyncMemoryOwner: await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared)) { Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); } break; case IteratorType.Sync: while (iter.GetNext(out byte[] result, out _, out _)) { Assert.IsTrue(result.SequenceEqual(entry)); } break; default: Assert.Fail("Unknown IteratorType"); break; } recoveredLog.Dispose(); }