public async Task InitializeAsync() { if (!System.IO.Directory.Exists(this._BaseFolder)) { System.IO.Directory.CreateDirectory(this._BaseFolder); } FasterLogSettings logSettings = new FasterLogSettings(); this._Log = await FasterLog.CreateAsync(logSettings); }
// This creates a separate FasterLog over the same log file, using RecoverReadOnly to continuously update // to the primary FasterLog's commits. private async Task ReadOnlyConsumerAsync(string deviceName, CancellationToken cancellationToken, bool isAsync) { using var device = Devices.CreateLogDevice(deviceName); var logSettings = new FasterLogSettings { LogDevice = device, ReadOnlyMode = true, PageSizeBits = 9, SegmentSizeBits = 9 }; using var log = isAsync ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); var _ = BeginRecoverAsyncLoop(); // This enumerator waits asynchronously when we have reached the committed tail of the duplicate FasterLog. When RecoverReadOnly // reads new data committed by the primary FasterLog, it signals commit completion to let iter continue to the new tail. using var iter = log.Scan(log.BeginAddress, long.MaxValue); var prevValue = -1L; try { await foreach (var(result, _, _, nextAddress) in iter.GetAsyncEnumerable(cancellationToken)) { var value = long.Parse(Encoding.UTF8.GetString(result)); Assert.AreEqual(prevValue + 1, value); prevValue = value; iter.CompleteUntil(nextAddress); if (prevValue == NumElements - 1) { done.Release(); } } } catch (OperationCanceledException) { } Assert.AreEqual(NumElements - 1, prevValue); async Task BeginRecoverAsyncLoop() { while (!cancellationToken.IsCancellationRequested) { // Delay for a while before recovering to the last commit by the primary FasterLog instance. await Task.Delay(TimeSpan.FromMilliseconds(RestorePeriodMs), cancellationToken); if (cancellationToken.IsCancellationRequested) { break; } if (isAsync) { await log.RecoverReadOnlyAsync(); } else { log.RecoverReadOnly(); } } } }
public async Task RecoverReadOnlyCheck1([Values] bool isAsync) { using var device = Devices.CreateLogDevice(deviceName); var logSettings = new FasterLogSettings { LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9 }; using var log = isAsync ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); await Task.WhenAll(ProducerAsync(log, cts), CommitterAsync(log, cts.Token), ReadOnlyConsumerAsync(deviceName, cts.Token, isAsync)); }
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType) { var logSettings = new FasterLogSettings { PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager }; log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } log.Commit(true); using (var iter = log.Scan(0, long.MaxValue)) { var counter = new FasterLogTests.Counter(log); switch (iteratorType) { case FasterLogTests.IteratorType.AsyncByteVector: await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable()) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.AsyncMemoryOwner: await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared)) { Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); counter.IncrementAndMaybeTruncateUntil(nextAddress); // MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add). // Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because // the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme. if (nextAddress == log.TailAddress) { break; } } break; case FasterLogTests.IteratorType.Sync: while (iter.GetNext(out byte[] result, out _, out _)) { Assert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(iter.NextAddress); } break; default: Assert.Fail("Unknown IteratorType"); break; } Assert.IsTrue(counter.count == numEntries); } log.Dispose(); }