public async Task InitializeAsync()
{
if (!System.IO.Directory.Exists(this._BaseFolder))
{
System.IO.Directory.CreateDirectory(this._BaseFolder);
}
FasterLogSettings logSettings = new FasterLogSettings();
this._Log = await FasterLog.CreateAsync(logSettings);
}
// This creates a separate FasterLog over the same log file, using RecoverReadOnly to continuously update
// to the primary FasterLog's commits.
private async Task ReadOnlyConsumerAsync(string deviceName, CancellationToken cancellationToken, bool isAsync)
{
using var device = Devices.CreateLogDevice(deviceName);
var logSettings = new FasterLogSettings {
LogDevice = device, ReadOnlyMode = true, PageSizeBits = 9, SegmentSizeBits = 9
};
using var log = isAsync ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
var _ = BeginRecoverAsyncLoop();
// This enumerator waits asynchronously when we have reached the committed tail of the duplicate FasterLog. When RecoverReadOnly
// reads new data committed by the primary FasterLog, it signals commit completion to let iter continue to the new tail.
using var iter = log.Scan(log.BeginAddress, long.MaxValue);
var prevValue = -1L;
try
{
await foreach (var(result, _, _, nextAddress) in iter.GetAsyncEnumerable(cancellationToken))
{
var value = long.Parse(Encoding.UTF8.GetString(result));
Assert.AreEqual(prevValue + 1, value);
prevValue = value;
iter.CompleteUntil(nextAddress);
if (prevValue == NumElements - 1)
{
done.Release();
}
}
} catch (OperationCanceledException) { }
Assert.AreEqual(NumElements - 1, prevValue);
async Task BeginRecoverAsyncLoop()
{
while (!cancellationToken.IsCancellationRequested)
{
// Delay for a while before recovering to the last commit by the primary FasterLog instance.
await Task.Delay(TimeSpan.FromMilliseconds(RestorePeriodMs), cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (isAsync)
{
await log.RecoverReadOnlyAsync();
}
else
{
log.RecoverReadOnly();
}
}
}
}
public async Task RecoverReadOnlyCheck1([Values] bool isAsync)
{
using var device = Devices.CreateLogDevice(deviceName);
var logSettings = new FasterLogSettings {
LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9
};
using var log = isAsync ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
await Task.WhenAll(ProducerAsync(log, cts),
CommitterAsync(log, cts.Token),
ReadOnlyConsumerAsync(deviceName, cts.Token, isAsync));
}
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType)
{
var logSettings = new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
};
log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
var counter = new FasterLogTests.Counter(log);
switch (iteratorType)
{
case FasterLogTests.IteratorType.AsyncByteVector:
await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(iter.NextAddress);
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
Assert.IsTrue(counter.count == numEntries);
}
log.Dispose();
}