static async Task CommiterAsync(FasterLog log, CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromMilliseconds(5000), cancellationToken);
Console.WriteLine("Committing...");
await log.CommitAsync();
}
}
private async Task CommitterAsync(FasterLog log, CancellationToken cancellationToken)
{
try
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromMilliseconds(CommitPeriodMs), cancellationToken);
await log.CommitAsync();
}
} catch (OperationCanceledException) { }
}
public async Task FasterLogTest2([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager
});
byte[] data1 = new byte[10000];
for (int i = 0; i < 10000; i++)
{
data1[i] = (byte)i;
}
using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering))
{
int i = 0;
while (i++ < 500)
{
var waitingReader = iter.WaitAsync();
Assert.IsTrue(!waitingReader.IsCompleted);
while (!log.TryEnqueue(data1, out _))
{
;
}
// We might have auto-committed at page boundary
// Ensure we don't find new entry in iterator
while (waitingReader.IsCompleted)
{
var _next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(_next);
waitingReader = iter.WaitAsync();
}
Assert.IsFalse(waitingReader.IsCompleted);
await log.CommitAsync();
while (!waitingReader.IsCompleted)
{
;
}
Assert.IsTrue(waitingReader.IsCompleted);
var curr = iter.GetNext(out byte[] result, out _, out _);
Assert.IsTrue(curr);
Assert.IsTrue(result.SequenceEqual(data1));
var next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(next);
}
}
log.Dispose();
}
public async Task FasterLogTest4([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
});
byte[] data1 = new byte[100];
for (int i = 0; i < 100; i++)
{
data1[i] = (byte)i;
}
for (int i = 0; i < 100; i++)
{
log.Enqueue(data1);
}
Assert.IsTrue(log.CommittedUntilAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
using (var iter = log.Scan(0, long.MaxValue))
{
// Should read the "hole" and return false
var iterResult = iter.GetNext(out byte[] entry, out _, out _);
log.TruncateUntil(iter.NextAddress);
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress < log.BeginAddress);
Assert.IsTrue(iter.NextAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
}
log.Dispose();
}
public async Task FasterLogResumePersistedReaderSpec([Values] LogChecksumType logChecksum)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
await l.EnqueueAsync(input1);
await l.EnqueueAsync(input2);
await l.EnqueueAsync(input3);
await l.CommitAsync();
using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
{
originalIterator.GetNext(out _, out _, out _, out long recoveryAddress);
originalIterator.CompleteUntil(recoveryAddress);
originalIterator.GetNext(out _, out _, out _, out _); // move the reader ahead
await l.CommitAsync();
}
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
{
recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);
Assert.True(input2.SequenceEqual(outBuf)); // we should have read in input2, not input1 or input3
}
}
}
public async void TestLog()
{
using var log = new FasterLog(new FasterLogSettings { LogDevice = _device });
await log.EnqueueAsync(new byte[2] {
0x12, 0x34
});
await log.CommitAsync();
// TODO:
Assert.True(File.Exists(_path + ".0"));
}
public async Task FasterLogResumeViaCompleteUntilRecordAtSpec([Values] LogChecksumType logChecksum)
{
CancellationToken cancellationToken = default;
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum
}))
{
await l.EnqueueAsync(input1, cancellationToken);
await l.EnqueueAsync(input2);
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out long recordAddress, out _));
await originalIterator.CompleteUntilRecordAtAsync(recordAddress);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out _, out _)); // move the reader ahead
await l.CommitAsync();
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum
}))
{
using var recoveredIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _));
Assert.True(input2.SequenceEqual(outBuf)); // we should have read in input2, not input1 or input3
}
}
static async Task CommitterAsync(FasterLog log, CancellationToken cancellationToken)
{
try
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromMilliseconds(commitPeriodMs), cancellationToken);
Console.WriteLine("Committing...");
await log.CommitAsync(cancellationToken);
}
}
catch (OperationCanceledException) { }
Console.WriteLine("Committer complete");
}
public async Task SaveManyAsync(IEnumerable <EventEntity> entities)
{
var source = new CancellationTokenSource();
foreach (var entity in entities)
{
var enqueueAsync = _log.EnqueueAsync(Encoding.UTF8.GetBytes(entity.ToString()), source.Token);
if (!enqueueAsync.IsCompleted)
{
await enqueueAsync;
}
}
var commitAsync = _log.CommitAsync(source.Token);
if (!commitAsync.IsCompleted)
{
await commitAsync;
}
}
public async Task FasterLogResumePersistedReader3([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abcd";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in originalIterator.GetAsyncEnumerable())
{
if (count < 2) // we complete 1st and 2nd item read
{
originalIterator.CompleteUntil(item.nextAddress);
}
if (count < 1) // we commit only 1st item read
{
await l.CommitAsync();
}
count++;
}
originalCompleted = originalIterator.CompletedUntilAddress;
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using var recoveredIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in recoveredIterator.GetAsyncEnumerable())
{
if (count == 0) // resumed iterator will start at item2
{
Assert.True(input2.SequenceEqual(item.entry), $"Original: {input2[0]}, Recovered: {item.entry[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
count++;
}
Assert.IsTrue(count == 2);
}
}
}
public async ValueTask CommitAsync(CancellationToken token = default)
{
await logger.CommitAsync(token);
}
public async ValueTask FlakyLogTestTolerateFailure([Values] IteratorType iteratorType)
{
var errorOptions = new ErrorSimulationOptions
{
readTransientErrorRate = 0,
readPermanentErrorRate = 0,
writeTransientErrorRate = 0,
writePermanentErrorRate = 0.1,
};
device = new SimulatedFlakyDevice(Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true),
errorOptions);
var logSettings = new FasterLogSettings
{
LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, TolerateDeviceFailure = true
};
log = new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
// Ensure we write enough to trigger errors
for (int i = 0; i < 1000; i++)
{
log.Enqueue(entry);
try
{
if (IsAsync(iteratorType))
{
await log.CommitAsync();
}
else
{
log.Commit();
}
}
catch (CommitFailureException)
{
// Ignore failure
}
}
// For surviving entries, scan should still work best-effort
// If endAddress > log.TailAddress then GetAsyncEnumerable() will wait until more entries are added.
var endAddress = IsAsync(iteratorType) ? log.CommittedUntilAddress : long.MaxValue;
var recoveredLog = new FasterLog(logSettings);
using var iter = recoveredLog.Scan(0, endAddress);
switch (iteratorType)
{
case IteratorType.AsyncByteVector:
await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
}
break;
case IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
}
break;
case IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
recoveredLog.Dispose();
}
public static async ValueTask CommitAsync(this FasterLog logger, CancellationToken token = default)
{
await logger.CommitAsync(token);
}
public async Task ResumePersistedReader2([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
var commitPath = TestContext.CurrentContext.TestDirectory + "\\ResumePersistedReader2";
if (Directory.Exists(commitPath))
{
DeleteDirectory(commitPath);
}
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
long recoveryAddress;
using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
{
originalIterator.GetNext(out _, out _, out _, out recoveryAddress);
originalIterator.CompleteUntil(recoveryAddress);
originalIterator.GetNext(out _, out _, out _, out _); // move the reader ahead
await l.CommitAsync();
originalCompleted = originalIterator.CompletedUntilAddress;
}
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
{
recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);
// we should have read in input2, not input1 or input3
Assert.True(input2.SequenceEqual(outBuf), $"Original: {input2[0]}, Recovered: {outBuf[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
// TestContext.Progress.WriteLine($"Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
}
}
DeleteDirectory(commitPath);
}
public async Task RecoverLog(bool useAzure, int numEntries, int maxBytesPerEntry)
{
List <byte[]> entries = new List <byte[]>();
List <long> positions = new List <long>();
var random = new Random(0);
var taskHubName = useAzure ? "test-taskhub" : Guid.NewGuid().ToString("N");
var account = useAzure ? CloudStorageAccount.Parse(Environment.GetEnvironmentVariable(TestConstants.StorageConnectionName)) : null;
var logger = this.loggerFactory.CreateLogger("testlogger");
await BlobManager.DeleteTaskhubStorageAsync(account, taskHubName);
// first, commit some number of random entries to the log and record the commit positions
{
var blobManager = new BlobManager(
account,
account,
taskHubName,
logger,
Microsoft.Extensions.Logging.LogLevel.Trace,
0,
new PartitionErrorHandler(0, logger, Microsoft.Extensions.Logging.LogLevel.Trace, "account", taskHubName));
await blobManager.StartAsync();
var log = new FasterLog(blobManager, new NetheriteOrchestrationServiceSettings());
for (int i = 0; i < numEntries; i++)
{
var bytes = new byte[1 + random.Next(maxBytesPerEntry)];
random.NextBytes(bytes);
entries.Add(bytes);
positions.Add(log.Enqueue(entries[i]));
}
await log.CommitAsync();
await blobManager.StopAsync();
}
// then, read back all the entries, and compare position and content
{
var blobManager = new BlobManager(
account,
account,
taskHubName,
logger,
Microsoft.Extensions.Logging.LogLevel.Trace,
0,
new PartitionErrorHandler(0, logger, Microsoft.Extensions.Logging.LogLevel.Trace, "account", taskHubName));
await blobManager.StartAsync();
var log = new FasterLog(blobManager, new NetheriteOrchestrationServiceSettings());
int iterationCount = 0;
await Iterate(0, positions[positions.Count - 1]);
async Task Iterate(long from, long to)
{
using (var iter = log.Scan(from, to + 1))
{
byte[] result;
int entryLength;
long currentAddress;
while (true)
{
var next = iter.NextAddress;
while (!iter.GetNext(out result, out entryLength, out currentAddress))
{
if (currentAddress >= to)
{
Assert.Equal(iterationCount, numEntries);
return;
}
await iter.WaitAsync();
}
// process entry
Assert.Equal(positions[iterationCount], next);
var reference = entries[iterationCount];
Assert.Equal(reference.Length, entryLength);
for (int i = 0; i < entryLength; i++)
{
Assert.Equal(reference[i], result[i]);
}
iterationCount++;
}
}
}
await blobManager.StopAsync();
}
await BlobManager.DeleteTaskhubStorageAsync(account, taskHubName);
}