public void FasterLogTest6([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager
});
byte[] data1 = new byte[1000];
for (int i = 0; i < 100; i++)
{
data1[i] = (byte)i;
}
for (int i = 0; i < 100; i++)
{
log.Enqueue(data1);
}
log.RefreshUncommitted();
Assert.IsTrue(log.SafeTailAddress == log.TailAddress);
Assert.IsTrue(log.CommittedUntilAddress < log.SafeTailAddress);
using (var iter = log.Scan(0, long.MaxValue, scanUncommitted: true))
{
while (iter.GetNext(out _, out _, out _))
{
log.TruncateUntil(iter.NextAddress);
}
Assert.IsTrue(iter.NextAddress == log.SafeTailAddress);
log.Enqueue(data1);
Assert.IsFalse(iter.GetNext(out _, out _, out _));
log.RefreshUncommitted();
Assert.IsTrue(iter.GetNext(out _, out _, out _));
}
log.Dispose();
}
private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager)
{
log = new FasterLog(new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
});
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
int count = 0;
while (iter.GetNext(out byte[] result, out int length, out long currentAddress))
{
count++;
Assert.IsTrue(result.SequenceEqual(entry));
if (count % 100 == 0)
{
log.TruncateUntil(iter.NextAddress);
}
}
Assert.IsTrue(count == numEntries);
}
log.Dispose();
}
public static void Main()
{
commitPath = "FasterLogStress/";
// Clean up log files from previous test runs in case they weren't cleaned up
// We loop to ensure clean-up as deleteOnClose does not always work for MLSD
while (Directory.Exists(commitPath))
{
Directory.Delete(commitPath, true);
}
// Create devices \ log for test
device = new ManagedLocalStorageDevice(commitPath + "ManagedLocalStore.log", deleteOnClose: true);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 12, MemorySizeBits = 14
});
ManagedLocalStoreBasicTest();
log.Dispose();
device.Dispose();
// Clean up log files
if (Directory.Exists(commitPath))
{
Directory.Delete(commitPath, true);
}
}
static async Task Main()
{
var path = Path.GetTempPath() + "FasterLogPubSub\\";
var device = Devices.CreateLogDevice(path + "mylog");
var log = new FasterLog(new FasterLogSettings {
LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9
});
using var cts = new CancellationTokenSource();
var producer = ProducerAsync(log, cts.Token);
var commiter = CommiterAsync(log, cts.Token);
var consumer = ConsumerAsync(log, cts.Token);
Console.CancelKeyPress += (o, eventArgs) =>
{
Console.WriteLine("Cancelling program...");
eventArgs.Cancel = true;
cts.Cancel();
};
await producer;
await consumer;
await commiter;
Console.WriteLine("Finished.");
log.Dispose();
try { new DirectoryInfo(path).Delete(true); } catch { }
}
public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.DeviceType deviceType)
{
string path = TestUtils.MethodTestDir + "/";
string filename = path + "TestDisposeRelease" + deviceType.ToString() + ".log";
DirectoryInfo di = Directory.CreateDirectory(path);
IDevice device = TestUtils.CreateTestDevice(deviceType, filename);
FasterLog fasterLog = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path, LogChecksum = LogChecksumType.PerEntry
});
Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out _));
fasterLog.Commit(spinWait: true);
fasterLog.Dispose();
device.Dispose();
while (true)
{
try
{
di.Delete(recursive: true);
break;
}
catch { }
}
}
static async Task Main()
{
var device = Devices.CreateLogDevice(path + "mylog");
var log = new FasterLog(new FasterLogSettings {
LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9
});
using var cts = new CancellationTokenSource();
var producer = ProducerAsync(log, cts.Token);
var commiter = CommitterAsync(log, cts.Token);
// Consumer on SAME FasterLog instance
var consumer = ConsumerAsync(log, true, cts.Token);
// Uncomment below to run consumer on SEPARATE read-only FasterLog instance
// var consumer = SeparateConsumerAsync(cts.Token);
Console.CancelKeyPress += (o, eventArgs) =>
{
Console.WriteLine("Cancelling program...");
eventArgs.Cancel = true;
cts.Cancel();
};
await producer;
await consumer;
await commiter;
Console.WriteLine("Finished.");
log.Dispose();
try { new DirectoryInfo(path).Delete(true); } catch { }
}
public void FasterLogTest1([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = logChecksum
});
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
int count = 0;
while (iter.GetNext(out byte[] result, out int length))
{
count++;
Assert.IsTrue(result.SequenceEqual(entry));
if (count % 100 == 0)
{
log.TruncateUntil(iter.CurrentAddress);
}
}
Assert.IsTrue(count == numEntries);
}
log.Dispose();
}
public void TearDown()
{
log.Dispose();
device.Dispose();
// Clean up log files
try { new DirectoryInfo(path).Delete(true); }
catch { }
}
public void TearDown()
{
log?.Dispose();
log = null;
device?.Dispose();
device = null;
// Clean up log files
TestUtils.DeleteDirectory(path, wait: true);
}
public void TearDown()
{
log?.Dispose();
log = null;
device?.Dispose();
device = null;
// Clean up log files
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
static async Task Main()
{
// This is two samples in one, enumerating over the same FasterLog instance that does commits, or over a separate
// FasterLog that opens the log file read-only and continuously catches up with the first intance's commits.
const bool sameInstance = true;
#pragma warning disable CS0162 // Unreachable code detected
if (!sameInstance)
{
// Because the SAME-instance iterator illustrates truncating the log, the SEPARATE-instance may encounter EOF
// issues if it is run after that truncation without cleaning up the directory first.
// In all other cases, the sample should run without needing to clean up the directory.
//if (Directory.Exists(path)) Directory.Delete(path, true);
}
var device = Devices.CreateLogDevice(path + "mylog");
var log = new FasterLog(new FasterLogSettings {
LogDevice = device, MemorySizeBits = 11, PageSizeBits = 9, MutableFraction = 0.5, SegmentSizeBits = 9
});
using var cts = new CancellationTokenSource();
var producer = ProducerAsync(log, cts.Token);
var committer = CommitterAsync(log, cts.Token);
Task consumer;
if (sameInstance)
{
// Consumer on SAME FasterLog instance
consumer = ConsumerAsync(log, true, cts.Token);
}
else
{
// Consumer on SEPARATE read-only FasterLog instance
consumer = SeparateConsumerAsync(cts.Token);
}
#pragma warning restore CS0162 // Unreachable code detected
Console.CancelKeyPress += (o, eventArgs) =>
{
Console.WriteLine("Cancelling program...");
eventArgs.Cancel = true;
cts.Cancel();
};
await producer;
await consumer;
await committer;
Console.WriteLine("Finished.");
log.Dispose();
device.Dispose();
try { new DirectoryInfo(path).Delete(true); } catch { }
}
public async Task FasterLogTest2([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager
});
byte[] data1 = new byte[10000];
for (int i = 0; i < 10000; i++)
{
data1[i] = (byte)i;
}
using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering))
{
int i = 0;
while (i++ < 500)
{
var waitingReader = iter.WaitAsync();
Assert.IsTrue(!waitingReader.IsCompleted);
while (!log.TryEnqueue(data1, out _))
{
;
}
// We might have auto-committed at page boundary
// Ensure we don't find new entry in iterator
while (waitingReader.IsCompleted)
{
var _next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(_next);
waitingReader = iter.WaitAsync();
}
Assert.IsFalse(waitingReader.IsCompleted);
await log.CommitAsync();
while (!waitingReader.IsCompleted)
{
;
}
Assert.IsTrue(waitingReader.IsCompleted);
var curr = iter.GetNext(out byte[] result, out _, out _);
Assert.IsTrue(curr);
Assert.IsTrue(result.SequenceEqual(data1));
var next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(next);
}
}
log.Dispose();
}
public void TearDown()
{
log.Dispose();
device.Dispose();
logFullParams.Dispose();
deviceFullParams.Dispose();
// Clean up log files
if (Directory.Exists(commitPath))
{
Directory.Delete(commitPath, true);
}
}
public void TearDown()
{
log?.Dispose();
log = null;
device?.Dispose();
device = null;
logFullParams?.Dispose();
logFullParams = null;
deviceFullParams?.Dispose();
deviceFullParams = null;
// Clean up log
TestUtils.DeleteDirectory(path);
}
public void TearDown()
{
log?.Dispose();
log = null;
device?.Dispose();
device = null;
deviceUnCommitted?.Dispose();
deviceUnCommitted = null;
logUncommitted?.Dispose();
logUncommitted = null;
// Clean up log files
TestUtils.DeleteDirectory(path);
}
public void TearDown()
{
log?.Dispose();
log = null;
device?.Dispose();
device = null;
logReadOnly?.Dispose();
logReadOnly = null;
deviceReadOnly?.Dispose();
deviceReadOnly = null;
// Clean up log files
TestUtils.DeleteDirectory(path);
}
public void TearDown()
{
log.Dispose();
device.Dispose();
// Clean up log files
try
{
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
}
catch { }
}
public async Task FasterLogTest3([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
});
byte[] data1 = new byte[10000];
for (int i = 0; i < 10000; i++)
{
data1[i] = (byte)i;
}
using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering))
{
var appendResult = log.TryEnqueue(data1, out _);
Assert.IsTrue(appendResult);
await log.CommitAsync();
await iter.WaitAsync();
var iterResult = iter.GetNext(out byte[] entry, out _, out _);
Assert.IsTrue(iterResult);
appendResult = log.TryEnqueue(data1, out _);
Assert.IsFalse(appendResult);
await iter.WaitAsync();
// Should read the "hole" and return false
iterResult = iter.GetNext(out entry, out _, out _);
Assert.IsFalse(iterResult);
// Should wait for next item
var task = iter.WaitAsync();
Assert.IsFalse(task.IsCompleted);
appendResult = log.TryEnqueue(data1, out _);
Assert.IsTrue(appendResult);
await log.CommitAsync();
await task;
iterResult = iter.GetNext(out entry, out _, out _);
Assert.IsTrue(iterResult);
}
log.Dispose();
}
public async Task FasterLogTest4([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
});
byte[] data1 = new byte[100];
for (int i = 0; i < 100; i++)
{
data1[i] = (byte)i;
}
for (int i = 0; i < 100; i++)
{
log.Enqueue(data1);
}
Assert.IsTrue(log.CommittedUntilAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
using (var iter = log.Scan(0, long.MaxValue))
{
// Should read the "hole" and return false
var iterResult = iter.GetNext(out byte[] entry, out _, out _);
log.TruncateUntil(iter.NextAddress);
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress < log.BeginAddress);
Assert.IsTrue(iter.NextAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
}
log.Dispose();
}
public void TestDisposeReleasesFileLocksWithInprogressCommit()
{
string commitPath = TestContext.CurrentContext.TestDirectory + "/" + TestContext.CurrentContext.Test.Name + "/";
DirectoryInfo di = Directory.CreateDirectory(commitPath);
IDevice device = Devices.CreateLogDevice(commitPath + "testDisposeReleasesFileLocksWithInprogressCommit.log", preallocateFile: true, deleteOnClose: false);
FasterLog fasterLog = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = LogChecksumType.PerEntry
});
Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out long beginAddress));
fasterLog.Commit(spinWait: false);
fasterLog.Dispose();
device.Dispose();
while (true)
{
try
{
di.Delete(recursive: true);
break;
}
catch { }
}
}
public async Task FasterLogTest5([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum
});
int headerSize = logChecksum == LogChecksumType.None ? 4 : 12;
bool _disposed = false;
var commit = new Thread(() => { while (!_disposed)
{
log.Commit(true); Thread.Sleep(1);
}
});
commit.Start();
// 65536=page size|headerSize|64=log header
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize - 64]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
_disposed = true;
commit.Join();
log.Dispose();
}
public void FasterLogSimpleFastCommitTest([Values] TestUtils.DeviceType deviceType)
{
var cookie = new byte[100];
new Random().NextBytes(cookie);
var filename = path + "fastCommit" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true);
var logSettings = new FasterLogSettings {
LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true, TryRecoverLatest = false
};
log = new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie1 = new byte[100];
new Random().NextBytes(cookie1);
var commitSuccessful = log.CommitStrongly(out var commit1Addr, out _, true, cookie1, 1);
Assert.IsTrue(commitSuccessful);
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie2 = new byte[100];
new Random().NextBytes(cookie2);
commitSuccessful = log.CommitStrongly(out var commit2Addr, out _, true, cookie2, 2);
Assert.IsTrue(commitSuccessful);
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie6 = new byte[100];
new Random().NextBytes(cookie6);
commitSuccessful = log.CommitStrongly(out var commit6Addr, out _, true, cookie6, 6);
Assert.IsTrue(commitSuccessful);
// Wait for all metadata writes to be complete to avoid a concurrent access exception
log.Dispose();
log = null;
// be a deviant and remove commit metadata files
manager.RemoveAllCommits();
// Recovery should still work
var recoveredLog = new FasterLog(logSettings);
recoveredLog.Recover(1);
Assert.AreEqual(cookie1, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit1Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
recoveredLog = new FasterLog(logSettings);
recoveredLog.Recover(2);
Assert.AreEqual(cookie2, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit2Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
// Default argument should recover to most recent, if TryRecoverLatest is set
logSettings.TryRecoverLatest = true;
recoveredLog = new FasterLog(logSettings);
Assert.AreEqual(cookie6, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit6Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
}
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType)
{
var logSettings = new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
};
log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
var counter = new FasterLogTests.Counter(log);
switch (iteratorType)
{
case FasterLogTests.IteratorType.AsyncByteVector:
await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(iter.NextAddress);
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
Assert.IsTrue(counter.count == numEntries);
}
log.Dispose();
}
public async ValueTask FlakyLogTestTolerateFailure([Values] IteratorType iteratorType)
{
var errorOptions = new ErrorSimulationOptions
{
readTransientErrorRate = 0,
readPermanentErrorRate = 0,
writeTransientErrorRate = 0,
writePermanentErrorRate = 0.1,
};
device = new SimulatedFlakyDevice(Devices.CreateLogDevice(path + "fasterlog.log", deleteOnClose: true),
errorOptions);
var logSettings = new FasterLogSettings
{
LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, TolerateDeviceFailure = true
};
log = new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
// Ensure we write enough to trigger errors
for (int i = 0; i < 1000; i++)
{
log.Enqueue(entry);
try
{
if (IsAsync(iteratorType))
{
await log.CommitAsync();
}
else
{
log.Commit();
}
}
catch (CommitFailureException)
{
// Ignore failure
}
}
// For surviving entries, scan should still work best-effort
// If endAddress > log.TailAddress then GetAsyncEnumerable() will wait until more entries are added.
var endAddress = IsAsync(iteratorType) ? log.CommittedUntilAddress : long.MaxValue;
var recoveredLog = new FasterLog(logSettings);
using var iter = recoveredLog.Scan(0, endAddress);
switch (iteratorType)
{
case IteratorType.AsyncByteVector:
await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
}
break;
case IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
}
break;
case IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
recoveredLog.Dispose();
}