private void FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager)
{
log = new FasterLog(new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
});
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
int count = 0;
while (iter.GetNext(out byte[] result, out int length, out long currentAddress))
{
count++;
Assert.IsTrue(result.SequenceEqual(entry));
if (count % 100 == 0)
{
log.TruncateUntil(iter.NextAddress);
}
}
Assert.IsTrue(count == numEntries);
}
log.Dispose();
}
public void FasterLogTest1([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = logChecksum
});
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
int count = 0;
while (iter.GetNext(out byte[] result, out int length))
{
count++;
Assert.IsTrue(result.SequenceEqual(entry));
if (count % 100 == 0)
{
log.TruncateUntil(iter.CurrentAddress);
}
}
Assert.IsTrue(count == numEntries);
}
log.Dispose();
}
public void FasterLogTest6([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager
});
byte[] data1 = new byte[1000];
for (int i = 0; i < 100; i++)
{
data1[i] = (byte)i;
}
for (int i = 0; i < 100; i++)
{
log.Enqueue(data1);
}
log.RefreshUncommitted();
Assert.IsTrue(log.SafeTailAddress == log.TailAddress);
Assert.IsTrue(log.CommittedUntilAddress < log.SafeTailAddress);
using (var iter = log.Scan(0, long.MaxValue, scanUncommitted: true))
{
while (iter.GetNext(out _, out _, out _))
{
log.TruncateUntil(iter.NextAddress);
}
Assert.IsTrue(iter.NextAddress == log.SafeTailAddress);
log.Enqueue(data1);
Assert.IsFalse(iter.GetNext(out _, out _, out _));
log.RefreshUncommitted();
Assert.IsTrue(iter.GetNext(out _, out _, out _));
}
log.Dispose();
}
public async Task FasterLogResumePersistedReaderSpec([Values] LogChecksumType logChecksum)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
await l.EnqueueAsync(input1);
await l.EnqueueAsync(input2);
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out _, out long recoveryAddress));
originalIterator.CompleteUntil(recoveryAddress);
Assert.IsTrue(originalIterator.GetNext(out _, out _, out _, out _)); // move the reader ahead
await l.CommitAsync();
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitFile = commitPath
}))
{
using var recoveredIterator = l.Scan(0, long.MaxValue, readerName);
Assert.IsTrue(recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _));
Assert.True(input2.SequenceEqual(outBuf)); // we should have read in input2, not input1 or input3
}
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
{
// Need this environment variable set AND Azure Storage Emulator running
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
// Set up the blob manager so can set lease to it
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async Task FasterLogResumePersistedReader2([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
{
originalIterator.GetNext(out _, out _, out _, out long recoveryAddress);
originalIterator.CompleteUntil(recoveryAddress);
originalIterator.GetNext(out _, out _, out _, out _); // move the reader ahead
await l.CommitAsync();
originalCompleted = originalIterator.CompletedUntilAddress;
}
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
{
recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);
// we should have read in input2, not input1 or input3
Assert.True(input2.SequenceEqual(outBuf), $"Original: {input2[0]}, Recovered: {outBuf[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
// TestContext.Progress.WriteLine($"Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
}
}
}
public async Task FasterLogTest2([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager
});
byte[] data1 = new byte[10000];
for (int i = 0; i < 10000; i++)
{
data1[i] = (byte)i;
}
using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering))
{
int i = 0;
while (i++ < 500)
{
var waitingReader = iter.WaitAsync();
Assert.IsTrue(!waitingReader.IsCompleted);
while (!log.TryEnqueue(data1, out _))
{
;
}
// We might have auto-committed at page boundary
// Ensure we don't find new entry in iterator
while (waitingReader.IsCompleted)
{
var _next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(_next);
waitingReader = iter.WaitAsync();
}
Assert.IsFalse(waitingReader.IsCompleted);
await log.CommitAsync();
while (!waitingReader.IsCompleted)
{
;
}
Assert.IsTrue(waitingReader.IsCompleted);
var curr = iter.GetNext(out byte[] result, out _, out _);
Assert.IsTrue(curr);
Assert.IsTrue(result.SequenceEqual(data1));
var next = iter.GetNext(out _, out _, out _);
Assert.IsFalse(next);
}
}
log.Dispose();
}
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
FasterLogTest1(logChecksum, device, checkpointManager);
device.Close();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
/// <summary>
/// Create new log instance
/// </summary>
/// <param name="logSettings"></param>
public FasterLog(FasterLogSettings logSettings)
{
bool oldCommitManager = false;
if (oldCommitManager)
{
logCommitManager = logSettings.LogCommitManager ??
new LocalLogCommitManager(logSettings.LogCommitFile ??
logSettings.LogDevice.FileName + ".commit");
}
else
{
logCommitManager = logSettings.LogCommitManager ??
new DeviceLogCommitCheckpointManager
(new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(
new FileInfo(logSettings.LogDevice.FileName).Directory.FullName));
}
if (logSettings.LogCommitManager == null)
{
disposeLogCommitManager = true;
}
// Reserve 8 byte checksum in header if requested
logChecksum = logSettings.LogChecksum;
headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4;
getMemory = logSettings.GetMemory;
epoch = new LightEpoch();
CommittedUntilAddress = Constants.kFirstValidAddress;
CommittedBeginAddress = Constants.kFirstValidAddress;
SafeTailAddress = Constants.kFirstValidAddress;
allocator = new BlittableAllocator <Empty, byte>(
logSettings.GetLogSettings(), null,
null, epoch, CommitCallback);
allocator.Initialize();
// FasterLog is used as a read-only iterator
if (logSettings.ReadOnlyMode)
{
readOnlyMode = true;
allocator.HeadAddress = long.MaxValue;
}
Restore(out RecoveredIterators);
}
public async Task FasterLogTest3([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
});
byte[] data1 = new byte[10000];
for (int i = 0; i < 10000; i++)
{
data1[i] = (byte)i;
}
using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering))
{
var appendResult = log.TryEnqueue(data1, out _);
Assert.IsTrue(appendResult);
await log.CommitAsync();
await iter.WaitAsync();
var iterResult = iter.GetNext(out byte[] entry, out _, out _);
Assert.IsTrue(iterResult);
appendResult = log.TryEnqueue(data1, out _);
Assert.IsFalse(appendResult);
await iter.WaitAsync();
// Should read the "hole" and return false
iterResult = iter.GetNext(out entry, out _, out _);
Assert.IsFalse(iterResult);
// Should wait for next item
var task = iter.WaitAsync();
Assert.IsFalse(task.IsCompleted);
appendResult = log.TryEnqueue(data1, out _);
Assert.IsTrue(appendResult);
await log.CommitAsync();
await task;
iterResult = iter.GetNext(out entry, out _, out _);
Assert.IsTrue(iterResult);
}
log.Dispose();
}
/// <summary>
/// Create new log instance
/// </summary>
/// <param name="logSettings"></param>
public FasterLog(FasterLogSettings logSettings)
{
logCommitManager = logSettings.LogCommitManager ??
new LocalLogCommitManager(logSettings.LogCommitFile ??
logSettings.LogDevice.FileName + ".commit");
// Reserve 8 byte checksum in header if requested
logChecksum = logSettings.LogChecksum;
headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4;
getMemory = logSettings.GetMemory;
epoch = new LightEpoch();
CommittedUntilAddress = Constants.kFirstValidAddress;
CommittedBeginAddress = Constants.kFirstValidAddress;
allocator = new BlittableAllocator <Empty, byte>(
logSettings.GetLogSettings(), null,
null, epoch, CommitCallback);
allocator.Initialize();
Restore(out RecoveredIterators);
}
public async Task FasterLogTest4([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 14, LogChecksum = logChecksum
});
byte[] data1 = new byte[100];
for (int i = 0; i < 100; i++)
{
data1[i] = (byte)i;
}
for (int i = 0; i < 100; i++)
{
log.Enqueue(data1);
}
Assert.IsTrue(log.CommittedUntilAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
using (var iter = log.Scan(0, long.MaxValue))
{
// Should read the "hole" and return false
var iterResult = iter.GetNext(out byte[] entry, out _, out _);
log.TruncateUntil(iter.NextAddress);
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress < log.BeginAddress);
Assert.IsTrue(iter.NextAddress == log.BeginAddress);
await log.CommitAsync();
Assert.IsTrue(log.CommittedUntilAddress == log.TailAddress);
Assert.IsTrue(log.CommittedBeginAddress == log.BeginAddress);
}
log.Dispose();
}
public async Task FasterLogTest5([Values] LogChecksumType logChecksum)
{
log = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum
});
int headerSize = logChecksum == LogChecksumType.None ? 4 : 12;
bool _disposed = false;
var commit = new Thread(() => { while (!_disposed)
{
log.Commit(true); Thread.Sleep(1);
}
});
commit.Start();
// 65536=page size|headerSize|64=log header
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize - 64]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
// 65536=page size|headerSize
await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]);
_disposed = true;
commit.Join();
log.Dispose();
}
public async Task FasterLogResumePersistedReader3([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abcd";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in originalIterator.GetAsyncEnumerable())
{
if (count < 2) // we complete 1st and 2nd item read
{
originalIterator.CompleteUntil(item.nextAddress);
}
if (count < 1) // we commit only 1st item read
{
await l.CommitAsync();
}
count++;
}
originalCompleted = originalIterator.CompletedUntilAddress;
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using var recoveredIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in recoveredIterator.GetAsyncEnumerable())
{
if (count == 0) // resumed iterator will start at item2
{
Assert.True(input2.SequenceEqual(item.entry), $"Original: {input2[0]}, Recovered: {item.entry[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
count++;
}
Assert.IsTrue(count == 2);
}
}
}
private async ValueTask FasterLogTest1(LogChecksumType logChecksum, IDevice device, ILogCommitManager logCommitManager, FasterLogTests.IteratorType iteratorType)
{
var logSettings = new FasterLogSettings {
PageSizeBits = 20, SegmentSizeBits = 20, LogDevice = device, LogChecksum = logChecksum, LogCommitManager = logCommitManager
};
log = FasterLogTests.IsAsync(iteratorType) ? await FasterLog.CreateAsync(logSettings) : new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
log.Commit(true);
using (var iter = log.Scan(0, long.MaxValue))
{
var counter = new FasterLogTests.Counter(log);
switch (iteratorType)
{
case FasterLogTests.IteratorType.AsyncByteVector:
await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable())
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.AsyncMemoryOwner:
await foreach ((IMemoryOwner <byte> result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool <byte> .Shared))
{
Assert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry));
result.Dispose();
counter.IncrementAndMaybeTruncateUntil(nextAddress);
// MoveNextAsync() would hang at TailAddress, waiting for more entries (that we don't add).
// Note: If this happens and the test has to be canceled, there may be a leftover blob from the log.Commit(), because
// the log device isn't Dispose()d; the symptom is currently a numeric string format error in DefaultCheckpointNamingScheme.
if (nextAddress == log.TailAddress)
{
break;
}
}
break;
case FasterLogTests.IteratorType.Sync:
while (iter.GetNext(out byte[] result, out _, out _))
{
Assert.IsTrue(result.SequenceEqual(entry));
counter.IncrementAndMaybeTruncateUntil(iter.NextAddress);
}
break;
default:
Assert.Fail("Unknown IteratorType");
break;
}
Assert.IsTrue(counter.count == numEntries);
}
log.Dispose();
}
