public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async ValueTask IncrSnapshotRecoveryCheck([Values] DeviceMode deviceMode)
{
ICheckpointManager checkpointManager;
if (deviceMode == DeviceMode.Local)
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + $"/RecoveryChecks/IncrSnapshotRecoveryCheck"));
}
else
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/IncrSnapshotRecoveryCheck"));
}
else
{
return;
}
}
await IncrSnapshotRecoveryCheck(checkpointManager);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
{
// Need this environment variable set AND Azure Storage Emulator running
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
// Set up the blob manager so can set lease to it
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
}
public void LocalDeviceSimpleRecoveryTest(CheckpointType checkpointType)
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
SimpleRecoveryTest1(checkpointType, checkpointManager);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async Task FasterLogResumePersistedReader2([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abc";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
{
originalIterator.GetNext(out _, out _, out _, out long recoveryAddress);
originalIterator.CompleteUntil(recoveryAddress);
originalIterator.GetNext(out _, out _, out _, out _); // move the reader ahead
await l.CommitAsync();
originalCompleted = originalIterator.CompletedUntilAddress;
}
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
{
recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);
// we should have read in input2, not input1 or input3
Assert.True(input2.SequenceEqual(outBuf), $"Original: {input2[0]}, Recovered: {outBuf[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
// TestContext.Progress.WriteLine($"Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
}
}
}
public void Setup()
{
commitPath = TestContext.CurrentContext.TestDirectory + "\\" + TestContext.CurrentContext.Test.Name + "\\";
if (Directory.Exists(commitPath))
{
DeleteDirectory(commitPath);
}
device = Devices.CreateLogDevice(commitPath + "fasterlog.log", deleteOnClose: true);
manager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(deleteOnClose: true), new DefaultCheckpointNamingScheme(commitPath));
}
public void PageBlobSimpleRecoveryTest(CheckpointType checkpointType)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
SimpleRecoveryTest1(checkpointType, checkpointManager);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
FasterLogTest1(logChecksum, device, checkpointManager);
device.Close();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async Task FasterLog_Test1Write()
{
const int entryLength = 1 << 10;
byte[] staticEntry1 = new byte[entryLength];
byte[] staticEntry1Brotli = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
staticEntry1[i] = (byte)i;
}
byte[] staticEntry2 = new byte[entryLength];
byte[] staticEntry2Brotli = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
staticEntry2[i] = (byte)(entryLength - i);
}
var path = GetPath();
using (var logCommitManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(path), true, false)
) {
using (IDevice device = Devices.CreateLogDevice(path + "hlog.log")) {
//FasterLogScanIterator iter;
var logSettings = new FASTER.core.FasterLogSettings()
{
LogDevice = device,
LogChecksum = LogChecksumType.PerEntry,
LogCommitManager = logCommitManager,
};
using (var log = await FASTER.core.FasterLog.CreateAsync(logSettings)) {
//log.TruncateUntilPageStart(0);
//await log.CommitAsync();
for (int i = 0; i < 1000; i++)
{
if (BrotliEncoder.TryCompress(staticEntry1, staticEntry1Brotli, out var bytesWritten1, 4, 10))
{
await log.EnqueueAsync(new Memory <byte>(staticEntry1Brotli).Slice(0, bytesWritten1));
}
public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode)
{
ICheckpointManager checkpointManager;
if (deviceMode == DeviceMode.Local)
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir +
"/checkpoints/")); // PurgeAll deletes this directory
}
else
{
TestUtils.IgnoreIfNotRunningAzureTests();
checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme(
$"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
}
var path = TestUtils.MethodTestDir + "/";
using var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true);
TestUtils.RecreateDirectory(path);
using var fht = new FasterKV <long, long>
(1 << 10,
logSettings: new LogSettings
{
LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20,
ReadCacheSettings = null
},
checkpointSettings: new CheckpointSettings { CheckpointManager = checkpointManager }
);
using var s = fht.NewSession(new SimpleFunctions <long, long>());
var logCheckpoints = new Dictionary <Guid, int>();
var indexCheckpoints = new Dictionary <Guid, int>();
var fullCheckpoints = new Dictionary <Guid, int>();
for (var i = 0; i < 10; i++)
{
// Do some dummy update
s.Upsert(0, random.Next());
var checkpointType = random.Next(5);
Guid result = default;
switch (checkpointType)
{
case 0:
fht.TakeHybridLogCheckpoint(out result, CheckpointType.FoldOver);
logCheckpoints.Add(result, 0);
break;
case 1:
fht.TakeHybridLogCheckpoint(out result, CheckpointType.Snapshot);
logCheckpoints.Add(result, 0);
break;
case 2:
fht.TakeIndexCheckpoint(out result);
indexCheckpoints.Add(result, 0);
break;
case 3:
fht.TakeFullCheckpoint(out result, CheckpointType.FoldOver);
fullCheckpoints.Add(result, 0);
break;
case 4:
fht.TakeFullCheckpoint(out result, CheckpointType.Snapshot);
fullCheckpoints.Add(result, 0);
break;
default:
Assert.True(false);
break;
}
await fht.CompleteCheckpointAsync();
}
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
if (logCheckpoints.Count != 0)
{
var guid = logCheckpoints.First().Key;
checkpointManager.Purge(guid);
logCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (indexCheckpoints.Count != 0)
{
var guid = indexCheckpoints.First().Key;
checkpointManager.Purge(guid);
indexCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (fullCheckpoints.Count != 0)
{
var guid = fullCheckpoints.First().Key;
checkpointManager.Purge(guid);
fullCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
checkpointManager.PurgeAll();
Assert.IsEmpty(checkpointManager.GetLogCheckpointTokens());
Assert.IsEmpty(checkpointManager.GetIndexCheckpointTokens());
checkpointManager.Dispose();
}
static void Main()
{
// Main hybrid log device
var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
// With non-blittable types, you need an object log device in addition to the
// main device. FASTER serializes the actual objects in the object log.
var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));
var store = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
using (var s = store.NewSession(new Functions()))
{
for (long i = 0; i < 20000; i++)
{
var _key = i;
var value = $"value-{i}";
s.Upsert(ref _key, ref value);
}
s.CompletePending(true);
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
// Take fold-over checkpoint of FASTER, wait to complete
store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
.GetAwaiter().GetResult();
// Dispose store instance
store.Dispose();
// Dispose logs
log.Dispose();
objlog.Dispose();
// Create new store
log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var store2 = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
// Recover store from latest checkpoint
store2.Recover();
using (var s = store2.NewSession(new Functions()))
{
// Test Read
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
store2.Dispose();
// Purge cloud log files
log.PurgeAll();
objlog.PurgeAll();
// Purge cloud checkpoints - warning all data under specified base path are removed
checkpointManager.PurgeAll();
// Dispose devices
log.Dispose();
objlog.Dispose();
// Dispose checkpoint manager
checkpointManager.Dispose();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
public async Task FasterLogResumePersistedReader3([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
{
var input1 = new byte[] { 0, 1, 2, 3 };
var input2 = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
var input3 = new byte[] { 11, 12 };
string readerName = "abcd";
using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
{
long originalCompleted;
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
await l.EnqueueAsync(input1);
await l.CommitAsync();
await l.EnqueueAsync(input2);
await l.CommitAsync();
await l.EnqueueAsync(input3);
await l.CommitAsync();
using var originalIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in originalIterator.GetAsyncEnumerable())
{
if (count < 2) // we complete 1st and 2nd item read
{
originalIterator.CompleteUntil(item.nextAddress);
}
if (count < 1) // we commit only 1st item read
{
await l.CommitAsync();
}
count++;
}
originalCompleted = originalIterator.CompletedUntilAddress;
}
using (var l = new FasterLog(new FasterLogSettings {
LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
}))
{
using var recoveredIterator = l.Scan(0, l.TailAddress, readerName);
int count = 0;
await foreach (var item in recoveredIterator.GetAsyncEnumerable())
{
if (count == 0) // resumed iterator will start at item2
{
Assert.True(input2.SequenceEqual(item.entry), $"Original: {input2[0]}, Recovered: {item.entry[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
}
count++;
}
Assert.IsTrue(count == 2);
}
}
}
public async ValueTask SimpleRecoveryTest2([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
var checkpointManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + "/checkpoints4"), false);
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/SimpleRecoveryTest2.log", deleteOnClose: true);
// Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "/checkpoints4");
fht1 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
fht2 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
int numOps = 5000;
var inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
NumClicks value;
AdInput inputArg = default;
Output output = default;
var session1 = fht1.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
session1.Dispose();
if (isAsync)
{
await fht2.RecoverAsync(token);
}
else
{
fht2.Recover(token);
}
var session2 = fht2.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);
if (status == Status.PENDING)
{
session2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
session2.Dispose();
log.Dispose();
fht1.Dispose();
fht2.Dispose();
checkpointManager.Dispose();
new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "/checkpoints4").Delete(true);
}