public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
{
// Need this environment variable set AND Azure Storage Emulator running
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
// Set up the blob manager so can set lease to it
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
}
public void TearDown()
{
manager.Dispose();
device.Dispose();
if (Directory.Exists(commitPath))
{
DeleteDirectory(commitPath);
}
}
public void LocalDeviceSimpleRecoveryTest(CheckpointType checkpointType)
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
SimpleRecoveryTest1(checkpointType, checkpointManager);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public void PageBlobSimpleRecoveryTest(CheckpointType checkpointType)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
SimpleRecoveryTest1(checkpointType, checkpointManager);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
FasterLogTest1(logChecksum, device, checkpointManager);
device.Close();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
static void Main()
{
// Main hybrid log device
var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
// With non-blittable types, you need an object log device in addition to the
// main device. FASTER serializes the actual objects in the object log.
var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));
var store = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
using (var s = store.NewSession(new Functions()))
{
for (long i = 0; i < 20000; i++)
{
var _key = i;
var value = $"value-{i}";
s.Upsert(ref _key, ref value);
}
s.CompletePending(true);
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
// Take fold-over checkpoint of FASTER, wait to complete
store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
.GetAwaiter().GetResult();
// Dispose store instance
store.Dispose();
// Dispose logs
log.Dispose();
objlog.Dispose();
// Create new store
log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var store2 = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
// Recover store from latest checkpoint
store2.Recover();
using (var s = store2.NewSession(new Functions()))
{
// Test Read
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
store2.Dispose();
// Purge cloud log files
log.PurgeAll();
objlog.PurgeAll();
// Purge cloud checkpoints - warning all data under specified base path are removed
checkpointManager.PurgeAll();
// Dispose devices
log.Dispose();
objlog.Dispose();
// Dispose checkpoint manager
checkpointManager.Dispose();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
public async ValueTask SimpleRecoveryTest2([Values] CheckpointType checkpointType, [Values] bool isAsync)
{
var checkpointManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + "/checkpoints4"), false);
log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/SimpleRecoveryTest2.log", deleteOnClose: true);
// Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "/checkpoints4");
fht1 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
fht2 = new FasterKV
<AdId, NumClicks>
(128,
logSettings: new LogSettings {
LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
},
checkpointSettings: new CheckpointSettings {
CheckpointManager = checkpointManager, CheckPointType = checkpointType
}
);
int numOps = 5000;
var inputArray = new AdId[numOps];
for (int i = 0; i < numOps; i++)
{
inputArray[i].adId = i;
}
NumClicks value;
AdInput inputArg = default;
Output output = default;
var session1 = fht1.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
value.numClicks = key;
session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
}
fht1.TakeFullCheckpoint(out Guid token);
fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
session1.Dispose();
if (isAsync)
{
await fht2.RecoverAsync(token);
}
else
{
fht2.Recover(token);
}
var session2 = fht2.NewSession(new AdSimpleFunctions());
for (int key = 0; key < numOps; key++)
{
var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);
if (status == Status.PENDING)
{
session2.CompletePending(true);
}
else
{
Assert.IsTrue(output.value.numClicks == key);
}
}
session2.Dispose();
log.Dispose();
fht1.Dispose();
fht2.Dispose();
checkpointManager.Dispose();
new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "/checkpoints4").Delete(true);
}
