public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
{
// Need this environment variable set AND Azure Storage Emulator running
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
// Set up the blob manager so can set lease to it
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
}
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
FasterLogTest1(logChecksum, device, checkpointManager);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
static void Main()
{
// Main hybrid log device
var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
// With non-blittable types, you need an object log device in addition to the
// main device. FASTER serializes the actual objects in the object log.
var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));
var store = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
using (var s = store.NewSession(new Functions()))
{
for (long i = 0; i < 20000; i++)
{
var _key = i;
var value = $"value-{i}";
s.Upsert(ref _key, ref value);
}
s.CompletePending(true);
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
// Take fold-over checkpoint of FASTER, wait to complete
store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
.GetAwaiter().GetResult();
// Dispose store instance
store.Dispose();
// Dispose logs
log.Dispose();
objlog.Dispose();
// Create new store
log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var store2 = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
// Recover store from latest checkpoint
store2.Recover();
using (var s = store2.NewSession(new Functions()))
{
// Test Read
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
store2.Dispose();
// Purge cloud log files
log.PurgeAll();
objlog.PurgeAll();
// Purge cloud checkpoints - warning all data under specified base path are removed
checkpointManager.PurgeAll();
// Dispose devices
log.Dispose();
objlog.Dispose();
// Dispose checkpoint manager
checkpointManager.Dispose();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}