public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
{
// Need this environment variable set AND Azure Storage Emulator running
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
// Set up the blob manager so can set lease to it
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
}
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
{
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
FasterLogTest1(logChecksum, device, checkpointManager);
device.Close();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public void TieredWriteRead()
{
IDevice tested;
IDevice localDevice = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\BasicDiskFASTERTests.log", deleteOnClose: true, capacity: 1 << 30);
if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
{
IDevice cloudDevice = new AzureStorageDevice(EMULATED_STORAGE_STRING, TEST_CONTAINER, "BasicDiskFASTERTests", false);
tested = new TieredStorageDevice(1, localDevice, cloudDevice);
}
else
{
// If no Azure is enabled, just use another disk
IDevice localDevice2 = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\BasicDiskFASTERTests2.log", deleteOnClose: true, capacity: 1 << 30);
tested = new TieredStorageDevice(1, localDevice, localDevice2);
}
TestDeviceWriteRead(tested);
}
public void TieredWriteRead()
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
IDevice tested;
IDevice localDevice = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/BasicDiskFASTERTests.log", deleteOnClose: true, capacity: 1 << 30);
if (TestUtils.IsRunningAzureTests)
{
IDevice cloudDevice = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskFASTERTests");
tested = new TieredStorageDevice(1, localDevice, cloudDevice);
}
else
{
// If no Azure is enabled, just use another disk
IDevice localDevice2 = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/BasicDiskFASTERTests2.log", deleteOnClose: true, capacity: 1 << 30);
tested = new TieredStorageDevice(1, localDevice, localDevice2);
}
TestDeviceWriteRead(tested);
}
internal static IDevice CreateTestDevice(DeviceType testDeviceType, string filename, int latencyMs = 20) // latencyMs works only for DeviceType = LocalMemory
{
IDevice device = null;
bool preallocateFile = false;
long capacity = -1; // Capacity unspecified
bool recoverDevice = false;
bool useIoCompletionPort = false;
bool disableFileBuffering = true;
bool deleteOnClose = false;
switch (testDeviceType)
{
#if WINDOWS
case DeviceType.LSD:
#if NETSTANDARD || NET
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // avoids CA1416 // Validate platform compatibility
#endif
device = new LocalStorageDevice(filename, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort);
break;
case DeviceType.EmulatedAzure:
IgnoreIfNotRunningAzureTests();
device = new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, Path.GetFileName(filename), deleteOnClose: false);
break;
#endif
case DeviceType.MLSD:
device = new ManagedLocalStorageDevice(filename, preallocateFile, deleteOnClose, capacity, recoverDevice);
break;
// Emulated higher latency storage device - takes a disk latency arg (latencyMs) and emulates an IDevice using main memory, serving data at specified latency
case DeviceType.LocalMemory:
device = new LocalMemoryDevice(1L << 26, 1L << 22, 2, sector_size: 512, latencyMs: latencyMs, fileName: filename); // 64 MB (1L << 26) is enough for our test cases
break;
}
return(device);
}
static void Main()
{
// Main hybrid log device
var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
// With non-blittable types, you need an object log device in addition to the
// main device. FASTER serializes the actual objects in the object log.
var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(STORAGE_STRING),
new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));
var store = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
using (var s = store.NewSession(new Functions()))
{
for (long i = 0; i < 20000; i++)
{
var _key = i;
var value = $"value-{i}";
s.Upsert(ref _key, ref value);
}
s.CompletePending(true);
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
// Take fold-over checkpoint of FASTER, wait to complete
store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
.GetAwaiter().GetResult();
// Dispose store instance
store.Dispose();
// Dispose logs
log.Dispose();
objlog.Dispose();
// Create new store
log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");
var store2 = new FasterKV <long, string>(
1L << 17,
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog
},
new CheckpointSettings {
CheckpointManager = checkpointManager
}
);
// Recover store from latest checkpoint
store2.Recover();
using (var s = store2.NewSession(new Functions()))
{
// Test Read
long key = 23;
string output = default;
string context = "value-23";
var status = s.Read(ref key, ref output, context);
if (status != Status.PENDING)
{
if (status == Status.OK && output == context)
{
Console.WriteLine("Success!");
}
else
{
Console.WriteLine("Error!");
}
}
}
store2.Dispose();
// Purge cloud log files
log.PurgeAll();
objlog.PurgeAll();
// Purge cloud checkpoints - warning all data under specified base path are removed
checkpointManager.PurgeAll();
// Dispose devices
log.Dispose();
objlog.Dispose();
// Dispose checkpoint manager
checkpointManager.Dispose();
Console.WriteLine("Press <ENTER> to end");
Console.ReadLine();
}
