public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
// Set up the blob manager so can set lease to it
TestUtils.IgnoreIfNotRunningAzureTests();
CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
blobContainer.CreateIfNotExists();
var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");
var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
blobContainer.Delete();
}
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
{
TestUtils.IgnoreIfNotRunningAzureTests();
var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
var checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);
device.Dispose();
checkpointManager.PurgeAll();
checkpointManager.Dispose();
}
public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode)
{
ICheckpointManager checkpointManager;
if (deviceMode == DeviceMode.Local)
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir +
"/checkpoints/")); // PurgeAll deletes this directory
}
else
{
TestUtils.IgnoreIfNotRunningAzureTests();
checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme(
$"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
}
var path = TestUtils.MethodTestDir + "/";
using var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true);
TestUtils.RecreateDirectory(path);
using var fht = new FasterKV <long, long>
(1 << 10,
logSettings: new LogSettings
{
LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20,
ReadCacheSettings = null
},
checkpointSettings: new CheckpointSettings { CheckpointManager = checkpointManager }
);
using var s = fht.NewSession(new SimpleFunctions <long, long>());
var logCheckpoints = new Dictionary <Guid, int>();
var indexCheckpoints = new Dictionary <Guid, int>();
var fullCheckpoints = new Dictionary <Guid, int>();
for (var i = 0; i < 10; i++)
{
// Do some dummy update
s.Upsert(0, random.Next());
var checkpointType = random.Next(5);
Guid result = default;
switch (checkpointType)
{
case 0:
fht.TakeHybridLogCheckpoint(out result, CheckpointType.FoldOver);
logCheckpoints.Add(result, 0);
break;
case 1:
fht.TakeHybridLogCheckpoint(out result, CheckpointType.Snapshot);
logCheckpoints.Add(result, 0);
break;
case 2:
fht.TakeIndexCheckpoint(out result);
indexCheckpoints.Add(result, 0);
break;
case 3:
fht.TakeFullCheckpoint(out result, CheckpointType.FoldOver);
fullCheckpoints.Add(result, 0);
break;
case 4:
fht.TakeFullCheckpoint(out result, CheckpointType.Snapshot);
fullCheckpoints.Add(result, 0);
break;
default:
Assert.True(false);
break;
}
await fht.CompleteCheckpointAsync();
}
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
if (logCheckpoints.Count != 0)
{
var guid = logCheckpoints.First().Key;
checkpointManager.Purge(guid);
logCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (indexCheckpoints.Count != 0)
{
var guid = indexCheckpoints.First().Key;
checkpointManager.Purge(guid);
indexCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (fullCheckpoints.Count != 0)
{
var guid = fullCheckpoints.First().Key;
checkpointManager.Purge(guid);
fullCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
checkpointManager.PurgeAll();
Assert.IsEmpty(checkpointManager.GetLogCheckpointTokens());
Assert.IsEmpty(checkpointManager.GetIndexCheckpointTokens());
checkpointManager.Dispose();
}
public void PageBlobWriteReadWithLease()
{
TestUtils.IgnoreIfNotRunningAzureTests();
TestDeviceWriteRead(new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskFASTERTests", null, true, true));
}
