public void TearDown()
{
device?.Dispose();
device = null;
TestUtils.DeleteDirectory(path);
}
void TestDeviceWriteRead(IDevice log)
{
fht = new FasterKV <KeyStruct, ValueStruct>
(1L << 20, new LogSettings {
LogDevice = log, MemorySizeBits = 15, PageSizeBits = 10
});
var session = fht.NewSession(new Functions());
InputStruct input = default;
for (int i = 0; i < 700; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
}
session.CompletePending(true);
// Update first 100 using RMW from storage
for (int i = 0; i < 100; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
input = new InputStruct {
ifield1 = 1, ifield2 = 1
};
var status = session.RMW(ref key1, ref input, Empty.Default, 0);
if (status == Status.PENDING)
{
session.CompletePending(true);
}
}
for (int i = 0; i < 700; i++)
{
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING)
{
session.CompletePending(true);
}
else
{
if (i < 100)
{
Assert.AreEqual(value.vfield1 + 1, output.value.vfield1);
Assert.AreEqual(value.vfield2 + 1, output.value.vfield2);
}
else
{
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
}
}
session.Dispose();
fht.Dispose();
fht = null;
log.Dispose();
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
public unsafe void MultiReadSpanByteKeyTest()
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
try
{
using var log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/MultiReadSpanByteKeyTest.log", deleteOnClose: true);
using var fht = new FasterKV <SpanByte, long>(
size: 1L << 10,
new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12 });
using var session = fht.For(new MultiReadSpanByteKeyTestFunctions()).NewSession <MultiReadSpanByteKeyTestFunctions>();
for (int i = 0; i < 200; i++)
{
var key = MemoryMarshal.Cast <char, byte>($"{i}".AsSpan());
fixed(byte *_ = key)
session.Upsert(SpanByte.FromFixedSpan(key), i);
}
// Evict all records to disk
fht.Log.FlushAndEvict(true);
for (long key = 0; key < 50; key++)
{
// read each key multiple times
for (int i = 0; i < 10; i++)
{
Assert.AreEqual(key, ReadKey($"{key}"));
}
}
long ReadKey(string keyString)
{
Status status;
var key = MemoryMarshal.Cast <char, byte>(keyString.AsSpan());
fixed(byte *_ = key)
status = session.Read(key: SpanByte.FromFixedSpan(key), out var unused);
// All keys need to be fetched from disk
Assert.AreEqual(Status.PENDING, status);
session.CompletePendingWithOutputs(out var completedOutputs, wait: true);
var count = 0;
var value = 0L;
using (completedOutputs)
{
while (completedOutputs.Next())
{
count++;
Assert.AreEqual(Status.OK, completedOutputs.Current.Status);
value = completedOutputs.Current.Output;
}
}
Assert.AreEqual(1, count);
return(value);
}
}
finally
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
}
public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode)
{
ICheckpointManager checkpointManager;
if (deviceMode == DeviceMode.Local)
{
checkpointManager = new DeviceLogCommitCheckpointManager(
new LocalStorageNamedDeviceFactory(),
new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir +
"/checkpoints/"), false); // PurgeAll deletes this directory
}
else
{
TestUtils.IgnoreIfNotRunningAzureTests();
checkpointManager = new DeviceLogCommitCheckpointManager(
new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
new DefaultCheckpointNamingScheme(
$"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"), false);
}
var path = TestUtils.MethodTestDir + "/";
using (var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true))
{
TestUtils.RecreateDirectory(path);
using var fht = new FasterKV <long, long>
(1 << 10,
logSettings: new LogSettings
{
LogDevice = log,
MutableFraction = 1,
PageSizeBits = 10,
MemorySizeBits = 20,
ReadCacheSettings = null
},
checkpointSettings: new CheckpointSettings { CheckpointManager = checkpointManager }
);
using var s = fht.NewSession(new SimpleFunctions <long, long>());
var logCheckpoints = new Dictionary <Guid, int>();
var indexCheckpoints = new Dictionary <Guid, int>();
var fullCheckpoints = new Dictionary <Guid, int>();
for (var i = 0; i < 10; i++)
{
// Do some dummy update
s.Upsert(0, random.Next());
var checkpointType = random.Next(5);
Guid result = default;
switch (checkpointType)
{
case 0:
fht.TryInitiateHybridLogCheckpoint(out result, CheckpointType.FoldOver);
logCheckpoints.Add(result, 0);
break;
case 1:
fht.TryInitiateHybridLogCheckpoint(out result, CheckpointType.Snapshot);
logCheckpoints.Add(result, 0);
break;
case 2:
fht.TryInitiateIndexCheckpoint(out result);
indexCheckpoints.Add(result, 0);
break;
case 3:
fht.TryInitiateFullCheckpoint(out result, CheckpointType.FoldOver);
fullCheckpoints.Add(result, 0);
break;
case 4:
fht.TryInitiateFullCheckpoint(out result, CheckpointType.Snapshot);
fullCheckpoints.Add(result, 0);
break;
default:
Assert.True(false);
break;
}
await fht.CompleteCheckpointAsync();
}
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
if (logCheckpoints.Count != 0)
{
var guid = logCheckpoints.First().Key;
checkpointManager.Purge(guid);
logCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (indexCheckpoints.Count != 0)
{
var guid = indexCheckpoints.First().Key;
checkpointManager.Purge(guid);
indexCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
if (fullCheckpoints.Count != 0)
{
var guid = fullCheckpoints.First().Key;
checkpointManager.Purge(guid);
fullCheckpoints.Remove(guid);
Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
}
checkpointManager.PurgeAll();
Assert.IsEmpty(checkpointManager.GetLogCheckpointTokens());
Assert.IsEmpty(checkpointManager.GetIndexCheckpointTokens());
}
checkpointManager.Dispose();
TestUtils.DeleteDirectory(path, wait: true);
}
