public void ScanWithoutRecoverTest([Values] TestUtils.DeviceType deviceType)
{
// You may also force an iterator to start at the specified begin address, i.e., without recovering: recover parameter = false
// Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup
string filename = path + "LogScanWithoutRecover" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path
});
PopulateLog(log);
// Read the log
int currentEntry = 9; // since starting at specified address of 1000, need to set current entry as 9 so verification starts at proper spot
using (var iter = log.Scan(1000, 100_000_000, recover: false))
{
while (iter.GetNext(out byte[] result, out _, out _))
{
if (currentEntry < entryLength)
{
// Span Batch only added first entry several times so have separate verification
Assert.AreEqual((byte)entryFlag, result[currentEntry]);
currentEntry++;
}
}
}
// Make sure expected length is same as current - also makes sure that data verification was not skipped
Assert.AreEqual(entryLength, currentEntry);
}
public void ScanByNameTest([Values] TestUtils.DeviceType deviceType)
{
//You can persist iterators(or more precisely, their CompletedUntilAddress) as part of a commit by simply naming them during their creation.
// Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup
string filename = path + "LogScanByName" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path
});
PopulateLog(log);
// Read the log - Look for the flag so know each entry is unique
int currentEntry = 0;
using (var iter = log.Scan(0, 100_000_000, name: "TestScan", recover: true))
{
while (iter.GetNext(out byte[] result, out _, out _))
{
if (currentEntry < entryLength)
{
// Span Batch only added first entry several times so have separate verification
Assert.AreEqual((byte)entryFlag, result[currentEntry]);
currentEntry++;
}
}
}
// Make sure expected length is same as current - also makes sure that data verification was not skipped
Assert.AreEqual(entryLength, currentEntry);
}
public void UpsertNoRefNoDefaultsTest()
{
// Just checking more parameter values so one device is enough
deviceType = TestUtils.DeviceType.MLSD;
string filename = path + "UpsertNoRefNoDefaultsTest" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
session.Upsert(key1, value, Empty.Default, 0);
var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
public void ScanUncommittedTest([Values] TestUtils.DeviceType deviceType)
{
// Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup
string filename = path + "LogScan" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path
});
PopulateUncommittedLog(log);
// Setting scanUnCommitted to true is actual test here.
// Read the log - Look for the flag so know each entry is unique and still reads uncommitted
int currentEntry = 0;
using (var iter = log.Scan(0, 100_000_000, scanUncommitted: true))
{
while (iter.GetNext(out byte[] result, out _, out _))
{
if (currentEntry < entryLength)
{
// Span Batch only added first entry several times so have separate verification
Assert.AreEqual((byte)entryFlag, result[currentEntry]);
currentEntry++;
}
}
}
// Make sure expected length is same as current - also makes sure that data verification was not skipped
Assert.AreEqual(entryLength, currentEntry);
}
public void ReadBareMinParams([Values] TestUtils.DeviceType deviceType)
{
string filename = path + "ReadBareMinParams" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10
});
session = fht.For(new Functions()).NewSession <Functions>();
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
var(status, output) = session.Read(key1);
AssertCompleted(Status.OK, status);
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
Assert.AreEqual(key1.kfield1, 13);
Assert.AreEqual(key1.kfield2, 14);
}
public void UpsertDefaultsTest([Values] TestUtils.DeviceType deviceType)
{
string filename = path + "UpsertDefaultsTest" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
Assert.AreEqual(0, fht.EntryCount);
session.Upsert(ref key1, ref value);
var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
Assert.AreEqual(1, fht.EntryCount);
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
public void ReadAtAddressReadFlagsNone()
{
// Just functional test of ReadFlag so one device is enough
deviceType = TestUtils.DeviceType.MLSD;
string filename = path + "ReadAtAddressReadFlagsNone" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
var readAtAddress = fht.Log.BeginAddress;
session.Upsert(ref key1, ref value, Empty.Default, 0);
var status = session.ReadAtAddress(readAtAddress, ref input, ref output, ReadFlags.None, Empty.Default, 0);
AssertCompleted(Status.OK, status);
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
Assert.AreEqual(key1.kfield1, 13);
Assert.AreEqual(key1.kfield2, 14);
}
public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.DeviceType deviceType)
{
string path = TestUtils.MethodTestDir + "/";
string filename = path + "TestDisposeRelease" + deviceType.ToString() + ".log";
DirectoryInfo di = Directory.CreateDirectory(path);
IDevice device = TestUtils.CreateTestDevice(deviceType, filename);
FasterLog fasterLog = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path, LogChecksum = LogChecksumType.PerEntry
});
Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out _));
fasterLog.Commit(spinWait: true);
fasterLog.Dispose();
device.Dispose();
while (true)
{
try
{
di.Delete(recursive: true);
break;
}
catch { }
}
}
public void ScanConsumerTest([Values] TestUtils.DeviceType deviceType)
{
// Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup
string filename = path + "LogScanDefault" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path
});
PopulateLog(log);
// Basic default scan from start to end
// Indirectly used in other tests, but good to have the basic test here for completeness
// Read the log - Look for the flag so know each entry is unique
var consumer = new TestConsumer();
using (var iter = log.Scan(0, 100_000_000))
{
while (iter.TryConsumeNext(consumer))
{
}
}
// Make sure expected length is same as current - also makes sure that data verification was not skipped
Assert.AreEqual(entryLength, consumer.currentEntry);
}
public void DiskWriteScanBasicTest([Values] TestUtils.DeviceType deviceType)
{
log = TestUtils.CreateTestDevice(deviceType, $"{path}DiskWriteScanBasicTest_{deviceType}.log");
objlog = TestUtils.CreateTestDevice(deviceType, $"{path}DiskWriteScanBasicTest_{deviceType}.obj.log");
fht = new (128,
logSettings : new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 15, PageSizeBits = 9, SegmentSizeBits = 22
},
checkpointSettings : new CheckpointSettings {
CheckPointType = CheckpointType.FoldOver
},
serializerSettings : new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
using var session = fht.For(new MyFunctions()).NewSession <MyFunctions>();
using var s = fht.Log.Subscribe(new LogObserver());
var start = fht.Log.TailAddress;
for (int i = 0; i < totalRecords; i++)
{
var _key = new MyKey {
key = i
};
var _value = new MyValue {
value = i
};
session.Upsert(ref _key, ref _value, Empty.Default, 0);
if (i % 100 == 0)
{
fht.Log.FlushAndEvict(true);
}
}
fht.Log.FlushAndEvict(true);
using (var iter = fht.Log.Scan(start, fht.Log.TailAddress, ScanBufferingMode.SinglePageBuffering))
{
int val;
for (val = 0; iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value); ++val)
{
Assert.AreEqual(val, key.key, $"log scan 1: key");
Assert.AreEqual(val, value.value, $"log scan 1: value");
}
Assert.AreEqual(val, totalRecords, $"log scan 1: totalRecords");
}
using (var iter = fht.Log.Scan(start, fht.Log.TailAddress, ScanBufferingMode.DoublePageBuffering))
{
int val;
for (val = 0; iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value); ++val)
{
Assert.AreEqual(val, key.key, $"log scan 2: key");
Assert.AreEqual(val, value.value, $"log scan 2: value");
}
Assert.AreEqual(val, totalRecords, $"log scan 2: totalRecords");
}
}
private void Setup(long size, LogSettings logSettings, TestUtils.DeviceType deviceType)
{
string filename = path + TestContext.CurrentContext.Test.Name + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
logSettings.LogDevice = log;
fht = new FasterKV <KeyStruct, ValueStruct>(size, logSettings);
session = fht.For(new Functions()).NewSession <Functions>();
}
public void DeltaLogTest1([Values] TestUtils.DeviceType deviceType)
{
const int TotalCount = 200;
string filename = $"{path}delta_{deviceType}.log";
TestUtils.RecreateDirectory(path);
device = TestUtils.CreateTestDevice(deviceType, filename);
device.Initialize(-1);
using DeltaLog deltaLog = new DeltaLog(device, 12, 0);
Random r = new (20);
int i;
SectorAlignedBufferPool bufferPool = new(1, (int)device.SectorSize);
deltaLog.InitializeForWrites(bufferPool);
for (i = 0; i < TotalCount; i++)
{
int _len = 1 + r.Next(254);
long address;
while (true)
{
deltaLog.Allocate(out int maxLen, out address);
if (_len <= maxLen)
{
break;
}
deltaLog.Seal(0);
}
for (int j = 0; j < _len; j++)
{
unsafe { *(byte *)(address + j) = (byte)_len; }
}
deltaLog.Seal(_len, i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA);
}
deltaLog.FlushAsync().Wait();
deltaLog.InitializeForReads();
r = new (20);
for (i = 0; deltaLog.GetNext(out long address, out int len, out var type); i++)
{
int _len = 1 + r.Next(254);
Assert.AreEqual(i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA, type);
Assert.AreEqual(len, _len);
for (int j = 0; j < len; j++)
{
unsafe { Assert.AreEqual((byte)_len, *(byte *)(address + j)); };
}
}
Assert.AreEqual(TotalCount, i, $"i={i} and TotalCount={TotalCount}");
bufferPool.Free();
}
public void Setup()
{
path = TestUtils.MethodTestDir + "/";
// Clean up log files from previous test runs in case they weren't cleaned up
TestUtils.DeleteDirectory(path, wait: true);
if (TestContext.CurrentContext.Test.Arguments.Length == 0)
{
// Default log creation
log = Devices.CreateLogDevice(path + "/GenericLogCompactionTests.log", deleteOnClose: true);
objlog = Devices.CreateLogDevice(path + "/GenericLogCompactionTests.obj.log", deleteOnClose: true);
fht = new FasterKV <MyKey, MyValue>
(128,
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9
},
checkpointSettings: new CheckpointSettings {
CheckPointType = CheckpointType.FoldOver
},
serializerSettings: new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
}
else
{
// For this class, deviceType is the only parameter. Using this to illustrate the approach; NUnit doesn't provide metadata for arguments,
// so for multi-parameter tests it is probably better to stay with the "separate SetUp method" approach.
var deviceType = (TestUtils.DeviceType)TestContext.CurrentContext.Test.Arguments[0];
log = TestUtils.CreateTestDevice(deviceType, $"{path}LogCompactBasicTest_{deviceType}.log");
objlog = TestUtils.CreateTestDevice(deviceType, $"{path}LogCompactBasicTest_{deviceType}.obj.log");
fht = new FasterKV <MyKey, MyValue>
(128,
logSettings: new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9, SegmentSizeBits = 22
},
checkpointSettings: new CheckpointSettings {
CheckPointType = CheckpointType.FoldOver
},
serializerSettings: new SerializerSettings <MyKey, MyValue> {
keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer()
}
);
}
session = fht.For(new MyFunctionsDelete()).NewSession <MyFunctionsDelete>();
}
public void NativeInMemWriteReadDelete([Values] TestUtils.DeviceType deviceType)
{
string filename = path + "NativeInMemWriteReadDelete" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, PageSizeBits = 10, MemorySizeBits = 12, SegmentSizeBits = 22
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
session.Delete(ref key1, Empty.Default, 0);
status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.NOTFOUND, status);
var key2 = new KeyStruct {
kfield1 = 14, kfield2 = 15
};
var value2 = new ValueStruct {
vfield1 = 24, vfield2 = 25
};
session.Upsert(ref key2, ref value2, Empty.Default, 0);
status = session.Read(ref key2, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
Assert.AreEqual(value2.vfield1, output.value.vfield1);
Assert.AreEqual(value2.vfield2, output.value.vfield2);
}
public void StringBasicTest([Values] TestUtils.DeviceType deviceType)
{
string logfilename = path + "GenericStringTests" + deviceType.ToString() + ".log";
string objlogfilename = path + "GenericStringTests" + deviceType.ToString() + ".obj.log";
log = TestUtils.CreateTestDevice(deviceType, logfilename);
objlog = TestUtils.CreateTestDevice(deviceType, objlogfilename);
fht = new FasterKV <string, string>(
1L << 20, // size of hash table in #cache lines; 64 bytes per cache line
new LogSettings {
LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9, SegmentSizeBits = 22
} // log device
);
session = fht.For(new MyFuncs()).NewSession <MyFuncs>();
const int totalRecords = 200;
for (int i = 0; i < totalRecords; i++)
{
var _key = $"{i}";
var _value = $"{i}";;
session.Upsert(ref _key, ref _value, Empty.Default, 0);
}
session.CompletePending(true);
Assert.AreEqual(totalRecords, fht.EntryCount);
for (int i = 0; i < totalRecords; i++)
{
string input = default;
string output = default;
var key = $"{i}";
var value = $"{i}";
if (session.Read(ref key, ref input, ref output, Empty.Default, 0) == Status.PENDING)
{
session.CompletePending(true);
}
else
{
Assert.AreEqual(value, output);
}
}
}
public void UpsertSerialNumberTest()
{
// Simple Upsert of Serial Number test so one device is enough
deviceType = TestUtils.DeviceType.MLSD;
string filename = path + "UpsertSerialNumberTest" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29
});
session = fht.For(new Functions()).NewSession <Functions>();
int numKeys = 100;
int keyMod = 10;
int maxLap = numKeys / keyMod;
InputStruct input = default;
OutputStruct output = default;
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
var key = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
for (int i = 0; i < numKeys; i++)
{
// lap is used to illustrate the changing values
var lap = i / keyMod;
session.Upsert(ref key, ref value, serialNo: lap);
}
// Now verify
for (int j = 0; j < numKeys; j++)
{
var status = session.Read(ref key, ref input, ref output, serialNo: maxLap + 1);
AssertCompleted(Status.OK, status);
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
}
public void MemoryLogCompactionTest1([Values] TestUtils.DeviceType deviceType)
{
string filename = path + "MemoryLogCompactionTests1" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <ReadOnlyMemory <int>, Memory <int> >
(1L << 20, new LogSettings {
LogDevice = log, MemorySizeBits = 12, PageSizeBits = 10, SegmentSizeBits = 22
});
using var session = fht.For(new MemoryCompaction()).NewSession <MemoryCompaction>();
var key = new Memory <int>(new int[20]);
var value = new Memory <int>(new int[20]);
const int totalRecords = 200;
var start = fht.Log.TailAddress;
for (int i = 0; i < totalRecords; i++)
{
key.Span.Fill(i);
value.Span.Fill(i);
session.Upsert(key, value);
if (i < 5)
{
session.Delete(key); // in-place delete
}
}
for (int i = 5; i < 10; i++)
{
key.Span.Fill(i);
value.Span.Fill(i);
session.Delete(key); // tombstone inserted
}
// Compact log
var compactUntil = fht.Log.BeginAddress + (fht.Log.TailAddress - fht.Log.BeginAddress) / 5;
compactUntil = session.Compact(compactUntil, true);
Assert.AreEqual(compactUntil, fht.Log.BeginAddress);
// Read total keys - all but first 5 (deleted) should be present
for (int i = 0; i < totalRecords; i++)
{
key.Span.Fill(i);
var(status, output) = session.Read(key, userContext: i < 10 ? 1 : 0);
if (status == Status.PENDING)
{
session.CompletePending(true);
}
else
{
if (i < 10)
{
Assert.AreEqual(Status.NOTFOUND, status);
}
else
{
Assert.AreEqual(Status.OK, status);
Assert.IsTrue(output.Item1.Memory.Span.Slice(0, output.Item2).SequenceEqual(key.Span));
output.Item1.Dispose();
}
}
}
// Test iteration of distinct live keys
using (var iter = session.Iterate())
{
int count = 0;
while (iter.GetNext(out RecordInfo recordInfo))
{
var k = iter.GetKey();
Assert.GreaterOrEqual(k.Span[0], 10);
count++;
}
Assert.AreEqual(190, count);
}
// Test iteration of all log records
using (var iter = fht.Log.Scan(fht.Log.BeginAddress, fht.Log.TailAddress))
{
int count = 0;
while (iter.GetNext(out RecordInfo recordInfo))
{
var k = iter.GetKey();
Assert.GreaterOrEqual(k.Span[0], 5);
count++;
}
// Includes 190 live records + 5 deleted records
Assert.AreEqual(195, count);
}
}
public void LogReadAsyncBasicTest([Values] ParameterDefaultsIteratorType iteratorType, [Values] TestUtils.DeviceType deviceType)
{
int entryLength = 20;
int numEntries = 500;
int entryFlag = 9999;
string filename = path + "LogReadAsync" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename);
log = new FasterLog(new FasterLogSettings {
LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path
});
byte[] entry = new byte[entryLength];
// Set Default entry data
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
// Enqueue but set each Entry in a way that can differentiate between entries
for (int i = 0; i < numEntries; i++)
{
// Flag one part of entry data that corresponds to index
if (i < entryLength)
{
entry[i] = (byte)entryFlag;
}
// puts back the previous entry value
if ((i > 0) && (i < entryLength))
{
entry[i - 1] = (byte)(i - 1);
}
log.Enqueue(entry);
}
// Commit to the log
log.Commit(true);
// Read one entry based on different parameters for AsyncReadOnly and verify
switch (iteratorType)
{
case ParameterDefaultsIteratorType.DefaultParams:
// Read one entry and verify
var record = log.ReadAsync(log.BeginAddress);
var foundFlagged = record.Result.Item1[0]; // 15
var foundEntry = record.Result.Item1[1]; // 1
var foundTotal = record.Result.Item2;
Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail reading Flagged Entry");
Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry");
Assert.AreEqual(entryLength, foundTotal, $"Fail reading Total");
break;
case ParameterDefaultsIteratorType.LengthParam:
// Read one entry and verify
record = log.ReadAsync(log.BeginAddress, 208);
foundFlagged = record.Result.Item1[0]; // 15
foundEntry = record.Result.Item1[1]; // 1
foundTotal = record.Result.Item2;
Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail reading Flagged Entry");
Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry");
Assert.AreEqual(entryLength, foundTotal, $"Fail readingTotal");
break;
case ParameterDefaultsIteratorType.TokenParam:
var cts = new CancellationToken();
// Read one entry and verify
record = log.ReadAsync(log.BeginAddress, 104, cts);
foundFlagged = record.Result.Item1[0]; // 15
foundEntry = record.Result.Item1[1]; // 1
foundTotal = record.Result.Item2;
Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail readingFlagged Entry");
Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry");
Assert.AreEqual(entryLength, foundTotal, $"Fail reading Total");
// Read one entry as IMemoryOwner and verify
var recordMemoryOwner = log.ReadAsync(log.BeginAddress, MemoryPool <byte> .Shared, 104, cts);
var foundFlaggedMem = recordMemoryOwner.Result.Item1.Memory.Span[0]; // 15
var foundEntryMem = recordMemoryOwner.Result.Item1.Memory.Span[1]; // 1
var foundTotalMem = recordMemoryOwner.Result.Item2;
Assert.IsTrue(foundFlagged == foundFlaggedMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundFlaggedMem} expected: {foundFlagged}");
Assert.IsTrue(foundEntry == foundEntryMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundEntryMem} expected: {foundEntry}");
Assert.IsTrue(foundTotal == foundTotalMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundTotalMem} expected: {foundTotal}");
break;
default:
Assert.Fail("Unknown case ParameterDefaultsIteratorType.DefaultParams:");
break;
}
}
public void BlittableIterationTest1([Values] TestUtils.DeviceType deviceType)
{
string filename = path + "BlittableIterationTest1" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(1L << 20, new LogSettings {
LogDevice = log, MemorySizeBits = 15, PageSizeBits = 9, SegmentSizeBits = 22
});
using var session = fht.For(new FunctionsCompaction()).NewSession <FunctionsCompaction>();
const int totalRecords = 500;
var start = fht.Log.TailAddress;
for (int i = 0; i < totalRecords; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, 0, 0);
}
int count = 0;
var iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
Assert.AreEqual(iter.GetKey().kfield1, iter.GetValue().vfield1);
}
iter.Dispose();
Assert.AreEqual(totalRecords, count);
for (int i = 0; i < totalRecords; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = 2 * i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, 0);
}
count = 0;
iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
Assert.AreEqual(iter.GetKey().kfield1 * 2, iter.GetValue().vfield1);
}
iter.Dispose();
Assert.AreEqual(totalRecords, count);
for (int i = totalRecords / 2; i < totalRecords; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, 0);
}
count = 0;
iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
}
iter.Dispose();
Assert.AreEqual(totalRecords, count);
for (int i = 0; i < totalRecords; i += 2)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, 0);
}
count = 0;
iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
}
iter.Dispose();
Assert.AreEqual(totalRecords, count);
for (int i = 0; i < totalRecords; i += 2)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Delete(ref key1, 0);
}
count = 0;
iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
}
iter.Dispose();
Assert.AreEqual(totalRecords / 2, count);
for (int i = 0; i < totalRecords; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = 3 * i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, 0);
}
count = 0;
iter = session.Iterate();
while (iter.GetNext(out var recordInfo))
{
count++;
Assert.AreEqual(iter.GetKey().kfield1 * 3, iter.GetValue().vfield1);
}
iter.Dispose();
Assert.AreEqual(totalRecords, count);
}
public unsafe void NativeInMemWriteRead2()
{
// Just use this one instead of all four devices since InMemWriteRead covers all four devices
deviceType = TestUtils.DeviceType.MLSD;
int count = 200;
string filename = path + "NativeInMemWriteRead2" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
// (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 });
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
Random r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
}
r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING)
{
session.CompletePending(true);
}
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
// Clean up and retry - should not find now
fht.Log.ShiftBeginAddress(fht.Log.TailAddress);
r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
Assert.AreEqual(Status.NOTFOUND, session.Read(ref key1, ref input, ref output, Empty.Default, 0));
}
}
public unsafe void TestShiftHeadAddress([Values] TestUtils.DeviceType deviceType)
{
InputStruct input = default;
int count = 200;
string filename = path + "TestShiftHeadAddress" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10
});
session = fht.For(new Functions()).NewSession <Functions>();
Random r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
}
r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
var value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING)
{
session.CompletePending(true);
}
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
}
// Shift head and retry - should not find in main memory now
fht.Log.FlushAndEvict(true);
r = new Random(10);
for (int c = 0; c < count; c++)
{
var i = r.Next(10000);
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
Status foundStatus = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
Assert.AreEqual(Status.PENDING, foundStatus);
session.CompletePending(true);
}
}
public unsafe void NativeInMemRMWNoRefKeys([Values] TestUtils.DeviceType deviceType)
{
InputStruct input = default;
string filename = path + "NativeInMemRMWNoRefKeys" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10
});
session = fht.For(new Functions()).NewSession <Functions>();
var nums = Enumerable.Range(0, 1000).ToArray();
var rnd = new Random(11);
for (int i = 0; i < nums.Length; ++i)
{
int randomIndex = rnd.Next(nums.Length);
int temp = nums[randomIndex];
nums[randomIndex] = nums[i];
nums[i] = temp;
}
for (int j = 0; j < nums.Length; ++j)
{
var i = nums[j];
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
input = new InputStruct {
ifield1 = i, ifield2 = i + 1
};
session.RMW(ref key1, ref input, Empty.Default, 0);
}
for (int j = 0; j < nums.Length; ++j)
{
var i = nums[j];
var key1 = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
input = new InputStruct {
ifield1 = i, ifield2 = i + 1
};
session.RMW(key1, input); // no ref and do not set any other params
}
OutputStruct output = default;
Status status;
KeyStruct key;
for (int j = 0; j < nums.Length; ++j)
{
var i = nums[j];
key = new KeyStruct {
kfield1 = i, kfield2 = i + 1
};
ValueStruct value = new ValueStruct {
vfield1 = i, vfield2 = i + 1
};
status = session.Read(ref key, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
Assert.AreEqual(2 * value.vfield1, output.value.vfield1);
Assert.AreEqual(2 * value.vfield2, output.value.vfield2);
}
key = new KeyStruct {
kfield1 = nums.Length, kfield2 = nums.Length + 1
};
status = session.Read(ref key, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.NOTFOUND, status);
}
public void ReadAtAddressReadFlagsSkipReadCache()
{
// Another ReadFlag functional test so one device is enough
deviceType = TestUtils.DeviceType.MLSD;
string filename = path + "ReadAtAddressReadFlagsSkipReadCache" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29, ReadCacheSettings = new ReadCacheSettings()
});
SkipReadCacheFunctions functions = new();
using var skipReadCacheSession = fht.For(functions).NewSession <SkipReadCacheFunctions>();
InputStruct input = default;
OutputStruct output = default;
var key1 = new KeyStruct {
kfield1 = 13, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = 23, vfield2 = 24
};
var readAtAddress = fht.Log.BeginAddress;
Status status;
skipReadCacheSession.Upsert(ref key1, ref value, Empty.Default, 0);
void VerifyOutput()
{
Assert.AreEqual(-1, functions.expectedReadAddress); // make sure the test executed
Assert.AreEqual(value.vfield1, output.value.vfield1);
Assert.AreEqual(value.vfield2, output.value.vfield2);
Assert.AreEqual(13, key1.kfield1);
Assert.AreEqual(14, key1.kfield2);
}
void VerifyResult()
{
if (status == Status.PENDING)
{
skipReadCacheSession.CompletePendingWithOutputs(out var completedOutputs, wait: true);
(status, output) = TestUtils.GetSinglePendingResult(completedOutputs);
}
Assert.AreEqual(Status.OK, status);
VerifyOutput();
}
// This will just be an ordinary read, as the record is in memory.
functions.expectedReadAddress = readAtAddress;
status = skipReadCacheSession.Read(ref key1, ref input, ref output);
Assert.AreEqual(Status.OK, status);
VerifyOutput();
// ReadCache is used when the record is read from disk.
fht.Log.FlushAndEvict(wait: true);
// SkipReadCache is primarily for indexing, so a read during index scan does not result in a readcache update.
// Reading at a normal logical address will not use the readcache, because the "readcache" bit is not set in that logical address.
// And we cannot get a readcache address, since reads satisfied from the readcache pass kInvalidAddress to functions.
// Therefore, we test here simply that we do not put it in the readcache when we tell it not to.
// Do not put it into the read cache.
functions.expectedReadAddress = readAtAddress;
RecordInfo recordInfo = new() { PreviousAddress = readAtAddress };
status = skipReadCacheSession.Read(ref key1, ref input, ref output, ref recordInfo, ReadFlags.SkipReadCache);
VerifyResult();
Assert.AreEqual(fht.ReadCache.BeginAddress, fht.ReadCache.TailAddress);
// Put it into the read cache.
functions.expectedReadAddress = readAtAddress;
recordInfo.PreviousAddress = readAtAddress; // Read*() sets this to the record's PreviousAddress (so caller can follow the chain), so reinitialize it.
status = skipReadCacheSession.Read(ref key1, ref input, ref output, ref recordInfo);
VerifyResult();
Assert.Less(fht.ReadCache.BeginAddress, fht.ReadCache.TailAddress);
// Now this will read from the read cache.
functions.expectedReadAddress = Constants.kInvalidAddress;
status = skipReadCacheSession.Read(ref key1, ref input, ref output);
Assert.AreEqual(Status.OK, status);
VerifyOutput();
}
public void CommitRecordBoundedGrowthTest([Values] TestUtils.DeviceType deviceType)
{
var cookie = new byte[100];
new Random().NextBytes(cookie);
var filename = path + "boundedGrowth" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true);
var logSettings = new FasterLogSettings {
LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true
};
log = new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < 5 * numEntries; i++)
{
log.Enqueue(entry);
}
// for comparison, insert some entries without any commit records
var referenceTailLength = log.TailAddress;
var enqueueDone = new ManualResetEventSlim();
var commitThreads = new List <Thread>();
// Make sure to not spin up too many commit threads, otherwise we might clog epochs and halt progress
for (var i = 0; i < Math.Max(1, Environment.ProcessorCount - 1); i++)
{
commitThreads.Add(new Thread(() =>
{
// Otherwise, absolutely clog the commit pipeline
while (!enqueueDone.IsSet)
{
log.Commit();
}
}));
}
foreach (var t in commitThreads)
{
t.Start();
}
for (int i = 0; i < 5 * numEntries; i++)
{
log.Enqueue(entry);
}
enqueueDone.Set();
foreach (var t in commitThreads)
{
t.Join();
}
// TODO: Hardcoded constant --- if this number changes in FasterLogRecoverInfo, it needs to be updated here too
var commitRecordSize = 44;
var logTailGrowth = log.TailAddress - referenceTailLength;
// Check that we are not growing the log more than one commit record per user entry
Assert.IsTrue(logTailGrowth - referenceTailLength <= commitRecordSize * 5 * numEntries);
// Ensure clean shutdown
log.Commit(true);
}
public void FasterLogSimpleFastCommitTest([Values] TestUtils.DeviceType deviceType)
{
var cookie = new byte[100];
new Random().NextBytes(cookie);
var filename = path + "fastCommit" + deviceType.ToString() + ".log";
device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true);
var logSettings = new FasterLogSettings {
LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true, TryRecoverLatest = false
};
log = new FasterLog(logSettings);
byte[] entry = new byte[entryLength];
for (int i = 0; i < entryLength; i++)
{
entry[i] = (byte)i;
}
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie1 = new byte[100];
new Random().NextBytes(cookie1);
var commitSuccessful = log.CommitStrongly(out var commit1Addr, out _, true, cookie1, 1);
Assert.IsTrue(commitSuccessful);
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie2 = new byte[100];
new Random().NextBytes(cookie2);
commitSuccessful = log.CommitStrongly(out var commit2Addr, out _, true, cookie2, 2);
Assert.IsTrue(commitSuccessful);
for (int i = 0; i < numEntries; i++)
{
log.Enqueue(entry);
}
var cookie6 = new byte[100];
new Random().NextBytes(cookie6);
commitSuccessful = log.CommitStrongly(out var commit6Addr, out _, true, cookie6, 6);
Assert.IsTrue(commitSuccessful);
// Wait for all metadata writes to be complete to avoid a concurrent access exception
log.Dispose();
log = null;
// be a deviant and remove commit metadata files
manager.RemoveAllCommits();
// Recovery should still work
var recoveredLog = new FasterLog(logSettings);
recoveredLog.Recover(1);
Assert.AreEqual(cookie1, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit1Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
recoveredLog = new FasterLog(logSettings);
recoveredLog.Recover(2);
Assert.AreEqual(cookie2, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit2Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
// Default argument should recover to most recent, if TryRecoverLatest is set
logSettings.TryRecoverLatest = true;
recoveredLog = new FasterLog(logSettings);
Assert.AreEqual(cookie6, recoveredLog.RecoveredCookie);
Assert.AreEqual(commit6Addr, recoveredLog.TailAddress);
recoveredLog.Dispose();
}
public void NativeInMemWriteReadDelete2()
{
// Just set this one since Write Read Delete already does all four devices
deviceType = TestUtils.DeviceType.MLSD;
const int count = 10;
string filename = path + "NativeInMemWriteReadDelete2" + deviceType.ToString() + ".log";
log = TestUtils.CreateTestDevice(deviceType, filename);
fht = new FasterKV <KeyStruct, ValueStruct>
// (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 });
(128, new LogSettings {
LogDevice = log, MemorySizeBits = 29
});
session = fht.For(new Functions()).NewSession <Functions>();
InputStruct input = default;
OutputStruct output = default;
for (int i = 0; i < 10 * count; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = i, vfield2 = 24
};
session.Upsert(ref key1, ref value, Empty.Default, 0);
}
for (int i = 0; i < 10 * count; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = 14
};
session.Delete(ref key1, Empty.Default, 0);
}
for (int i = 0; i < 10 * count; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = 14
};
var value = new ValueStruct {
vfield1 = i, vfield2 = 24
};
var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.NOTFOUND, status);
session.Upsert(ref key1, ref value, Empty.Default, 0);
}
for (int i = 0; i < 10 * count; i++)
{
var key1 = new KeyStruct {
kfield1 = i, kfield2 = 14
};
var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0);
AssertCompleted(Status.OK, status);
}
}
