public void ScanWithoutRecoverTest([Values] TestUtils.DeviceType deviceType) { // You may also force an iterator to start at the specified begin address, i.e., without recovering: recover parameter = false // Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup string filename = path + "LogScanWithoutRecover" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); log = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path }); PopulateLog(log); // Read the log int currentEntry = 9; // since starting at specified address of 1000, need to set current entry as 9 so verification starts at proper spot using (var iter = log.Scan(1000, 100_000_000, recover: false)) { while (iter.GetNext(out byte[] result, out _, out _)) { if (currentEntry < entryLength) { // Span Batch only added first entry several times so have separate verification Assert.AreEqual((byte)entryFlag, result[currentEntry]); currentEntry++; } } } // Make sure expected length is same as current - also makes sure that data verification was not skipped Assert.AreEqual(entryLength, currentEntry); }
public void ScanByNameTest([Values] TestUtils.DeviceType deviceType) { //You can persist iterators(or more precisely, their CompletedUntilAddress) as part of a commit by simply naming them during their creation. // Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup string filename = path + "LogScanByName" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); log = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path }); PopulateLog(log); // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; using (var iter = log.Scan(0, 100_000_000, name: "TestScan", recover: true)) { while (iter.GetNext(out byte[] result, out _, out _)) { if (currentEntry < entryLength) { // Span Batch only added first entry several times so have separate verification Assert.AreEqual((byte)entryFlag, result[currentEntry]); currentEntry++; } } } // Make sure expected length is same as current - also makes sure that data verification was not skipped Assert.AreEqual(entryLength, currentEntry); }
public void UpsertNoRefNoDefaultsTest() { // Just checking more parameter values so one device is enough deviceType = TestUtils.DeviceType.MLSD; string filename = path + "UpsertNoRefNoDefaultsTest" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 29 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; session.Upsert(key1, value, Empty.Default, 0); var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); }
public void ScanUncommittedTest([Values] TestUtils.DeviceType deviceType) { // Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup string filename = path + "LogScan" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); log = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path }); PopulateUncommittedLog(log); // Setting scanUnCommitted to true is actual test here. // Read the log - Look for the flag so know each entry is unique and still reads uncommitted int currentEntry = 0; using (var iter = log.Scan(0, 100_000_000, scanUncommitted: true)) { while (iter.GetNext(out byte[] result, out _, out _)) { if (currentEntry < entryLength) { // Span Batch only added first entry several times so have separate verification Assert.AreEqual((byte)entryFlag, result[currentEntry]); currentEntry++; } } } // Make sure expected length is same as current - also makes sure that data verification was not skipped Assert.AreEqual(entryLength, currentEntry); }
public void ReadBareMinParams([Values] TestUtils.DeviceType deviceType) { string filename = path + "ReadBareMinParams" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); session = fht.For(new Functions()).NewSession <Functions>(); var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; session.Upsert(ref key1, ref value, Empty.Default, 0); var(status, output) = session.Read(key1); AssertCompleted(Status.OK, status); Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); Assert.AreEqual(key1.kfield1, 13); Assert.AreEqual(key1.kfield2, 14); }
public void UpsertDefaultsTest([Values] TestUtils.DeviceType deviceType) { string filename = path + "UpsertDefaultsTest" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; Assert.AreEqual(0, fht.EntryCount); session.Upsert(ref key1, ref value); var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); Assert.AreEqual(1, fht.EntryCount); Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); }
public void ReadAtAddressReadFlagsNone() { // Just functional test of ReadFlag so one device is enough deviceType = TestUtils.DeviceType.MLSD; string filename = path + "ReadAtAddressReadFlagsNone" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 29 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; var readAtAddress = fht.Log.BeginAddress; session.Upsert(ref key1, ref value, Empty.Default, 0); var status = session.ReadAtAddress(readAtAddress, ref input, ref output, ReadFlags.None, Empty.Default, 0); AssertCompleted(Status.OK, status); Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); Assert.AreEqual(key1.kfield1, 13); Assert.AreEqual(key1.kfield2, 14); }
public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.DeviceType deviceType) { string path = TestUtils.MethodTestDir + "/"; string filename = path + "TestDisposeRelease" + deviceType.ToString() + ".log"; DirectoryInfo di = Directory.CreateDirectory(path); IDevice device = TestUtils.CreateTestDevice(deviceType, filename); FasterLog fasterLog = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path, LogChecksum = LogChecksumType.PerEntry }); Assert.IsTrue(fasterLog.TryEnqueue(new byte[100], out _)); fasterLog.Commit(spinWait: true); fasterLog.Dispose(); device.Dispose(); while (true) { try { di.Delete(recursive: true); break; } catch { } } }
public void ScanConsumerTest([Values] TestUtils.DeviceType deviceType) { // Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup string filename = path + "LogScanDefault" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); log = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path }); PopulateLog(log); // Basic default scan from start to end // Indirectly used in other tests, but good to have the basic test here for completeness // Read the log - Look for the flag so know each entry is unique var consumer = new TestConsumer(); using (var iter = log.Scan(0, 100_000_000)) { while (iter.TryConsumeNext(consumer)) { } } // Make sure expected length is same as current - also makes sure that data verification was not skipped Assert.AreEqual(entryLength, consumer.currentEntry); }
public void DiskWriteScanBasicTest([Values] TestUtils.DeviceType deviceType) { log = TestUtils.CreateTestDevice(deviceType, $"{path}DiskWriteScanBasicTest_{deviceType}.log"); objlog = TestUtils.CreateTestDevice(deviceType, $"{path}DiskWriteScanBasicTest_{deviceType}.obj.log"); fht = new (128, logSettings : new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 15, PageSizeBits = 9, SegmentSizeBits = 22 }, checkpointSettings : new CheckpointSettings { CheckPointType = CheckpointType.FoldOver }, serializerSettings : new SerializerSettings <MyKey, MyValue> { keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer() } ); using var session = fht.For(new MyFunctions()).NewSession <MyFunctions>(); using var s = fht.Log.Subscribe(new LogObserver()); var start = fht.Log.TailAddress; for (int i = 0; i < totalRecords; i++) { var _key = new MyKey { key = i }; var _value = new MyValue { value = i }; session.Upsert(ref _key, ref _value, Empty.Default, 0); if (i % 100 == 0) { fht.Log.FlushAndEvict(true); } } fht.Log.FlushAndEvict(true); using (var iter = fht.Log.Scan(start, fht.Log.TailAddress, ScanBufferingMode.SinglePageBuffering)) { int val; for (val = 0; iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value); ++val) { Assert.AreEqual(val, key.key, $"log scan 1: key"); Assert.AreEqual(val, value.value, $"log scan 1: value"); } Assert.AreEqual(val, totalRecords, $"log scan 1: totalRecords"); } using (var iter = fht.Log.Scan(start, fht.Log.TailAddress, ScanBufferingMode.DoublePageBuffering)) { int val; for (val = 0; iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value); ++val) { Assert.AreEqual(val, key.key, $"log scan 2: key"); Assert.AreEqual(val, value.value, $"log scan 2: value"); } Assert.AreEqual(val, totalRecords, $"log scan 2: totalRecords"); } }
private void Setup(long size, LogSettings logSettings, TestUtils.DeviceType deviceType) { string filename = path + TestContext.CurrentContext.Test.Name + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); logSettings.LogDevice = log; fht = new FasterKV <KeyStruct, ValueStruct>(size, logSettings); session = fht.For(new Functions()).NewSession <Functions>(); }
public void DeltaLogTest1([Values] TestUtils.DeviceType deviceType) { const int TotalCount = 200; string filename = $"{path}delta_{deviceType}.log"; TestUtils.RecreateDirectory(path); device = TestUtils.CreateTestDevice(deviceType, filename); device.Initialize(-1); using DeltaLog deltaLog = new DeltaLog(device, 12, 0); Random r = new (20); int i; SectorAlignedBufferPool bufferPool = new(1, (int)device.SectorSize); deltaLog.InitializeForWrites(bufferPool); for (i = 0; i < TotalCount; i++) { int _len = 1 + r.Next(254); long address; while (true) { deltaLog.Allocate(out int maxLen, out address); if (_len <= maxLen) { break; } deltaLog.Seal(0); } for (int j = 0; j < _len; j++) { unsafe { *(byte *)(address + j) = (byte)_len; } } deltaLog.Seal(_len, i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA); } deltaLog.FlushAsync().Wait(); deltaLog.InitializeForReads(); r = new (20); for (i = 0; deltaLog.GetNext(out long address, out int len, out var type); i++) { int _len = 1 + r.Next(254); Assert.AreEqual(i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA, type); Assert.AreEqual(len, _len); for (int j = 0; j < len; j++) { unsafe { Assert.AreEqual((byte)_len, *(byte *)(address + j)); }; } } Assert.AreEqual(TotalCount, i, $"i={i} and TotalCount={TotalCount}"); bufferPool.Free(); }
public void Setup() { path = TestUtils.MethodTestDir + "/"; // Clean up log files from previous test runs in case they weren't cleaned up TestUtils.DeleteDirectory(path, wait: true); if (TestContext.CurrentContext.Test.Arguments.Length == 0) { // Default log creation log = Devices.CreateLogDevice(path + "/GenericLogCompactionTests.log", deleteOnClose: true); objlog = Devices.CreateLogDevice(path + "/GenericLogCompactionTests.obj.log", deleteOnClose: true); fht = new FasterKV <MyKey, MyValue> (128, logSettings: new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9 }, checkpointSettings: new CheckpointSettings { CheckPointType = CheckpointType.FoldOver }, serializerSettings: new SerializerSettings <MyKey, MyValue> { keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer() } ); } else { // For this class, deviceType is the only parameter. Using this to illustrate the approach; NUnit doesn't provide metadata for arguments, // so for multi-parameter tests it is probably better to stay with the "separate SetUp method" approach. var deviceType = (TestUtils.DeviceType)TestContext.CurrentContext.Test.Arguments[0]; log = TestUtils.CreateTestDevice(deviceType, $"{path}LogCompactBasicTest_{deviceType}.log"); objlog = TestUtils.CreateTestDevice(deviceType, $"{path}LogCompactBasicTest_{deviceType}.obj.log"); fht = new FasterKV <MyKey, MyValue> (128, logSettings: new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9, SegmentSizeBits = 22 }, checkpointSettings: new CheckpointSettings { CheckPointType = CheckpointType.FoldOver }, serializerSettings: new SerializerSettings <MyKey, MyValue> { keySerializer = () => new MyKeySerializer(), valueSerializer = () => new MyValueSerializer() } ); } session = fht.For(new MyFunctionsDelete()).NewSession <MyFunctionsDelete>(); }
public void NativeInMemWriteReadDelete([Values] TestUtils.DeviceType deviceType) { string filename = path + "NativeInMemWriteReadDelete" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, PageSizeBits = 10, MemorySizeBits = 12, SegmentSizeBits = 22 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; session.Upsert(ref key1, ref value, Empty.Default, 0); var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); session.Delete(ref key1, Empty.Default, 0); status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.NOTFOUND, status); var key2 = new KeyStruct { kfield1 = 14, kfield2 = 15 }; var value2 = new ValueStruct { vfield1 = 24, vfield2 = 25 }; session.Upsert(ref key2, ref value2, Empty.Default, 0); status = session.Read(ref key2, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); Assert.AreEqual(value2.vfield1, output.value.vfield1); Assert.AreEqual(value2.vfield2, output.value.vfield2); }
public void StringBasicTest([Values] TestUtils.DeviceType deviceType) { string logfilename = path + "GenericStringTests" + deviceType.ToString() + ".log"; string objlogfilename = path + "GenericStringTests" + deviceType.ToString() + ".obj.log"; log = TestUtils.CreateTestDevice(deviceType, logfilename); objlog = TestUtils.CreateTestDevice(deviceType, objlogfilename); fht = new FasterKV <string, string>( 1L << 20, // size of hash table in #cache lines; 64 bytes per cache line new LogSettings { LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, MemorySizeBits = 14, PageSizeBits = 9, SegmentSizeBits = 22 } // log device ); session = fht.For(new MyFuncs()).NewSession <MyFuncs>(); const int totalRecords = 200; for (int i = 0; i < totalRecords; i++) { var _key = $"{i}"; var _value = $"{i}";; session.Upsert(ref _key, ref _value, Empty.Default, 0); } session.CompletePending(true); Assert.AreEqual(totalRecords, fht.EntryCount); for (int i = 0; i < totalRecords; i++) { string input = default; string output = default; var key = $"{i}"; var value = $"{i}"; if (session.Read(ref key, ref input, ref output, Empty.Default, 0) == Status.PENDING) { session.CompletePending(true); } else { Assert.AreEqual(value, output); } } }
public void UpsertSerialNumberTest() { // Simple Upsert of Serial Number test so one device is enough deviceType = TestUtils.DeviceType.MLSD; string filename = path + "UpsertSerialNumberTest" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 29 }); session = fht.For(new Functions()).NewSession <Functions>(); int numKeys = 100; int keyMod = 10; int maxLap = numKeys / keyMod; InputStruct input = default; OutputStruct output = default; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; var key = new KeyStruct { kfield1 = 13, kfield2 = 14 }; for (int i = 0; i < numKeys; i++) { // lap is used to illustrate the changing values var lap = i / keyMod; session.Upsert(ref key, ref value, serialNo: lap); } // Now verify for (int j = 0; j < numKeys; j++) { var status = session.Read(ref key, ref input, ref output, serialNo: maxLap + 1); AssertCompleted(Status.OK, status); Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); } }
public void MemoryLogCompactionTest1([Values] TestUtils.DeviceType deviceType) { string filename = path + "MemoryLogCompactionTests1" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <ReadOnlyMemory <int>, Memory <int> > (1L << 20, new LogSettings { LogDevice = log, MemorySizeBits = 12, PageSizeBits = 10, SegmentSizeBits = 22 }); using var session = fht.For(new MemoryCompaction()).NewSession <MemoryCompaction>(); var key = new Memory <int>(new int[20]); var value = new Memory <int>(new int[20]); const int totalRecords = 200; var start = fht.Log.TailAddress; for (int i = 0; i < totalRecords; i++) { key.Span.Fill(i); value.Span.Fill(i); session.Upsert(key, value); if (i < 5) { session.Delete(key); // in-place delete } } for (int i = 5; i < 10; i++) { key.Span.Fill(i); value.Span.Fill(i); session.Delete(key); // tombstone inserted } // Compact log var compactUntil = fht.Log.BeginAddress + (fht.Log.TailAddress - fht.Log.BeginAddress) / 5; compactUntil = session.Compact(compactUntil, true); Assert.AreEqual(compactUntil, fht.Log.BeginAddress); // Read total keys - all but first 5 (deleted) should be present for (int i = 0; i < totalRecords; i++) { key.Span.Fill(i); var(status, output) = session.Read(key, userContext: i < 10 ? 1 : 0); if (status == Status.PENDING) { session.CompletePending(true); } else { if (i < 10) { Assert.AreEqual(Status.NOTFOUND, status); } else { Assert.AreEqual(Status.OK, status); Assert.IsTrue(output.Item1.Memory.Span.Slice(0, output.Item2).SequenceEqual(key.Span)); output.Item1.Dispose(); } } } // Test iteration of distinct live keys using (var iter = session.Iterate()) { int count = 0; while (iter.GetNext(out RecordInfo recordInfo)) { var k = iter.GetKey(); Assert.GreaterOrEqual(k.Span[0], 10); count++; } Assert.AreEqual(190, count); } // Test iteration of all log records using (var iter = fht.Log.Scan(fht.Log.BeginAddress, fht.Log.TailAddress)) { int count = 0; while (iter.GetNext(out RecordInfo recordInfo)) { var k = iter.GetKey(); Assert.GreaterOrEqual(k.Span[0], 5); count++; } // Includes 190 live records + 5 deleted records Assert.AreEqual(195, count); } }
public void LogReadAsyncBasicTest([Values] ParameterDefaultsIteratorType iteratorType, [Values] TestUtils.DeviceType deviceType) { int entryLength = 20; int numEntries = 500; int entryFlag = 9999; string filename = path + "LogReadAsync" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename); log = new FasterLog(new FasterLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = path }); byte[] entry = new byte[entryLength]; // Set Default entry data for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } // Enqueue but set each Entry in a way that can differentiate between entries for (int i = 0; i < numEntries; i++) { // Flag one part of entry data that corresponds to index if (i < entryLength) { entry[i] = (byte)entryFlag; } // puts back the previous entry value if ((i > 0) && (i < entryLength)) { entry[i - 1] = (byte)(i - 1); } log.Enqueue(entry); } // Commit to the log log.Commit(true); // Read one entry based on different parameters for AsyncReadOnly and verify switch (iteratorType) { case ParameterDefaultsIteratorType.DefaultParams: // Read one entry and verify var record = log.ReadAsync(log.BeginAddress); var foundFlagged = record.Result.Item1[0]; // 15 var foundEntry = record.Result.Item1[1]; // 1 var foundTotal = record.Result.Item2; Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail reading Flagged Entry"); Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry"); Assert.AreEqual(entryLength, foundTotal, $"Fail reading Total"); break; case ParameterDefaultsIteratorType.LengthParam: // Read one entry and verify record = log.ReadAsync(log.BeginAddress, 208); foundFlagged = record.Result.Item1[0]; // 15 foundEntry = record.Result.Item1[1]; // 1 foundTotal = record.Result.Item2; Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail reading Flagged Entry"); Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry"); Assert.AreEqual(entryLength, foundTotal, $"Fail readingTotal"); break; case ParameterDefaultsIteratorType.TokenParam: var cts = new CancellationToken(); // Read one entry and verify record = log.ReadAsync(log.BeginAddress, 104, cts); foundFlagged = record.Result.Item1[0]; // 15 foundEntry = record.Result.Item1[1]; // 1 foundTotal = record.Result.Item2; Assert.AreEqual((byte)entryFlag, foundFlagged, $"Fail readingFlagged Entry"); Assert.AreEqual(1, foundEntry, $"Fail reading Normal Entry"); Assert.AreEqual(entryLength, foundTotal, $"Fail reading Total"); // Read one entry as IMemoryOwner and verify var recordMemoryOwner = log.ReadAsync(log.BeginAddress, MemoryPool <byte> .Shared, 104, cts); var foundFlaggedMem = recordMemoryOwner.Result.Item1.Memory.Span[0]; // 15 var foundEntryMem = recordMemoryOwner.Result.Item1.Memory.Span[1]; // 1 var foundTotalMem = recordMemoryOwner.Result.Item2; Assert.IsTrue(foundFlagged == foundFlaggedMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundFlaggedMem} expected: {foundFlagged}"); Assert.IsTrue(foundEntry == foundEntryMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundEntryMem} expected: {foundEntry}"); Assert.IsTrue(foundTotal == foundTotalMem, $"MemoryPool-based ReadAsync result does not match that of the byte array one. value: {foundTotalMem} expected: {foundTotal}"); break; default: Assert.Fail("Unknown case ParameterDefaultsIteratorType.DefaultParams:"); break; } }
public void BlittableIterationTest1([Values] TestUtils.DeviceType deviceType) { string filename = path + "BlittableIterationTest1" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (1L << 20, new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 9, SegmentSizeBits = 22 }); using var session = fht.For(new FunctionsCompaction()).NewSession <FunctionsCompaction>(); const int totalRecords = 500; var start = fht.Log.TailAddress; for (int i = 0; i < totalRecords; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, 0, 0); } int count = 0; var iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; Assert.AreEqual(iter.GetKey().kfield1, iter.GetValue().vfield1); } iter.Dispose(); Assert.AreEqual(totalRecords, count); for (int i = 0; i < totalRecords; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = 2 * i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, 0); } count = 0; iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; Assert.AreEqual(iter.GetKey().kfield1 * 2, iter.GetValue().vfield1); } iter.Dispose(); Assert.AreEqual(totalRecords, count); for (int i = totalRecords / 2; i < totalRecords; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, 0); } count = 0; iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; } iter.Dispose(); Assert.AreEqual(totalRecords, count); for (int i = 0; i < totalRecords; i += 2) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, 0); } count = 0; iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; } iter.Dispose(); Assert.AreEqual(totalRecords, count); for (int i = 0; i < totalRecords; i += 2) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Delete(ref key1, 0); } count = 0; iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; } iter.Dispose(); Assert.AreEqual(totalRecords / 2, count); for (int i = 0; i < totalRecords; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = 3 * i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, 0); } count = 0; iter = session.Iterate(); while (iter.GetNext(out var recordInfo)) { count++; Assert.AreEqual(iter.GetKey().kfield1 * 3, iter.GetValue().vfield1); } iter.Dispose(); Assert.AreEqual(totalRecords, count); }
public unsafe void NativeInMemWriteRead2() { // Just use this one instead of all four devices since InMemWriteRead covers all four devices deviceType = TestUtils.DeviceType.MLSD; int count = 200; string filename = path + "NativeInMemWriteRead2" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> // (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); (128, new LogSettings { LogDevice = log, MemorySizeBits = 29 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; Random r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, Empty.Default, 0); } r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING) { session.CompletePending(true); } Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); } // Clean up and retry - should not find now fht.Log.ShiftBeginAddress(fht.Log.TailAddress); r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; Assert.AreEqual(Status.NOTFOUND, session.Read(ref key1, ref input, ref output, Empty.Default, 0)); } }
public unsafe void TestShiftHeadAddress([Values] TestUtils.DeviceType deviceType) { InputStruct input = default; int count = 200; string filename = path + "TestShiftHeadAddress" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); session = fht.For(new Functions()).NewSession <Functions>(); Random r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, Empty.Default, 0); } r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING) { session.CompletePending(true); } Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); } // Shift head and retry - should not find in main memory now fht.Log.FlushAndEvict(true); r = new Random(10); for (int c = 0; c < count; c++) { var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; Status foundStatus = session.Read(ref key1, ref input, ref output, Empty.Default, 0); Assert.AreEqual(Status.PENDING, foundStatus); session.CompletePending(true); } }
public unsafe void NativeInMemRMWNoRefKeys([Values] TestUtils.DeviceType deviceType) { InputStruct input = default; string filename = path + "NativeInMemRMWNoRefKeys" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); session = fht.For(new Functions()).NewSession <Functions>(); var nums = Enumerable.Range(0, 1000).ToArray(); var rnd = new Random(11); for (int i = 0; i < nums.Length; ++i) { int randomIndex = rnd.Next(nums.Length); int temp = nums[randomIndex]; nums[randomIndex] = nums[i]; nums[i] = temp; } for (int j = 0; j < nums.Length; ++j) { var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; session.RMW(ref key1, ref input, Empty.Default, 0); } for (int j = 0; j < nums.Length; ++j) { var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; session.RMW(key1, input); // no ref and do not set any other params } OutputStruct output = default; Status status; KeyStruct key; for (int j = 0; j < nums.Length; ++j) { var i = nums[j]; key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; ValueStruct value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; status = session.Read(ref key, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); Assert.AreEqual(2 * value.vfield1, output.value.vfield1); Assert.AreEqual(2 * value.vfield2, output.value.vfield2); } key = new KeyStruct { kfield1 = nums.Length, kfield2 = nums.Length + 1 }; status = session.Read(ref key, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.NOTFOUND, status); }
public void ReadAtAddressReadFlagsSkipReadCache() { // Another ReadFlag functional test so one device is enough deviceType = TestUtils.DeviceType.MLSD; string filename = path + "ReadAtAddressReadFlagsSkipReadCache" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> (128, new LogSettings { LogDevice = log, MemorySizeBits = 29, ReadCacheSettings = new ReadCacheSettings() }); SkipReadCacheFunctions functions = new(); using var skipReadCacheSession = fht.For(functions).NewSession <SkipReadCacheFunctions>(); InputStruct input = default; OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; var readAtAddress = fht.Log.BeginAddress; Status status; skipReadCacheSession.Upsert(ref key1, ref value, Empty.Default, 0); void VerifyOutput() { Assert.AreEqual(-1, functions.expectedReadAddress); // make sure the test executed Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); Assert.AreEqual(13, key1.kfield1); Assert.AreEqual(14, key1.kfield2); } void VerifyResult() { if (status == Status.PENDING) { skipReadCacheSession.CompletePendingWithOutputs(out var completedOutputs, wait: true); (status, output) = TestUtils.GetSinglePendingResult(completedOutputs); } Assert.AreEqual(Status.OK, status); VerifyOutput(); } // This will just be an ordinary read, as the record is in memory. functions.expectedReadAddress = readAtAddress; status = skipReadCacheSession.Read(ref key1, ref input, ref output); Assert.AreEqual(Status.OK, status); VerifyOutput(); // ReadCache is used when the record is read from disk. fht.Log.FlushAndEvict(wait: true); // SkipReadCache is primarily for indexing, so a read during index scan does not result in a readcache update. // Reading at a normal logical address will not use the readcache, because the "readcache" bit is not set in that logical address. // And we cannot get a readcache address, since reads satisfied from the readcache pass kInvalidAddress to functions. // Therefore, we test here simply that we do not put it in the readcache when we tell it not to. // Do not put it into the read cache. functions.expectedReadAddress = readAtAddress; RecordInfo recordInfo = new() { PreviousAddress = readAtAddress }; status = skipReadCacheSession.Read(ref key1, ref input, ref output, ref recordInfo, ReadFlags.SkipReadCache); VerifyResult(); Assert.AreEqual(fht.ReadCache.BeginAddress, fht.ReadCache.TailAddress); // Put it into the read cache. functions.expectedReadAddress = readAtAddress; recordInfo.PreviousAddress = readAtAddress; // Read*() sets this to the record's PreviousAddress (so caller can follow the chain), so reinitialize it. status = skipReadCacheSession.Read(ref key1, ref input, ref output, ref recordInfo); VerifyResult(); Assert.Less(fht.ReadCache.BeginAddress, fht.ReadCache.TailAddress); // Now this will read from the read cache. functions.expectedReadAddress = Constants.kInvalidAddress; status = skipReadCacheSession.Read(ref key1, ref input, ref output); Assert.AreEqual(Status.OK, status); VerifyOutput(); }
public void CommitRecordBoundedGrowthTest([Values] TestUtils.DeviceType deviceType) { var cookie = new byte[100]; new Random().NextBytes(cookie); var filename = path + "boundedGrowth" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true); var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true }; log = new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < 5 * numEntries; i++) { log.Enqueue(entry); } // for comparison, insert some entries without any commit records var referenceTailLength = log.TailAddress; var enqueueDone = new ManualResetEventSlim(); var commitThreads = new List <Thread>(); // Make sure to not spin up too many commit threads, otherwise we might clog epochs and halt progress for (var i = 0; i < Math.Max(1, Environment.ProcessorCount - 1); i++) { commitThreads.Add(new Thread(() => { // Otherwise, absolutely clog the commit pipeline while (!enqueueDone.IsSet) { log.Commit(); } })); } foreach (var t in commitThreads) { t.Start(); } for (int i = 0; i < 5 * numEntries; i++) { log.Enqueue(entry); } enqueueDone.Set(); foreach (var t in commitThreads) { t.Join(); } // TODO: Hardcoded constant --- if this number changes in FasterLogRecoverInfo, it needs to be updated here too var commitRecordSize = 44; var logTailGrowth = log.TailAddress - referenceTailLength; // Check that we are not growing the log more than one commit record per user entry Assert.IsTrue(logTailGrowth - referenceTailLength <= commitRecordSize * 5 * numEntries); // Ensure clean shutdown log.Commit(true); }
public void FasterLogSimpleFastCommitTest([Values] TestUtils.DeviceType deviceType) { var cookie = new byte[100]; new Random().NextBytes(cookie); var filename = path + "fastCommit" + deviceType.ToString() + ".log"; device = TestUtils.CreateTestDevice(deviceType, filename, deleteOnClose: true); var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.PerEntry, LogCommitManager = manager, FastCommitMode = true, TryRecoverLatest = false }; log = new FasterLog(logSettings); byte[] entry = new byte[entryLength]; for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; } for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie1 = new byte[100]; new Random().NextBytes(cookie1); var commitSuccessful = log.CommitStrongly(out var commit1Addr, out _, true, cookie1, 1); Assert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie2 = new byte[100]; new Random().NextBytes(cookie2); commitSuccessful = log.CommitStrongly(out var commit2Addr, out _, true, cookie2, 2); Assert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) { log.Enqueue(entry); } var cookie6 = new byte[100]; new Random().NextBytes(cookie6); commitSuccessful = log.CommitStrongly(out var commit6Addr, out _, true, cookie6, 6); Assert.IsTrue(commitSuccessful); // Wait for all metadata writes to be complete to avoid a concurrent access exception log.Dispose(); log = null; // be a deviant and remove commit metadata files manager.RemoveAllCommits(); // Recovery should still work var recoveredLog = new FasterLog(logSettings); recoveredLog.Recover(1); Assert.AreEqual(cookie1, recoveredLog.RecoveredCookie); Assert.AreEqual(commit1Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); recoveredLog = new FasterLog(logSettings); recoveredLog.Recover(2); Assert.AreEqual(cookie2, recoveredLog.RecoveredCookie); Assert.AreEqual(commit2Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); // Default argument should recover to most recent, if TryRecoverLatest is set logSettings.TryRecoverLatest = true; recoveredLog = new FasterLog(logSettings); Assert.AreEqual(cookie6, recoveredLog.RecoveredCookie); Assert.AreEqual(commit6Addr, recoveredLog.TailAddress); recoveredLog.Dispose(); }
public void NativeInMemWriteReadDelete2() { // Just set this one since Write Read Delete already does all four devices deviceType = TestUtils.DeviceType.MLSD; const int count = 10; string filename = path + "NativeInMemWriteReadDelete2" + deviceType.ToString() + ".log"; log = TestUtils.CreateTestDevice(deviceType, filename); fht = new FasterKV <KeyStruct, ValueStruct> // (128, new LogSettings { LogDevice = log, MemorySizeBits = 22, SegmentSizeBits = 22, PageSizeBits = 10 }); (128, new LogSettings { LogDevice = log, MemorySizeBits = 29 }); session = fht.For(new Functions()).NewSession <Functions>(); InputStruct input = default; OutputStruct output = default; for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; session.Upsert(ref key1, ref value, Empty.Default, 0); } for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; session.Delete(ref key1, Empty.Default, 0); } for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.NOTFOUND, status); session.Upsert(ref key1, ref value, Empty.Default, 0); } for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var status = session.Read(ref key1, ref input, ref output, Empty.Default, 0); AssertCompleted(Status.OK, status); } }