public void TearDown() { device?.Dispose(); device = null; TestUtils.DeleteDirectory(path); }
void TestDeviceWriteRead(IDevice log) { fht = new FasterKV <KeyStruct, ValueStruct> (1L << 20, new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 10 }); var session = fht.NewSession(new Functions()); InputStruct input = default; for (int i = 0; i < 700; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; session.Upsert(ref key1, ref value, Empty.Default, 0); } session.CompletePending(true); // Update first 100 using RMW from storage for (int i = 0; i < 100; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = 1, ifield2 = 1 }; var status = session.RMW(ref key1, ref input, Empty.Default, 0); if (status == Status.PENDING) { session.CompletePending(true); } } for (int i = 0; i < 700; i++) { OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; if (session.Read(ref key1, ref input, ref output, Empty.Default, 0) == Status.PENDING) { session.CompletePending(true); } else { if (i < 100) { Assert.AreEqual(value.vfield1 + 1, output.value.vfield1); Assert.AreEqual(value.vfield2 + 1, output.value.vfield2); } else { Assert.AreEqual(value.vfield1, output.value.vfield1); Assert.AreEqual(value.vfield2, output.value.vfield2); } } } session.Dispose(); fht.Dispose(); fht = null; log.Dispose(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); }
public unsafe void MultiReadSpanByteKeyTest() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); try { using var log = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/MultiReadSpanByteKeyTest.log", deleteOnClose: true); using var fht = new FasterKV <SpanByte, long>( size: 1L << 10, new LogSettings { LogDevice = log, MemorySizeBits = 15, PageSizeBits = 12 }); using var session = fht.For(new MultiReadSpanByteKeyTestFunctions()).NewSession <MultiReadSpanByteKeyTestFunctions>(); for (int i = 0; i < 200; i++) { var key = MemoryMarshal.Cast <char, byte>($"{i}".AsSpan()); fixed(byte *_ = key) session.Upsert(SpanByte.FromFixedSpan(key), i); } // Evict all records to disk fht.Log.FlushAndEvict(true); for (long key = 0; key < 50; key++) { // read each key multiple times for (int i = 0; i < 10; i++) { Assert.AreEqual(key, ReadKey($"{key}")); } } long ReadKey(string keyString) { Status status; var key = MemoryMarshal.Cast <char, byte>(keyString.AsSpan()); fixed(byte *_ = key) status = session.Read(key: SpanByte.FromFixedSpan(key), out var unused); // All keys need to be fetched from disk Assert.AreEqual(Status.PENDING, status); session.CompletePendingWithOutputs(out var completedOutputs, wait: true); var count = 0; var value = 0L; using (completedOutputs) { while (completedOutputs.Next()) { count++; Assert.AreEqual(Status.OK, completedOutputs.Current.Status); value = completedOutputs.Current.Output; } } Assert.AreEqual(1, count); return(value); } } finally { TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } }
public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode) { ICheckpointManager checkpointManager; if (deviceMode == DeviceMode.Local) { checkpointManager = new DeviceLogCommitCheckpointManager( new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir + "/checkpoints/"), false); // PurgeAll deletes this directory } else { TestUtils.IgnoreIfNotRunningAzureTests(); checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString), new DefaultCheckpointNamingScheme( $"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"), false); } var path = TestUtils.MethodTestDir + "/"; using (var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true)) { TestUtils.RecreateDirectory(path); using var fht = new FasterKV <long, long> (1 << 10, logSettings: new LogSettings { LogDevice = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20, ReadCacheSettings = null }, checkpointSettings: new CheckpointSettings { CheckpointManager = checkpointManager } ); using var s = fht.NewSession(new SimpleFunctions <long, long>()); var logCheckpoints = new Dictionary <Guid, int>(); var indexCheckpoints = new Dictionary <Guid, int>(); var fullCheckpoints = new Dictionary <Guid, int>(); for (var i = 0; i < 10; i++) { // Do some dummy update s.Upsert(0, random.Next()); var checkpointType = random.Next(5); Guid result = default; switch (checkpointType) { case 0: fht.TryInitiateHybridLogCheckpoint(out result, CheckpointType.FoldOver); logCheckpoints.Add(result, 0); break; case 1: fht.TryInitiateHybridLogCheckpoint(out result, CheckpointType.Snapshot); logCheckpoints.Add(result, 0); break; case 2: fht.TryInitiateIndexCheckpoint(out result); indexCheckpoints.Add(result, 0); break; case 3: fht.TryInitiateFullCheckpoint(out result, CheckpointType.FoldOver); fullCheckpoints.Add(result, 0); break; case 4: fht.TryInitiateFullCheckpoint(out result, CheckpointType.Snapshot); fullCheckpoints.Add(result, 0); break; default: Assert.True(false); break; } await fht.CompleteCheckpointAsync(); } Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0), logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0), indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); if (logCheckpoints.Count != 0) { var guid = logCheckpoints.First().Key; checkpointManager.Purge(guid); logCheckpoints.Remove(guid); Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0), logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0), indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); } if (indexCheckpoints.Count != 0) { var guid = indexCheckpoints.First().Key; checkpointManager.Purge(guid); indexCheckpoints.Remove(guid); Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0), logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0), indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); } if (fullCheckpoints.Count != 0) { var guid = fullCheckpoints.First().Key; checkpointManager.Purge(guid); fullCheckpoints.Remove(guid); Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0), logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0), indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value)); } checkpointManager.PurgeAll(); Assert.IsEmpty(checkpointManager.GetLogCheckpointTokens()); Assert.IsEmpty(checkpointManager.GetIndexCheckpointTokens()); } checkpointManager.Dispose(); TestUtils.DeleteDirectory(path, wait: true); }