Esempio n. 1
0
        public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
        {
            // Set up the blob manager so can set lease to it
            TestUtils.IgnoreIfNotRunningAzureTests();
            CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
            var cloudBlobClient = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");

            blobContainer.CreateIfNotExists();
            var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");

            var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
            var device  = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);

            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));

            await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

            device.Dispose();
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
            blobContainer.Delete();
        }
Esempio n. 2
0
        public async ValueTask IncrSnapshotRecoveryCheck([Values] DeviceMode deviceMode)
        {
            ICheckpointManager checkpointManager;

            if (deviceMode == DeviceMode.Local)
            {
                checkpointManager = new DeviceLogCommitCheckpointManager(
                    new LocalStorageNamedDeviceFactory(),
                    new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + $"/RecoveryChecks/IncrSnapshotRecoveryCheck"));
            }
            else
            {
                if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
                {
                    checkpointManager = new DeviceLogCommitCheckpointManager(
                        new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                        new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/IncrSnapshotRecoveryCheck"));
                }
                else
                {
                    return;
                }
            }

            await IncrSnapshotRecoveryCheck(checkpointManager);

            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Esempio n. 3
0
        public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
        {
            // Need this environment variable set AND Azure Storage Emulator running
            if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
            {
                // Set up the blob manager so can set lease to it
                CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
                var cloudBlobClient = storageAccount.CreateCloudBlobClient();
                CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
                blobContainer.CreateIfNotExists();
                var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");

                var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
                var device  = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);

                var checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                    new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
                await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

                device.Dispose();
                checkpointManager.PurgeAll();
                checkpointManager.Dispose();
                blobContainer.Delete();
            }
        }
Esempio n. 4
0
        public void LocalDeviceSimpleRecoveryTest(CheckpointType checkpointType)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            SimpleRecoveryTest1(checkpointType, checkpointManager);
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Esempio n. 5
0
        public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Esempio n. 6
0
        public async Task FasterLogResumePersistedReader2([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
        {
            var    input1     = new byte[] { 0, 1, 2, 3 };
            var    input2     = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
            var    input3     = new byte[] { 11, 12 };
            string readerName = "abc";

            using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
            {
                long originalCompleted;

                using (var l = new FasterLog(new FasterLogSettings {
                    LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
                }))
                {
                    await l.EnqueueAsync(input1);

                    await l.CommitAsync();

                    await l.EnqueueAsync(input2);

                    await l.CommitAsync();

                    await l.EnqueueAsync(input3);

                    await l.CommitAsync();

                    using (var originalIterator = l.Scan(0, long.MaxValue, readerName))
                    {
                        originalIterator.GetNext(out _, out _, out _, out long recoveryAddress);
                        originalIterator.CompleteUntil(recoveryAddress);
                        originalIterator.GetNext(out _, out _, out _, out _);  // move the reader ahead
                        await l.CommitAsync();

                        originalCompleted = originalIterator.CompletedUntilAddress;
                    }
                }

                using (var l = new FasterLog(new FasterLogSettings {
                    LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
                }))
                {
                    using (var recoveredIterator = l.Scan(0, long.MaxValue, readerName))
                    {
                        recoveredIterator.GetNext(out byte[] outBuf, out _, out _, out _);

                        // we should have read in input2, not input1 or input3
                        Assert.True(input2.SequenceEqual(outBuf), $"Original: {input2[0]}, Recovered: {outBuf[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");

                        // TestContext.Progress.WriteLine($"Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
                    }
                }
            }
        }
Esempio n. 7
0
        public void Setup()
        {
            commitPath = TestContext.CurrentContext.TestDirectory + "\\" + TestContext.CurrentContext.Test.Name + "\\";

            if (Directory.Exists(commitPath))
            {
                DeleteDirectory(commitPath);
            }

            device  = Devices.CreateLogDevice(commitPath + "fasterlog.log", deleteOnClose: true);
            manager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(deleteOnClose: true), new DefaultCheckpointNamingScheme(commitPath));
        }
Esempio n. 8
0
 public void PageBlobSimpleRecoveryTest(CheckpointType checkpointType)
 {
     if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
     {
         ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
             new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
             new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
         SimpleRecoveryTest1(checkpointType, checkpointManager);
         checkpointManager.PurgeAll();
         checkpointManager.Dispose();
     }
 }
Esempio n. 9
0
        public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
            {
                ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                    new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
                await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

                checkpointManager.PurgeAll();
                checkpointManager.Dispose();
            }
        }
Esempio n. 10
0
 public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
 {
     if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
     {
         var device            = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
         var checkpointManager = new DeviceLogCommitCheckpointManager(
             new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
             new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
         FasterLogTest1(logChecksum, device, checkpointManager);
         device.Close();
         checkpointManager.PurgeAll();
         checkpointManager.Dispose();
     }
 }
Esempio n. 11
0
        public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
        {
            TestUtils.IgnoreIfNotRunningAzureTests();
            var device            = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));

            await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

            device.Dispose();
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Esempio n. 12
0
        public async Task FasterLog_Test1Write()
        {
            const int entryLength = 1 << 10;

            byte[] staticEntry1       = new byte[entryLength];
            byte[] staticEntry1Brotli = new byte[entryLength];
            for (int i = 0; i < entryLength; i++)
            {
                staticEntry1[i] = (byte)i;
            }
            byte[] staticEntry2       = new byte[entryLength];
            byte[] staticEntry2Brotli = new byte[entryLength];
            for (int i = 0; i < entryLength; i++)
            {
                staticEntry2[i] = (byte)(entryLength - i);
            }
            var path = GetPath();

            using (var logCommitManager = new DeviceLogCommitCheckpointManager(
                       new LocalStorageNamedDeviceFactory(),
                       new DefaultCheckpointNamingScheme(path), true, false)
                   ) {
                using (IDevice device = Devices.CreateLogDevice(path + "hlog.log")) {
                    //FasterLogScanIterator iter;
                    var logSettings = new FASTER.core.FasterLogSettings()
                    {
                        LogDevice        = device,
                        LogChecksum      = LogChecksumType.PerEntry,
                        LogCommitManager = logCommitManager,
                    };

                    using (var log = await FASTER.core.FasterLog.CreateAsync(logSettings)) {
                        //log.TruncateUntilPageStart(0);
                        //await log.CommitAsync();
                        for (int i = 0; i < 1000; i++)
                        {
                            if (BrotliEncoder.TryCompress(staticEntry1, staticEntry1Brotli, out var bytesWritten1, 4, 10))
                            {
                                await log.EnqueueAsync(new Memory <byte>(staticEntry1Brotli).Slice(0, bytesWritten1));
                            }
Esempio n. 13
0
        public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode)
        {
            ICheckpointManager checkpointManager;

            if (deviceMode == DeviceMode.Local)
            {
                checkpointManager = new DeviceLogCommitCheckpointManager(
                    new LocalStorageNamedDeviceFactory(),
                    new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir +
                                                      "/checkpoints/")); // PurgeAll deletes this directory
            }
            else
            {
                TestUtils.IgnoreIfNotRunningAzureTests();
                checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                    new DefaultCheckpointNamingScheme(
                        $"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));
            }

            var path = TestUtils.MethodTestDir + "/";

            using var log = Devices.CreateLogDevice(path + "hlog.log", deleteOnClose: true);
            TestUtils.RecreateDirectory(path);

            using var fht = new FasterKV <long, long>
                                (1 << 10,
                                logSettings: new LogSettings
                    {
                    LogDevice         = log, MutableFraction = 1, PageSizeBits = 10, MemorySizeBits = 20,
                    ReadCacheSettings = null
                },
                                checkpointSettings: new CheckpointSettings { CheckpointManager = checkpointManager }
                                );
            using var s = fht.NewSession(new SimpleFunctions <long, long>());

            var logCheckpoints   = new Dictionary <Guid, int>();
            var indexCheckpoints = new Dictionary <Guid, int>();
            var fullCheckpoints  = new Dictionary <Guid, int>();

            for (var i = 0; i < 10; i++)
            {
                // Do some dummy update
                s.Upsert(0, random.Next());

                var  checkpointType = random.Next(5);
                Guid result         = default;
                switch (checkpointType)
                {
                case 0:
                    fht.TakeHybridLogCheckpoint(out result, CheckpointType.FoldOver);
                    logCheckpoints.Add(result, 0);
                    break;

                case 1:
                    fht.TakeHybridLogCheckpoint(out result, CheckpointType.Snapshot);
                    logCheckpoints.Add(result, 0);
                    break;

                case 2:
                    fht.TakeIndexCheckpoint(out result);
                    indexCheckpoints.Add(result, 0);
                    break;

                case 3:
                    fht.TakeFullCheckpoint(out result, CheckpointType.FoldOver);
                    fullCheckpoints.Add(result, 0);
                    break;

                case 4:
                    fht.TakeFullCheckpoint(out result, CheckpointType.Snapshot);
                    fullCheckpoints.Add(result, 0);
                    break;

                default:
                    Assert.True(false);
                    break;
                }

                await fht.CompleteCheckpointAsync();
            }

            Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                            logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
            Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                            indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));

            if (logCheckpoints.Count != 0)
            {
                var guid = logCheckpoints.First().Key;
                checkpointManager.Purge(guid);
                logCheckpoints.Remove(guid);
                Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
                Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
            }

            if (indexCheckpoints.Count != 0)
            {
                var guid = indexCheckpoints.First().Key;
                checkpointManager.Purge(guid);
                indexCheckpoints.Remove(guid);
                Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
                Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
            }


            if (fullCheckpoints.Count != 0)
            {
                var guid = fullCheckpoints.First().Key;
                checkpointManager.Purge(guid);
                fullCheckpoints.Remove(guid);
                Assert.AreEqual(checkpointManager.GetLogCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                logCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
                Assert.AreEqual(checkpointManager.GetIndexCheckpointTokens().ToDictionary(guid => guid, _ => 0),
                                indexCheckpoints.Union(fullCheckpoints).ToDictionary(e => e.Key, e => e.Value));
            }

            checkpointManager.PurgeAll();
            Assert.IsEmpty(checkpointManager.GetLogCheckpointTokens());
            Assert.IsEmpty(checkpointManager.GetIndexCheckpointTokens());

            checkpointManager.Dispose();
        }
Esempio n. 14
0
        static void Main()
        {
            // Main hybrid log device
            var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");

            // With non-blittable types, you need an object log device in addition to the
            // main device. FASTER serializes the actual objects in the object log.
            var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");

            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(STORAGE_STRING),
                new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));


            var store = new FasterKV <long, string>(
                1L << 17,
                    new LogSettings {
                LogDevice = log, ObjectLogDevice = objlog
                    },
                    new CheckpointSettings {
                CheckpointManager = checkpointManager
                    }
                );

            using (var s = store.NewSession(new Functions()))
            {
                for (long i = 0; i < 20000; i++)
                {
                    var _key  = i;
                    var value = $"value-{i}";
                    s.Upsert(ref _key, ref value);
                }
                s.CompletePending(true);

                long   key     = 23;
                string output  = default;
                string context = "value-23";

                var status = s.Read(ref key, ref output, context);

                if (status != Status.PENDING)
                {
                    if (status == Status.OK && output == context)
                    {
                        Console.WriteLine("Success!");
                    }
                    else
                    {
                        Console.WriteLine("Error!");
                    }
                }
            }

            // Take fold-over checkpoint of FASTER, wait to complete
            store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
            .GetAwaiter().GetResult();

            // Dispose store instance
            store.Dispose();

            // Dispose logs
            log.Dispose();
            objlog.Dispose();

            // Create new store
            log    = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
            objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");

            var store2 = new FasterKV <long, string>(
                1L << 17,
                    new LogSettings {
                LogDevice = log, ObjectLogDevice = objlog
                    },
                    new CheckpointSettings {
                CheckpointManager = checkpointManager
                    }
                );

            // Recover store from latest checkpoint
            store2.Recover();

            using (var s = store2.NewSession(new Functions()))
            {
                // Test Read
                long   key     = 23;
                string output  = default;
                string context = "value-23";

                var status = s.Read(ref key, ref output, context);

                if (status != Status.PENDING)
                {
                    if (status == Status.OK && output == context)
                    {
                        Console.WriteLine("Success!");
                    }
                    else
                    {
                        Console.WriteLine("Error!");
                    }
                }
            }

            store2.Dispose();

            // Purge cloud log files
            log.PurgeAll();
            objlog.PurgeAll();

            // Purge cloud checkpoints - warning all data under specified base path are removed
            checkpointManager.PurgeAll();

            // Dispose devices
            log.Dispose();
            objlog.Dispose();

            // Dispose checkpoint manager
            checkpointManager.Dispose();

            Console.WriteLine("Press <ENTER> to end");
            Console.ReadLine();
        }
Esempio n. 15
0
        public async Task FasterLogResumePersistedReader3([Values] LogChecksumType logChecksum, [Values] bool overwriteLogCommits, [Values] bool removeOutdated)
        {
            var    input1     = new byte[] { 0, 1, 2, 3 };
            var    input2     = new byte[] { 4, 5, 6, 7, 8, 9, 10 };
            var    input3     = new byte[] { 11, 12 };
            string readerName = "abcd";

            using (var logCommitManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(commitPath), overwriteLogCommits, removeOutdated))
            {
                long originalCompleted;

                using (var l = new FasterLog(new FasterLogSettings {
                    LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
                }))
                {
                    await l.EnqueueAsync(input1);

                    await l.CommitAsync();

                    await l.EnqueueAsync(input2);

                    await l.CommitAsync();

                    await l.EnqueueAsync(input3);

                    await l.CommitAsync();

                    using var originalIterator = l.Scan(0, l.TailAddress, readerName);

                    int count = 0;
                    await foreach (var item in originalIterator.GetAsyncEnumerable())
                    {
                        if (count < 2) // we complete 1st and 2nd item read
                        {
                            originalIterator.CompleteUntil(item.nextAddress);
                        }

                        if (count < 1) // we commit only 1st item read
                        {
                            await l.CommitAsync();
                        }

                        count++;
                    }
                    originalCompleted = originalIterator.CompletedUntilAddress;
                }

                using (var l = new FasterLog(new FasterLogSettings {
                    LogDevice = device, PageSizeBits = 16, MemorySizeBits = 16, LogChecksum = logChecksum, LogCommitManager = logCommitManager
                }))
                {
                    using var recoveredIterator = l.Scan(0, l.TailAddress, readerName);

                    int count = 0;
                    await foreach (var item in recoveredIterator.GetAsyncEnumerable())
                    {
                        if (count == 0) // resumed iterator will start at item2
                        {
                            Assert.True(input2.SequenceEqual(item.entry), $"Original: {input2[0]}, Recovered: {item.entry[0]}, Original: {originalCompleted}, Recovered: {recoveredIterator.CompletedUntilAddress}");
                        }
                        count++;
                    }
                    Assert.IsTrue(count == 2);
                }
            }
        }
Esempio n. 16
0
        public async ValueTask SimpleRecoveryTest2([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            var checkpointManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + "/checkpoints4"), false);

            log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/SimpleRecoveryTest2.log", deleteOnClose: true);

            // Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "/checkpoints4");

            fht1 = new FasterKV
                   <AdId, NumClicks>
                       (128,
                       logSettings: new LogSettings {
                LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
            },
                       checkpointSettings: new CheckpointSettings {
                CheckpointManager = checkpointManager, CheckPointType = checkpointType
            }
                       );

            fht2 = new FasterKV
                   <AdId, NumClicks>
                       (128,
                       logSettings: new LogSettings {
                LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
            },
                       checkpointSettings: new CheckpointSettings {
                CheckpointManager = checkpointManager, CheckPointType = checkpointType
            }
                       );


            int numOps     = 5000;
            var inputArray = new AdId[numOps];

            for (int i = 0; i < numOps; i++)
            {
                inputArray[i].adId = i;
            }

            NumClicks value;
            AdInput   inputArg = default;
            Output    output   = default;

            var session1 = fht1.NewSession(new AdSimpleFunctions());

            for (int key = 0; key < numOps; key++)
            {
                value.numClicks = key;
                session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
            }
            fht1.TakeFullCheckpoint(out Guid token);
            fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
            session1.Dispose();

            if (isAsync)
            {
                await fht2.RecoverAsync(token);
            }
            else
            {
                fht2.Recover(token);
            }

            var session2 = fht2.NewSession(new AdSimpleFunctions());

            for (int key = 0; key < numOps; key++)
            {
                var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);

                if (status == Status.PENDING)
                {
                    session2.CompletePending(true);
                }
                else
                {
                    Assert.IsTrue(output.value.numClicks == key);
                }
            }
            session2.Dispose();

            log.Dispose();
            fht1.Dispose();
            fht2.Dispose();
            checkpointManager.Dispose();

            new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "/checkpoints4").Delete(true);
        }