Пример #1
0
        public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
        {
            // Set up the blob manager so can set lease to it
            TestUtils.IgnoreIfNotRunningAzureTests();
            CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
            var cloudBlobClient = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");

            blobContainer.CreateIfNotExists();
            var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");

            var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
            var device  = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);

            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));

            await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

            device.Dispose();
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
            blobContainer.Delete();
        }
Пример #2
0
        public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType)
        {
            // Need this environment variable set AND Azure Storage Emulator running
            if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
            {
                // Set up the blob manager so can set lease to it
                CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
                var cloudBlobClient = storageAccount.CreateCloudBlobClient();
                CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container");
                blobContainer.CreateIfNotExists();
                var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1");

                var blobMgr = new DefaultBlobManager(true, mycloudBlobDir);
                var device  = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr);

                var checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                    new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease"));
                await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

                device.Dispose();
                checkpointManager.PurgeAll();
                checkpointManager.Dispose();
                blobContainer.Delete();
            }
        }
Пример #3
0
        public void TearDown()
        {
            manager.Dispose();
            device.Dispose();

            if (Directory.Exists(commitPath))
            {
                DeleteDirectory(commitPath);
            }
        }
Пример #4
0
        public void LocalDeviceSimpleRecoveryTest(CheckpointType checkpointType)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            SimpleRecoveryTest1(checkpointType, checkpointManager);
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Пример #5
0
        public async ValueTask LocalDeviceSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                new LocalStorageNamedDeviceFactory(),
                new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));

            await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Пример #6
0
 public void PageBlobSimpleRecoveryTest(CheckpointType checkpointType)
 {
     if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
     {
         ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
             new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
             new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
         SimpleRecoveryTest1(checkpointType, checkpointManager);
         checkpointManager.PurgeAll();
         checkpointManager.Dispose();
     }
 }
Пример #7
0
        public async ValueTask PageBlobSimpleRecoveryTest([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
            {
                ICheckpointManager checkpointManager = new DeviceLogCommitCheckpointManager(
                    new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
                    new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobSimpleRecoveryTest"));
                await SimpleRecoveryTest1_Worker(checkpointType, checkpointManager, isAsync);

                checkpointManager.PurgeAll();
                checkpointManager.Dispose();
            }
        }
Пример #8
0
 public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum)
 {
     if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests")))
     {
         var device            = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true);
         var checkpointManager = new DeviceLogCommitCheckpointManager(
             new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING),
             new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1"));
         FasterLogTest1(logChecksum, device, checkpointManager);
         device.Close();
         checkpointManager.PurgeAll();
         checkpointManager.Dispose();
     }
 }
Пример #9
0
        public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType)
        {
            TestUtils.IgnoreIfNotRunningAzureTests();
            var device            = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true);
            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString),
                new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}"));

            await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType);

            device.Dispose();
            checkpointManager.PurgeAll();
            checkpointManager.Dispose();
        }
Пример #10
0
        static void Main()
        {
            // Main hybrid log device
            var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");

            // With non-blittable types, you need an object log device in addition to the
            // main device. FASTER serializes the actual objects in the object log.
            var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");

            var checkpointManager = new DeviceLogCommitCheckpointManager(
                new AzureStorageNamedDeviceFactory(STORAGE_STRING),
                new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/"));


            var store = new FasterKV <long, string>(
                1L << 17,
                    new LogSettings {
                LogDevice = log, ObjectLogDevice = objlog
                    },
                    new CheckpointSettings {
                CheckpointManager = checkpointManager
                    }
                );

            using (var s = store.NewSession(new Functions()))
            {
                for (long i = 0; i < 20000; i++)
                {
                    var _key  = i;
                    var value = $"value-{i}";
                    s.Upsert(ref _key, ref value);
                }
                s.CompletePending(true);

                long   key     = 23;
                string output  = default;
                string context = "value-23";

                var status = s.Read(ref key, ref output, context);

                if (status != Status.PENDING)
                {
                    if (status == Status.OK && output == context)
                    {
                        Console.WriteLine("Success!");
                    }
                    else
                    {
                        Console.WriteLine("Error!");
                    }
                }
            }

            // Take fold-over checkpoint of FASTER, wait to complete
            store.TakeFullCheckpointAsync(CheckpointType.FoldOver)
            .GetAwaiter().GetResult();

            // Dispose store instance
            store.Dispose();

            // Dispose logs
            log.Dispose();
            objlog.Dispose();

            // Create new store
            log    = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log");
            objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log");

            var store2 = new FasterKV <long, string>(
                1L << 17,
                    new LogSettings {
                LogDevice = log, ObjectLogDevice = objlog
                    },
                    new CheckpointSettings {
                CheckpointManager = checkpointManager
                    }
                );

            // Recover store from latest checkpoint
            store2.Recover();

            using (var s = store2.NewSession(new Functions()))
            {
                // Test Read
                long   key     = 23;
                string output  = default;
                string context = "value-23";

                var status = s.Read(ref key, ref output, context);

                if (status != Status.PENDING)
                {
                    if (status == Status.OK && output == context)
                    {
                        Console.WriteLine("Success!");
                    }
                    else
                    {
                        Console.WriteLine("Error!");
                    }
                }
            }

            store2.Dispose();

            // Purge cloud log files
            log.PurgeAll();
            objlog.PurgeAll();

            // Purge cloud checkpoints - warning all data under specified base path are removed
            checkpointManager.PurgeAll();

            // Dispose devices
            log.Dispose();
            objlog.Dispose();

            // Dispose checkpoint manager
            checkpointManager.Dispose();

            Console.WriteLine("Press <ENTER> to end");
            Console.ReadLine();
        }
Пример #11
0
        public async ValueTask SimpleRecoveryTest2([Values] CheckpointType checkpointType, [Values] bool isAsync)
        {
            var checkpointManager = new DeviceLogCommitCheckpointManager(new LocalStorageNamedDeviceFactory(), new DefaultCheckpointNamingScheme(TestContext.CurrentContext.TestDirectory + "/checkpoints4"), false);

            log = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "/SimpleRecoveryTest2.log", deleteOnClose: true);

            // Directory.CreateDirectory(TestContext.CurrentContext.TestDirectory + "/checkpoints4");

            fht1 = new FasterKV
                   <AdId, NumClicks>
                       (128,
                       logSettings: new LogSettings {
                LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
            },
                       checkpointSettings: new CheckpointSettings {
                CheckpointManager = checkpointManager, CheckPointType = checkpointType
            }
                       );

            fht2 = new FasterKV
                   <AdId, NumClicks>
                       (128,
                       logSettings: new LogSettings {
                LogDevice = log, MutableFraction = 0.1, MemorySizeBits = 29
            },
                       checkpointSettings: new CheckpointSettings {
                CheckpointManager = checkpointManager, CheckPointType = checkpointType
            }
                       );


            int numOps     = 5000;
            var inputArray = new AdId[numOps];

            for (int i = 0; i < numOps; i++)
            {
                inputArray[i].adId = i;
            }

            NumClicks value;
            AdInput   inputArg = default;
            Output    output   = default;

            var session1 = fht1.NewSession(new AdSimpleFunctions());

            for (int key = 0; key < numOps; key++)
            {
                value.numClicks = key;
                session1.Upsert(ref inputArray[key], ref value, Empty.Default, 0);
            }
            fht1.TakeFullCheckpoint(out Guid token);
            fht1.CompleteCheckpointAsync().GetAwaiter().GetResult();
            session1.Dispose();

            if (isAsync)
            {
                await fht2.RecoverAsync(token);
            }
            else
            {
                fht2.Recover(token);
            }

            var session2 = fht2.NewSession(new AdSimpleFunctions());

            for (int key = 0; key < numOps; key++)
            {
                var status = session2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default, 0);

                if (status == Status.PENDING)
                {
                    session2.CompletePending(true);
                }
                else
                {
                    Assert.IsTrue(output.value.numClicks == key);
                }
            }
            session2.Dispose();

            log.Dispose();
            fht1.Dispose();
            fht2.Dispose();
            checkpointManager.Dispose();

            new DirectoryInfo(TestContext.CurrentContext.TestDirectory + "/checkpoints4").Delete(true);
        }