public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType) { // Set up the blob manager so can set lease to it TestUtils.IgnoreIfNotRunningAzureTests(); CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount; var cloudBlobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container"); blobContainer.CreateIfNotExists(); var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1"); var blobMgr = new DefaultBlobManager(true, mycloudBlobDir); var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr); var checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString), new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}")); await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType); device.Dispose(); checkpointManager.PurgeAll(); checkpointManager.Dispose(); blobContainer.Delete(); }
public async ValueTask PageBlobFasterLogTestWithLease([Values] LogChecksumType logChecksum, [Values] FasterLogTests.IteratorType iteratorType) { // Need this environment variable set AND Azure Storage Emulator running if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests"))) { // Set up the blob manager so can set lease to it CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount; var cloudBlobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = cloudBlobClient.GetContainerReference("test-container"); blobContainer.CreateIfNotExists(); var mycloudBlobDir = blobContainer.GetDirectoryReference(@"BlobManager/MyLeaseTest1"); var blobMgr = new DefaultBlobManager(true, mycloudBlobDir); var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTestWithLease", "fasterlogLease.log", deleteOnClose: true, underLease: true, blobManager: blobMgr); var checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING), new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTestWithLease")); await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType); device.Dispose(); checkpointManager.PurgeAll(); checkpointManager.Dispose(); blobContainer.Delete(); } }
public void PageBlobFasterLogTest1([Values] LogChecksumType logChecksum) { if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests"))) { var device = new AzureStorageDevice(EMULATED_STORAGE_STRING, $"{TEST_CONTAINER}", "PageBlobFasterLogTest1", "fasterlog.log", deleteOnClose: true); var checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(EMULATED_STORAGE_STRING), new DefaultCheckpointNamingScheme($"{TEST_CONTAINER}/PageBlobFasterLogTest1")); FasterLogTest1(logChecksum, device, checkpointManager); device.Close(); checkpointManager.PurgeAll(); checkpointManager.Dispose(); } }
public async ValueTask PageBlobFasterLogTest1([Values] LogChecksumType logChecksum, [Values] FasterLogTestBase.IteratorType iteratorType) { TestUtils.IgnoreIfNotRunningAzureTests(); var device = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, $"{TestUtils.AzureTestContainer}", TestUtils.AzureTestDirectory, "fasterlog.log", deleteOnClose: true); var checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(TestUtils.AzureEmulatedStorageString), new DefaultCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}")); await FasterLogTest1(logChecksum, device, checkpointManager, iteratorType); device.Dispose(); checkpointManager.PurgeAll(); checkpointManager.Dispose(); }
public void TieredWriteRead() { IDevice tested; IDevice localDevice = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\BasicDiskFASTERTests.log", deleteOnClose: true, capacity: 1 << 30); if ("yes".Equals(Environment.GetEnvironmentVariable("RunAzureTests"))) { IDevice cloudDevice = new AzureStorageDevice(EMULATED_STORAGE_STRING, TEST_CONTAINER, "BasicDiskFASTERTests", false); tested = new TieredStorageDevice(1, localDevice, cloudDevice); } else { // If no Azure is enabled, just use another disk IDevice localDevice2 = Devices.CreateLogDevice(TestContext.CurrentContext.TestDirectory + "\\BasicDiskFASTERTests2.log", deleteOnClose: true, capacity: 1 << 30); tested = new TieredStorageDevice(1, localDevice, localDevice2); } TestDeviceWriteRead(tested); }
public void TieredWriteRead() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir); IDevice tested; IDevice localDevice = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/BasicDiskFASTERTests.log", deleteOnClose: true, capacity: 1 << 30); if (TestUtils.IsRunningAzureTests) { IDevice cloudDevice = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskFASTERTests"); tested = new TieredStorageDevice(1, localDevice, cloudDevice); } else { // If no Azure is enabled, just use another disk IDevice localDevice2 = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/BasicDiskFASTERTests2.log", deleteOnClose: true, capacity: 1 << 30); tested = new TieredStorageDevice(1, localDevice, localDevice2); } TestDeviceWriteRead(tested); }
internal static IDevice CreateTestDevice(DeviceType testDeviceType, string filename, int latencyMs = 20) // latencyMs works only for DeviceType = LocalMemory { IDevice device = null; bool preallocateFile = false; long capacity = -1; // Capacity unspecified bool recoverDevice = false; bool useIoCompletionPort = false; bool disableFileBuffering = true; bool deleteOnClose = false; switch (testDeviceType) { #if WINDOWS case DeviceType.LSD: #if NETSTANDARD || NET if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // avoids CA1416 // Validate platform compatibility #endif device = new LocalStorageDevice(filename, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort); break; case DeviceType.EmulatedAzure: IgnoreIfNotRunningAzureTests(); device = new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, Path.GetFileName(filename), deleteOnClose: false); break; #endif case DeviceType.MLSD: device = new ManagedLocalStorageDevice(filename, preallocateFile, deleteOnClose, capacity, recoverDevice); break; // Emulated higher latency storage device - takes a disk latency arg (latencyMs) and emulates an IDevice using main memory, serving data at specified latency case DeviceType.LocalMemory: device = new LocalMemoryDevice(1L << 26, 1L << 22, 2, sector_size: 512, latencyMs: latencyMs, fileName: filename); // 64 MB (1L << 26) is enough for our test cases break; } return(device); }
static void Main() { // Main hybrid log device var log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log"); // With non-blittable types, you need an object log device in addition to the // main device. FASTER serializes the actual objects in the object log. var objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log"); var checkpointManager = new DeviceLogCommitCheckpointManager( new AzureStorageNamedDeviceFactory(STORAGE_STRING), new DefaultCheckpointNamingScheme($"{BASE_CONTAINER}/checkpoints/")); var store = new FasterKV <long, string>( 1L << 17, new LogSettings { LogDevice = log, ObjectLogDevice = objlog }, new CheckpointSettings { CheckpointManager = checkpointManager } ); using (var s = store.NewSession(new Functions())) { for (long i = 0; i < 20000; i++) { var _key = i; var value = $"value-{i}"; s.Upsert(ref _key, ref value); } s.CompletePending(true); long key = 23; string output = default; string context = "value-23"; var status = s.Read(ref key, ref output, context); if (status != Status.PENDING) { if (status == Status.OK && output == context) { Console.WriteLine("Success!"); } else { Console.WriteLine("Error!"); } } } // Take fold-over checkpoint of FASTER, wait to complete store.TakeFullCheckpointAsync(CheckpointType.FoldOver) .GetAwaiter().GetResult(); // Dispose store instance store.Dispose(); // Dispose logs log.Dispose(); objlog.Dispose(); // Create new store log = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.log"); objlog = new AzureStorageDevice(STORAGE_STRING, BASE_CONTAINER, "", "hlog.obj.log"); var store2 = new FasterKV <long, string>( 1L << 17, new LogSettings { LogDevice = log, ObjectLogDevice = objlog }, new CheckpointSettings { CheckpointManager = checkpointManager } ); // Recover store from latest checkpoint store2.Recover(); using (var s = store2.NewSession(new Functions())) { // Test Read long key = 23; string output = default; string context = "value-23"; var status = s.Read(ref key, ref output, context); if (status != Status.PENDING) { if (status == Status.OK && output == context) { Console.WriteLine("Success!"); } else { Console.WriteLine("Error!"); } } } store2.Dispose(); // Purge cloud log files log.PurgeAll(); objlog.PurgeAll(); // Purge cloud checkpoints - warning all data under specified base path are removed checkpointManager.PurgeAll(); // Dispose devices log.Dispose(); objlog.Dispose(); // Dispose checkpoint manager checkpointManager.Dispose(); Console.WriteLine("Press <ENTER> to end"); Console.ReadLine(); }