public void Initialize(string checkpointDirectory, string logDirectory, bool populateLogHandles = false) { #if NETCOREAPP || NET if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) { populateLogHandles = false; } #endif this.CheckpointDirectory = checkpointDirectory; this.LogDirectory = logDirectory; string logFileName = "log"; string deviceFileName = $"{this.LogDirectory}/{logFileName}"; KeyValuePair <int, SafeFileHandle>[] initialHandles = null; if (populateLogHandles) { var segmentIds = new List <int>(); foreach (FileInfo item in new DirectoryInfo(logDirectory).GetFiles(logFileName + "*")) { segmentIds.Add(int.Parse(item.Name.Replace(logFileName, "").Replace(".", ""))); } segmentIds.Sort(); initialHandles = new KeyValuePair <int, SafeFileHandle> [segmentIds.Count]; for (int i = 0; i < segmentIds.Count; i++) { var segmentId = segmentIds[i]; #pragma warning disable CA1416 // populateLogHandles will be false for non-windows var handle = LocalStorageDevice.CreateHandle(segmentId, disableFileBuffering: false, deleteOnClose: true, preallocateFile: false, segmentSize: -1, fileName: deviceFileName, IntPtr.Zero); #pragma warning restore CA1416 initialHandles[i] = new KeyValuePair <int, SafeFileHandle>(segmentId, handle); } } #if NETCOREAPP || NET if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) { this.LogDevice = new ManagedLocalStorageDevice(deviceFileName, deleteOnClose: true); } else #endif { this.LogDevice = new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles); } this.Faster = new FasterKV <AdId, NumClicks>( keySpace, new LogSettings { LogDevice = this.LogDevice }, new CheckpointSettings { CheckpointDir = this.CheckpointDirectory, CheckPointType = CheckpointType.FoldOver }); }
public unsafe void MallocFixedPageSizeRecoveryTest() { int seed = 123; var rand1 = new Random(seed); IDevice device = new LocalStorageDevice("test_ofb.dat", false, false, true, true); var allocator = new MallocFixedPageSize <HashBucket>(); //do something int numBucketsToAdd = 16 * MallocFixedPageSize <HashBucket> .PageSize; long[] logicalAddresses = new long[numBucketsToAdd]; for (int i = 0; i < numBucketsToAdd; i++) { long logicalAddress = allocator.Allocate(); logicalAddresses[i] = logicalAddress; var bucket = (HashBucket *)allocator.GetPhysicalAddress(logicalAddress); for (int j = 0; j < Constants.kOverflowBucketIndex; j++) { bucket->bucket_entries[j] = rand1.Next(); } } //issue call to checkpoint allocator.begin_checkpoint(device, 0, out ulong numBytesWritten); //wait until complete allocator.IsCheckpointCompleted(true); var recoveredAllocator = new MallocFixedPageSize <HashBucket>(); //issue call to recover recoveredAllocator.begin_recovery(device, 0, numBucketsToAdd, numBytesWritten, out ulong numBytesRead); //wait until complete recoveredAllocator.IsRecoveryCompleted(true); Assert.IsTrue(numBytesWritten == numBytesRead); var rand2 = new Random(seed); for (int i = 0; i < numBucketsToAdd; i++) { var logicalAddress = logicalAddresses[i]; var bucket = (HashBucket *)recoveredAllocator.GetPhysicalAddress(logicalAddress); for (int j = 0; j < Constants.kOverflowBucketIndex; j++) { Assert.IsTrue(bucket->bucket_entries[j] == rand2.Next()); } } }
internal static IDevice CreateTestDevice(DeviceType testDeviceType, string filename, int latencyMs = 20) // latencyMs works only for DeviceType = LocalMemory { IDevice device = null; bool preallocateFile = false; long capacity = -1; // Capacity unspecified bool recoverDevice = false; bool useIoCompletionPort = false; bool disableFileBuffering = true; bool deleteOnClose = false; switch (testDeviceType) { #if WINDOWS case DeviceType.LSD: #if NETSTANDARD || NET if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // avoids CA1416 // Validate platform compatibility #endif device = new LocalStorageDevice(filename, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort); break; case DeviceType.EmulatedAzure: IgnoreIfNotRunningAzureTests(); device = new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, Path.GetFileName(filename), deleteOnClose: false); break; #endif case DeviceType.MLSD: device = new ManagedLocalStorageDevice(filename, preallocateFile, deleteOnClose, capacity, recoverDevice); break; // Emulated higher latency storage device - takes a disk latency arg (latencyMs) and emulates an IDevice using main memory, serving data at specified latency case DeviceType.LocalMemory: device = new LocalMemoryDevice(1L << 26, 1L << 22, 2, sector_size: 512, latencyMs: latencyMs, fileName: filename); // 64 MB (1L << 26) is enough for our test cases break; } return(device); }
public void Initialize(string checkpointDirectory, string logDirectory, bool populateLogHandles = false) { this.CheckpointDirectory = checkpointDirectory; this.LogDirectory = logDirectory; string logFileName = "log"; string deviceFileName = $"{this.LogDirectory}\\{logFileName}"; KeyValuePair <int, SafeFileHandle>[] initialHandles = null; if (populateLogHandles) { var segmentIds = new List <int>(); foreach (FileInfo item in new DirectoryInfo(logDirectory).GetFiles(logFileName + "*")) { segmentIds.Add(int.Parse(item.Name.Replace(logFileName, "").Replace(".", ""))); } segmentIds.Sort(); initialHandles = new KeyValuePair <int, SafeFileHandle> [segmentIds.Count]; for (int i = 0; i < segmentIds.Count; i++) { var segmentId = segmentIds[i]; var handle = LocalStorageDevice.CreateHandle(segmentId, disableFileBuffering: false, deleteOnClose: true, preallocateFile: false, segmentSize: -1, fileName: deviceFileName); initialHandles[i] = new KeyValuePair <int, SafeFileHandle>(segmentId, handle); } } this.LogDevice = new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles); this.Faster = new FasterKV <AdId, NumClicks, AdInput, Output, Empty, Functions>( keySpace, new Functions(), new LogSettings { LogDevice = this.LogDevice }, new CheckpointSettings { CheckpointDir = this.CheckpointDirectory, CheckPointType = CheckpointType.FoldOver }); }
public unsafe void TestFuzzyIndexRecovery() { int seed = 123; int size = 1 << 16; long numAdds = 1 << 18; IDevice ht_device = new LocalStorageDevice("ht.dat", false, false, true, true); IDevice ofb_device = new LocalStorageDevice("ofb.dat", false, false, true, true); var hash_table1 = new FASTERBase(); hash_table1.Initialize(size, 512); //do something var bucket = default(HashBucket *); var slot = default(int); var keyGenerator1 = new Random(seed); var valueGenerator = new Random(seed + 1); for (int i = 0; i < numAdds; i++) { long key = keyGenerator1.Next(); var hash = Utility.GetHashCode(key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); var entry = default(HashBucketEntry); hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry); hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word); } //issue checkpoint call hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ulong ht_num_bytes_written, ofb_device, out ulong ofb_num_bytes_written, out int num_ofb_buckets); //wait until complete hash_table1.IsIndexFuzzyCheckpointCompleted(true); var hash_table2 = new FASTERBase(); hash_table2.Initialize(size, 512); //issue recover call hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written); //wait until complete hash_table2.IsFuzzyIndexRecoveryComplete(true); //verify var keyGenerator2 = new Random(seed); var bucket1 = default(HashBucket *); var bucket2 = default(HashBucket *); var slot1 = default(int); var slot2 = default(int); var entry1 = default(HashBucketEntry); var entry2 = default(HashBucketEntry); for (int i = 0; i < 2 * numAdds; i++) { long key = keyGenerator2.Next(); var hash = Utility.GetHashCode(key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1); var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2); Assert.IsTrue(exists1 == exists2); if (exists1) { Assert.IsTrue(entry1.word == entry2.word); } } }