public void Initialize(string checkpointDirectory, string logDirectory, bool populateLogHandles = false)
{
#if NETCOREAPP || NET
if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows))
{
populateLogHandles = false;
}
#endif
this.CheckpointDirectory = checkpointDirectory;
this.LogDirectory = logDirectory;
string logFileName = "log";
string deviceFileName = $"{this.LogDirectory}/{logFileName}";
KeyValuePair <int, SafeFileHandle>[] initialHandles = null;
if (populateLogHandles)
{
var segmentIds = new List <int>();
foreach (FileInfo item in new DirectoryInfo(logDirectory).GetFiles(logFileName + "*"))
{
segmentIds.Add(int.Parse(item.Name.Replace(logFileName, "").Replace(".", "")));
}
segmentIds.Sort();
initialHandles = new KeyValuePair <int, SafeFileHandle> [segmentIds.Count];
for (int i = 0; i < segmentIds.Count; i++)
{
var segmentId = segmentIds[i];
#pragma warning disable CA1416 // populateLogHandles will be false for non-windows
var handle = LocalStorageDevice.CreateHandle(segmentId, disableFileBuffering: false, deleteOnClose: true, preallocateFile: false, segmentSize: -1, fileName: deviceFileName, IntPtr.Zero);
#pragma warning restore CA1416
initialHandles[i] = new KeyValuePair <int, SafeFileHandle>(segmentId, handle);
}
}
#if NETCOREAPP || NET
if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows))
{
this.LogDevice = new ManagedLocalStorageDevice(deviceFileName, deleteOnClose: true);
}
else
#endif
{
this.LogDevice = new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles);
}
this.Faster = new FasterKV <AdId, NumClicks>(
keySpace,
new LogSettings {
LogDevice = this.LogDevice
},
new CheckpointSettings {
CheckpointDir = this.CheckpointDirectory, CheckPointType = CheckpointType.FoldOver
});
}
public unsafe void MallocFixedPageSizeRecoveryTest()
{
int seed = 123;
var rand1 = new Random(seed);
IDevice device = new LocalStorageDevice("test_ofb.dat", false, false, true, true);
var allocator = new MallocFixedPageSize <HashBucket>();
//do something
int numBucketsToAdd = 16 * MallocFixedPageSize <HashBucket> .PageSize;
long[] logicalAddresses = new long[numBucketsToAdd];
for (int i = 0; i < numBucketsToAdd; i++)
{
long logicalAddress = allocator.Allocate();
logicalAddresses[i] = logicalAddress;
var bucket = (HashBucket *)allocator.GetPhysicalAddress(logicalAddress);
for (int j = 0; j < Constants.kOverflowBucketIndex; j++)
{
bucket->bucket_entries[j] = rand1.Next();
}
}
//issue call to checkpoint
allocator.begin_checkpoint(device, 0, out ulong numBytesWritten);
//wait until complete
allocator.IsCheckpointCompleted(true);
var recoveredAllocator = new MallocFixedPageSize <HashBucket>();
//issue call to recover
recoveredAllocator.begin_recovery(device, 0, numBucketsToAdd, numBytesWritten, out ulong numBytesRead);
//wait until complete
recoveredAllocator.IsRecoveryCompleted(true);
Assert.IsTrue(numBytesWritten == numBytesRead);
var rand2 = new Random(seed);
for (int i = 0; i < numBucketsToAdd; i++)
{
var logicalAddress = logicalAddresses[i];
var bucket = (HashBucket *)recoveredAllocator.GetPhysicalAddress(logicalAddress);
for (int j = 0; j < Constants.kOverflowBucketIndex; j++)
{
Assert.IsTrue(bucket->bucket_entries[j] == rand2.Next());
}
}
}
internal static IDevice CreateTestDevice(DeviceType testDeviceType, string filename, int latencyMs = 20) // latencyMs works only for DeviceType = LocalMemory
{
IDevice device = null;
bool preallocateFile = false;
long capacity = -1; // Capacity unspecified
bool recoverDevice = false;
bool useIoCompletionPort = false;
bool disableFileBuffering = true;
bool deleteOnClose = false;
switch (testDeviceType)
{
#if WINDOWS
case DeviceType.LSD:
#if NETSTANDARD || NET
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // avoids CA1416 // Validate platform compatibility
#endif
device = new LocalStorageDevice(filename, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort);
break;
case DeviceType.EmulatedAzure:
IgnoreIfNotRunningAzureTests();
device = new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, Path.GetFileName(filename), deleteOnClose: false);
break;
#endif
case DeviceType.MLSD:
device = new ManagedLocalStorageDevice(filename, preallocateFile, deleteOnClose, capacity, recoverDevice);
break;
// Emulated higher latency storage device - takes a disk latency arg (latencyMs) and emulates an IDevice using main memory, serving data at specified latency
case DeviceType.LocalMemory:
device = new LocalMemoryDevice(1L << 26, 1L << 22, 2, sector_size: 512, latencyMs: latencyMs, fileName: filename); // 64 MB (1L << 26) is enough for our test cases
break;
}
return(device);
}
public void Initialize(string checkpointDirectory, string logDirectory, bool populateLogHandles = false)
{
this.CheckpointDirectory = checkpointDirectory;
this.LogDirectory = logDirectory;
string logFileName = "log";
string deviceFileName = $"{this.LogDirectory}\\{logFileName}";
KeyValuePair <int, SafeFileHandle>[] initialHandles = null;
if (populateLogHandles)
{
var segmentIds = new List <int>();
foreach (FileInfo item in new DirectoryInfo(logDirectory).GetFiles(logFileName + "*"))
{
segmentIds.Add(int.Parse(item.Name.Replace(logFileName, "").Replace(".", "")));
}
segmentIds.Sort();
initialHandles = new KeyValuePair <int, SafeFileHandle> [segmentIds.Count];
for (int i = 0; i < segmentIds.Count; i++)
{
var segmentId = segmentIds[i];
var handle = LocalStorageDevice.CreateHandle(segmentId, disableFileBuffering: false, deleteOnClose: true, preallocateFile: false, segmentSize: -1, fileName: deviceFileName);
initialHandles[i] = new KeyValuePair <int, SafeFileHandle>(segmentId, handle);
}
}
this.LogDevice = new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles);
this.Faster = new FasterKV <AdId, NumClicks, AdInput, Output, Empty, Functions>(
keySpace,
new Functions(),
new LogSettings {
LogDevice = this.LogDevice
},
new CheckpointSettings {
CheckpointDir = this.CheckpointDirectory, CheckPointType = CheckpointType.FoldOver
});
}
public unsafe void TestFuzzyIndexRecovery()
{
int seed = 123;
int size = 1 << 16;
long numAdds = 1 << 18;
IDevice ht_device = new LocalStorageDevice("ht.dat", false, false, true, true);
IDevice ofb_device = new LocalStorageDevice("ofb.dat", false, false, true, true);
var hash_table1 = new FASTERBase();
hash_table1.Initialize(size, 512);
//do something
var bucket = default(HashBucket *);
var slot = default(int);
var keyGenerator1 = new Random(seed);
var valueGenerator = new Random(seed + 1);
for (int i = 0; i < numAdds; i++)
{
long key = keyGenerator1.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var entry = default(HashBucketEntry);
hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry);
hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word);
}
//issue checkpoint call
hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ulong ht_num_bytes_written,
ofb_device, out ulong ofb_num_bytes_written, out int num_ofb_buckets);
//wait until complete
hash_table1.IsIndexFuzzyCheckpointCompleted(true);
var hash_table2 = new FASTERBase();
hash_table2.Initialize(size, 512);
//issue recover call
hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written);
//wait until complete
hash_table2.IsFuzzyIndexRecoveryComplete(true);
//verify
var keyGenerator2 = new Random(seed);
var bucket1 = default(HashBucket *);
var bucket2 = default(HashBucket *);
var slot1 = default(int);
var slot2 = default(int);
var entry1 = default(HashBucketEntry);
var entry2 = default(HashBucketEntry);
for (int i = 0; i < 2 * numAdds; i++)
{
long key = keyGenerator2.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);
Assert.IsTrue(exists1 == exists2);
if (exists1)
{
Assert.IsTrue(entry1.word == entry2.word);
}
}
}
