public async Task FuzzyIndexRecoveryAsyncTest()
{
Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1,
out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets);
var hash_table2 = new FasterBase();
hash_table2.Initialize(size, 512);
await hash_table2.RecoverFuzzyIndexAsync(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written, cancellationToken : default);
Finish_FuzzyIndexRecoveryTest(seed, numAdds, ht_device, ofb_device, hash_table1, hash_table2);
}
public unsafe void FuzzyIndexRecoveryTest()
{
Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1,
out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets);
var hash_table2 = new FasterBase();
hash_table2.Initialize(size, 512);
//issue recover call
hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written);
//wait until complete
hash_table2.IsFuzzyIndexRecoveryComplete(true);
Finish_FuzzyIndexRecoveryTest(seed, numAdds, ht_device, ofb_device, hash_table1, hash_table2);
}
private static unsafe void Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1, out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets)
{
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
seed = 123;
size = 1 << 16;
numAdds = 1 << 18;
ht_device = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/TestFuzzyIndexRecoveryht.dat", deleteOnClose: true);
ofb_device = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/TestFuzzyIndexRecoveryofb.dat", deleteOnClose: true);
hash_table1 = new FasterBase();
hash_table1.Initialize(size, 512);
//do something
var bucket = default(HashBucket *);
var slot = default(int);
var keyGenerator1 = new Random(seed);
var valueGenerator = new Random(seed + 1);
for (int i = 0; i < numAdds; i++)
{
long key = keyGenerator1.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var entry = default(HashBucketEntry);
hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, 0);
hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word);
}
//issue checkpoint call
hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ht_num_bytes_written,
ofb_device, out ofb_num_bytes_written, out num_ofb_buckets);
//wait until complete
hash_table1.IsIndexFuzzyCheckpointCompletedAsync().AsTask().Wait();
}
private static unsafe void Finish_FuzzyIndexRecoveryTest(int seed, long numAdds, IDevice ht_device, IDevice ofb_device, FasterBase hash_table1, FasterBase hash_table2)
{
var keyGenerator2 = new Random(seed);
var bucket1 = default(HashBucket *);
var bucket2 = default(HashBucket *);
var slot1 = default(int);
var slot2 = default(int);
var entry1 = default(HashBucketEntry);
var entry2 = default(HashBucketEntry);
for (int i = 0; i < 2 * numAdds; i++)
{
long key = keyGenerator2.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);
Assert.AreEqual(exists2, exists1);
if (exists1)
{
Assert.AreEqual(entry2.word, entry1.word);
}
}
hash_table1.Free();
hash_table2.Free();
ht_device.Dispose();
ofb_device.Dispose();
TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
}
public unsafe void TestFuzzyIndexRecovery()
{
int seed = 123;
int size = 1 << 16;
long numAdds = 1 << 18;
IDevice ht_device = new LocalStorageDevice("ht.dat", deleteOnClose: true);
IDevice ofb_device = new LocalStorageDevice("ofb.dat", deleteOnClose: true);
var hash_table1 = new FasterBase();
hash_table1.Initialize(size, 512);
//do something
var bucket = default(HashBucket *);
var slot = default(int);
var keyGenerator1 = new Random(seed);
var valueGenerator = new Random(seed + 1);
for (int i = 0; i < numAdds; i++)
{
long key = keyGenerator1.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var entry = default(HashBucketEntry);
hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry);
hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word);
}
//issue checkpoint call
hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ulong ht_num_bytes_written,
ofb_device, out ulong ofb_num_bytes_written, out int num_ofb_buckets);
//wait until complete
hash_table1.IsIndexFuzzyCheckpointCompleted(true);
var hash_table2 = new FasterBase();
hash_table2.Initialize(size, 512);
//issue recover call
hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written);
//wait until complete
hash_table2.IsFuzzyIndexRecoveryComplete(true);
//verify
var keyGenerator2 = new Random(seed);
var bucket1 = default(HashBucket *);
var bucket2 = default(HashBucket *);
var slot1 = default(int);
var slot2 = default(int);
var entry1 = default(HashBucketEntry);
var entry2 = default(HashBucketEntry);
for (int i = 0; i < 2 * numAdds; i++)
{
long key = keyGenerator2.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);
Assert.IsTrue(exists1 == exists2);
if (exists1)
{
Assert.IsTrue(entry1.word == entry2.word);
}
}
}
private static unsafe void Finish_FuzzyIndexRecoveryTest(int seed, long numAdds, FasterBase hash_table1, FasterBase hash_table2)
{
var keyGenerator2 = new Random(seed);
var bucket1 = default(HashBucket *);
var bucket2 = default(HashBucket *);
var slot1 = default(int);
var slot2 = default(int);
var entry1 = default(HashBucketEntry);
var entry2 = default(HashBucketEntry);
for (int i = 0; i < 2 * numAdds; i++)
{
long key = keyGenerator2.Next();
var hash = Utility.GetHashCode(key);
var tag = (ushort)((ulong)hash >> Constants.kHashTagShift);
var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);
Assert.IsTrue(exists1 == exists2);
if (exists1)
{
Assert.IsTrue(entry1.word == entry2.word);
}
}
hash_table1.Free();
hash_table2.Free();
}
