示例#1
0
        public async Task FuzzyIndexRecoveryAsyncTest()
        {
            Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1,
                                         out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets);

            var hash_table2 = new FasterBase();

            hash_table2.Initialize(size, 512);

            await hash_table2.RecoverFuzzyIndexAsync(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written, cancellationToken : default);

            Finish_FuzzyIndexRecoveryTest(seed, numAdds, ht_device, ofb_device, hash_table1, hash_table2);
        }
示例#2
0
        public unsafe void FuzzyIndexRecoveryTest()
        {
            Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1,
                                         out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets);

            var hash_table2 = new FasterBase();

            hash_table2.Initialize(size, 512);

            //issue recover call
            hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written);
            //wait until complete
            hash_table2.IsFuzzyIndexRecoveryComplete(true);

            Finish_FuzzyIndexRecoveryTest(seed, numAdds, ht_device, ofb_device, hash_table1, hash_table2);
        }
示例#3
0
        private static unsafe void Setup_FuzzyIndexRecoveryTest(out int seed, out int size, out long numAdds, out IDevice ht_device, out IDevice ofb_device, out FasterBase hash_table1, out ulong ht_num_bytes_written, out ulong ofb_num_bytes_written, out int num_ofb_buckets)
        {
            TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);

            seed        = 123;
            size        = 1 << 16;
            numAdds     = 1 << 18;
            ht_device   = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/TestFuzzyIndexRecoveryht.dat", deleteOnClose: true);
            ofb_device  = Devices.CreateLogDevice(TestUtils.MethodTestDir + "/TestFuzzyIndexRecoveryofb.dat", deleteOnClose: true);
            hash_table1 = new FasterBase();
            hash_table1.Initialize(size, 512);

            //do something
            var bucket = default(HashBucket *);
            var slot   = default(int);

            var keyGenerator1  = new Random(seed);
            var valueGenerator = new Random(seed + 1);

            for (int i = 0; i < numAdds; i++)
            {
                long key  = keyGenerator1.Next();
                var  hash = Utility.GetHashCode(key);
                var  tag  = (ushort)((ulong)hash >> Constants.kHashTagShift);

                var entry = default(HashBucketEntry);
                hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry, 0);

                hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word);
            }

            //issue checkpoint call
            hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ht_num_bytes_written,
                                                 ofb_device, out ofb_num_bytes_written, out num_ofb_buckets);

            //wait until complete
            hash_table1.IsIndexFuzzyCheckpointCompletedAsync().AsTask().Wait();
        }
示例#4
0
        private static unsafe void Finish_FuzzyIndexRecoveryTest(int seed, long numAdds, IDevice ht_device, IDevice ofb_device, FasterBase hash_table1, FasterBase hash_table2)
        {
            var keyGenerator2 = new Random(seed);

            var bucket1 = default(HashBucket *);
            var bucket2 = default(HashBucket *);
            var slot1   = default(int);
            var slot2   = default(int);

            var entry1 = default(HashBucketEntry);
            var entry2 = default(HashBucketEntry);

            for (int i = 0; i < 2 * numAdds; i++)
            {
                long key  = keyGenerator2.Next();
                var  hash = Utility.GetHashCode(key);
                var  tag  = (ushort)((ulong)hash >> Constants.kHashTagShift);

                var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
                var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);

                Assert.AreEqual(exists2, exists1);

                if (exists1)
                {
                    Assert.AreEqual(entry2.word, entry1.word);
                }
            }

            hash_table1.Free();
            hash_table2.Free();

            ht_device.Dispose();
            ofb_device.Dispose();
            TestUtils.DeleteDirectory(TestUtils.MethodTestDir);
        }
示例#5
0
        public unsafe void TestFuzzyIndexRecovery()
        {
            int  seed    = 123;
            int  size    = 1 << 16;
            long numAdds = 1 << 18;

            IDevice ht_device  = new LocalStorageDevice("ht.dat", deleteOnClose: true);
            IDevice ofb_device = new LocalStorageDevice("ofb.dat", deleteOnClose: true);

            var hash_table1 = new FasterBase();

            hash_table1.Initialize(size, 512);

            //do something
            var bucket = default(HashBucket *);
            var slot   = default(int);

            var keyGenerator1  = new Random(seed);
            var valueGenerator = new Random(seed + 1);

            for (int i = 0; i < numAdds; i++)
            {
                long key  = keyGenerator1.Next();
                var  hash = Utility.GetHashCode(key);
                var  tag  = (ushort)((ulong)hash >> Constants.kHashTagShift);

                var entry = default(HashBucketEntry);
                hash_table1.FindOrCreateTag(hash, tag, ref bucket, ref slot, ref entry);

                hash_table1.UpdateSlot(bucket, slot, entry.word, valueGenerator.Next(), out long found_word);
            }

            //issue checkpoint call
            hash_table1.TakeIndexFuzzyCheckpoint(0, ht_device, out ulong ht_num_bytes_written,
                                                 ofb_device, out ulong ofb_num_bytes_written, out int num_ofb_buckets);

            //wait until complete
            hash_table1.IsIndexFuzzyCheckpointCompleted(true);

            var hash_table2 = new FasterBase();

            hash_table2.Initialize(size, 512);

            //issue recover call
            hash_table2.RecoverFuzzyIndex(0, ht_device, ht_num_bytes_written, ofb_device, num_ofb_buckets, ofb_num_bytes_written);
            //wait until complete
            hash_table2.IsFuzzyIndexRecoveryComplete(true);

            //verify
            var keyGenerator2 = new Random(seed);

            var bucket1 = default(HashBucket *);
            var bucket2 = default(HashBucket *);
            var slot1   = default(int);
            var slot2   = default(int);

            var entry1 = default(HashBucketEntry);
            var entry2 = default(HashBucketEntry);

            for (int i = 0; i < 2 * numAdds; i++)
            {
                long key  = keyGenerator2.Next();
                var  hash = Utility.GetHashCode(key);
                var  tag  = (ushort)((ulong)hash >> Constants.kHashTagShift);

                var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
                var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);

                Assert.IsTrue(exists1 == exists2);

                if (exists1)
                {
                    Assert.IsTrue(entry1.word == entry2.word);
                }
            }
        }
示例#6
0
        private static unsafe void Finish_FuzzyIndexRecoveryTest(int seed, long numAdds, FasterBase hash_table1, FasterBase hash_table2)
        {
            var keyGenerator2 = new Random(seed);

            var bucket1 = default(HashBucket *);
            var bucket2 = default(HashBucket *);
            var slot1   = default(int);
            var slot2   = default(int);

            var entry1 = default(HashBucketEntry);
            var entry2 = default(HashBucketEntry);

            for (int i = 0; i < 2 * numAdds; i++)
            {
                long key  = keyGenerator2.Next();
                var  hash = Utility.GetHashCode(key);
                var  tag  = (ushort)((ulong)hash >> Constants.kHashTagShift);

                var exists1 = hash_table1.FindTag(hash, tag, ref bucket1, ref slot1, ref entry1);
                var exists2 = hash_table2.FindTag(hash, tag, ref bucket2, ref slot2, ref entry2);

                Assert.IsTrue(exists1 == exists2);

                if (exists1)
                {
                    Assert.IsTrue(entry1.word == entry2.word);
                }
            }

            hash_table1.Free();
            hash_table2.Free();
        }