public void LevelMergeDuplicateValuesTest() { string path = Path.GetFullPath ("TestData\\LevelMergeDuplicateValuesTest"); if (!Directory.Exists (path)) Directory.CreateDirectory (path); foreach (string file in Directory.GetFiles(path)) File.Delete (file); int num_tables_to_merge = 4; int items_per_table = 2500; int totalData = 0; for (int i = 0; i < num_tables_to_merge; i++) { var mt = new MemTable (); for (int j = 0; j < items_per_table; j++) { int numToStore = j % 100; var key = new Key (new ByteArray (BitConverter.GetBytes (numToStore))); var value = new Value (BitConverter.GetBytes (j)); mt.Add (key, value); } mt.WriteToSortedBlockTable ("TestData\\LevelMergeDuplicateValuesTest", 0, i); totalData += mt.Size; } var cache = new RazorCache (); var timer = new Stopwatch (); timer.Start (); Manifest mf = new Manifest ("TestData\\LevelMergeDuplicateValuesTest"); SortedBlockTable.MergeTables (cache, mf, 1, new List<PageRef> { new PageRef { Level = 0, Version = 0}, new PageRef { Level = 0, Version = 1}, new PageRef { Level = 0, Version = 2}, new PageRef { Level = 0, Version = 3} }, ExceptionHandling.ThrowAll, null); timer.Stop (); // Open the block table and scan it to check the stored values var sbt = new SortedBlockTable (cache, mf.BaseFileName, 1, 1); try { var pairs = sbt.Enumerate ().ToList (); Assert.AreEqual (100, pairs.Count ()); Assert.AreEqual (2400, BitConverter.ToInt32 (pairs.First ().Value.ValueBytes, 0)); } finally { sbt.Close (); } Console.WriteLine ("Wrote a multilevel merge with duplicates at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); }
public void LevelMergeReadTest() { string path = Path.GetFullPath ("TestData\\LevelMergeReadTest"); if (!Directory.Exists (path)) Directory.CreateDirectory (path); int num_tables_to_merge = 4; int items_per_table = 2500; int totalData = 0; for (int i = 0; i < num_tables_to_merge; i++) { var mt = new MemTable (); for (int j = 0; j < items_per_table; j++) { var randKey = Key.Random (40); var randVal = Value.Random (512); mt.Add (randKey, randVal); } mt.WriteToSortedBlockTable ("TestData\\LevelMergeReadTest", 0, i); totalData += mt.Size; } var tables = new List<IEnumerable<KeyValuePair<Key, Value>>> (); var sbts = new List<SortedBlockTable> (); var cache = new RazorCache (); for (int j = 0; j < num_tables_to_merge; j++) { var sbt = new SortedBlockTable (cache, "TestData\\LevelMergeReadTest", 0, j); tables.Add (sbt.Enumerate ()); sbts.Add (sbt); } int ct = 0; Key key = Key.FromBytes (new byte[] { 0, 0 }); var timer = new Stopwatch (); timer.Start (); foreach (var pair in MergeEnumerator.Merge(tables, p => p.Key)) { Assert.True (key.CompareTo (pair.Key) < 0); key = pair.Key; ct++; } timer.Stop (); sbts.ForEach (s => s.Close ()); Console.WriteLine ("Scanned through a multilevel merge at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); }
public void ReadKeys() { string path = Path.GetFullPath ("TestData\\ReadKeys"); if (!Directory.Exists (path)) Directory.CreateDirectory (path); var mt = new MemTable (); for (int i = 0; i < 10000; i++) { var k0 = Key.Random (40); var v0 = Value.Random (200); mt.Add (k0, v0); } mt.WriteToSortedBlockTable ("TestData\\ReadKeys", 0, 10); var cache = new RazorCache (); var sbt = new SortedBlockTable (cache, "TestData\\ReadKeys", 0, 10); var timer = new Stopwatch (); timer.Start (); Assert.AreEqual (10000, sbt.Enumerate ().Count ()); timer.Stop (); Console.WriteLine ("Counted sorted table at a throughput of {0} MB/s", (double)mt.Size / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); // Confirm that the items are sorted. Key lastKey = Key.Empty; timer.Reset (); timer.Start (); foreach (var pair in sbt.Enumerate()) { Assert.IsTrue (lastKey.CompareTo (pair.Key) < 0); lastKey = pair.Key; } timer.Stop (); Console.WriteLine ("Read & verify sorted table at a throughput of {0} MB/s", (double)mt.Size / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); sbt.Close (); }