public static void Test_DuplicateHandelingOptions() { var options = new BPlusTree <double, string> .OptionsV2(PrimitiveSerializer.Double, PrimitiveSerializer.String); options.CalcBTreeOrder(16, 24); options.FileBlockSize = 8192; options.CreateFile = CreatePolicy.Always; options.FileName = "I:\\test.tmp"; BulkInsertOptions opts = new BulkInsertOptions(); opts.CommitOnCompletion = true; // check how to properly set this value using Roger examples. opts.DuplicateHandling = DuplicateHandling.FirstValueWins; opts.InputIsSorted = true; opts.ReplaceContents = false; var sortedContent = new SortedDictionary <double, string>(); sortedContent.Add(10.0, "Demah"); using (var tree = new BPlusTree <double, string>(options)) { tree.Add(10.0, "Hamed"); tree.BulkInsert(sortedContent, opts); } }
public Dictionary <int, string> TestMergeRandom(BPlusTreeOptions <int, string> options, int nsets, int nsize) { Dictionary <int, string> test = new Dictionary <int, string>(); IEnumerable <KeyValuePair <int, string> >[] sets = new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(nsets, nsize, test)).ToArray(); using (BPlusTree <int, string> tree = new BPlusTree <int, string>(options)) { foreach (IEnumerable <KeyValuePair <int, string> > set in sets) { tree.BulkInsert(set, new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins }); } VerifyDictionary(test, tree); tree.UnloadCache(); tree.Add(int.MaxValue, "max"); tree.Remove(int.MaxValue); VerifyDictionary(test, tree); } return(test); }
public static void Run() { Console.WriteLine("Now generating sorted items ..."); int nNewItems = 100000; int nOldItems = 100000; var rnd = new Random(); var sortedItems = new SortedDictionary <double, string>(); int newItem = 0; for (int i = 1; i <= nNewItems; i++) { do { newItem = rnd.Next(0, Int32.MaxValue); }while (sortedItems.ContainsKey(newItem)); sortedItems.Add(newItem, Convert.ToString(rnd.Next(0, Int32.MaxValue))); Console.Write("\rAdding {0,5}% : {1,10:N0}/{2:N0}", Math.Round((((double)i / (double)nNewItems) * 100)), i, nNewItems); } Stopwatch stp = new Stopwatch(); var options = new BPlusTree <double, string> .OptionsV2(PrimitiveSerializer.Double, PrimitiveSerializer.String); options.CalcBTreeOrder(16, 24); options.FileBlockSize = 8192; options.CreateFile = CreatePolicy.Always; options.FileName = "I:\\test.tmp"; BulkInsertOptions opts = new BulkInsertOptions(); opts.CommitOnCompletion = true; // check how to properly set this value using Roger examples. opts.DuplicateHandling = DuplicateHandling.LastValueWins; opts.InputIsSorted = true; opts.ReplaceContents = false; AddUpdateValue update = new AddUpdateValue(); Console.WriteLine(); Console.WriteLine("Now creating tree ..."); using (var tree = new BPlusTree <double, string>(options)) { stp.Start(); for (int i = 0; i < nOldItems; i++) { tree.AddOrUpdate(rnd.Next(0, Int32.MaxValue), ref update); } stp.Stop(); Console.WriteLine("Initial <{0:N0}> items => ET : {1} Speed : {2:N0} item/sec", nOldItems, stp.Elapsed, Math.Round((double)(nOldItems / stp.Elapsed.TotalSeconds), 5)); stp.Restart(); tree.BulkInsert(sortedItems, opts); stp.Stop(); Console.WriteLine("Bulk <{0:N0}> items => ET : {1} Speed : {2:N0} item/sec", nNewItems, stp.Elapsed, Math.Round((double)(nNewItems / stp.Elapsed.TotalSeconds), 5)); } Console.ReadLine(); }
void BulkyInserts(BPlusTree <Guid, TestInfo> tree) { while (!mreStop.WaitOne(0, false)) { tree.BulkInsert(CreateData(100)); AddIdle(tree); Thread.Sleep(100); } }
private static void FlushToDisk(BPlusTree <string, int> map, Dictionary <string, int> dict) { int takenCount = 0; while (takenCount < dict.Count) { int count = 0; var keysToUpdate = new List <Tuple <string, int> >(); // Only update in blocks of 10000 because we don't want to // Pop our max memory when calculating diffs. int amountToTake = 10000; // We might not be able to take exactly 10000. if (dict.Count - count < 10000) { amountToTake = dict.Count - count; count = 0; } // Create list of diffs foreach (var kvp in dict.Skip(takenCount).Take(amountToTake)) { int value; if (map.TryGetValue(kvp.Key, out value)) { keysToUpdate.Add(new Tuple <string, int>(kvp.Key, kvp.Value + value)); } } // Apply diffs (can't modify dictionary during enumeration) foreach (var update in keysToUpdate) { dict[update.Item1] = update.Item2; } count += amountToTake; takenCount += amountToTake; } BulkInsertOptions bio = new BulkInsertOptions(); //bio.ReplaceContents = true; bio.DuplicateHandling = DuplicateHandling.LastValueWins; // BPlusTree is optimized to insert in batches. map.BulkInsert(dict, bio); }
public void TestReplaceContents() { Dictionary <int, string> test = new Dictionary <int, string>(); IEnumerable <KeyValuePair <int, string> >[] sets = new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 1000, test)).ToArray(); using (BPlusTree <int, string> tree = new BPlusTree <int, string>(Options)) { tree.BulkInsert( new OrderedKeyValuePairs <int, string>(sets[0]), new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true } ); VerifyDictionary(test, tree); // Use bulk insert to overwrite the contents of tree test = new Dictionary <int, string>(); sets = new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 100, test)).ToArray(); tree.BulkInsert( new OrderedKeyValuePairs <int, string>(sets[0]), new BulkInsertOptions { CommitOnCompletion = false, InputIsSorted = true, ReplaceContents = true, DuplicateHandling = DuplicateHandling.RaisesException, } ); VerifyDictionary(test, tree); } }
void TestMergeSequenceInFile(BPlusTreeOptions <int, string> options, int count) { Dictionary <int, string> expected = new Dictionary <int, string>(); for (int i = 0; i < count; i++) { expected.Add(i + 1, i.ToString()); } using (BPlusTree <int, string> tree = new BPlusTree <int, string>(options)) { Assert.AreEqual(expected.Count, tree.BulkInsert(expected)); VerifyDictionary(expected, tree); } }
public void TestBulkInsertSorted() { Dictionary <int, string> test = new Dictionary <int, string>(); IEnumerable <KeyValuePair <int, string> >[] sets = new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 1000, test)).ToArray(); using (BPlusTree <int, string> tree = new BPlusTree <int, string>(Options)) { tree.BulkInsert( new OrderedKeyValuePairs <int, string>(sets[0]), new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true } ); VerifyDictionary(test, tree); } }
/// <summary> /// </summary> /// <param name="data">The data.</param> public void BulkInsert(IEnumerable <T> data) { tree.BulkInsert(data.Select(arg => new KeyValuePair <Guid, T>(arg.Id, arg))); }
void BulkyInserts(BPlusTree<Guid, TestInfo> tree) { while (!mreStop.WaitOne(0, false)) { tree.BulkInsert(CreateData(100)); AddIdle(tree); Thread.Sleep(100); } }
public void TestReplaceContents() { Dictionary<int, string> test = new Dictionary<int, string>(); IEnumerable<KeyValuePair<int, string>>[] sets = new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 1000, test)).ToArray(); using (BPlusTree<int, string> tree = new BPlusTree<int, string>(Options)) { tree.BulkInsert( new OrderedKeyValuePairs<int, string>(sets[0]), new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true } ); VerifyDictionary(test, tree); // Use bulk insert to overwrite the contents of tree test = new Dictionary<int, string>(); sets = new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 100, test)).ToArray(); tree.BulkInsert( new OrderedKeyValuePairs<int, string>(sets[0]), new BulkInsertOptions { CommitOnCompletion = false, InputIsSorted = true, ReplaceContents = true, DuplicateHandling = DuplicateHandling.RaisesException, } ); VerifyDictionary(test, tree); } }
public void TestBulkInsertSorted() { Dictionary<int, string> test = new Dictionary<int, string>(); IEnumerable<KeyValuePair<int, string>>[] sets = new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 1000, test)).ToArray(); using (BPlusTree<int, string> tree = new BPlusTree<int, string>(Options)) { tree.BulkInsert( new OrderedKeyValuePairs<int, string>(sets[0]), new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true } ); VerifyDictionary(test, tree); } }
public Dictionary<int, string> TestMergeRandom(BPlusTreeOptions<int, string> options, int nsets, int nsize) { Dictionary<int, string> test = new Dictionary<int, string>(); IEnumerable<KeyValuePair<int, string>>[] sets = new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(nsets, nsize, test)).ToArray(); using (BPlusTree<int, string> tree = new BPlusTree<int, string>(options)) { foreach(IEnumerable<KeyValuePair<int, string>> set in sets) tree.BulkInsert(set, new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins }); VerifyDictionary(test, tree); tree.UnloadCache(); tree.Add(int.MaxValue, "max"); tree.Remove(int.MaxValue); VerifyDictionary(test, tree); } return test; }
void TestMergeSequenceInFile(BPlusTreeOptions<int, string> options, int count) { Dictionary<int, string> expected = new Dictionary<int, string>(); for (int i = 0; i < count; i++) expected.Add(i + 1, i.ToString()); using (BPlusTree<int, string> tree = new BPlusTree<int, string>(options)) { Assert.AreEqual(expected.Count, tree.BulkInsert(expected)); VerifyDictionary(expected, tree); } }
public void TestSyncFromLogging() { using (var tempFile = new TempFile()) using (var logfile = new TempFile()) using (var tempCopy = new TempFile()) { var options = new BPlusTree<int, string>.OptionsV2(new PrimitiveSerializer(), new PrimitiveSerializer()) { CreateFile = CreatePolicy.Always, FileName = tempFile.TempPath, TransactionLogFileName = logfile.TempPath, }.CalcBTreeOrder(4, 10); var readcopy = options.Clone(); readcopy.FileName = tempCopy.TempPath; readcopy.StoragePerformance = StoragePerformance.Fastest; using (var tree = new BPlusTree<int, string>(options)) using (var copy = new BPlusTree<int, string>(readcopy)) using (var tlog = new TransactionLog<int, string>( new TransactionLogOptions<int, string>(logfile.TempPath, PrimitiveSerializer.Int32, PrimitiveSerializer.String) { ReadOnly = true })) { tree.Add(0, "0"); tree.Commit(); long logpos = 0; copy.EnableCount(); //start by copying the data from tree's file into the copy instance: copy.BulkInsert( BPlusTree<int, string>.EnumerateFile(options), new BulkInsertOptions { InputIsSorted = true, CommitOnCompletion = false, ReplaceContents = true } ); Assert.AreEqual(1, copy.Count); Assert.AreEqual("0", copy[0]); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(1, copy.Count); //insert some data... tree.AddRange(MakeValues(1, 99)); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(100, copy.Count); //insert some data... for (int i = 0; i < 100; i++) tree.Remove(i); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(0, copy.Count); tree.AddRange(MakeValues(1000, 1000)); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(1000, copy.Count); } } }
private void btnTestCreateTree_Click(object sender, EventArgs e) { string fileName = Toolkit.ResolveFilePath("./test.bpt", true); if (File.Exists(fileName)) { File.Delete(fileName); } //using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Create(fileName + "1", Encoding.UTF8, 3)) { // tree1.Insert("aardvark", new BPlusListLong(new long[] { 13, 47, 22 })); // tree1.Insert("byproduct", new BPlusListLong(new long[] { 5 })); // tree1.Insert("charactercode", new BPlusListLong(new long[] { 21 })); // // this one should split the root into [ab] [c] [d] // tree1.Insert("delaware", new BPlusListLong(new long[] { 32, 14 })); // tree1.Insert("elephant", new BPlusListLong(new long[] { 41 })); // tree1.Insert("foliage", new BPlusListLong(new long[] { 59 })); //} //using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Load(fileName + "1", true)){ // foreach(BPlusTreeLeafNode<BPlusString, BPlusListLong> leaf in tree1.GetAllLeaves()){ // Debug.WriteLine(leaf.ToString()); // } //} //using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Load(fileName + "1", true)) { // foreach (BPlusTreeNode<BPlusString, BPlusListLong> node in tree1.TraverseBreadthFirst() ) { // Debug.WriteLine(node.ToString()); // } //} SortedDictionary <string, List <long> > dic = new SortedDictionary <string, List <long> >(); using (BPlusTree <BPlusString, BPlusListLong> bulkTree = BPlusTree <BPlusString, BPlusListLong> .Create(fileName, Encoding.UTF8, 3, 10, 5, 5)) { using (BinaryWriter bw = new BinaryWriter(new MemoryStream())) { bw.Write("aardvark"); bw.Write(3); bw.Write((long)13); bw.Write((long)47); bw.Write((long)22); bw.Write("byproduct"); bw.Write(1); bw.Write((long)5); bw.Write("charactercode"); bw.Write(1); bw.Write((long)21); bw.Write("delaware"); bw.Write(2); bw.Write((long)32); bw.Write((long)14); bw.Write("elephant"); bw.Write(1); bw.Write((long)41); bw.Write("foliage"); bw.Write(1); bw.Write((long)59); bw.Write("georgia"); bw.Write(1); bw.Write((long)63); bw.Write("homeworkism"); bw.Write(1); bw.Write((long)131); bw.Write("indigo"); bw.Write(1); bw.Write((long)217); bw.Write("jello"); bw.Write(1); bw.Write((long)142); bw.Write("kangaroo"); bw.Write(1); bw.Write((long)150); bw.Write("literal"); bw.Write(1); bw.Write((long)167); bw.Write("mommy"); bw.Write(1); bw.Write((long)169); bw.Write("nuance"); bw.Write(1); bw.Write((long)173); bw.Write("omaha"); bw.Write(1); bw.Write((long)180); bw.Write("penny"); bw.Write(1); bw.Write((long)233); bw.Write("qwerty"); bw.Write(1); bw.Write((long)250); bw.Write("rollin'"); bw.Write(1); bw.Write((long)261); bw.Write("semi-professional"); bw.Write(1); bw.Write((long)99); bw.Write("trouble"); bw.Write(1); bw.Write((long)269); bw.Write("umpire"); bw.Write(1); bw.Write((long)271); bw.Write("vampire"); bw.Write(1); bw.Write((long)280); bw.Write("weaver"); bw.Write(1); bw.Write((long)290); bw.Write("xylophone"); bw.Write(1); bw.Write((long)102); bw.Write("yellowbelly"); bw.Write(1); bw.Write((long)120); bw.Write("zebra"); bw.Write(1); bw.Write((long)210); int max = 10000; for (int i = 1; i < max; i++) { bw.Write("zebra" + i.ToString().PadLeft(4, '0')); bw.Write(1); bw.Write((long)(200 + i)); } bw.BaseStream.Position = 0; bulkTree.BulkInsert(bw.BaseStream, 1.0M); } } using (BPlusTree <BPlusString, BPlusListLong> tree = BPlusTree <BPlusString, BPlusListLong> .Load(fileName, true)) { foreach (BPlusTreeNode <BPlusString, BPlusListLong> node in tree.TraverseBreadthFirst(true)) { Debug.WriteLine(node.ToString()); } List <KeywordMatch <BPlusString, BPlusListLong> > matches = tree.Search("zebra", KeywordMatchMode.ExactMatch, true); foreach (var match in matches) { Debug.WriteLine(match.ToString()); } } using (BPlusTree <BPlusString, BPlusListLong> tree = BPlusTree <BPlusString, BPlusListLong> .Load(fileName, false)) { tree.Insert("abdomen", new BPlusListLong(new long[] { 1, 2, 3 }.ToList())); // tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 14, 98 }.ToList()), UpdateMode.Add); // tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 14, 98 }.ToList()), UpdateMode.Replace); // tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 3 }.ToList()), UpdateMode.Subtract); tree.Update("a", KeywordMatchMode.StartsWith, new BPlusListLong(new long[] { 3 }.ToList()), UpdateMode.Add); //foreach (KeywordMatch km in tree.GetAllMatches()) { // MessageBox.Show(km.Keyword + "=" + km.Value.ToString()); //} //foreach (BPlusString keyword in tree.GetAllKeywords()) { // MessageBox.Show(keyword.Value); //} // ExactMatch tests List <KeywordMatch <BPlusString, BPlusListLong> > matches = tree.Search("a", KeywordMatchMode.StartsWith, false); // should return 2 items foreach (string key in dic.Keys) { matches = tree.Search(new BPlusString(key), KeywordMatchMode.ExactMatch, false); if (matches.Count != 1) { MessageBox.Show("Invalid! did not find value '" + key + "' in the tree!"); } } matches = tree.Search("homeworkism", KeywordMatchMode.ExactMatch, false); matches = tree.Search("Elephant", KeywordMatchMode.ExactMatch, false); // should return no items matches = tree.Search("CHaractercode", KeywordMatchMode.ExactMatch, true); // should return 1 item matches = tree.Search("semi-professional", KeywordMatchMode.ExactMatch, true); // should return 1 item matches = tree.Search("nonexistantkeyhere", KeywordMatchMode.StartsWith, true); // should return no items // StartsWith tests matches = tree.Search("byp", KeywordMatchMode.StartsWith, false); // should return 1 item matches = tree.Search("Elepha", KeywordMatchMode.StartsWith, false); // should return no items matches = tree.Search("CHar", KeywordMatchMode.StartsWith, true); // should return 1 item matches = tree.Search("nonexistantkeyhere", KeywordMatchMode.StartsWith, true); // should return no items matches = tree.Search("semi-pro", KeywordMatchMode.StartsWith, true); // should return 1 item // EndsWith tests matches = tree.Search("duct", KeywordMatchMode.EndsWith, false); // should return 1 item matches = tree.Search("ANt", KeywordMatchMode.EndsWith, false); // should return no items matches = tree.Search("CoDe", KeywordMatchMode.EndsWith, true); // should return 1 item matches = tree.Search("ISM", KeywordMatchMode.EndsWith, true); // should return 1 item matches = tree.Search("ssional", KeywordMatchMode.EndsWith, true); // should return 1 item // Contains tests matches = tree.Search("pro", KeywordMatchMode.Contains, true); // should return 2 items matches = tree.Search("RAc", KeywordMatchMode.Contains, true); // should return 1 items matches = tree.Search("ARA", KeywordMatchMode.Contains, false); // should return no items matches = tree.Search("-", KeywordMatchMode.Contains, false); // should return 1 item matches = tree.Search("zebra0030", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0012", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0032", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0039", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0040", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0060", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0051", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0058", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0099", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0072", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("zebra0080", KeywordMatchMode.ExactMatch, false); // should return 1 item matches = tree.Search("semi-professional", KeywordMatchMode.Contains, true); // should return 1 item } }
public void TestSyncFromLogging() { using (var tempFile = new TempFile()) using (var logfile = new TempFile()) using (var tempCopy = new TempFile()) { var options = new BPlusTree <int, string> .OptionsV2(new PrimitiveSerializer(), new PrimitiveSerializer()) { CreateFile = CreatePolicy.Always, FileName = tempFile.TempPath, TransactionLogFileName = logfile.TempPath, } .CalcBTreeOrder(4, 10); var readcopy = options.Clone(); readcopy.FileName = tempCopy.TempPath; readcopy.StoragePerformance = StoragePerformance.Fastest; using (var tree = new BPlusTree <int, string>(options)) using (var copy = new BPlusTree <int, string>(readcopy)) using (var tlog = new TransactionLog <int, string>( new TransactionLogOptions <int, string>(logfile.TempPath, PrimitiveSerializer.Int32, PrimitiveSerializer.String) { ReadOnly = true })) { tree.Add(0, "0"); tree.Commit(); long logpos = 0; copy.EnableCount(); //start by copying the data from tree's file into the copy instance: copy.BulkInsert( BPlusTree <int, string> .EnumerateFile(options), new BulkInsertOptions { InputIsSorted = true, CommitOnCompletion = false, ReplaceContents = true } ); Assert.AreEqual(1, copy.Count); Assert.AreEqual("0", copy[0]); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(1, copy.Count); //insert some data... tree.AddRange(MakeValues(1, 99)); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(100, copy.Count); //insert some data... for (int i = 0; i < 100; i++) { tree.Remove(i); } tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(0, copy.Count); tree.AddRange(MakeValues(1000, 1000)); tlog.ReplayLog(copy, ref logpos); Assert.AreEqual(1000, copy.Count); } } }