public static void Test_DuplicateHandelingOptions()
{
var options = new BPlusTree <double, string> .OptionsV2(PrimitiveSerializer.Double, PrimitiveSerializer.String);
options.CalcBTreeOrder(16, 24);
options.FileBlockSize = 8192;
options.CreateFile = CreatePolicy.Always;
options.FileName = "I:\\test.tmp";
BulkInsertOptions opts = new BulkInsertOptions();
opts.CommitOnCompletion = true; // check how to properly set this value using Roger examples.
opts.DuplicateHandling = DuplicateHandling.FirstValueWins;
opts.InputIsSorted = true;
opts.ReplaceContents = false;
var sortedContent = new SortedDictionary <double, string>();
sortedContent.Add(10.0, "Demah");
using (var tree = new BPlusTree <double, string>(options))
{
tree.Add(10.0, "Hamed");
tree.BulkInsert(sortedContent, opts);
}
}
public Dictionary <int, string> TestMergeRandom(BPlusTreeOptions <int, string> options, int nsets, int nsize)
{
Dictionary <int, string> test = new Dictionary <int, string>();
IEnumerable <KeyValuePair <int, string> >[] sets =
new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(nsets, nsize, test)).ToArray();
using (BPlusTree <int, string> tree = new BPlusTree <int, string>(options))
{
foreach (IEnumerable <KeyValuePair <int, string> > set in sets)
{
tree.BulkInsert(set, new BulkInsertOptions {
DuplicateHandling = DuplicateHandling.LastValueWins
});
}
VerifyDictionary(test, tree);
tree.UnloadCache();
tree.Add(int.MaxValue, "max");
tree.Remove(int.MaxValue);
VerifyDictionary(test, tree);
}
return(test);
}
public static void Run()
{
Console.WriteLine("Now generating sorted items ...");
int nNewItems = 100000;
int nOldItems = 100000;
var rnd = new Random();
var sortedItems = new SortedDictionary <double, string>();
int newItem = 0;
for (int i = 1; i <= nNewItems; i++)
{
do
{
newItem = rnd.Next(0, Int32.MaxValue);
}while (sortedItems.ContainsKey(newItem));
sortedItems.Add(newItem, Convert.ToString(rnd.Next(0, Int32.MaxValue)));
Console.Write("\rAdding {0,5}% : {1,10:N0}/{2:N0}", Math.Round((((double)i / (double)nNewItems) * 100)), i, nNewItems);
}
Stopwatch stp = new Stopwatch();
var options = new BPlusTree <double, string> .OptionsV2(PrimitiveSerializer.Double, PrimitiveSerializer.String);
options.CalcBTreeOrder(16, 24);
options.FileBlockSize = 8192;
options.CreateFile = CreatePolicy.Always;
options.FileName = "I:\\test.tmp";
BulkInsertOptions opts = new BulkInsertOptions();
opts.CommitOnCompletion = true; // check how to properly set this value using Roger examples.
opts.DuplicateHandling = DuplicateHandling.LastValueWins;
opts.InputIsSorted = true;
opts.ReplaceContents = false;
AddUpdateValue update = new AddUpdateValue();
Console.WriteLine();
Console.WriteLine("Now creating tree ...");
using (var tree = new BPlusTree <double, string>(options))
{
stp.Start();
for (int i = 0; i < nOldItems; i++)
{
tree.AddOrUpdate(rnd.Next(0, Int32.MaxValue), ref update);
}
stp.Stop();
Console.WriteLine("Initial <{0:N0}> items => ET : {1} Speed : {2:N0} item/sec", nOldItems, stp.Elapsed, Math.Round((double)(nOldItems / stp.Elapsed.TotalSeconds), 5));
stp.Restart();
tree.BulkInsert(sortedItems, opts);
stp.Stop();
Console.WriteLine("Bulk <{0:N0}> items => ET : {1} Speed : {2:N0} item/sec", nNewItems, stp.Elapsed, Math.Round((double)(nNewItems / stp.Elapsed.TotalSeconds), 5));
}
Console.ReadLine();
}
void BulkyInserts(BPlusTree <Guid, TestInfo> tree)
{
while (!mreStop.WaitOne(0, false))
{
tree.BulkInsert(CreateData(100));
AddIdle(tree);
Thread.Sleep(100);
}
}
private static void FlushToDisk(BPlusTree <string, int> map, Dictionary <string, int> dict)
{
int takenCount = 0;
while (takenCount < dict.Count)
{
int count = 0;
var keysToUpdate = new List <Tuple <string, int> >();
// Only update in blocks of 10000 because we don't want to
// Pop our max memory when calculating diffs.
int amountToTake = 10000;
// We might not be able to take exactly 10000.
if (dict.Count - count < 10000)
{
amountToTake = dict.Count - count;
count = 0;
}
// Create list of diffs
foreach (var kvp in dict.Skip(takenCount).Take(amountToTake))
{
int value;
if (map.TryGetValue(kvp.Key, out value))
{
keysToUpdate.Add(new Tuple <string, int>(kvp.Key, kvp.Value + value));
}
}
// Apply diffs (can't modify dictionary during enumeration)
foreach (var update in keysToUpdate)
{
dict[update.Item1] = update.Item2;
}
count += amountToTake;
takenCount += amountToTake;
}
BulkInsertOptions bio = new BulkInsertOptions();
//bio.ReplaceContents = true;
bio.DuplicateHandling = DuplicateHandling.LastValueWins;
// BPlusTree is optimized to insert in batches.
map.BulkInsert(dict, bio);
}
public void TestReplaceContents()
{
Dictionary <int, string> test = new Dictionary <int, string>();
IEnumerable <KeyValuePair <int, string> >[] sets =
new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 1000, test)).ToArray();
using (BPlusTree <int, string> tree = new BPlusTree <int, string>(Options))
{
tree.BulkInsert(
new OrderedKeyValuePairs <int, string>(sets[0]),
new BulkInsertOptions {
DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true
}
);
VerifyDictionary(test, tree);
// Use bulk insert to overwrite the contents of tree
test = new Dictionary <int, string>();
sets = new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 100, test)).ToArray();
tree.BulkInsert(
new OrderedKeyValuePairs <int, string>(sets[0]),
new BulkInsertOptions
{
CommitOnCompletion = false,
InputIsSorted = true,
ReplaceContents = true,
DuplicateHandling = DuplicateHandling.RaisesException,
}
);
VerifyDictionary(test, tree);
}
}
void TestMergeSequenceInFile(BPlusTreeOptions <int, string> options, int count)
{
Dictionary <int, string> expected = new Dictionary <int, string>();
for (int i = 0; i < count; i++)
{
expected.Add(i + 1, i.ToString());
}
using (BPlusTree <int, string> tree = new BPlusTree <int, string>(options))
{
Assert.AreEqual(expected.Count, tree.BulkInsert(expected));
VerifyDictionary(expected, tree);
}
}
public void TestBulkInsertSorted()
{
Dictionary <int, string> test = new Dictionary <int, string>();
IEnumerable <KeyValuePair <int, string> >[] sets =
new List <IEnumerable <KeyValuePair <int, string> > >(CreateSets(1, 1000, test)).ToArray();
using (BPlusTree <int, string> tree = new BPlusTree <int, string>(Options))
{
tree.BulkInsert(
new OrderedKeyValuePairs <int, string>(sets[0]),
new BulkInsertOptions {
DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true
}
);
VerifyDictionary(test, tree);
}
}
/// <summary>
/// </summary>
/// <param name="data">The data.</param>
public void BulkInsert(IEnumerable <T> data)
{
tree.BulkInsert(data.Select(arg => new KeyValuePair <Guid, T>(arg.Id, arg)));
}
void BulkyInserts(BPlusTree<Guid, TestInfo> tree)
{
while (!mreStop.WaitOne(0, false))
{
tree.BulkInsert(CreateData(100));
AddIdle(tree);
Thread.Sleep(100);
}
}
public void TestReplaceContents()
{
Dictionary<int, string> test = new Dictionary<int, string>();
IEnumerable<KeyValuePair<int, string>>[] sets =
new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 1000, test)).ToArray();
using (BPlusTree<int, string> tree = new BPlusTree<int, string>(Options))
{
tree.BulkInsert(
new OrderedKeyValuePairs<int, string>(sets[0]),
new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true }
);
VerifyDictionary(test, tree);
// Use bulk insert to overwrite the contents of tree
test = new Dictionary<int, string>();
sets = new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 100, test)).ToArray();
tree.BulkInsert(
new OrderedKeyValuePairs<int, string>(sets[0]),
new BulkInsertOptions
{
CommitOnCompletion = false,
InputIsSorted = true,
ReplaceContents = true,
DuplicateHandling = DuplicateHandling.RaisesException,
}
);
VerifyDictionary(test, tree);
}
}
public void TestBulkInsertSorted()
{
Dictionary<int, string> test = new Dictionary<int, string>();
IEnumerable<KeyValuePair<int, string>>[] sets =
new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(1, 1000, test)).ToArray();
using (BPlusTree<int, string> tree = new BPlusTree<int, string>(Options))
{
tree.BulkInsert(
new OrderedKeyValuePairs<int, string>(sets[0]),
new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins, InputIsSorted = true }
);
VerifyDictionary(test, tree);
}
}
public Dictionary<int, string> TestMergeRandom(BPlusTreeOptions<int, string> options, int nsets, int nsize)
{
Dictionary<int, string> test = new Dictionary<int, string>();
IEnumerable<KeyValuePair<int, string>>[] sets =
new List<IEnumerable<KeyValuePair<int, string>>>(CreateSets(nsets, nsize, test)).ToArray();
using (BPlusTree<int, string> tree = new BPlusTree<int, string>(options))
{
foreach(IEnumerable<KeyValuePair<int, string>> set in sets)
tree.BulkInsert(set, new BulkInsertOptions { DuplicateHandling = DuplicateHandling.LastValueWins });
VerifyDictionary(test, tree);
tree.UnloadCache();
tree.Add(int.MaxValue, "max");
tree.Remove(int.MaxValue);
VerifyDictionary(test, tree);
}
return test;
}
void TestMergeSequenceInFile(BPlusTreeOptions<int, string> options, int count)
{
Dictionary<int, string> expected = new Dictionary<int, string>();
for (int i = 0; i < count; i++)
expected.Add(i + 1, i.ToString());
using (BPlusTree<int, string> tree = new BPlusTree<int, string>(options))
{
Assert.AreEqual(expected.Count, tree.BulkInsert(expected));
VerifyDictionary(expected, tree);
}
}
public void TestSyncFromLogging()
{
using (var tempFile = new TempFile())
using (var logfile = new TempFile())
using (var tempCopy = new TempFile())
{
var options = new BPlusTree<int, string>.OptionsV2(new PrimitiveSerializer(), new PrimitiveSerializer())
{
CreateFile = CreatePolicy.Always,
FileName = tempFile.TempPath,
TransactionLogFileName = logfile.TempPath,
}.CalcBTreeOrder(4, 10);
var readcopy = options.Clone();
readcopy.FileName = tempCopy.TempPath;
readcopy.StoragePerformance = StoragePerformance.Fastest;
using (var tree = new BPlusTree<int, string>(options))
using (var copy = new BPlusTree<int, string>(readcopy))
using (var tlog = new TransactionLog<int, string>(
new TransactionLogOptions<int, string>(logfile.TempPath, PrimitiveSerializer.Int32, PrimitiveSerializer.String) { ReadOnly = true }))
{
tree.Add(0, "0");
tree.Commit();
long logpos = 0;
copy.EnableCount();
//start by copying the data from tree's file into the copy instance:
copy.BulkInsert(
BPlusTree<int, string>.EnumerateFile(options),
new BulkInsertOptions { InputIsSorted = true, CommitOnCompletion = false, ReplaceContents = true }
);
Assert.AreEqual(1, copy.Count);
Assert.AreEqual("0", copy[0]);
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(1, copy.Count);
//insert some data...
tree.AddRange(MakeValues(1, 99));
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(100, copy.Count);
//insert some data...
for (int i = 0; i < 100; i++)
tree.Remove(i);
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(0, copy.Count);
tree.AddRange(MakeValues(1000, 1000));
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(1000, copy.Count);
}
}
}
private void btnTestCreateTree_Click(object sender, EventArgs e)
{
string fileName = Toolkit.ResolveFilePath("./test.bpt", true);
if (File.Exists(fileName))
{
File.Delete(fileName);
}
//using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Create(fileName + "1", Encoding.UTF8, 3)) {
// tree1.Insert("aardvark", new BPlusListLong(new long[] { 13, 47, 22 }));
// tree1.Insert("byproduct", new BPlusListLong(new long[] { 5 }));
// tree1.Insert("charactercode", new BPlusListLong(new long[] { 21 }));
// // this one should split the root into [ab] [c] [d]
// tree1.Insert("delaware", new BPlusListLong(new long[] { 32, 14 }));
// tree1.Insert("elephant", new BPlusListLong(new long[] { 41 }));
// tree1.Insert("foliage", new BPlusListLong(new long[] { 59 }));
//}
//using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Load(fileName + "1", true)){
// foreach(BPlusTreeLeafNode<BPlusString, BPlusListLong> leaf in tree1.GetAllLeaves()){
// Debug.WriteLine(leaf.ToString());
// }
//}
//using (BPlusTree<BPlusString, BPlusListLong> tree1 = BPlusTree<BPlusString, BPlusListLong>.Load(fileName + "1", true)) {
// foreach (BPlusTreeNode<BPlusString, BPlusListLong> node in tree1.TraverseBreadthFirst() ) {
// Debug.WriteLine(node.ToString());
// }
//}
SortedDictionary <string, List <long> > dic = new SortedDictionary <string, List <long> >();
using (BPlusTree <BPlusString, BPlusListLong> bulkTree = BPlusTree <BPlusString, BPlusListLong> .Create(fileName, Encoding.UTF8, 3, 10, 5, 5)) {
using (BinaryWriter bw = new BinaryWriter(new MemoryStream())) {
bw.Write("aardvark");
bw.Write(3);
bw.Write((long)13);
bw.Write((long)47);
bw.Write((long)22);
bw.Write("byproduct");
bw.Write(1);
bw.Write((long)5);
bw.Write("charactercode");
bw.Write(1);
bw.Write((long)21);
bw.Write("delaware");
bw.Write(2);
bw.Write((long)32);
bw.Write((long)14);
bw.Write("elephant");
bw.Write(1);
bw.Write((long)41);
bw.Write("foliage");
bw.Write(1);
bw.Write((long)59);
bw.Write("georgia");
bw.Write(1);
bw.Write((long)63);
bw.Write("homeworkism");
bw.Write(1);
bw.Write((long)131);
bw.Write("indigo");
bw.Write(1);
bw.Write((long)217);
bw.Write("jello");
bw.Write(1);
bw.Write((long)142);
bw.Write("kangaroo");
bw.Write(1);
bw.Write((long)150);
bw.Write("literal");
bw.Write(1);
bw.Write((long)167);
bw.Write("mommy");
bw.Write(1);
bw.Write((long)169);
bw.Write("nuance");
bw.Write(1);
bw.Write((long)173);
bw.Write("omaha");
bw.Write(1);
bw.Write((long)180);
bw.Write("penny");
bw.Write(1);
bw.Write((long)233);
bw.Write("qwerty");
bw.Write(1);
bw.Write((long)250);
bw.Write("rollin'");
bw.Write(1);
bw.Write((long)261);
bw.Write("semi-professional");
bw.Write(1);
bw.Write((long)99);
bw.Write("trouble");
bw.Write(1);
bw.Write((long)269);
bw.Write("umpire");
bw.Write(1);
bw.Write((long)271);
bw.Write("vampire");
bw.Write(1);
bw.Write((long)280);
bw.Write("weaver");
bw.Write(1);
bw.Write((long)290);
bw.Write("xylophone");
bw.Write(1);
bw.Write((long)102);
bw.Write("yellowbelly");
bw.Write(1);
bw.Write((long)120);
bw.Write("zebra");
bw.Write(1);
bw.Write((long)210);
int max = 10000;
for (int i = 1; i < max; i++)
{
bw.Write("zebra" + i.ToString().PadLeft(4, '0'));
bw.Write(1);
bw.Write((long)(200 + i));
}
bw.BaseStream.Position = 0;
bulkTree.BulkInsert(bw.BaseStream, 1.0M);
}
}
using (BPlusTree <BPlusString, BPlusListLong> tree = BPlusTree <BPlusString, BPlusListLong> .Load(fileName, true)) {
foreach (BPlusTreeNode <BPlusString, BPlusListLong> node in tree.TraverseBreadthFirst(true))
{
Debug.WriteLine(node.ToString());
}
List <KeywordMatch <BPlusString, BPlusListLong> > matches = tree.Search("zebra", KeywordMatchMode.ExactMatch, true);
foreach (var match in matches)
{
Debug.WriteLine(match.ToString());
}
}
using (BPlusTree <BPlusString, BPlusListLong> tree = BPlusTree <BPlusString, BPlusListLong> .Load(fileName, false)) {
tree.Insert("abdomen", new BPlusListLong(new long[] { 1, 2, 3 }.ToList()));
// tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 14, 98 }.ToList()), UpdateMode.Add);
// tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 14, 98 }.ToList()), UpdateMode.Replace);
// tree.Update("abdomen", KeywordMatchMode.ExactMatch, new BPlusListLong(new long[] { 3 }.ToList()), UpdateMode.Subtract);
tree.Update("a", KeywordMatchMode.StartsWith, new BPlusListLong(new long[] { 3 }.ToList()), UpdateMode.Add);
//foreach (KeywordMatch km in tree.GetAllMatches()) {
// MessageBox.Show(km.Keyword + "=" + km.Value.ToString());
//}
//foreach (BPlusString keyword in tree.GetAllKeywords()) {
// MessageBox.Show(keyword.Value);
//}
// ExactMatch tests
List <KeywordMatch <BPlusString, BPlusListLong> > matches = tree.Search("a", KeywordMatchMode.StartsWith, false); // should return 2 items
foreach (string key in dic.Keys)
{
matches = tree.Search(new BPlusString(key), KeywordMatchMode.ExactMatch, false);
if (matches.Count != 1)
{
MessageBox.Show("Invalid! did not find value '" + key + "' in the tree!");
}
}
matches = tree.Search("homeworkism", KeywordMatchMode.ExactMatch, false);
matches = tree.Search("Elephant", KeywordMatchMode.ExactMatch, false); // should return no items
matches = tree.Search("CHaractercode", KeywordMatchMode.ExactMatch, true); // should return 1 item
matches = tree.Search("semi-professional", KeywordMatchMode.ExactMatch, true); // should return 1 item
matches = tree.Search("nonexistantkeyhere", KeywordMatchMode.StartsWith, true); // should return no items
// StartsWith tests
matches = tree.Search("byp", KeywordMatchMode.StartsWith, false); // should return 1 item
matches = tree.Search("Elepha", KeywordMatchMode.StartsWith, false); // should return no items
matches = tree.Search("CHar", KeywordMatchMode.StartsWith, true); // should return 1 item
matches = tree.Search("nonexistantkeyhere", KeywordMatchMode.StartsWith, true); // should return no items
matches = tree.Search("semi-pro", KeywordMatchMode.StartsWith, true); // should return 1 item
// EndsWith tests
matches = tree.Search("duct", KeywordMatchMode.EndsWith, false); // should return 1 item
matches = tree.Search("ANt", KeywordMatchMode.EndsWith, false); // should return no items
matches = tree.Search("CoDe", KeywordMatchMode.EndsWith, true); // should return 1 item
matches = tree.Search("ISM", KeywordMatchMode.EndsWith, true); // should return 1 item
matches = tree.Search("ssional", KeywordMatchMode.EndsWith, true); // should return 1 item
// Contains tests
matches = tree.Search("pro", KeywordMatchMode.Contains, true); // should return 2 items
matches = tree.Search("RAc", KeywordMatchMode.Contains, true); // should return 1 items
matches = tree.Search("ARA", KeywordMatchMode.Contains, false); // should return no items
matches = tree.Search("-", KeywordMatchMode.Contains, false); // should return 1 item
matches = tree.Search("zebra0030", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0012", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0032", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0039", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0040", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0060", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0051", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0058", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0099", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0072", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("zebra0080", KeywordMatchMode.ExactMatch, false); // should return 1 item
matches = tree.Search("semi-professional", KeywordMatchMode.Contains, true); // should return 1 item
}
}
public void TestSyncFromLogging()
{
using (var tempFile = new TempFile())
using (var logfile = new TempFile())
using (var tempCopy = new TempFile())
{
var options = new BPlusTree <int, string> .OptionsV2(new PrimitiveSerializer(), new PrimitiveSerializer())
{
CreateFile = CreatePolicy.Always,
FileName = tempFile.TempPath,
TransactionLogFileName = logfile.TempPath,
}
.CalcBTreeOrder(4, 10);
var readcopy = options.Clone();
readcopy.FileName = tempCopy.TempPath;
readcopy.StoragePerformance = StoragePerformance.Fastest;
using (var tree = new BPlusTree <int, string>(options))
using (var copy = new BPlusTree <int, string>(readcopy))
using (var tlog = new TransactionLog <int, string>(
new TransactionLogOptions <int, string>(logfile.TempPath, PrimitiveSerializer.Int32, PrimitiveSerializer.String)
{
ReadOnly = true
}))
{
tree.Add(0, "0");
tree.Commit();
long logpos = 0;
copy.EnableCount();
//start by copying the data from tree's file into the copy instance:
copy.BulkInsert(
BPlusTree <int, string> .EnumerateFile(options),
new BulkInsertOptions {
InputIsSorted = true, CommitOnCompletion = false, ReplaceContents = true
}
);
Assert.AreEqual(1, copy.Count);
Assert.AreEqual("0", copy[0]);
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(1, copy.Count);
//insert some data...
tree.AddRange(MakeValues(1, 99));
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(100, copy.Count);
//insert some data...
for (int i = 0; i < 100; i++)
{
tree.Remove(i);
}
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(0, copy.Count);
tree.AddRange(MakeValues(1000, 1000));
tlog.ReplayLog(copy, ref logpos);
Assert.AreEqual(1000, copy.Count);
}
}
}
