public void Can_find_within_range()
{
var fileName = Path.Combine(CreateDir(), "MappedTrieReaderTests.Can_find_within_range.tri");
var tree = new LcrsTrie();
tree.Add("ape");
tree.Add("app");
tree.Add("apple");
tree.Add("banana");
tree.Add("bananas");
tree.Add("xanax");
tree.Add("xxx");
foreach (var node in tree.EndOfWordNodes())
{
node.PostingsAddress = new BlockInfo(long.MinValue, int.MinValue);
}
File.WriteAllText("Can_find_within_range.log", tree.Visualize(), Encoding.UTF8);
tree.Serialize(fileName);
IList <Word> words;
using (var reader = new MappedTrieReader(fileName))
{
words = reader.Range("azz", "xerox").ToList();
}
Assert.AreEqual(3, words.Count);
Assert.AreEqual("banana", words[0].Value);
Assert.AreEqual("bananas", words[1].Value);
Assert.AreEqual("xanax", words[2].Value);
}
public void Can_find_exact()
{
var tree = new LcrsTrie('\0', false);
Word word;
Assert.False(tree.HasWord("xxx", out word));
tree.Add("xxx");
Assert.True(tree.HasWord("xxx", out word));
Assert.False(tree.HasWord("baby", out word));
Assert.False(tree.HasWord("dad", out word));
tree.Add("baby");
Assert.True(tree.HasWord("xxx", out word));
Assert.True(tree.HasWord("baby", out word));
Assert.False(tree.HasWord("dad", out word));
tree.Add("dad");
Assert.True(tree.HasWord("xxx", out word));
Assert.True(tree.HasWord("baby", out word));
Assert.True(tree.HasWord("dad", out word));
}
public void Can_find_near()
{
var tree = new LcrsTrie('\0', false);
var near = tree.SemanticallyNear("ba", 1).Select(w => w.Value).ToList();
Assert.IsFalse(near.Any());
tree.Add("bad");
near = tree.SemanticallyNear("ba", 1).Select(w => w.Value).ToList();
Assert.AreEqual(1, near.Count);
Assert.IsTrue(near.Contains("bad"));
tree.Add("baby");
near = tree.SemanticallyNear("ba", 1).Select(w => w.Value).ToList();
Assert.AreEqual(1, near.Count);
Assert.IsTrue(near.Contains("bad"));
tree.Add("b");
near = tree.SemanticallyNear("ba", 1).Select(w => w.Value).ToList();
Assert.AreEqual(2, near.Count);
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("b"));
near = tree.SemanticallyNear("ba", 2).Select(w => w.Value).ToList();
Assert.AreEqual(3, near.Count);
Assert.IsTrue(near.Contains("b"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("baby"));
near = tree.SemanticallyNear("ba", 0).Select(w => w.Value).ToList();
Assert.AreEqual(0, near.Count);
tree.Add("bananas");
near = tree.SemanticallyNear("ba", 6).Select(w => w.Value).ToList();
Assert.AreEqual(4, near.Count);
Assert.IsTrue(near.Contains("b"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("baby"));
Assert.IsTrue(near.Contains("bananas"));
near = tree.SemanticallyNear("bazy", 1).Select(w => w.Value).ToList();
Assert.AreEqual(1, near.Count);
Assert.IsTrue(near.Contains("baby"));
tree.Add("bank");
near = tree.SemanticallyNear("bazy", 3).Select(w => w.Value).ToList();
Assert.AreEqual(4, near.Count);
Assert.IsTrue(near.Contains("baby"));
Assert.IsTrue(near.Contains("bank"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("b"));
}
public void Can_find_exact()
{
var tree = new LcrsTrie('\0', false);
Word word;
Assert.IsFalse(tree.IsWord("xxx").Any());
tree.Add("xxx");
Assert.IsTrue(tree.IsWord("xxx").Any());
Assert.IsFalse(tree.IsWord("baby").Any());
Assert.IsFalse(tree.IsWord("dad").Any());
tree.Add("baby");
Assert.IsTrue(tree.IsWord("xxx").Any());
Assert.IsTrue(tree.IsWord("baby").Any());
Assert.IsFalse(tree.IsWord("dad").Any());
tree.Add("dad");
Assert.IsTrue(tree.IsWord("xxx").Any());
Assert.IsTrue(tree.IsWord("baby").Any());
Assert.IsTrue(tree.IsWord("dad").Any());
}
public void Can_find_near()
{
var tree = new LcrsTrie('\0', false);
var near = tree.Near("ba", 1).Select(w => w.Value).ToList();
Assert.That(near, Is.Empty);
tree.Add("bad");
near = tree.Near("ba", 1).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(1));
Assert.IsTrue(near.Contains("bad"));
tree.Add("baby");
near = tree.Near("ba", 1).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(1));
Assert.IsTrue(near.Contains("bad"));
tree.Add("b");
near = tree.Near("ba", 1).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(2));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("b"));
near = tree.Near("ba", 2).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(3));
Assert.IsTrue(near.Contains("b"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("baby"));
near = tree.Near("ba", 0).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(0));
tree.Add("bananas");
near = tree.Near("ba", 6).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(4));
Assert.IsTrue(near.Contains("b"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("baby"));
Assert.IsTrue(near.Contains("bananas"));
near = tree.Near("bazy", 1).Select(w => w.Value).ToList();
Assert.That(near.Count, Is.EqualTo(1));
Assert.IsTrue(near.Contains("baby"));
tree.Add("bank");
near = tree.Near("bazy", 3).Select(w => w.Value).ToList();
Assert.AreEqual(4, near.Count);
Assert.IsTrue(near.Contains("baby"));
Assert.IsTrue(near.Contains("bank"));
Assert.IsTrue(near.Contains("bad"));
Assert.IsTrue(near.Contains("b"));
}
public void Can_find_prefixed()
{
var tree = new LcrsTrie('\0', false);
tree.Add("rambo");
tree.Add("rambo");
tree.Add("2");
tree.Add("rocky");
tree.Add("2");
tree.Add("raiders");
tree.Add("of");
tree.Add("the");
tree.Add("lost");
tree.Add("ark");
tree.Add("rain");
tree.Add("man");
var prefixed = tree.StartsWith("ra").Select(w => w.Value).ToList();
Assert.That(prefixed.Count, Is.EqualTo(3));
Assert.IsTrue(prefixed.Contains("rambo"));
Assert.IsTrue(prefixed.Contains("raiders"));
Assert.IsTrue(prefixed.Contains("rain"));
}
public void Can_find_prefixed()
{
var fileName = Path.Combine(Dir, "Can_find_prefixed.tri");
var tree = new LcrsTrie('\0', false);
tree.Add("rambo");
tree.Add("rambo");
tree.Add("2");
tree.Add("rocky");
tree.Add("2");
tree.Add("raiders");
tree.Add("of");
tree.Add("the");
tree.Add("lost");
tree.Add("ark");
tree.Add("rain");
tree.Add("man");
tree.Serialize(fileName);
var prefixed = new MappedTrieReader(fileName).StartsWith("ra").Select(w => w.Value).ToList();
Assert.AreEqual(3, prefixed.Count);
Assert.IsTrue(prefixed.Contains("rambo"));
Assert.IsTrue(prefixed.Contains("raiders"));
Assert.IsTrue(prefixed.Contains("rain"));
}
public void Can_deserialize_whole_file()
{
var fileName = Path.Combine(Dir, "Can_serialize_whole_file.tri");
var tree = new LcrsTrie('\0', false);
tree.Add("baby");
tree.Add("bad");
tree.Add("bank");
tree.Add("box");
tree.Add("dad");
tree.Add("dance");
Word found;
Assert.IsTrue(tree.HasWord("baby", out found));
Assert.IsTrue(tree.HasWord("bad", out found));
Assert.IsTrue(tree.HasWord("bank", out found));
Assert.IsTrue(tree.HasWord("box", out found));
Assert.IsTrue(tree.HasWord("dad", out found));
Assert.IsTrue(tree.HasWord("dance", out found));
tree.Serialize(fileName);
var recreated = Serializer.DeserializeTrie(Dir, new FileInfo(fileName).Name);
Assert.IsTrue(recreated.HasWord("baby", out found));
Assert.IsTrue(recreated.HasWord("bad", out found));
Assert.IsTrue(recreated.HasWord("bank", out found));
Assert.IsTrue(recreated.HasWord("box", out found));
Assert.IsTrue(recreated.HasWord("dad", out found));
Assert.IsTrue(recreated.HasWord("dance", out found));
}
public void Can_deserialize_whole_file()
{
var dir = CreateDir();
var fileName = Path.Combine(dir, "MappedTrieReaderTests.Can_deserialize_whole_file.tri");
var tree = new LcrsTrie('\0', false);
tree.Add("baby");
tree.Add("bad");
tree.Add("badness");
tree.Add("bank");
tree.Add("box");
tree.Add("dad");
tree.Add("dance");
tree.Add("flower");
tree.Add("flowers");
tree.Add("globe");
tree.Add("global");
File.WriteAllText("Can_deserialize_whole_file_orig.log", tree.Visualize(), Encoding.UTF8);
foreach (var node in tree.EndOfWordNodes())
{
node.PostingsAddress = new BlockInfo(long.MinValue, int.MinValue);
}
Assert.IsTrue(tree.IsWord("baby") != null);
Assert.IsTrue(tree.IsWord("bad") != null);
Assert.IsTrue(tree.IsWord("badness") != null);
Assert.IsTrue(tree.IsWord("bank") != null);
Assert.IsTrue(tree.IsWord("box") != null);
Assert.IsTrue(tree.IsWord("dad") != null);
Assert.IsTrue(tree.IsWord("dance") != null);
Assert.IsTrue(tree.IsWord("flower") != null);
Assert.IsTrue(tree.IsWord("flowers") != null);
Assert.IsTrue(tree.IsWord("globe") != null);
Assert.IsTrue(tree.IsWord("global") != null);
tree.Serialize(fileName);
var recreated = Serializer.DeserializeTrie(fileName, 0);
File.WriteAllText("Can_deserialize_whole_file_recreated.log", recreated.Visualize(), Encoding.UTF8);
var result = recreated.IsWord("baby");
Assert.IsTrue(result != null);
Assert.IsTrue(recreated.IsWord("bad") != null);
Assert.IsTrue(recreated.IsWord("badness") != null);
Assert.IsTrue(recreated.IsWord("bank") != null);
Assert.IsTrue(recreated.IsWord("box") != null);
Assert.IsTrue(recreated.IsWord("dad") != null);
Assert.IsTrue(recreated.IsWord("dance") != null);
Assert.IsTrue(recreated.IsWord("flower") != null);
Assert.IsTrue(recreated.IsWord("flowers") != null);
Assert.IsTrue(recreated.IsWord("globe") != null);
Assert.IsTrue(recreated.IsWord("global") != null);
}
public LcrsNode(LcrsTrie trie, int depth, int weight)
{
Value = trie.Value;
HaveSibling = trie.RightSibling != null;
HaveChild = trie.LeftChild != null;
EndOfWord = trie.EndOfWord;
Depth = depth;
Weight = weight;
}
public static void SerializeMapped(this LcrsTrie node, string fileName)
{
using (var stream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None, 4800))
{
if (node.LeftChild != null)
{
node.LeftChild.SerializeMappedDepthFirst(stream, 0);
}
}
}
public void Commit()
{
var deleteSet = new LcrsTrie();
foreach (var value in _pks)
{
var hashString = value.ToHash().ToString(CultureInfo.InvariantCulture);
deleteSet.Add(hashString);
}
foreach (var ix in _ixs)
{
var docHashFileName = Path.Combine(_directory, string.Format("{0}.{1}", ix.VersionId, "pk"));
var tmpDocHashFileName = Path.Combine(_directory, string.Format("{0}.{1}", ix.VersionId, "pk.tmp"));
var tmpIxFileName = Path.Combine(_directory, ix.VersionId + ".ix.tmp");
var ixFileName = Path.Combine(_directory, ix.VersionId + ".ix");
var deleted = 0;
using (var stream = new FileStream(tmpDocHashFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
foreach (var document in Serializer.DeserializeDocHashes(docHashFileName))
{
var hash = document.Hash.ToString(CultureInfo.InvariantCulture);
IList <Word> found = deleteSet.IsWord(hash).ToList();
if (found.Any())
{
if (!document.IsObsolete)
{
document.IsObsolete = true;
deleted++;
}
}
document.Serialize(stream);
}
}
if (deleted > 0)
{
ix.DocumentCount -= deleted;
ix.Serialize(tmpIxFileName);
File.Copy(tmpIxFileName, ixFileName, overwrite: true);
File.Copy(tmpDocHashFileName, docHashFileName, overwrite: true);
File.Delete(tmpIxFileName);
File.Delete(tmpDocHashFileName);
}
}
}
public void Can_build_one_leg()
{
var tree = new LcrsTrie('\0', false);
tree.Add("baby");
Assert.That(tree.LeftChild.Value, Is.EqualTo('b'));
Assert.That(tree.LeftChild.LeftChild.Value, Is.EqualTo('a'));
Assert.That(tree.LeftChild.LeftChild.LeftChild.Value, Is.EqualTo('b'));
Assert.That(tree.LeftChild.LeftChild.LeftChild.LeftChild.Value, Is.EqualTo('y'));
Assert.True(tree.HasWord("baby"));
}
public LcrsTrie ReadWholeFile()
{
var words = Words();
var root = new LcrsTrie();
// TODO: assemble trie node by node
foreach (var word in words)
{
root.Add(word.Value);
}
return(root);
}
public void Can_build_one_leg()
{
var tree = new LcrsTrie('\0', false);
Word word;
tree.Add("baby");
Assert.AreEqual('b', tree.LeftChild.Value);
Assert.AreEqual('a', tree.LeftChild.LeftChild.Value);
Assert.AreEqual('b', tree.LeftChild.LeftChild.LeftChild.Value);
Assert.AreEqual('y', tree.LeftChild.LeftChild.LeftChild.LeftChild.Value);
Assert.IsTrue(tree.IsWord("baby").Any());
}
public void Can_find_exact()
{
var fileName = Path.Combine(CreateDir(), "MappedTrieReaderTests.Can_find_exact.tri");
var tree = new LcrsTrie('\0', false);
tree.Add("xor");
tree.Add("xxx");
tree.Add("donkey");
tree.Add("xavier");
tree.Add("baby");
tree.Add("dad");
tree.Add("daddy");
foreach (var node in tree.EndOfWordNodes())
{
node.PostingsAddress = new BlockInfo(long.MinValue, int.MinValue);
}
tree.Serialize(fileName);
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("xxx") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("xxx") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("baby") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("xxx") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("baby") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("dad") != null);
}
using (var reader = new MappedTrieReader(fileName))
{
Assert.IsTrue(reader.IsWord("daddy") != null);
}
}
private LcrsTrie GetTrie(string field)
{
var key = field.ToHash().ToString();
LcrsTrie trie;
if (!_tries.TryGetValue(key, out trie))
{
trie = new LcrsTrie();
_tries[key] = trie;
}
return(trie);
}
public LcrsTrie ReadWholeFile()
{
var words = new List <Word>();
DepthFirst(string.Empty, new List <char>(), words, -1);
var root = new LcrsTrie();
foreach (var word in words)
{
root.Add(word.Value);
}
return(root.LeftChild);
}
public void Can_get_weight()
{
var tree = new LcrsTrie('\0', false);
tree.Add("pap");
tree.Add("papp");
tree.Add("papaya");
Assert.AreEqual(8, tree.Weight);
tree.Add("ape");
tree.Add("apelsin");
Assert.AreEqual(15, tree.Weight);
}
public LcrsTrie ReadWholeFile()
{
var words = new List <Word>();
DepthFirst(string.Empty, new List <char>(), words, -1);
var root = new LcrsTrie();
// TODO: assemble trie node by node
foreach (var word in words)
{
root.Add(word.Value);
}
return(root);
}
private static void SerializeMappedDepthFirst(this LcrsTrie node, Stream stream, int depth)
{
var bytes = TypeToBytes(new LcrsNode(node, depth, node.GetWeight()));
stream.Write(bytes, 0, bytes.Length);
if (node.LeftChild != null)
{
node.LeftChild.SerializeMappedDepthFirst(stream, depth + 1);
}
if (node.RightSibling != null)
{
node.RightSibling.SerializeMappedDepthFirst(stream, depth);
}
}
public void Can_append()
{
var root = new LcrsTrie('\0', false);
root.Add("baby");
root.Add("bad");
Assert.That(root.LeftChild.Value, Is.EqualTo('b'));
Assert.That(root.LeftChild.LeftChild.Value, Is.EqualTo('a'));
Assert.That(root.LeftChild.LeftChild.LeftChild.Value, Is.EqualTo('d'));
Assert.That(root.LeftChild.LeftChild.LeftChild.RightSibling.Value, Is.EqualTo('b'));
Assert.That(root.LeftChild.LeftChild.LeftChild.RightSibling.LeftChild.Value, Is.EqualTo('y'));
Assert.True(root.HasWord("baby"));
Assert.True(root.HasWord("bad"));
}
private LcrsTrie GetTrie(string field, string token)
{
var key = string.Format("{0}-{1}", field.ToTrieFileId(), token.ToBucketName());
LcrsTrie trie;
if (!_tries.TryGetValue(key, out trie))
{
lock (_sync)
{
if (!_tries.TryGetValue(key, out trie))
{
trie = new LcrsTrie('\0', false);
_tries[key] = trie;
}
}
}
return(trie);
}
public void Can_append()
{
var root = new LcrsTrie('\0', false);
root.Add("baby");
root.Add("bad");
Word word;
Assert.AreEqual('b', root.LeftChild.Value);
Assert.AreEqual('a', root.LeftChild.LeftChild.Value);
Assert.AreEqual('d', root.LeftChild.LeftChild.LeftChild.RightSibling.Value);
Assert.AreEqual('b', root.LeftChild.LeftChild.LeftChild.Value);
Assert.AreEqual('y', root.LeftChild.LeftChild.LeftChild.LeftChild.Value);
Assert.IsTrue(root.IsWord("baby").Any());
Assert.IsTrue(root.IsWord("bad").Any());
}
public void Can_find_greater_than()
{
var trie = new LcrsTrie();
trie.Add("ape");
trie.Add("app");
trie.Add("apple");
trie.Add("banana");
trie.Add("bananas");
var words = trie.GreaterThan("ape").ToList();
Assert.AreEqual(4, words.Count);
Assert.AreEqual("app", words[0]);
Assert.AreEqual("apple", words[1]);
Assert.AreEqual("banana", words[2]);
Assert.AreEqual("bananas", words[3]);
}
public void Can_build_two_legs()
{
var root = new LcrsTrie('\0', false);
root.Add("baby");
root.Add("dad");
Word word;
Assert.AreEqual('d', root.LeftChild.RightSibling.Value);
Assert.AreEqual('a', root.LeftChild.LeftChild.Value);
Assert.AreEqual('d', root.LeftChild.RightSibling.LeftChild.LeftChild.Value);
Assert.AreEqual('b', root.LeftChild.Value);
Assert.AreEqual('a', root.LeftChild.RightSibling.LeftChild.Value);
Assert.AreEqual('b', root.LeftChild.LeftChild.LeftChild.Value);
Assert.AreEqual('y', root.LeftChild.LeftChild.LeftChild.LeftChild.Value);
Assert.IsTrue(root.IsWord("baby") != null);
Assert.IsTrue(root.IsWord("dad") != null);
}
public void Can_build_two_legs()
{
var root = new LcrsTrie('\0', false);
root.Add("baby");
root.Add("dad");
Word word;
Assert.That(root.LeftChild.RightSibling.Value, Is.EqualTo('d'));
Assert.That(root.LeftChild.LeftChild.Value, Is.EqualTo('a'));
Assert.That(root.LeftChild.RightSibling.LeftChild.LeftChild.Value, Is.EqualTo('d'));
Assert.That(root.LeftChild.Value, Is.EqualTo('b'));
Assert.That(root.LeftChild.RightSibling.LeftChild.Value, Is.EqualTo('a'));
Assert.That(root.LeftChild.LeftChild.LeftChild.Value, Is.EqualTo('b'));
Assert.That(root.LeftChild.LeftChild.LeftChild.LeftChild.Value, Is.EqualTo('y'));
Assert.True(root.HasWord("baby", out word));
Assert.True(root.HasWord("dad", out word));
}
public void Can_find_prefixed()
{
var fileName = Path.Combine(CreateDir(), "MappedTrieReaderTests.Can_find_prefixed.tri");
var tree = new LcrsTrie('\0', false);
tree.Add("rambo");
tree.Add("rambo");
tree.Add("2");
tree.Add("rocky");
tree.Add("2");
tree.Add("raiders");
tree.Add("of");
tree.Add("the");
tree.Add("lost");
tree.Add("ark");
tree.Add("rain");
tree.Add("man");
foreach (var node in tree.EndOfWordNodes())
{
node.PostingsAddress = new BlockInfo(long.MinValue, int.MinValue);
}
tree.Serialize(fileName);
var prefixed = new MappedTrieReader(fileName).StartsWith("ra").Select(w => w.Value).ToList();
Assert.AreEqual(3, prefixed.Count);
Assert.IsTrue(prefixed.Contains("rambo"));
Assert.IsTrue(prefixed.Contains("raiders"));
Assert.IsTrue(prefixed.Contains("rain"));
}
public void Can_append_tries()
{
var one = new LcrsTrie('\0', false);
one.Add("ape");
one.Add("app");
one.Add("banana");
var two = new LcrsTrie('\0', false);
two.Add("apple");
two.Add("banana");
one.Merge(two);
Word found;
Assert.IsTrue(one.HasWord("ape", out found));
Assert.IsTrue(one.HasWord("app", out found));
Assert.IsTrue(one.HasWord("apple", out found));
Assert.IsTrue(one.HasWord("banana", out found));
}
public void Can_merge_tries()
{
var one = new LcrsTrie('\0', false);
one.Add("ape");
one.Add("app");
one.Add("bananas");
var two = new LcrsTrie('\0', false);
two.Add("apple");
two.Add("banana");
two.Add("citron");
one.Merge(two);
Assert.IsTrue(one.IsWord("ape").Any());
Assert.IsTrue(one.IsWord("app").Any());
Assert.IsTrue(one.IsWord("apple").Any());
Assert.IsTrue(one.IsWord("banana").Any());
Assert.IsTrue(one.IsWord("bananas").Any());
Assert.IsTrue(one.IsWord("citron").Any());
}
