void BuildTokenInfoDictionary(string inputDirname, string outputDirname, Encoding encoding, bool compactTries)
{
ProgressLog.Begin("compiling tokeninfo dict");
var tokenInfoCompiler = GetTokenInfoDictionaryCompiler(encoding);
ProgressLog.Println("analyzing dictionary features");
using (var stream = tokenInfoCompiler.CombinedSequentialFileInputStream(inputDirname))
{
tokenInfoCompiler.AnalyzeTokenInfo(stream);
}
ProgressLog.Println("reading tokeninfo");
using (var stream = tokenInfoCompiler.CombinedSequentialFileInputStream(inputDirname))
{
tokenInfoCompiler.ReadTokenInfo(stream);
}
tokenInfoCompiler.Compile();
var surfaces = tokenInfoCompiler.Surfaces;
ProgressLog.Begin("compiling double array trie");
using (var fs = new FileStream(Path.Combine(outputDirname, DoubleArrayTrie.DoubleArrayTrieFileName), FileMode.Create, FileAccess.ReadWrite))
{
var trie = DoubleArrayTrieCompiler.Build(surfaces, compactTries);
trie.Write(fs);
}
ProgressLog.Println("validating saved double array trie");
DoubleArrayTrie daTrie;
using (var fs = new FileStream(Path.Combine(outputDirname, DoubleArrayTrie.DoubleArrayTrieFileName), FileMode.Open, FileAccess.Read))
{
daTrie = DoubleArrayTrie.Read(fs);
foreach (var surface in surfaces)
{
if (daTrie.Lookup(surface) < 0)
{
ProgressLog.Println("failed to look up [" + surface + "]");
}
}
}
ProgressLog.End();
ProgressLog.Begin("processing target map");
for (var i = 0; i < surfaces.Count; i++)
{
int id = daTrie.Lookup(surfaces[i]);
tokenInfoCompiler.AddMapping(id, i);
}
tokenInfoCompiler.Write(outputDirname); // TODO: Should be refactored -Christian
ProgressLog.End();
ProgressLog.End();
}
void TestSimpleTrie(bool compact)
{
var trie = MakeTrie();
var doubleArrayTrie = new DoubleArrayTrie(compact);
doubleArrayTrie.Build(trie);
using (var ms = new MemoryStream())
{
doubleArrayTrie.Write(ms);
ms.Seek(0, SeekOrigin.Begin);
doubleArrayTrie = DoubleArrayTrie.Read(ms);
}
doubleArrayTrie.Lookup("a").Is(0);
(doubleArrayTrie.Lookup("abc") > 0).IsTrue();
(doubleArrayTrie.Lookup("あいう") > 0).IsTrue();
(doubleArrayTrie.Lookup("xyz") < 0).IsTrue();
}
