void BuildTokenInfoDictionary(string inputDirname, string outputDirname, Encoding encoding, bool compactTries) { ProgressLog.Begin("compiling tokeninfo dict"); var tokenInfoCompiler = GetTokenInfoDictionaryCompiler(encoding); ProgressLog.Println("analyzing dictionary features"); using (var stream = tokenInfoCompiler.CombinedSequentialFileInputStream(inputDirname)) { tokenInfoCompiler.AnalyzeTokenInfo(stream); } ProgressLog.Println("reading tokeninfo"); using (var stream = tokenInfoCompiler.CombinedSequentialFileInputStream(inputDirname)) { tokenInfoCompiler.ReadTokenInfo(stream); } tokenInfoCompiler.Compile(); var surfaces = tokenInfoCompiler.Surfaces; ProgressLog.Begin("compiling double array trie"); using (var fs = new FileStream(Path.Combine(outputDirname, DoubleArrayTrie.DoubleArrayTrieFileName), FileMode.Create, FileAccess.ReadWrite)) { var trie = DoubleArrayTrieCompiler.Build(surfaces, compactTries); trie.Write(fs); } ProgressLog.Println("validating saved double array trie"); DoubleArrayTrie daTrie; using (var fs = new FileStream(Path.Combine(outputDirname, DoubleArrayTrie.DoubleArrayTrieFileName), FileMode.Open, FileAccess.Read)) { daTrie = DoubleArrayTrie.Read(fs); foreach (var surface in surfaces) { if (daTrie.Lookup(surface) < 0) { ProgressLog.Println("failed to look up [" + surface + "]"); } } } ProgressLog.End(); ProgressLog.Begin("processing target map"); for (var i = 0; i < surfaces.Count; i++) { int id = daTrie.Lookup(surfaces[i]); tokenInfoCompiler.AddMapping(id, i); } tokenInfoCompiler.Write(outputDirname); // TODO: Should be refactored -Christian ProgressLog.End(); ProgressLog.End(); }
void BuildConnectionCosts(string inputDirname, string outputDirname) { ProgressLog.Begin("compiling connection costs"); using (var output = new FileStream(Path.Combine(outputDirname, ConnectionCosts.ConnectionCostsFileName), FileMode.Create, FileAccess.ReadWrite)) using (var matrix = new FileStream(Path.Combine(inputDirname, "matrix.def"), FileMode.Open, FileAccess.Read)) { var connectionCostsCompiler = new ConnectionCostsCompiler(output); connectionCostsCompiler.ReadCosts(matrix); connectionCostsCompiler.Compile(); } ProgressLog.End(); }
protected void BuildUnknownWordDictionary(string inputDirname, string outputDirname, Encoding encoding) { ProgressLog.Begin("compiling unknown word dict"); using (var charDefOutput = new FileStream(Path.Combine(outputDirname, CharacterDefinitions.CharacterDefinitionsFileName), FileMode.Create, FileAccess.ReadWrite)) using (var charDefInput = new FileStream(Path.Combine(inputDirname, "char.def"), FileMode.Open, FileAccess.Read)) using (var unkDicOutput = new FileStream(Path.Combine(outputDirname, UnknownDictionary.UnknownDictionaryFileName), FileMode.Create, FileAccess.ReadWrite)) using (var unkDicInput = new FileStream(Path.Combine(inputDirname, "unk.def"), FileMode.Open, FileAccess.Read)) { var charDefCompiler = new CharacterDefinitionsCompiler(charDefOutput); charDefCompiler.ReadCharacterDefinition(charDefInput, encoding); charDefCompiler.Compile(); var unkDicCompiler = new UnknownDictionaryCompiler(unkDicOutput, charDefCompiler.MakeCharacterCategoryMap()); unkDicCompiler.ReadUnknownDefinition(unkDicInput, encoding); unkDicCompiler.Compile(); } ProgressLog.End(); }