protected void Configure <V, K>(V builder) where V : Builder <K> where K : TokenizerBase <T> { builder.LoadDictionaries(); this.tokenFactory = builder.TokenFactory; this.tokenInfoDictionary = builder.TokenInfoDictionary; this.unknownDictionary = builder.UnknownDictionary; this.userDictionary = builder.UserDictionary; this.insertedDictionary = builder.InsertedDictionary; this.viterbiBuilder = new ViterbiBuilder( builder.Fst, tokenInfoDictionary, unknownDictionary, userDictionary, builder.Mode ); this.viterbiSearcher = new ViterbiSearcher( builder.Mode, builder.ConnectionCosts, unknownDictionary, builder.Penalties ); this.viterbiFormatter = new ViterbiFormatter(builder.ConnectionCosts); this.split = builder.Split; InitDictionaryMap(); }
/// <summary> /// Writes the Viterbi lattice for the provided text to an output stream /// /// The output is written in <a href="https://en.wikipedia.org/wiki/DOT_(graph_description_language)">DOT</a> format. /// /// This method is not thread safe /// </summary> /// <param name="output">output stream to write to</param> /// <param name="text">text to create lattice for</param> public void DebugLattice(Stream output, string text) { var lattice = ViterbiBuilder.Build(text); using (var writer = new StreamWriter(output, Encoding.UTF8, 1024, true)) { writer.Write(ViterbiFormatter.Format(lattice)); } }
/// <summary> /// Tokenizes the provided text and outputs the corresponding Viterbi lattice and the Viterbi path to the provided output stream /// /// The output is written in <a href="https://en.wikipedia.org/wiki/DOT_(graph_description_language)">DOT</a> format. /// /// This method is not thread safe /// </summary> /// <param name="output">output stream to write to</param> /// <param name="text">text to tokenize</param> public void DebugTokenize(Stream output, string text) { var lattice = ViterbiBuilder.Build(text); var bestPath = ViterbiSearcher.Search(lattice); using (var writer = new StreamWriter(output, Encoding.UTF8, 1024, true)) { writer.Write(ViterbiFormatter.Format(lattice, bestPath)); } }
protected void Configure(BuilderBase builder) { builder.LoadDictionaries(); TokenFactory = builder.TokenFactory; TokenInfoDictionary = builder.TokenInfoDictionary; UnknownDictionary = builder.UnknownDictionary; UserDictionary = builder.UserDictionary; InsertedDictionary = builder.InsertedDictionary; ViterbiBuilder = new ViterbiBuilder(builder.DoubleArrayTrie, TokenInfoDictionary, UnknownDictionary, UserDictionary, builder.Mode); ViterbiSearcher = new ViterbiSearcher(builder.Mode, builder.ConnectionCosts, UnknownDictionary, builder.Penalties); ViterbiFormatter = new ViterbiFormatter(builder.ConnectionCosts); Split = builder.Split; InitDictionaryMap(); }
public void Dispose() { if (viterbiBuilder != null) { viterbiBuilder.Dispose(); } if (viterbiSearcher != null) { viterbiSearcher.Dispose(); } if (viterbiFormatter != null) { viterbiFormatter.Dispose(); } if (tokenInfoDictionary != null) { tokenInfoDictionary.Dispose(); } viterbiBuilder = null; viterbiSearcher = null; viterbiFormatter = null; tokenInfoDictionary = null; }