public override void LoadDictionaries() { Penalties = new List <int>(); Penalties.Add(kanjiPenaltyLengthTreshold); Penalties.Add(kanjiPenalty); Penalties.Add(otherPenaltyLengthThreshold); Penalties.Add(otherPenalty); try { Fst = FST.NewInstance(AbsoluteFolderPath); ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath); TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath); CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath); if (IsSplitOnNakaguro) { CharacterDefinitions.SetCategories('・', new string[] { "SYMBOL" }); } UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures); InsertedDictionary = new InsertedDictionary(totalFeatures); } catch (Exception ouch) { throw new Exception("Could not load dictionaries: " + ouch.Message); } }
protected void Configure <V, K>(V builder) where V : Builder <K> where K : TokenizerBase <T> { builder.LoadDictionaries(); this.tokenFactory = builder.TokenFactory; this.tokenInfoDictionary = builder.TokenInfoDictionary; this.unknownDictionary = builder.UnknownDictionary; this.userDictionary = builder.UserDictionary; this.insertedDictionary = builder.InsertedDictionary; this.viterbiBuilder = new ViterbiBuilder( builder.Fst, tokenInfoDictionary, unknownDictionary, userDictionary, builder.Mode ); this.viterbiSearcher = new ViterbiSearcher( builder.Mode, builder.ConnectionCosts, unknownDictionary, builder.Penalties ); this.viterbiFormatter = new ViterbiFormatter(builder.ConnectionCosts); this.split = builder.Split; InitDictionaryMap(); }
protected internal virtual void LoadDictionaries() { DoubleArrayTrie = DoubleArrayTrie.NewInstance(Resolver); ConnectionCosts = ConnectionCosts.NewInstance(Resolver); TokenInfoDictionary = TokenInfoDictionary.NewInstance(Resolver); CharacterDefinitions = CharacterDefinitions.NewInstance(Resolver); UnknownDictionary = UnknownDictionary.NewInstance(Resolver, CharacterDefinitions, TotalFeatures); InsertedDictionary = new InsertedDictionary(TotalFeatures); }
public void Dispose() { if (ConnectionCosts != null) { ConnectionCosts.Dispose(); } if (TokenInfoDictionary != null) { TokenInfoDictionary.Dispose(); } }
public ViterbiBuilder(DoubleArrayTrie doubleArrayTrie, TokenInfoDictionary dictionary, UnknownDictionary unknownDictionary, UserDictionary userDictionary, TokenizerMode mode) { DoubleArrayTrie = doubleArrayTrie; Dictionary = dictionary; UnknownDictionary = unknownDictionary; UserDictionary = userDictionary; UseUserDictionary = userDictionary != null; SearchMode = mode == TokenizerMode.Search || mode == TokenizerMode.Extended; CharacterDefinitions = UnknownDictionary.CharacterDefinition; }
public virtual void LoadDictionaries() { try { Fst = FST.FST.NewInstance(AbsoluteFolderPath); ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath); TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath); CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath); UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures); InsertedDictionary = new InsertedDictionary(totalFeatures); } catch (Exception ouch) { throw new Exception("Could not load dictionaries.", ouch); } }
/** * Constructor * * @param fst FST with surface forms * @param dictionary token info dictionary * @param unknownDictionary unknown word dictionary * @param userDictionary user dictionary * @param mode tokenization {@link Mode mode} */ public ViterbiBuilder(FST.FST fst, TokenInfoDictionary dictionary, UnknownDictionary unknownDictionary, UserDictionary userDictionary, Mode mode) { this.fst = fst; this.dictionary = dictionary; this.unknownDictionary = unknownDictionary; this.userDictionary = userDictionary; this.useUserDictionary = (userDictionary != null); if (mode == Mode.SEARCH || mode == Mode.EXTENDED) { searchMode = true; } this.characterDefinitions = unknownDictionary.GetCharacterDefinition(); }
public void Dispose() { if (viterbiBuilder != null) { viterbiBuilder.Dispose(); } if (viterbiSearcher != null) { viterbiSearcher.Dispose(); } if (viterbiFormatter != null) { viterbiFormatter.Dispose(); } if (tokenInfoDictionary != null) { tokenInfoDictionary.Dispose(); } viterbiBuilder = null; viterbiSearcher = null; viterbiFormatter = null; tokenInfoDictionary = null; }