public override void LoadDictionaries()
{
Penalties = new List <int>();
Penalties.Add(kanjiPenaltyLengthTreshold);
Penalties.Add(kanjiPenalty);
Penalties.Add(otherPenaltyLengthThreshold);
Penalties.Add(otherPenalty);
try
{
Fst = FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
if (IsSplitOnNakaguro)
{
CharacterDefinitions.SetCategories('・', new string[] { "SYMBOL" });
}
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries: " + ouch.Message);
}
}
protected void Configure <V, K>(V builder) where V : Builder <K> where K : TokenizerBase <T>
{
builder.LoadDictionaries();
this.tokenFactory = builder.TokenFactory;
this.tokenInfoDictionary = builder.TokenInfoDictionary;
this.unknownDictionary = builder.UnknownDictionary;
this.userDictionary = builder.UserDictionary;
this.insertedDictionary = builder.InsertedDictionary;
this.viterbiBuilder = new ViterbiBuilder(
builder.Fst,
tokenInfoDictionary,
unknownDictionary,
userDictionary,
builder.Mode
);
this.viterbiSearcher = new ViterbiSearcher(
builder.Mode,
builder.ConnectionCosts,
unknownDictionary,
builder.Penalties
);
this.viterbiFormatter = new ViterbiFormatter(builder.ConnectionCosts);
this.split = builder.Split;
InitDictionaryMap();
}
protected internal virtual void LoadDictionaries()
{
DoubleArrayTrie = DoubleArrayTrie.NewInstance(Resolver);
ConnectionCosts = ConnectionCosts.NewInstance(Resolver);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(Resolver);
CharacterDefinitions = CharacterDefinitions.NewInstance(Resolver);
UnknownDictionary = UnknownDictionary.NewInstance(Resolver, CharacterDefinitions, TotalFeatures);
InsertedDictionary = new InsertedDictionary(TotalFeatures);
}
public void Dispose()
{
if (ConnectionCosts != null)
{
ConnectionCosts.Dispose();
}
if (TokenInfoDictionary != null)
{
TokenInfoDictionary.Dispose();
}
}
public ViterbiBuilder(DoubleArrayTrie doubleArrayTrie, TokenInfoDictionary dictionary, UnknownDictionary unknownDictionary, UserDictionary userDictionary, TokenizerMode mode)
{
DoubleArrayTrie = doubleArrayTrie;
Dictionary = dictionary;
UnknownDictionary = unknownDictionary;
UserDictionary = userDictionary;
UseUserDictionary = userDictionary != null;
SearchMode = mode == TokenizerMode.Search || mode == TokenizerMode.Extended;
CharacterDefinitions = UnknownDictionary.CharacterDefinition;
}
public virtual void LoadDictionaries()
{
try
{
Fst = FST.FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries.", ouch);
}
}
/**
* Constructor
*
* @param fst FST with surface forms
* @param dictionary token info dictionary
* @param unknownDictionary unknown word dictionary
* @param userDictionary user dictionary
* @param mode tokenization {@link Mode mode}
*/
public ViterbiBuilder(FST.FST fst,
TokenInfoDictionary dictionary,
UnknownDictionary unknownDictionary,
UserDictionary userDictionary,
Mode mode)
{
this.fst = fst;
this.dictionary = dictionary;
this.unknownDictionary = unknownDictionary;
this.userDictionary = userDictionary;
this.useUserDictionary = (userDictionary != null);
if (mode == Mode.SEARCH || mode == Mode.EXTENDED)
{
searchMode = true;
}
this.characterDefinitions = unknownDictionary.GetCharacterDefinition();
}
public void Dispose()
{
if (viterbiBuilder != null)
{
viterbiBuilder.Dispose();
}
if (viterbiSearcher != null)
{
viterbiSearcher.Dispose();
}
if (viterbiFormatter != null)
{
viterbiFormatter.Dispose();
}
if (tokenInfoDictionary != null)
{
tokenInfoDictionary.Dispose();
}
viterbiBuilder = null;
viterbiSearcher = null;
viterbiFormatter = null;
tokenInfoDictionary = null;
}
