protected void Configure <V, K>(V builder) where V : Builder <K> where K : TokenizerBase <T>
{
builder.LoadDictionaries();
this.tokenFactory = builder.TokenFactory;
this.tokenInfoDictionary = builder.TokenInfoDictionary;
this.unknownDictionary = builder.UnknownDictionary;
this.userDictionary = builder.UserDictionary;
this.insertedDictionary = builder.InsertedDictionary;
this.viterbiBuilder = new ViterbiBuilder(
builder.Fst,
tokenInfoDictionary,
unknownDictionary,
userDictionary,
builder.Mode
);
this.viterbiSearcher = new ViterbiSearcher(
builder.Mode,
builder.ConnectionCosts,
unknownDictionary,
builder.Penalties
);
this.viterbiFormatter = new ViterbiFormatter(builder.ConnectionCosts);
this.split = builder.Split;
InitDictionaryMap();
}
public override void LoadDictionaries()
{
Penalties = new List <int>();
Penalties.Add(kanjiPenaltyLengthTreshold);
Penalties.Add(kanjiPenalty);
Penalties.Add(otherPenaltyLengthThreshold);
Penalties.Add(otherPenalty);
try
{
Fst = FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
if (IsSplitOnNakaguro)
{
CharacterDefinitions.SetCategories('・', new string[] { "SYMBOL" });
}
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries: " + ouch.Message);
}
}
protected internal virtual void LoadDictionaries()
{
DoubleArrayTrie = DoubleArrayTrie.NewInstance(Resolver);
ConnectionCosts = ConnectionCosts.NewInstance(Resolver);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(Resolver);
CharacterDefinitions = CharacterDefinitions.NewInstance(Resolver);
UnknownDictionary = UnknownDictionary.NewInstance(Resolver, CharacterDefinitions, TotalFeatures);
InsertedDictionary = new InsertedDictionary(TotalFeatures);
}
public void TestFeatureSize()
{
var dictionary1 = new InsertedDictionary(9);
var dictionary2 = new InsertedDictionary(5);
dictionary1.GetAllFeatures(0).Is("*,*,*,*,*,*,*,*,*");
dictionary2.GetAllFeatures(0).Is("*,*,*,*,*");
dictionary1.GetAllFeaturesArray(0).SequenceEqual(new string[] { "*", "*", "*", "*", "*", "*", "*", "*", "*" }).IsTrue();
dictionary2.GetAllFeaturesArray(0).SequenceEqual(new string[] { "*", "*", "*", "*", "*" }).IsTrue();
}
public virtual void LoadDictionaries()
{
try
{
Fst = FST.FST.NewInstance(AbsoluteFolderPath);
ConnectionCosts = ConnectionCosts.NewInstance(AbsoluteFolderPath);
TokenInfoDictionary = TokenInfoDictionary.NewInstance(AbsoluteFolderPath);
CharacterDefinitions = CharacterDefinitions.NewInstance(AbsoluteFolderPath);
UnknownDictionary = UnknownDictionary.NewInstance(AbsoluteFolderPath, CharacterDefinitions, totalFeatures);
InsertedDictionary = new InsertedDictionary(totalFeatures);
}
catch (Exception ouch)
{
throw new Exception("Could not load dictionaries.", ouch);
}
}
public void TestFeatureSize()
{
InsertedDictionary dictionary1 = new InsertedDictionary(9);
InsertedDictionary dictionary2 = new InsertedDictionary(5);
Assert.AreEqual("*,*,*,*,*,*,*,*,*", dictionary1.GetAllFeatures(0));
Assert.AreEqual("*,*,*,*,*", dictionary2.GetAllFeatures(0));
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new string[] { "*", "*", "*", "*", "*", "*", "*", "*", "*" },
dictionary1.GetAllFeaturesArray(0)
));
Assert.IsTrue(NLPJDictTest.TestUtils.IsArrayEqual(
new string[] { "*", "*", "*", "*", "*" },
dictionary2.GetAllFeaturesArray(0)
));
}