private Tokenizer MakeTokenizer(string userDictionaryEntry)
{
using (var stream = MakeUserDictionaryStream(userDictionaryEntry))
{
var builder = new Tokenizer.Builder(TestUtils.AbsoluteIpadicResourcePath);
builder.LoadUserDictionary(stream);
return(new Tokenizer(builder));
}
}
private Tokenizer MakeTokenizer()
{
using (var file = File.OpenRead(Locations.ABS_DICT_COMPILED_PATH + System.IO.Path.DirectorySeparatorChar + "userDict.txt"))
{
var builder = new Tokenizer.Builder(Locations.ABS_DICT_COMPILED_PATH);
builder.LoadUserDictionary(file);
builder.IsSplitOnNakaguro = true;
return(new Tokenizer(builder));
}
}
public void TestFeatureLengths()
{
string userDictionary = "" +
"gsf,gsf,ジーエスーエフ,カスタム名詞\n";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(userDictionary)))
{
var builder = new Tokenizer.Builder(TestUtils.DictResourcedPath);
builder.LoadUserDictionary(stream);
using (Tokenizer tokenizer = new Tokenizer(builder))
TestUtils.AssertEqualTokenFeatureLengths("ahgsfdajhgsfdこの丘はアクロポリスと呼ばれている。", tokenizer);
}
}
public void TestMultiThreadedUserDictionary()
{
var filePath = "./Core/Resource/userdict.txt";
using (var stream = File.OpenRead(filePath))
{
using (var builder = new Tokenizer.Builder(TestUtils.AbsoluteIpadicResourcePath))
{
builder.LoadUserDictionary(stream);
TestUtils.AssertMultiThreadedTokenizedStreamEquals(
5,
10,
TestUtils.AbsoluteIpadicResourcePath + "jawikisentences-ipadic-features.txt",
TestUtils.AbsoluteIpadicResourcePath + "jawikisentences.txt",
new Tokenizer(builder)
);
}
}
}
