private TokenInfoDictionary(IResourceResolver resolver)
{
using (var tokenInfoBufferResource = resolver.Resolve(TokenInfoDictionaryFileName))
using (var stringValueResource = resolver.Resolve(FeatureMapFileName))
using (var posValueResource = resolver.Resolve(PosMapFileName))
using (var wordIdMapResource = resolver.Resolve(TargetMapFileName))
{
TokenInfoBuffer = new TokenInfoBuffer(tokenInfoBufferResource);
StringValues = new StringValueMapBuffer(stringValueResource);
PosValues = new StringValueMapBuffer(posValueResource);
WordIdMap = new WordIdMap(wordIdMapResource);
}
}
private void Setup(string absoluteResourcePath)
{
try
{
using (var tokenStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TOKEN_INFO_DICTIONARY_FILENAME))
{ tokenInfoBuffer = new TokenInfoBuffer(tokenStream); }
using (var stringStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + FEATURE_MAP_FILENAME))
{ stringValues = new StringValueMapBuffer(stringStream); }
using (var posStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + POS_MAP_FILENAME))
{ posValues = new StringValueMapBuffer(posStream); }
using (var stream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TARGETMAP_FILENAME))
{ wordIdMap = new WordIdMap(stream); }
}
catch (IOException ex)
{
throw new Exception("TokenInfoDictionary.Setup: " + ex.Message);
}
}
public void TestCompiler()
{
var compiler = new WordIdMapCompiler();
compiler.AddMapping(3, 1);
compiler.AddMapping(3, 2);
compiler.AddMapping(3, 3);
compiler.AddMapping(10, 0);
using (var ms = new MemoryStream())
{
compiler.Write(ms);
ms.Seek(0, SeekOrigin.Begin);
var wordIds = new WordIdMap(ms);
string.Join(", ", wordIds.LookUp(3)).Is("1, 2, 3");
string.Join(", ", wordIds.LookUp(10)).Is("0");
string.Join(", ", wordIds.LookUp(1)).Is("");
}
}
public void TestCompiler()
{
WordIdMapCompiler compiler = new WordIdMapCompiler();
compiler.AddMapping(3, 1);
compiler.AddMapping(3, 2);
compiler.AddMapping(3, 3);
compiler.AddMapping(10, 0);
var fileName = TestUtils.CompiledPath + Path.DirectorySeparatorChar + "kuromoji-wordid-.bin";
using (var output = File.Create(fileName))
{
compiler.Write(output);
}
using (var input = File.OpenRead(fileName))
{
WordIdMap wordIds = new WordIdMap(input);
Assert.AreEqual("[1, 2, 3]", wordIds.LookUp(3).Array2String());
Assert.AreEqual("[0]", wordIds.LookUp(10).Array2String());
Assert.AreEqual("[]", wordIds.LookUp(1).Array2String());
}
}
