public void TestReadAndLookUpTokenInfo()
{
List <short> tokenInfo = new List <short>();
List <int> features = new List <int>();
short[] tokenInfos = new short[3];
tokenInfos[0] = 1;
tokenInfos[1] = 2;
tokenInfos[2] = 3;
int[] featureInfos = new int[2];
featureInfos[0] = 73;
featureInfos[1] = 99;
tokenInfo.Add((short)1);
tokenInfo.Add((short)2);
tokenInfo.Add((short)3);
features.Add(73);
features.Add(99);
BufferEntry entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
List <BufferEntry> bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
{
using (var outputStream = File.Create(tokenInfoFile))
{
compiler.Compile(outputStream);
}
}
using (var inputStream = File.OpenRead(tokenInfoFile))
{
using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream))
{
Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1));
Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0));
}
}
}
public void TestCompleteLookUp()
{
Dictionary <int, string> resultMap = new Dictionary <int, string>();
resultMap[73] = "hello";
resultMap[42] = "今日は";
resultMap[99] = "素敵な世界";
List <short> tokenInfo = new List <short>();
List <int> features = new List <int>();
tokenInfo.Add((short)1);
tokenInfo.Add((short)2);
tokenInfo.Add((short)3);
features.Add(73);
features.Add(99);
BufferEntry entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
List <BufferEntry> bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var outStream = File.Create(tokenInfoFile))
{
using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
{
compiler.Compile(outStream);
}
}
using (var inStream = File.OpenRead(tokenInfoFile))
{
using (TokenInfoBuffer tokenInfoBuffer = new TokenInfoBuffer(inStream))
{
BufferEntry result = tokenInfoBuffer.LookupEntry(0);
Assert.AreEqual("hello", resultMap[result.FeatureInfos[0]]);
Assert.AreEqual("素敵な世界", resultMap[result.FeatureInfos[1]]);
}
}
}
public void TestCompleteLookUp()
{
var resultMap = new Dictionary <int, string>();
resultMap.Add(73, "hello");
resultMap.Add(42, "今日は");
resultMap.Add(99, "素敵な世界");
var tokenInfo = new List <short>();
var features = new List <int>();
tokenInfo.Add(1);
tokenInfo.Add(2);
tokenInfo.Add(3);
features.Add(73);
features.Add(99);
var entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
var bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var ms = new MemoryStream())
{
var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
compiler.Compile();
ms.Seek(0, SeekOrigin.Begin);
var tokenInfoBuffer = new TokenInfoBuffer(ms);
var result = tokenInfoBuffer.LookupEntry(0);
resultMap[result.FeatureInfos[0]].Is("hello");
resultMap[result.FeatureInfos[1]].Is("素敵な世界");
}
}
public void TestReadAndLookUpTokenInfo()
{
var tokenInfo = new List <short>();
var features = new List <int>();
var tokenInfos = new short[] { 1, 2, 3 };
var featureInfos = new int[] { 73, 99 };
tokenInfo.Add(1);
tokenInfo.Add(2);
tokenInfo.Add(3);
features.Add(73);
features.Add(99);
var entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
var bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var ms = new MemoryStream())
{
var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
compiler.Compile();
ms.Seek(0, SeekOrigin.Begin);
var tokenInfoBuffer2 = new TokenInfoBuffer(ms);
tokenInfoBuffer2.LookupFeature(0, 1).Is(99);
tokenInfoBuffer2.LookupFeature(0, 0).Is(73);
}
}
