public void TestReadAndLookUpTokenInfo()
{
List <short> tokenInfo = new List <short>();
List <int> features = new List <int>();
short[] tokenInfos = new short[3];
tokenInfos[0] = 1;
tokenInfos[1] = 2;
tokenInfos[2] = 3;
int[] featureInfos = new int[2];
featureInfos[0] = 73;
featureInfos[1] = 99;
tokenInfo.Add((short)1);
tokenInfo.Add((short)2);
tokenInfo.Add((short)3);
features.Add(73);
features.Add(99);
BufferEntry entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
List <BufferEntry> bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
{
using (var outputStream = File.Create(tokenInfoFile))
{
compiler.Compile(outputStream);
}
}
using (var inputStream = File.OpenRead(tokenInfoFile))
{
using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream))
{
Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1));
Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0));
}
}
}
public void TestReadAndLookUpTokenInfo()
{
var tokenInfo = new List <short>();
var features = new List <int>();
var tokenInfos = new short[] { 1, 2, 3 };
var featureInfos = new int[] { 73, 99 };
tokenInfo.Add(1);
tokenInfo.Add(2);
tokenInfo.Add(3);
features.Add(73);
features.Add(99);
var entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
var bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var ms = new MemoryStream())
{
var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
compiler.Compile();
ms.Seek(0, SeekOrigin.Begin);
var tokenInfoBuffer2 = new TokenInfoBuffer(ms);
tokenInfoBuffer2.LookupFeature(0, 1).Is(99);
tokenInfoBuffer2.LookupFeature(0, 0).Is(73);
}
}
string ExtractSingleFeature(int wordId, int field)
{
if (TokenInfoBuffer.isPartOfSpeechFeature(field))
{
var featureId = TokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field);
return(PosValues[featureId]);
}
else
{
var featureId = TokenInfoBuffer.LookupFeature(wordId, field);
return(StringValues[featureId]);
}
}
private string ExtractSingleFeature(int wordId, int field)
{
int featureId;
if (tokenInfoBuffer.IsPartOfSpeechFeature(field))
{
featureId = tokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field);
return(posValues.Get(featureId));
}
featureId = tokenInfoBuffer.LookupFeature(wordId, field);
return(stringValues.Get(featureId));
}