string ExtractSingleFeature(int wordId, int field)
{
if (TokenInfoBuffer.isPartOfSpeechFeature(field))
{
var featureId = TokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field);
return(PosValues[featureId]);
}
else
{
var featureId = TokenInfoBuffer.LookupFeature(wordId, field);
return(StringValues[featureId]);
}
}
private TokenInfoDictionary(IResourceResolver resolver)
{
using (var tokenInfoBufferResource = resolver.Resolve(TokenInfoDictionaryFileName))
using (var stringValueResource = resolver.Resolve(FeatureMapFileName))
using (var posValueResource = resolver.Resolve(PosMapFileName))
using (var wordIdMapResource = resolver.Resolve(TargetMapFileName))
{
TokenInfoBuffer = new TokenInfoBuffer(tokenInfoBufferResource);
StringValues = new StringValueMapBuffer(stringValueResource);
PosValues = new StringValueMapBuffer(posValueResource);
WordIdMap = new WordIdMap(wordIdMapResource);
}
}
public void TestReadAndLookUpTokenInfo()
{
List <short> tokenInfo = new List <short>();
List <int> features = new List <int>();
short[] tokenInfos = new short[3];
tokenInfos[0] = 1;
tokenInfos[1] = 2;
tokenInfos[2] = 3;
int[] featureInfos = new int[2];
featureInfos[0] = 73;
featureInfos[1] = 99;
tokenInfo.Add((short)1);
tokenInfo.Add((short)2);
tokenInfo.Add((short)3);
features.Add(73);
features.Add(99);
BufferEntry entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
List <BufferEntry> bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
{
using (var outputStream = File.Create(tokenInfoFile))
{
compiler.Compile(outputStream);
}
}
using (var inputStream = File.OpenRead(tokenInfoFile))
{
using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream))
{
Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1));
Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0));
}
}
}
public void TestCompleteLookUp()
{
Dictionary <int, string> resultMap = new Dictionary <int, string>();
resultMap[73] = "hello";
resultMap[42] = "今日は";
resultMap[99] = "素敵な世界";
List <short> tokenInfo = new List <short>();
List <int> features = new List <int>();
tokenInfo.Add((short)1);
tokenInfo.Add((short)2);
tokenInfo.Add((short)3);
features.Add(73);
features.Add(99);
BufferEntry entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
List <BufferEntry> bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var outStream = File.Create(tokenInfoFile))
{
using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries))
{
compiler.Compile(outStream);
}
}
using (var inStream = File.OpenRead(tokenInfoFile))
{
using (TokenInfoBuffer tokenInfoBuffer = new TokenInfoBuffer(inStream))
{
BufferEntry result = tokenInfoBuffer.LookupEntry(0);
Assert.AreEqual("hello", resultMap[result.FeatureInfos[0]]);
Assert.AreEqual("素敵な世界", resultMap[result.FeatureInfos[1]]);
}
}
}
public string[] GetAllFeaturesArray(int wordId)
{
var bufferEntry = TokenInfoBuffer.LookupEntry(wordId);
var posLength = bufferEntry.PosInfos.Length;
var featureLength = bufferEntry.FeatureInfos.Length;
var partOfSpeechAsShorts = false;
if (posLength == 0)
{
posLength = bufferEntry.TokenInfos.Length - TokenInfoOffset;
partOfSpeechAsShorts = true;
}
var result = new string[posLength + featureLength];
if (partOfSpeechAsShorts)
{
for (var i = 0; i < posLength; i++)
{
var feature = bufferEntry.TokenInfos[i + TokenInfoOffset];
result[i] = PosValues[feature];
}
}
else
{
for (var i = 0; i < posLength; i++)
{
var feature = bufferEntry.PosInfos[i] & 0xff;
result[i] = PosValues[feature];
}
}
for (var i = 0; i < featureLength; i++)
{
var feature = bufferEntry.FeatureInfos[i];
var s = StringValues[feature];
result[i + posLength] = s;
}
return(result);
}
public void TestCompleteLookUp()
{
var resultMap = new Dictionary <int, string>();
resultMap.Add(73, "hello");
resultMap.Add(42, "今日は");
resultMap.Add(99, "素敵な世界");
var tokenInfo = new List <short>();
var features = new List <int>();
tokenInfo.Add(1);
tokenInfo.Add(2);
tokenInfo.Add(3);
features.Add(73);
features.Add(99);
var entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
var bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var ms = new MemoryStream())
{
var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
compiler.Compile();
ms.Seek(0, SeekOrigin.Begin);
var tokenInfoBuffer = new TokenInfoBuffer(ms);
var result = tokenInfoBuffer.LookupEntry(0);
resultMap[result.FeatureInfos[0]].Is("hello");
resultMap[result.FeatureInfos[1]].Is("素敵な世界");
}
}
public void TestReadAndLookUpTokenInfo()
{
var tokenInfo = new List <short>();
var features = new List <int>();
var tokenInfos = new short[] { 1, 2, 3 };
var featureInfos = new int[] { 73, 99 };
tokenInfo.Add(1);
tokenInfo.Add(2);
tokenInfo.Add(3);
features.Add(73);
features.Add(99);
var entry = new BufferEntry();
entry.TokenInfo = tokenInfo;
entry.Features = features;
entry.TokenInfos = tokenInfos;
entry.FeatureInfos = featureInfos;
var bufferEntries = new List <BufferEntry>();
bufferEntries.Add(entry);
using (var ms = new MemoryStream())
{
var compiler = new TokenInfoBufferCompiler(ms, bufferEntries);
compiler.Compile();
ms.Seek(0, SeekOrigin.Begin);
var tokenInfoBuffer2 = new TokenInfoBuffer(ms);
tokenInfoBuffer2.LookupFeature(0, 1).Is(99);
tokenInfoBuffer2.LookupFeature(0, 0).Is(73);
}
}
private void Setup(string absoluteResourcePath)
{
try
{
using (var tokenStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TOKEN_INFO_DICTIONARY_FILENAME))
{ tokenInfoBuffer = new TokenInfoBuffer(tokenStream); }
using (var stringStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + FEATURE_MAP_FILENAME))
{ stringValues = new StringValueMapBuffer(stringStream); }
using (var posStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + POS_MAP_FILENAME))
{ posValues = new StringValueMapBuffer(posStream); }
using (var stream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TARGETMAP_FILENAME))
{ wordIdMap = new WordIdMap(stream); }
}
catch (IOException ex)
{
throw new Exception("TokenInfoDictionary.Setup: " + ex.Message);
}
}
public int GetWordCost(int wordId)
{
return(TokenInfoBuffer.LookupTokenInfo(wordId, WordCost));
}
public int GetRightId(int wordId)
{
return(TokenInfoBuffer.LookupTokenInfo(wordId, RightId));
}