public void TestReadAndLookUpTokenInfo() { List <short> tokenInfo = new List <short>(); List <int> features = new List <int>(); short[] tokenInfos = new short[3]; tokenInfos[0] = 1; tokenInfos[1] = 2; tokenInfos[2] = 3; int[] featureInfos = new int[2]; featureInfos[0] = 73; featureInfos[1] = 99; tokenInfo.Add((short)1); tokenInfo.Add((short)2); tokenInfo.Add((short)3); features.Add(73); features.Add(99); BufferEntry entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; entry.TokenInfos = tokenInfos; entry.FeatureInfos = featureInfos; List <BufferEntry> bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries)) { using (var outputStream = File.Create(tokenInfoFile)) { compiler.Compile(outputStream); } } using (var inputStream = File.OpenRead(tokenInfoFile)) { using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream)) { Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1)); Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0)); } } }
public void TestReadAndLookUpTokenInfo() { var tokenInfo = new List <short>(); var features = new List <int>(); var tokenInfos = new short[] { 1, 2, 3 }; var featureInfos = new int[] { 73, 99 }; tokenInfo.Add(1); tokenInfo.Add(2); tokenInfo.Add(3); features.Add(73); features.Add(99); var entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; entry.TokenInfos = tokenInfos; entry.FeatureInfos = featureInfos; var bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (var ms = new MemoryStream()) { var compiler = new TokenInfoBufferCompiler(ms, bufferEntries); compiler.Compile(); ms.Seek(0, SeekOrigin.Begin); var tokenInfoBuffer2 = new TokenInfoBuffer(ms); tokenInfoBuffer2.LookupFeature(0, 1).Is(99); tokenInfoBuffer2.LookupFeature(0, 0).Is(73); } }
string ExtractSingleFeature(int wordId, int field) { if (TokenInfoBuffer.isPartOfSpeechFeature(field)) { var featureId = TokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field); return(PosValues[featureId]); } else { var featureId = TokenInfoBuffer.LookupFeature(wordId, field); return(StringValues[featureId]); } }
private string ExtractSingleFeature(int wordId, int field) { int featureId; if (tokenInfoBuffer.IsPartOfSpeechFeature(field)) { featureId = tokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field); return(posValues.Get(featureId)); } featureId = tokenInfoBuffer.LookupFeature(wordId, field); return(stringValues.Get(featureId)); }