string ExtractSingleFeature(int wordId, int field) { if (TokenInfoBuffer.isPartOfSpeechFeature(field)) { var featureId = TokenInfoBuffer.LookupPartOfSpeechFeature(wordId, field); return(PosValues[featureId]); } else { var featureId = TokenInfoBuffer.LookupFeature(wordId, field); return(StringValues[featureId]); } }
private TokenInfoDictionary(IResourceResolver resolver) { using (var tokenInfoBufferResource = resolver.Resolve(TokenInfoDictionaryFileName)) using (var stringValueResource = resolver.Resolve(FeatureMapFileName)) using (var posValueResource = resolver.Resolve(PosMapFileName)) using (var wordIdMapResource = resolver.Resolve(TargetMapFileName)) { TokenInfoBuffer = new TokenInfoBuffer(tokenInfoBufferResource); StringValues = new StringValueMapBuffer(stringValueResource); PosValues = new StringValueMapBuffer(posValueResource); WordIdMap = new WordIdMap(wordIdMapResource); } }
public void TestReadAndLookUpTokenInfo() { List <short> tokenInfo = new List <short>(); List <int> features = new List <int>(); short[] tokenInfos = new short[3]; tokenInfos[0] = 1; tokenInfos[1] = 2; tokenInfos[2] = 3; int[] featureInfos = new int[2]; featureInfos[0] = 73; featureInfos[1] = 99; tokenInfo.Add((short)1); tokenInfo.Add((short)2); tokenInfo.Add((short)3); features.Add(73); features.Add(99); BufferEntry entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; entry.TokenInfos = tokenInfos; entry.FeatureInfos = featureInfos; List <BufferEntry> bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries)) { using (var outputStream = File.Create(tokenInfoFile)) { compiler.Compile(outputStream); } } using (var inputStream = File.OpenRead(tokenInfoFile)) { using (TokenInfoBuffer tokenInfoBuffer2 = new TokenInfoBuffer(inputStream)) { Assert.AreEqual(99, tokenInfoBuffer2.LookupFeature(0, 1)); Assert.AreEqual(73, tokenInfoBuffer2.LookupFeature(0, 0)); } } }
public void TestCompleteLookUp() { Dictionary <int, string> resultMap = new Dictionary <int, string>(); resultMap[73] = "hello"; resultMap[42] = "今日は"; resultMap[99] = "素敵な世界"; List <short> tokenInfo = new List <short>(); List <int> features = new List <int>(); tokenInfo.Add((short)1); tokenInfo.Add((short)2); tokenInfo.Add((short)3); features.Add(73); features.Add(99); BufferEntry entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; List <BufferEntry> bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (var outStream = File.Create(tokenInfoFile)) { using (TokenInfoBufferCompiler compiler = new TokenInfoBufferCompiler(bufferEntries)) { compiler.Compile(outStream); } } using (var inStream = File.OpenRead(tokenInfoFile)) { using (TokenInfoBuffer tokenInfoBuffer = new TokenInfoBuffer(inStream)) { BufferEntry result = tokenInfoBuffer.LookupEntry(0); Assert.AreEqual("hello", resultMap[result.FeatureInfos[0]]); Assert.AreEqual("素敵な世界", resultMap[result.FeatureInfos[1]]); } } }
public string[] GetAllFeaturesArray(int wordId) { var bufferEntry = TokenInfoBuffer.LookupEntry(wordId); var posLength = bufferEntry.PosInfos.Length; var featureLength = bufferEntry.FeatureInfos.Length; var partOfSpeechAsShorts = false; if (posLength == 0) { posLength = bufferEntry.TokenInfos.Length - TokenInfoOffset; partOfSpeechAsShorts = true; } var result = new string[posLength + featureLength]; if (partOfSpeechAsShorts) { for (var i = 0; i < posLength; i++) { var feature = bufferEntry.TokenInfos[i + TokenInfoOffset]; result[i] = PosValues[feature]; } } else { for (var i = 0; i < posLength; i++) { var feature = bufferEntry.PosInfos[i] & 0xff; result[i] = PosValues[feature]; } } for (var i = 0; i < featureLength; i++) { var feature = bufferEntry.FeatureInfos[i]; var s = StringValues[feature]; result[i + posLength] = s; } return(result); }
public void TestCompleteLookUp() { var resultMap = new Dictionary <int, string>(); resultMap.Add(73, "hello"); resultMap.Add(42, "今日は"); resultMap.Add(99, "素敵な世界"); var tokenInfo = new List <short>(); var features = new List <int>(); tokenInfo.Add(1); tokenInfo.Add(2); tokenInfo.Add(3); features.Add(73); features.Add(99); var entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; var bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (var ms = new MemoryStream()) { var compiler = new TokenInfoBufferCompiler(ms, bufferEntries); compiler.Compile(); ms.Seek(0, SeekOrigin.Begin); var tokenInfoBuffer = new TokenInfoBuffer(ms); var result = tokenInfoBuffer.LookupEntry(0); resultMap[result.FeatureInfos[0]].Is("hello"); resultMap[result.FeatureInfos[1]].Is("素敵な世界"); } }
public void TestReadAndLookUpTokenInfo() { var tokenInfo = new List <short>(); var features = new List <int>(); var tokenInfos = new short[] { 1, 2, 3 }; var featureInfos = new int[] { 73, 99 }; tokenInfo.Add(1); tokenInfo.Add(2); tokenInfo.Add(3); features.Add(73); features.Add(99); var entry = new BufferEntry(); entry.TokenInfo = tokenInfo; entry.Features = features; entry.TokenInfos = tokenInfos; entry.FeatureInfos = featureInfos; var bufferEntries = new List <BufferEntry>(); bufferEntries.Add(entry); using (var ms = new MemoryStream()) { var compiler = new TokenInfoBufferCompiler(ms, bufferEntries); compiler.Compile(); ms.Seek(0, SeekOrigin.Begin); var tokenInfoBuffer2 = new TokenInfoBuffer(ms); tokenInfoBuffer2.LookupFeature(0, 1).Is(99); tokenInfoBuffer2.LookupFeature(0, 0).Is(73); } }
private void Setup(string absoluteResourcePath) { try { using (var tokenStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TOKEN_INFO_DICTIONARY_FILENAME)) { tokenInfoBuffer = new TokenInfoBuffer(tokenStream); } using (var stringStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + FEATURE_MAP_FILENAME)) { stringValues = new StringValueMapBuffer(stringStream); } using (var posStream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + POS_MAP_FILENAME)) { posValues = new StringValueMapBuffer(posStream); } using (var stream = File.OpenRead(absoluteResourcePath + Path.DirectorySeparatorChar + TARGETMAP_FILENAME)) { wordIdMap = new WordIdMap(stream); } } catch (IOException ex) { throw new Exception("TokenInfoDictionary.Setup: " + ex.Message); } }
public int GetWordCost(int wordId) { return(TokenInfoBuffer.LookupTokenInfo(wordId, WordCost)); }
public int GetRightId(int wordId) { return(TokenInfoBuffer.LookupTokenInfo(wordId, RightId)); }