public void ReadTokenInfo(Stream input) { try { input.Position = 0; var reader = new StreamReader(input, Encoding.GetEncoding(encoding)); int entryCount = posInfo.GetEntryCount(); while (!reader.EndOfStream) { T entry = Parse(reader.ReadLine().RemapCharIfNeeded()); GenericDictionaryEntry dictionaryEntry = MakeGenericDictionaryEntry(entry); short leftId = dictionaryEntry.GetLeftId(); short rightId = dictionaryEntry.GetRightId(); short wordCost = dictionaryEntry.GetWordCost(); string[] allPosFeatures = dictionaryEntry.GetPartOfSpeechFeatures(); List <int> posFeatureIds = posInfo.MapFeatures(allPosFeatures); string[] featureList = dictionaryEntry.GetOtherFeatures(); List <int> otherFeatureIds = otherInfo.MapFeatures(featureList); BufferEntry bufferEntry = new BufferEntry(); bufferEntry.TokenInfo.Add(leftId); bufferEntry.TokenInfo.Add(rightId); bufferEntry.TokenInfo.Add(wordCost); if (EntriesFitInAByte(entryCount)) { List <Byte> posFeatureIdBytes = CreatePosFeatureIds(posFeatureIds); bufferEntry.PosInfo.AddRange(posFeatureIdBytes); } else { foreach (int posFeatureId in posFeatureIds) { bufferEntry.TokenInfo.Add((short)posFeatureId); } } bufferEntry.Features.AddRange(otherFeatureIds); bufferEntries.Add(bufferEntry); surfaces.Add(dictionaryEntry.GetSurface()); if (dictionaryEntries != null) { dictionaryEntries.Add(dictionaryEntry); } } } catch (IOException ex) { throw new IOException("TokenInfoDictionaryCompilerBase.AnalyzeTokenInfo: " + ex.Message); } }
public void AnalyzeTokenInfo(Stream input) { try { input.Position = 0; var reader = new StreamReader(input, Encoding.GetEncoding(encoding)); string line; while (!reader.EndOfStream) { line = reader.ReadLine().RemapCharIfNeeded(); T entry = Parse(line); GenericDictionaryEntry dictionaryEntry = MakeGenericDictionaryEntry(entry); posInfo.MapFeatures(dictionaryEntry.GetPartOfSpeechFeatures()); } } catch (IOException ex) { throw new IOException("TokenInfoDictionaryCompilerBase.AnalyzeTokenInfo: " + ex.Message); } }