public void ReadTokenInfo(Stream input) { try { input.Position = 0; var reader = new StreamReader(input, Encoding.GetEncoding(encoding)); int entryCount = posInfo.GetEntryCount(); while (!reader.EndOfStream) { T entry = Parse(reader.ReadLine().RemapCharIfNeeded()); GenericDictionaryEntry dictionaryEntry = MakeGenericDictionaryEntry(entry); short leftId = dictionaryEntry.GetLeftId(); short rightId = dictionaryEntry.GetRightId(); short wordCost = dictionaryEntry.GetWordCost(); string[] allPosFeatures = dictionaryEntry.GetPartOfSpeechFeatures(); List <int> posFeatureIds = posInfo.MapFeatures(allPosFeatures); string[] featureList = dictionaryEntry.GetOtherFeatures(); List <int> otherFeatureIds = otherInfo.MapFeatures(featureList); BufferEntry bufferEntry = new BufferEntry(); bufferEntry.TokenInfo.Add(leftId); bufferEntry.TokenInfo.Add(rightId); bufferEntry.TokenInfo.Add(wordCost); if (EntriesFitInAByte(entryCount)) { List <Byte> posFeatureIdBytes = CreatePosFeatureIds(posFeatureIds); bufferEntry.PosInfo.AddRange(posFeatureIdBytes); } else { foreach (int posFeatureId in posFeatureIds) { bufferEntry.TokenInfo.Add((short)posFeatureId); } } bufferEntry.Features.AddRange(otherFeatureIds); bufferEntries.Add(bufferEntry); surfaces.Add(dictionaryEntry.GetSurface()); if (dictionaryEntries != null) { dictionaryEntries.Add(dictionaryEntry); } } } catch (IOException ex) { throw new IOException("TokenInfoDictionaryCompilerBase.AnalyzeTokenInfo: " + ex.Message); } }
public int[][] MakeCosts() { int[][] costs = new int[dictionaryEntries.Count][]; for (int i = 0; i < dictionaryEntries.Count; i++) { GenericDictionaryEntry entry = dictionaryEntries[i]; costs[i] = new int[] { entry.GetLeftId(), entry.GetRightId(), entry.GetWordCost() }; } return(costs); }