public void AnalyzeTokenInfo(Stream input)
{
foreach (var line in input.ReadLines(Encoding))
{
var entry = Parse(line);
var dictionaryEntry = MakeGenericDictionaryEntry(entry);
PosInfo.MapFeatures(dictionaryEntry.PartOfSpeechFeatures);
}
}
public void ReadTokenInfo(Stream input)
{
var entryCount = PosInfo.EntryCount;
foreach (var line in input.ReadLines(Encoding))
{
var entry = Parse(line);
var dictionaryEntry = MakeGenericDictionaryEntry(entry);
var leftId = dictionaryEntry.LeftId;
var rightId = dictionaryEntry.RightId;
var wordCost = dictionaryEntry.WordCost;
var allPosFeatures = dictionaryEntry.PartOfSpeechFeatures;
var posFeatureIds = PosInfo.MapFeatures(allPosFeatures);
var featureList = dictionaryEntry.OtherFeatures;
var otherFeatureIds = OtherInfo.MapFeatures(featureList);
var bufferEntry = new BufferEntry();
bufferEntry.TokenInfo.Add(leftId);
bufferEntry.TokenInfo.Add(rightId);
bufferEntry.TokenInfo.Add(wordCost);
if (EntriesFitInAByte(entryCount))
{
var posFeatureIdBytes = CreatePosFeatureIds(posFeatureIds);
bufferEntry.PosInfo.AddRange(posFeatureIdBytes);
}
else
{
foreach (var posFeatureId in posFeatureIds)
{
bufferEntry.TokenInfo.Add((short)posFeatureId);
}
}
bufferEntry.Features.AddRange(otherFeatureIds);
BufferEntries.Add(bufferEntry);
Surfaces.Add(dictionaryEntry.Surface);
if (DictionaryEntries != null)
{
DictionaryEntries.Add(dictionaryEntry);
}
}
}