public void ReadTokenInfo(Stream input)
{
try
{
input.Position = 0;
var reader = new StreamReader(input, Encoding.GetEncoding(encoding));
int entryCount = posInfo.GetEntryCount();
while (!reader.EndOfStream)
{
T entry = Parse(reader.ReadLine().RemapCharIfNeeded());
GenericDictionaryEntry dictionaryEntry = MakeGenericDictionaryEntry(entry);
short leftId = dictionaryEntry.GetLeftId();
short rightId = dictionaryEntry.GetRightId();
short wordCost = dictionaryEntry.GetWordCost();
string[] allPosFeatures = dictionaryEntry.GetPartOfSpeechFeatures();
List <int> posFeatureIds = posInfo.MapFeatures(allPosFeatures);
string[] featureList = dictionaryEntry.GetOtherFeatures();
List <int> otherFeatureIds = otherInfo.MapFeatures(featureList);
BufferEntry bufferEntry = new BufferEntry();
bufferEntry.TokenInfo.Add(leftId);
bufferEntry.TokenInfo.Add(rightId);
bufferEntry.TokenInfo.Add(wordCost);
if (EntriesFitInAByte(entryCount))
{
List <Byte> posFeatureIdBytes = CreatePosFeatureIds(posFeatureIds);
bufferEntry.PosInfo.AddRange(posFeatureIdBytes);
}
else
{
foreach (int posFeatureId in posFeatureIds)
{
bufferEntry.TokenInfo.Add((short)posFeatureId);
}
}
bufferEntry.Features.AddRange(otherFeatureIds);
bufferEntries.Add(bufferEntry);
surfaces.Add(dictionaryEntry.GetSurface());
if (dictionaryEntries != null)
{
dictionaryEntries.Add(dictionaryEntry);
}
}
}
catch (IOException ex)
{
throw new IOException("TokenInfoDictionaryCompilerBase.AnalyzeTokenInfo: " + ex.Message);
}
}
public void AnalyzeTokenInfo(Stream input)
{
try
{
input.Position = 0;
var reader = new StreamReader(input, Encoding.GetEncoding(encoding));
string line;
while (!reader.EndOfStream)
{
line = reader.ReadLine().RemapCharIfNeeded();
T entry = Parse(line);
GenericDictionaryEntry dictionaryEntry = MakeGenericDictionaryEntry(entry);
posInfo.MapFeatures(dictionaryEntry.GetPartOfSpeechFeatures());
}
}
catch (IOException ex)
{
throw new IOException("TokenInfoDictionaryCompilerBase.AnalyzeTokenInfo: " + ex.Message);
}
}
