public virtual KnownToken Score(string previousToken, string token)
{
if (KnownTokens.TryGetValue(token, out var returnToken))
{
return(returnToken);
}
return(new KnownToken(token));
}
private static IEnumerable <string> GetContentLines(TextReader reader)
{
var currentLine = new StringBuilder();
while (true)
{
var nextLine = reader.ReadLine();
if (nextLine == null)
{
break;
}
if (nextLine.Length <= 0)
{
continue;
}
if (!KnownTokens.IsKnownToken(nextLine))
{
if (!currentLine.ToString().StartsWith("END"))
{
nextLine = " " + nextLine;
}
}
if ((nextLine[0] == ' ' || nextLine[0] == '\t'))
{
currentLine.Append(nextLine, 1, nextLine.Length - 1);
}
else
{
if (currentLine.Length > 0)
{
yield return(currentLine.ToString());
}
currentLine.Clear();
currentLine.Append(nextLine);
}
}
if (currentLine.Length > 0)
{
yield return(currentLine.ToString());
}
}
