// Parsing
private static IEnumerable <Token> CoreParse(IEnumerable <string> source, ITokenDescriptionRules rule)
{
Token current = null;
string prefix = null;
int line = -1;
List <Token> context = new List <Token>();
TokenDescription.TokenDescriptionMatch match;
StringBuilder sb = new StringBuilder();
foreach (var lineOfSource in source)
{
line += 1;
int column = 0;
while (column < lineOfSource.Length)
{
// Long lexeme
if (current != null)
{
if (current.Description.TryMatchStop(lineOfSource, column, context, out match, prefix))
{
current.StopLine = line;
current.StopColumn = match.To;
sb.AppendLine();
sb.Append(lineOfSource.Substring(0, match.To));
//sb.Append(match.Extract(lineOfSource));
current.Text = sb.ToString();
yield return(current);
current = null;
sb.Clear();
column = match.To;
prefix = null;
continue;
}
else
{
sb.AppendLine();
sb.Append(lineOfSource);
break;
}
}
// white space to skip
if (char.IsWhiteSpace(lineOfSource[column]))
{
column += 1;
continue;
}
// Test for lexeme
match = rule.Match(lineOfSource, column, context);
if (!match.IsMatch)
{
throw new TokenSyntaxException($"Syntax error at {line + 1:00000} : {column + 1:000}", line, column);
}
current = new Token(match.Description, line, match.From);
if (match.Kind == TokenMatchKind.Entire)
{
current.StopLine = line;
current.StopColumn = match.To;
current.Text = match.Extract(lineOfSource);
context.Add(current);
yield return(current);
current = null;
sb.Clear();
column = match.To;
prefix = null;
continue;
}
prefix = match.Extract(lineOfSource);
sb.Append(lineOfSource.Substring(match.From));
break;
}
}
if (null != current)
{
throw new TokenSyntaxException($"Dangling token at {current.StartLine + 1:00000} : {current.StartColumn + 1:000}",
current.StartLine,
current.StartColumn);
}
yield break;
}