private string ReadSyntax(Tokenizer tokenizer, out TokenType?symbolType)
{
symbolType = null;
var sb = new StringBuilder().Append(tokenizer.NextChar());
for (var i = 0; i < 2; i++)
{
var ch = tokenizer.Peek();
if (!_syntax.IsSyntaxChar(ch) || _syntax.IsParen(ch))
{
break;
}
sb.Append(tokenizer.NextChar());
}
var probe = sb.ToString();
if (probe.Length == 1)
{
// one char read, let the lexer decide about the token type
return(probe);
}
while (true)
{
TokenType st;
if (_grammar.Syntax.Symbols.TryGetValue(probe, out st))
{
symbolType = st;
return(probe);
}
if (probe.Length == 1)
{
return(probe);
}
probe = probe.Substring(0, probe.Length - 1);
tokenizer.Position--;
}
}