} // method
#region Scanning tokens
private void NextToken()
{
//1. Check if there are buffered tokens
if (Context.BufferedTokens.Count > 0)
{
Context.CurrentToken = Context.BufferedTokens.Pop();
return;
}
//2. Skip whitespace.
_grammar.SkipWhitespace(Context.Source);
//3. That's the token start, calc location (line and column)
Context.Source.Position = Context.Source.PreviewPosition;
//4. Check for EOF
if (Context.Source.EOF())
{
Context.CurrentToken = new Token(_grammar.Eof, Context.Source.Location, string.Empty, _grammar.Eof.Name);;
return;
}
//5. Actually scan the source text and construct a new token
ScanToken();
}//method
private bool ReadStartSymbol(ISourceStream source, CompoundTokenDetails details)
{
if (!_startSymbolsFirsts.Contains(source.PreviewChar))
{
return(false);
}
foreach (var subType in _subTypes)
{
if (!source.MatchSymbol(subType.Start))
{
continue;
}
var previewPos = source.PreviewPosition;
source.PreviewPosition += subType.Start.Length;
Grammar.SkipWhitespace(source, true);
string quote = null;
// Search if there should be a quote.
if (subType.Quotes.Count > 0)
{
// Must be quoted.
var quoteMatchSuccessful = false;
foreach (var q in subType.Quotes)
{
// TODO: what if not case sensitive?
if (!source.MatchSymbol(q))
{
continue;
}
quoteMatchSuccessful = true;
quote = q;
}
if (!quoteMatchSuccessful)
{
// Revert, revert!
source.PreviewPosition = previewPos;
continue;
}
}
// Now the preview position is at the beginning of quotes, or the name.
var sb = new StringBuilder();
var previewChar = source.PreviewChar;
while (previewChar != '\r' && previewChar != '\n')
{
sb.Append(previewChar);
++source.PreviewPosition;
previewChar = source.PreviewChar;
}
var endLiteral = sb.ToString();
if (quote != null)
{
var comparisonType = QuoteCaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
// Check the quotes.
if (!(endLiteral.StartsWith(quote, comparisonType) && endLiteral.EndsWith(quote, comparisonType)))
{
// Malformed end literal.
return(false);
}
else
{
endLiteral = endLiteral.Substring(quote.Length, endLiteral.Length - quote.Length * 2);
}
}
int elStart = 0, elEnd = endLiteral.Length - 1;
// Trim
while (elStart < endLiteral.Length && Grammar.IsWhitespace(endLiteral[elStart]))
{
elStart++;
}
while (elEnd >= 0 && Grammar.IsWhitespace(endLiteral[elEnd]))
{
elStart--;
}
if (elEnd <= elStart)
{
// Malformed end literal.
return(false);
}
if (elStart != 0 || elEnd != endLiteral.Length - 1)
{
endLiteral = endLiteral.Substring(elStart, elEnd - elStart + 1);
}
//We found start symbol
details.StartSymbol = subType.Start;
details.Flags |= (short)subType.Flags;
details.TokenName = endLiteral;
details.TokenQuote = quote;
details.SubTypeIndex = subType.Index;
// No need to set source.PreviewPosition, we've done it.
return(true);
}
return(false);
}
