public override Token TryMatch(ParsingContext context, ISourceStream source)
{
var text = source.Text;
foreach (var entry in Constants)
{
source.PreviewPosition = source.Position;
var constant = entry.Key;
if (source.PreviewPosition + constant.Length > text.Length)
{
continue;
}
if (source.MatchSymbol(constant))
{
source.PreviewPosition += constant.Length;
if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
{
continue; //make sure it is delimiter
}
return(source.CreateToken(OutputTerminal, entry.Value));
}
}
return(null);
}
protected override Token QuickParse(ParsingContext context, ISourceStream source)
{
if (!_allFirstCharsSet.Contains(source.PreviewChar))
{
return(null);
}
source.PreviewPosition++;
while (_allCharsSet.Contains(source.PreviewChar) && !source.EOF)
{
source.PreviewPosition++;
}
//if it is not a terminator then cancel; we need to go through full algorithm
if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar))
{
return(null);
}
var token = source.CreateToken(OutputTerminal);
if (CaseRestriction != CaseRestriction.None && !CheckCaseRestriction(token.ValueString))
{
return(null);
}
//!!! Do not convert to common case (all-lower) for case-insensitive grammar. Let identifiers remain as is,
// it is responsibility of interpreter to provide case-insensitive read/write operations for identifiers
// if (!this.GrammarData.Grammar.CaseSensitive)
// token.Value = token.Text.ToLower(CultureInfo.InvariantCulture);
CheckReservedWord(token);
return(token);
}
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
int start = source.PreviewPosition;
bool allowEscapes = details.IsSet((short)IdOptions.AllowsEscapes);
CharList outputChars = new CharList();
while (!source.EOF())
{
char current = source.PreviewChar;
if (Grammar.IsWhitespaceOrDelimiter(current))
{
break;
}
if (allowEscapes && current == this.EscapeChar)
{
current = ReadUnicodeEscape(source, details);
//We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
//This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
// char is at position of last digit of escape sequence.
source.PreviewPosition--;
if (details.Error != null)
{
return(false);
}
}
//Check if current character is OK
if (!CharOk(current, source.PreviewPosition == start))
{
break;
}
//Check if we need to skip this char
#if NETSTANDARD
UnicodeCategory currCat = CharUnicodeInfo.GetUnicodeCategory(current);
#else
UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
#endif
if (!this.CharsToRemoveCategories.Contains(currCat))
{
outputChars.Add(current); //add it to output (identifier)
}
source.PreviewPosition++;
}//while
if (outputChars.Count == 0)
{
return(false);
}
//Convert collected chars to string
details.Body = new string(outputChars.ToArray());
if (!CheckCaseRestriction(details.Body))
{
return(false);
}
return(!string.IsNullOrEmpty(details.Body));
}
//Simply skip until whitespace or delimiter character
private bool Recover()
{
var src = Context.Source;
src.PreviewPosition++;
while (!Context.Source.EOF())
{
if (_grammar.IsWhitespaceOrDelimiter(src.PreviewChar))
{
src.Position = src.PreviewPosition;
return(true);
}
src.PreviewPosition++;
}
return(false);
}
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
// so we try to do a quick parse for these, without starting the whole general process
protected override Token QuickParse(ParsingContext context, ISourceStream source)
{
if (IsSet(NumberOptions.DisableQuickParse))
{
return(null);
}
char current = source.PreviewChar;
//it must be a digit followed by a whitespace or delimiter
if (!char.IsDigit(current))
{
return(null);
}
if (!Grammar.IsWhitespaceOrDelimiter(source.NextPreviewChar))
{
return(null);
}
int iValue = current - '0';
object value = null;
switch (DefaultIntTypes[0])
{
case TypeCode.Int32: value = iValue; break;
case TypeCode.UInt32: value = (UInt32)iValue; break;
case TypeCode.Byte: value = (byte)iValue; break;
case TypeCode.SByte: value = (sbyte)iValue; break;
case TypeCode.Int16: value = (Int16)iValue; break;
case TypeCode.UInt16: value = (UInt16)iValue; break;
default: return(null);
}
source.PreviewPosition++;
return(source.CreateToken(this.OutputTerminal, value));
}
private Token CompleteMatch(ISourceStream source)
{
if (source.EOF)
{
return(null);
}
do
{
// Match NewLine
var lookAhead = source.PreviewChar;
if (LineTerminators.Contains(lookAhead))
{
source.PreviewPosition++;
// Treat \r\n as single NewLine
if (!source.EOF && lookAhead == '\r' && source.PreviewChar == '\n')
{
source.PreviewPosition++;
}
break;
}
// Eat up whitespace
if (Grammar.IsWhitespaceOrDelimiter(lookAhead))
{
source.PreviewPosition++;
continue;
}
// Fail on anything else
return(null);
}while (!source.EOF);
// Create output token
return(source.CreateToken(OutputTerminal));
}
