protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
int start = source.Position;
bool allowEscapes = !details.IsSet(ScanFlags.DisableEscapes);
CharList outputChars = new CharList();
while (!source.EOF())
{
char current = source.CurrentChar;
if (_terminators.IndexOf(current) >= 0)
{
break;
}
if (allowEscapes && current == this.EscapeChar)
{
current = ReadUnicodeEscape(source, details);
//We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
//This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
// char is at position of last digit of escape sequence.
source.Position--;
if (details.HasError())
{
return(false);
}
}
//Check if current character is OK
if (!CharOk(current, source.Position == start))
{
break;
}
//Check if we need to skip this char
UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
if (!this.CharsToRemoveCategories.Contains(currCat))
{
outputChars.Add(current); //add it to output (identifier)
}
source.Position++;
}//while
if (outputChars.Count == 0)
{
return(false);
}
//Convert collected chars to string
details.Body = new string(outputChars.ToArray());
return(!string.IsNullOrEmpty(details.Body));
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token token = null;
if (IsSet(TermOptions.EnableQuickParse))
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
}
source.Position = source.TokenStart.Position;
ScanDetails details = new ScanDetails();
details.Flags = DefaultFlags;
details.TypeCodes = _defaultTypes;
ReadPrefix(source, details);
if (!ReadBody(source, details))
{
return(null);
}
if (details.HasError())
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, details.Error));
}
ReadSuffix(source, details);
if (!ConvertValue(details))
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value: " + details.Error));
}
token = CreateToken(context, source, details);
return(token);
}