protected override bool ReadBody(ISourceStream source, ScanDetails details) { int start = source.Position; bool allowEscapes = !details.IsSet(ScanFlags.DisableEscapes); CharList outputChars = new CharList(); while (!source.EOF()) { char current = source.CurrentChar; if (_terminators.IndexOf(current) >= 0) { break; } if (allowEscapes && current == this.EscapeChar) { current = ReadUnicodeEscape(source, details); //We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits. //This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped // char is at position of last digit of escape sequence. source.Position--; if (details.HasError()) { return(false); } } //Check if current character is OK if (!CharOk(current, source.Position == start)) { break; } //Check if we need to skip this char UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later if (!this.CharsToRemoveCategories.Contains(currCat)) { outputChars.Add(current); //add it to output (identifier) } source.Position++; }//while if (outputChars.Count == 0) { return(false); } //Convert collected chars to string details.Body = new string(outputChars.ToArray()); return(!string.IsNullOrEmpty(details.Body)); }
public override Token TryMatch(CompilerContext context, ISourceStream source) { Token token = null; if (IsSet(TermOptions.EnableQuickParse)) { token = QuickParse(context, source); if (token != null) { return(token); } } source.Position = source.TokenStart.Position; ScanDetails details = new ScanDetails(); details.Flags = DefaultFlags; details.TypeCodes = _defaultTypes; ReadPrefix(source, details); if (!ReadBody(source, details)) { return(null); } if (details.HasError()) { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, details.Error)); } ReadSuffix(source, details); if (!ConvertValue(details)) { return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value: " + details.Error)); } token = CreateToken(context, source, details); return(token); }