public override Token TryMatch(ParsingContext context, ISourceStream source) { var text = source.Text; foreach (var entry in Constants) { source.PreviewPosition = source.Position; var constant = entry.Key; if (source.PreviewPosition + constant.Length > text.Length) { continue; } if (source.MatchSymbol(constant)) { source.PreviewPosition += constant.Length; if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar)) { continue; //make sure it is delimiter } return(source.CreateToken(OutputTerminal, entry.Value)); } } return(null); }
protected override Token QuickParse(ParsingContext context, ISourceStream source) { if (!_allFirstCharsSet.Contains(source.PreviewChar)) { return(null); } source.PreviewPosition++; while (_allCharsSet.Contains(source.PreviewChar) && !source.EOF) { source.PreviewPosition++; } //if it is not a terminator then cancel; we need to go through full algorithm if (!Grammar.IsWhitespaceOrDelimiter(source.PreviewChar)) { return(null); } var token = source.CreateToken(OutputTerminal); if (CaseRestriction != CaseRestriction.None && !CheckCaseRestriction(token.ValueString)) { return(null); } //!!! Do not convert to common case (all-lower) for case-insensitive grammar. Let identifiers remain as is, // it is responsibility of interpreter to provide case-insensitive read/write operations for identifiers // if (!this.GrammarData.Grammar.CaseSensitive) // token.Value = token.Text.ToLower(CultureInfo.InvariantCulture); CheckReservedWord(token); return(token); }
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details) { int start = source.PreviewPosition; bool allowEscapes = details.IsSet((short)IdOptions.AllowsEscapes); CharList outputChars = new CharList(); while (!source.EOF()) { char current = source.PreviewChar; if (Grammar.IsWhitespaceOrDelimiter(current)) { break; } if (allowEscapes && current == this.EscapeChar) { current = ReadUnicodeEscape(source, details); //We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits. //This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped // char is at position of last digit of escape sequence. source.PreviewPosition--; if (details.Error != null) { return(false); } } //Check if current character is OK if (!CharOk(current, source.PreviewPosition == start)) { break; } //Check if we need to skip this char #if NETSTANDARD UnicodeCategory currCat = CharUnicodeInfo.GetUnicodeCategory(current); #else UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later #endif if (!this.CharsToRemoveCategories.Contains(currCat)) { outputChars.Add(current); //add it to output (identifier) } source.PreviewPosition++; }//while if (outputChars.Count == 0) { return(false); } //Convert collected chars to string details.Body = new string(outputChars.ToArray()); if (!CheckCaseRestriction(details.Body)) { return(false); } return(!string.IsNullOrEmpty(details.Body)); }
//Simply skip until whitespace or delimiter character private bool Recover() { var src = Context.Source; src.PreviewPosition++; while (!Context.Source.EOF()) { if (_grammar.IsWhitespaceOrDelimiter(src.PreviewChar)) { src.Position = src.PreviewPosition; return(true); } src.PreviewPosition++; } return(false); }
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9 // so we try to do a quick parse for these, without starting the whole general process protected override Token QuickParse(ParsingContext context, ISourceStream source) { if (IsSet(NumberOptions.DisableQuickParse)) { return(null); } char current = source.PreviewChar; //it must be a digit followed by a whitespace or delimiter if (!char.IsDigit(current)) { return(null); } if (!Grammar.IsWhitespaceOrDelimiter(source.NextPreviewChar)) { return(null); } int iValue = current - '0'; object value = null; switch (DefaultIntTypes[0]) { case TypeCode.Int32: value = iValue; break; case TypeCode.UInt32: value = (UInt32)iValue; break; case TypeCode.Byte: value = (byte)iValue; break; case TypeCode.SByte: value = (sbyte)iValue; break; case TypeCode.Int16: value = (Int16)iValue; break; case TypeCode.UInt16: value = (UInt16)iValue; break; default: return(null); } source.PreviewPosition++; return(source.CreateToken(this.OutputTerminal, value)); }
private Token CompleteMatch(ISourceStream source) { if (source.EOF) { return(null); } do { // Match NewLine var lookAhead = source.PreviewChar; if (LineTerminators.Contains(lookAhead)) { source.PreviewPosition++; // Treat \r\n as single NewLine if (!source.EOF && lookAhead == '\r' && source.PreviewChar == '\n') { source.PreviewPosition++; } break; } // Eat up whitespace if (Grammar.IsWhitespaceOrDelimiter(lookAhead)) { source.PreviewPosition++; continue; } // Fail on anything else return(null); }while (!source.EOF); // Create output token return(source.CreateToken(OutputTerminal)); }