} // method #region Scanning tokens private void NextToken() { //1. Check if there are buffered tokens if (Context.BufferedTokens.Count > 0) { Context.CurrentToken = Context.BufferedTokens.Pop(); return; } //2. Skip whitespace. _grammar.SkipWhitespace(Context.Source); //3. That's the token start, calc location (line and column) Context.Source.Position = Context.Source.PreviewPosition; //4. Check for EOF if (Context.Source.EOF()) { Context.CurrentToken = new Token(_grammar.Eof, Context.Source.Location, string.Empty, _grammar.Eof.Name);; return; } //5. Actually scan the source text and construct a new token ScanToken(); }//method
private bool ReadStartSymbol(ISourceStream source, CompoundTokenDetails details) { if (!_startSymbolsFirsts.Contains(source.PreviewChar)) { return(false); } foreach (var subType in _subTypes) { if (!source.MatchSymbol(subType.Start)) { continue; } var previewPos = source.PreviewPosition; source.PreviewPosition += subType.Start.Length; Grammar.SkipWhitespace(source, true); string quote = null; // Search if there should be a quote. if (subType.Quotes.Count > 0) { // Must be quoted. var quoteMatchSuccessful = false; foreach (var q in subType.Quotes) { // TODO: what if not case sensitive? if (!source.MatchSymbol(q)) { continue; } quoteMatchSuccessful = true; quote = q; } if (!quoteMatchSuccessful) { // Revert, revert! source.PreviewPosition = previewPos; continue; } } // Now the preview position is at the beginning of quotes, or the name. var sb = new StringBuilder(); var previewChar = source.PreviewChar; while (previewChar != '\r' && previewChar != '\n') { sb.Append(previewChar); ++source.PreviewPosition; previewChar = source.PreviewChar; } var endLiteral = sb.ToString(); if (quote != null) { var comparisonType = QuoteCaseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase; // Check the quotes. if (!(endLiteral.StartsWith(quote, comparisonType) && endLiteral.EndsWith(quote, comparisonType))) { // Malformed end literal. return(false); } else { endLiteral = endLiteral.Substring(quote.Length, endLiteral.Length - quote.Length * 2); } } int elStart = 0, elEnd = endLiteral.Length - 1; // Trim while (elStart < endLiteral.Length && Grammar.IsWhitespace(endLiteral[elStart])) { elStart++; } while (elEnd >= 0 && Grammar.IsWhitespace(endLiteral[elEnd])) { elStart--; } if (elEnd <= elStart) { // Malformed end literal. return(false); } if (elStart != 0 || elEnd != endLiteral.Length - 1) { endLiteral = endLiteral.Substring(elStart, elEnd - elStart + 1); } //We found start symbol details.StartSymbol = subType.Start; details.Flags |= (short)subType.Flags; details.TokenName = endLiteral; details.TokenQuote = quote; details.SubTypeIndex = subType.Index; // No need to set source.PreviewPosition, we've done it. return(true); } return(false); }