/* * Method: SinkOctalDigits * * Sink multiple octal digits. */ internal bool SinkMultipleOctalDigits() { int count = 0; while (TokenChar.IsOctalDigit(CurrentCharacter)) { ++count; Skip(); } return(count > 0); // Must match at least one }
internal bool SinkIdentifierPart() { // From 2.4.2 of the C# Language Specification // identifier-part-letter-character: if ( TokenChar.IsLetter(CurrentCharacter) || TokenChar.IsDecimalDigit(CurrentCharacter) || TokenChar.IsConnecting(CurrentCharacter) || TokenChar.IsCombining(CurrentCharacter) || TokenChar.IsFormatting(CurrentCharacter) ) { Skip(); return(true); } return(false); }
private bool ReadNextPage() { if (this.currentPageNumber == this.finalPageNumber) { return(false); } this.ReadBlockStripEOF(); this.SwapPages(); this.currentPageNumber++; if (this.charactersRead < this.pageSize) { this.finalPageNumber = this.currentPageNumber; if (!this.IsZeroLengthStream() && !TokenChar.IsNewLine(this.LastCharacterInStream())) { this.AppendCharacterToStream('\r'); } } return(this.charactersRead > 0); }
internal bool SinkNewLine() { if (!this.EndOfLines) { int position = this.position; if (this.Sink("\r\n")) { this.currentLine++; Microsoft.Build.Shared.ErrorUtilities.VerifyThrow(position != this.position, "Expected position to be incremented."); return(true); } if (TokenChar.IsNewLine(this.CurrentCharacter)) { this.Skip(); Microsoft.Build.Shared.ErrorUtilities.VerifyThrow(position != this.position, "Expected position to be incremented."); return(true); } } return(false); }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // Dealing with whitespace? if (_reader.SinkMultipleWhiteSpace()) { current = new WhitespaceToken(); return(true); } // Check for one-line comment else if (_reader.Sink("//")) { // Looks like a one-line comment. Follow it to the End-of-line _reader.SinkToEndOfLine(); current = new CommentToken(); return(true); } // Check for multi-line comment else if (_reader.Sink("/*")) { _reader.SinkUntil("*/"); // Was the ending */ found? if (_reader.EndOfLines) { // No. There was a /* without a */. Return this a syntax error token. current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return(true); } current = new CommentToken(); return(true); } // Handle chars else if (_reader.Sink("\'")) { while (_reader.CurrentCharacter != '\'') { if (_reader.Sink("\\")) { /* reader.Skip the escape sequence. * This isn't exactly right. We should detect: * * simple-escape-sequence: one of \' \" \\ \0 \a \b \f \n \r \t \v * * hexadecimal-escape-sequence: * \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt] */ } _reader.SinkCharacter(); } if (_reader.SinkCharacter() != '\'') { Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick."); } current = new CSharpTokenizer.CharLiteralToken(); return(true); } // Check for verbatim string else if (_reader.Sink("@\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return(true); } // reader.Skip the ending quote. current = new StringLiteralToken(); current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1); return(true); } // Check for a quoted string. else if (_reader.Sink("\"")) { while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral()) { // See if we have an escape sequence. if (_reader.SinkCharacter() == '\\') { // This is probably an escape character. if (_reader.SinkStringEscape()) { // This isn't nearly right. We just do barely enough to make a string // with an embedded escape sequence return _some_ string whose start and // end match the real bounds of the string. } else { // This is a compiler error. _reader.SinkCharacter(); current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return(true); } } } // Is it a newline? if (TokenChar.IsNewLine(_reader.CurrentCharacter)) { current = new CSharpTokenizer.NewlineInsideStringToken(); return(true); } // Create the token. if (_reader.SinkCharacter() != '\"') { Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote."); } current = new StringLiteralToken(); return(true); } // Identifier or keyword? else if ( // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier. _reader.CurrentCharacter == '@' || _reader.MatchNextIdentifierStart() ) { if (_reader.CurrentCharacter == '@') { _reader.SinkCharacter(); } // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return(true); } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); switch (identifierOrKeyword) { default: if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0) { current = new KeywordToken(); return(true); } // If the identifier starts with '@' then we need to strip it off. // The '@' is for escaping so that we can have an identifier called // the same thing as a reserved keyword (i.e. class, if, foreach, etc) string identifier = _reader.GetCurrentMatchedString(startPosition); if (identifier.StartsWith("@", StringComparison.Ordinal)) { identifier = identifier.Substring(1); } // Create the token. current = new IdentifierToken(); current.InnerText = identifier; return(true); case "false": case "true": current = new BooleanLiteralToken(); return(true); case "null": current = new CSharpTokenizer.NullLiteralToken(); return(true); } } // Open scope else if (_reader.Sink("{")) { current = new CSharpTokenizer.OpenScopeToken(); return(true); } // Close scope else if (_reader.Sink("}")) { current = new CSharpTokenizer.CloseScopeToken(); return(true); } // Hexidecimal integer literal else if (_reader.SinkIgnoreCase("0x")) { // Sink the hex digits. if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return(true); } // Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new HexIntegerLiteralToken(); return(true); } // Decimal integer literal else if (_reader.SinkMultipleDecimalDigits()) { // reader.Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return(true); } // Check for single-digit operators and punctuators else if (_reader.SinkOperatorOrPunctuator()) { current = new OperatorOrPunctuatorToken(); return(true); } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.Sink("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.Sink("#endif")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return(true); } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return(true); }
internal override bool FindNextToken() { int position = this.reader.Position; if (this.reader.SinkMultipleWhiteSpace()) { base.current = new WhitespaceToken(); return(true); } if (this.reader.Sink("//")) { this.reader.SinkToEndOfLine(); base.current = new CommentToken(); return(true); } if (this.reader.Sink("/*")) { this.reader.SinkUntil("*/"); if (this.reader.EndOfLines) { base.current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return(true); } base.current = new CommentToken(); return(true); } if (this.reader.Sink("'")) { while (this.reader.CurrentCharacter != '\'') { this.reader.Sink(@"\"); this.reader.SinkCharacter(); } this.reader.SinkCharacter(); base.current = new CSharpTokenizer.CharLiteralToken(); return(true); } if (this.reader.Sink("@\"")) { while (this.reader.Sink("\"\"") || (!this.reader.EndOfLines && (this.reader.SinkCharacter() != '"'))) { } if (this.reader.EndOfLines) { base.current = new Microsoft.Build.Shared.LanguageParser.EndOfFileInsideStringToken(); return(true); } base.current = new StringLiteralToken(); base.current.InnerText = this.reader.GetCurrentMatchedString(position).Substring(1); return(true); } if (this.reader.Sink("\"")) { while ((this.reader.CurrentCharacter == '\\') || this.reader.MatchRegularStringLiteral()) { if ((this.reader.SinkCharacter() == '\\') && !this.reader.SinkStringEscape()) { this.reader.SinkCharacter(); base.current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return(true); } } if (TokenChar.IsNewLine(this.reader.CurrentCharacter)) { base.current = new CSharpTokenizer.NewlineInsideStringToken(); return(true); } this.reader.SinkCharacter(); base.current = new StringLiteralToken(); return(true); } if ((this.reader.CurrentCharacter == '@') || this.reader.MatchNextIdentifierStart()) { if (this.reader.CurrentCharacter == '@') { this.reader.SinkCharacter(); } if (!this.reader.SinkIdentifierStart()) { base.current = new ExpectedIdentifierToken(); return(true); } while (this.reader.SinkIdentifierPart()) { } string currentMatchedString = this.reader.GetCurrentMatchedString(position); switch (currentMatchedString) { case "false": case "true": base.current = new BooleanLiteralToken(); return(true); case "null": base.current = new CSharpTokenizer.NullLiteralToken(); return(true); } if (Array.IndexOf <string>(keywordList, currentMatchedString) >= 0) { base.current = new KeywordToken(); return(true); } string str2 = this.reader.GetCurrentMatchedString(position); if (str2.StartsWith("@", StringComparison.Ordinal)) { str2 = str2.Substring(1); } base.current = new IdentifierToken(); base.current.InnerText = str2; return(true); } if (this.reader.Sink("{")) { base.current = new CSharpTokenizer.OpenScopeToken(); return(true); } if (this.reader.Sink("}")) { base.current = new CSharpTokenizer.CloseScopeToken(); return(true); } if (this.reader.SinkIgnoreCase("0x")) { if (!this.reader.SinkMultipleHexDigits()) { base.current = new ExpectedValidHexDigitToken(); return(true); } this.reader.SinkLongIntegerSuffix(); base.current = new HexIntegerLiteralToken(); return(true); } if (this.reader.SinkMultipleDecimalDigits()) { this.reader.SinkLongIntegerSuffix(); base.current = new DecimalIntegerLiteralToken(); return(true); } if (this.reader.SinkOperatorOrPunctuator()) { base.current = new OperatorOrPunctuatorToken(); return(true); } if (this.reader.CurrentCharacter == '#') { if (this.reader.Sink("#if")) { base.current = new OpenConditionalDirectiveToken(); } else if (this.reader.Sink("#endif")) { base.current = new CloseConditionalDirectiveToken(); } else { base.current = new PreprocessorToken(); } this.reader.SinkToEndOfLine(); return(true); } this.reader.SinkCharacter(); base.current = new UnrecognizedToken(); return(true); }
internal bool MatchRegularStringLiteral() { return(((base.CurrentCharacter != '"') && (base.CurrentCharacter != '\\')) && !TokenChar.IsNewLine(base.CurrentCharacter)); }
internal bool SinkIdentifierPart() { if ((!TokenChar.IsLetter(this.CurrentCharacter) && !TokenChar.IsDecimalDigit(this.CurrentCharacter)) && ((!TokenChar.IsConnecting(this.CurrentCharacter) && !TokenChar.IsCombining(this.CurrentCharacter)) && !TokenChar.IsFormatting(this.CurrentCharacter))) { return(false); } this.Skip(); return(true); }