/// <summary>
/// Read the next page.
/// </summary>
/// <returns></returns>
private bool ReadNextPage()
{
// Is this already known to be the last page?
if (_currentPageNumber == _finalPageNumber)
{
// 0x0d should already be appended.
return(false);
}
// If so, read it in to the lastBuffer...
ReadBlockStripEOF();
// ...and then swap lastBuffer for buffer.
SwapPages();
++_currentPageNumber;
// If the number of bytesRead is less than the number requested
// then this is the last page.
if (_charactersRead < _pageSize)
{
// Mark this as the last page.
_finalPageNumber = _currentPageNumber;
// Add a 0xd if the last character is not a newline.
if (!IsZeroLengthStream() && !TokenChar.IsNewLine(LastCharacterInStream()))
{
AppendCharacterToStream('\xd');
}
}
return(_charactersRead > 0);
}
internal bool SinkNewLine()
{
if (EndOfLines)
{
return(false);
}
int originalPosition = _position;
if (Sink("\xd\xa")) // This sequence is treated as a single new line.
{
++_currentLine;
ErrorUtilities.VerifyThrow(originalPosition != _position, "Expected position to be incremented.");
return(true);
}
if (TokenChar.IsNewLine(CurrentCharacter))
{
Skip();
ErrorUtilities.VerifyThrow(originalPosition != _position, "Expected position to be incremented.");
return(true);
}
return(false);
}
protected void Skip()
{
if (TokenChar.IsNewLine(CurrentCharacter))
{
++_currentLine;
}
++_position;
}
internal bool SinkToEndOfLine()
{
while (!TokenChar.IsNewLine(CurrentCharacter))
{
Skip();
}
return(true); // Matching zero characters is ok.
}
protected void Skip()
{
if (TokenChar.IsNewLine(this.CurrentCharacter))
{
this.currentLine++;
}
this.position++;
}
internal bool SinkToEndOfLine()
{
while (!TokenChar.IsNewLine(this.CurrentCharacter))
{
this.Skip();
}
return(true);
}
/*
* Method: MatchRegularStringLiteral
*
* Determine whether this is a regular C# string literal character
*/
internal bool MatchRegularStringLiteral()
{
if (CurrentCharacter == '\"' || CurrentCharacter == '\\' || TokenChar.IsNewLine(CurrentCharacter))
{
return(false);
}
return(true);
}
internal bool SinkWhiteSpace()
{
if (Char.IsWhiteSpace(CurrentCharacter) && !TokenChar.IsNewLine(CurrentCharacter))
{
Skip();
return(true);
}
return(false);
}
private bool ReadNextPage()
{
if (this.currentPageNumber == this.finalPageNumber)
{
return(false);
}
this.ReadBlockStripEOF();
this.SwapPages();
this.currentPageNumber++;
if (this.charactersRead < this.pageSize)
{
this.finalPageNumber = this.currentPageNumber;
if (!this.IsZeroLengthStream() && !TokenChar.IsNewLine(this.LastCharacterInStream()))
{
this.AppendCharacterToStream('\r');
}
}
return(this.charactersRead > 0);
}
internal bool SinkNewLine()
{
if (!this.EndOfLines)
{
int position = this.position;
if (this.Sink("\r\n"))
{
this.currentLine++;
Microsoft.Build.Shared.ErrorUtilities.VerifyThrow(position != this.position, "Expected position to be incremented.");
return(true);
}
if (TokenChar.IsNewLine(this.CurrentCharacter))
{
this.Skip();
Microsoft.Build.Shared.ErrorUtilities.VerifyThrow(position != this.position, "Expected position to be incremented.");
return(true);
}
}
return(false);
}
/*
* Method: FindNextToken
*
* Find the next token. Return 'true' if one was found. False, otherwise.
*/
override internal bool FindNextToken()
{
int startPosition = _reader.Position;
// Dealing with whitespace?
if (_reader.SinkMultipleWhiteSpace())
{
current = new WhitespaceToken();
return(true);
}
// Check for one-line comment
else if (_reader.Sink("//"))
{
// Looks like a one-line comment. Follow it to the End-of-line
_reader.SinkToEndOfLine();
current = new CommentToken();
return(true);
}
// Check for multi-line comment
else if (_reader.Sink("/*"))
{
_reader.SinkUntil("*/");
// Was the ending */ found?
if (_reader.EndOfLines)
{
// No. There was a /* without a */. Return this a syntax error token.
current = new CSharpTokenizer.EndOfFileInsideCommentToken();
return(true);
}
current = new CommentToken();
return(true);
}
// Handle chars
else if (_reader.Sink("\'"))
{
while (_reader.CurrentCharacter != '\'')
{
if (_reader.Sink("\\"))
{
/* reader.Skip the escape sequence.
* This isn't exactly right. We should detect:
*
* simple-escape-sequence: one of
\' \" \\ \0 \a \b \f \n \r \t \v
*
* hexadecimal-escape-sequence:
* \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt]
*/
}
_reader.SinkCharacter();
}
if (_reader.SinkCharacter() != '\'')
{
Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick.");
}
current = new CSharpTokenizer.CharLiteralToken();
return(true);
}
// Check for verbatim string
else if (_reader.Sink("@\""))
{
do
{
// Inside a verbatim string "" is treated as a special character
while (_reader.Sink("\"\""))
{
}
}while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');
// Can't end a file inside a string
if (_reader.EndOfLines)
{
current = new EndOfFileInsideStringToken();
return(true);
}
// reader.Skip the ending quote.
current = new StringLiteralToken();
current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1);
return(true);
}
// Check for a quoted string.
else if (_reader.Sink("\""))
{
while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral())
{
// See if we have an escape sequence.
if (_reader.SinkCharacter() == '\\')
{
// This is probably an escape character.
if (_reader.SinkStringEscape())
{
// This isn't nearly right. We just do barely enough to make a string
// with an embedded escape sequence return _some_ string whose start and
// end match the real bounds of the string.
}
else
{
// This is a compiler error.
_reader.SinkCharacter();
current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
return(true);
}
}
}
// Is it a newline?
if (TokenChar.IsNewLine(_reader.CurrentCharacter))
{
current = new CSharpTokenizer.NewlineInsideStringToken();
return(true);
}
// Create the token.
if (_reader.SinkCharacter() != '\"')
{
Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote.");
}
current = new StringLiteralToken();
return(true);
}
// Identifier or keyword?
else if
(
// From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier.
_reader.CurrentCharacter == '@' ||
_reader.MatchNextIdentifierStart()
)
{
if (_reader.CurrentCharacter == '@')
{
_reader.SinkCharacter();
}
// Now, the next character must be an identifier start.
if (!_reader.SinkIdentifierStart())
{
current = new ExpectedIdentifierToken();
return(true);
}
// Sink the rest of the identifier.
while (_reader.SinkIdentifierPart())
{
}
string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
switch (identifierOrKeyword)
{
default:
if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0)
{
current = new KeywordToken();
return(true);
}
// If the identifier starts with '@' then we need to strip it off.
// The '@' is for escaping so that we can have an identifier called
// the same thing as a reserved keyword (i.e. class, if, foreach, etc)
string identifier = _reader.GetCurrentMatchedString(startPosition);
if (identifier.StartsWith("@", StringComparison.Ordinal))
{
identifier = identifier.Substring(1);
}
// Create the token.
current = new IdentifierToken();
current.InnerText = identifier;
return(true);
case "false":
case "true":
current = new BooleanLiteralToken();
return(true);
case "null":
current = new CSharpTokenizer.NullLiteralToken();
return(true);
}
}
// Open scope
else if (_reader.Sink("{"))
{
current = new CSharpTokenizer.OpenScopeToken();
return(true);
}
// Close scope
else if (_reader.Sink("}"))
{
current = new CSharpTokenizer.CloseScopeToken();
return(true);
}
// Hexidecimal integer literal
else if (_reader.SinkIgnoreCase("0x"))
{
// Sink the hex digits.
if (!_reader.SinkMultipleHexDigits())
{
current = new ExpectedValidHexDigitToken();
return(true);
}
// Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new HexIntegerLiteralToken();
return(true);
}
// Decimal integer literal
else if (_reader.SinkMultipleDecimalDigits())
{
// reader.Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new DecimalIntegerLiteralToken();
return(true);
}
// Check for single-digit operators and punctuators
else if (_reader.SinkOperatorOrPunctuator())
{
current = new OperatorOrPunctuatorToken();
return(true);
}
// Preprocessor line
else if (_reader.CurrentCharacter == '#')
{
if (_reader.Sink("#if"))
{
current = new OpenConditionalDirectiveToken();
}
else if (_reader.Sink("#endif"))
{
current = new CloseConditionalDirectiveToken();
}
else
{
current = new PreprocessorToken();
}
_reader.SinkToEndOfLine();
return(true);
}
// We didn't recognize the token, so this is a syntax error.
_reader.SinkCharacter();
current = new UnrecognizedToken();
return(true);
}
internal override bool FindNextToken()
{
int position = this.reader.Position;
if (this.reader.SinkMultipleWhiteSpace())
{
base.current = new WhitespaceToken();
return(true);
}
if (this.reader.Sink("//"))
{
this.reader.SinkToEndOfLine();
base.current = new CommentToken();
return(true);
}
if (this.reader.Sink("/*"))
{
this.reader.SinkUntil("*/");
if (this.reader.EndOfLines)
{
base.current = new CSharpTokenizer.EndOfFileInsideCommentToken();
return(true);
}
base.current = new CommentToken();
return(true);
}
if (this.reader.Sink("'"))
{
while (this.reader.CurrentCharacter != '\'')
{
this.reader.Sink(@"\");
this.reader.SinkCharacter();
}
this.reader.SinkCharacter();
base.current = new CSharpTokenizer.CharLiteralToken();
return(true);
}
if (this.reader.Sink("@\""))
{
while (this.reader.Sink("\"\"") || (!this.reader.EndOfLines && (this.reader.SinkCharacter() != '"')))
{
}
if (this.reader.EndOfLines)
{
base.current = new Microsoft.Build.Shared.LanguageParser.EndOfFileInsideStringToken();
return(true);
}
base.current = new StringLiteralToken();
base.current.InnerText = this.reader.GetCurrentMatchedString(position).Substring(1);
return(true);
}
if (this.reader.Sink("\""))
{
while ((this.reader.CurrentCharacter == '\\') || this.reader.MatchRegularStringLiteral())
{
if ((this.reader.SinkCharacter() == '\\') && !this.reader.SinkStringEscape())
{
this.reader.SinkCharacter();
base.current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
return(true);
}
}
if (TokenChar.IsNewLine(this.reader.CurrentCharacter))
{
base.current = new CSharpTokenizer.NewlineInsideStringToken();
return(true);
}
this.reader.SinkCharacter();
base.current = new StringLiteralToken();
return(true);
}
if ((this.reader.CurrentCharacter == '@') || this.reader.MatchNextIdentifierStart())
{
if (this.reader.CurrentCharacter == '@')
{
this.reader.SinkCharacter();
}
if (!this.reader.SinkIdentifierStart())
{
base.current = new ExpectedIdentifierToken();
return(true);
}
while (this.reader.SinkIdentifierPart())
{
}
string currentMatchedString = this.reader.GetCurrentMatchedString(position);
switch (currentMatchedString)
{
case "false":
case "true":
base.current = new BooleanLiteralToken();
return(true);
case "null":
base.current = new CSharpTokenizer.NullLiteralToken();
return(true);
}
if (Array.IndexOf <string>(keywordList, currentMatchedString) >= 0)
{
base.current = new KeywordToken();
return(true);
}
string str2 = this.reader.GetCurrentMatchedString(position);
if (str2.StartsWith("@", StringComparison.Ordinal))
{
str2 = str2.Substring(1);
}
base.current = new IdentifierToken();
base.current.InnerText = str2;
return(true);
}
if (this.reader.Sink("{"))
{
base.current = new CSharpTokenizer.OpenScopeToken();
return(true);
}
if (this.reader.Sink("}"))
{
base.current = new CSharpTokenizer.CloseScopeToken();
return(true);
}
if (this.reader.SinkIgnoreCase("0x"))
{
if (!this.reader.SinkMultipleHexDigits())
{
base.current = new ExpectedValidHexDigitToken();
return(true);
}
this.reader.SinkLongIntegerSuffix();
base.current = new HexIntegerLiteralToken();
return(true);
}
if (this.reader.SinkMultipleDecimalDigits())
{
this.reader.SinkLongIntegerSuffix();
base.current = new DecimalIntegerLiteralToken();
return(true);
}
if (this.reader.SinkOperatorOrPunctuator())
{
base.current = new OperatorOrPunctuatorToken();
return(true);
}
if (this.reader.CurrentCharacter == '#')
{
if (this.reader.Sink("#if"))
{
base.current = new OpenConditionalDirectiveToken();
}
else if (this.reader.Sink("#endif"))
{
base.current = new CloseConditionalDirectiveToken();
}
else
{
base.current = new PreprocessorToken();
}
this.reader.SinkToEndOfLine();
return(true);
}
this.reader.SinkCharacter();
base.current = new UnrecognizedToken();
return(true);
}
internal bool MatchRegularStringLiteral()
{
return(((base.CurrentCharacter != '"') && (base.CurrentCharacter != '\\')) && !TokenChar.IsNewLine(base.CurrentCharacter));
}
