private static void GetResult(CodeProcessorContext context, PreprocessorToken token, TokenStream nodes)
{
while (token != null)
{
switch ((TokenTypes)token.Type)
{
case TokenTypes.IfDefined: Ifdef(context, token as ConditionToken, nodes); break;
case TokenTypes.IfNotDefined: Ifndef(context, token as ConditionToken, nodes); break;
case TokenTypes.If: If(context, token as ConditionToken, nodes, false); break;
case TokenTypes.ElseIf: If(context, token as ConditionToken, nodes, true); break;
case TokenTypes.Else: Else(context, token, nodes); break;
case TokenTypes.Define:
if (!Define(context, (token as TextToken).Text, nodes))
{
ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Undefine:
if (!Undefine(context, (token as TextToken).Text, nodes))
{
ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Include:
if (!Include(context, (token as TextToken).Text, nodes))
{
ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Pragma:
if (!Pragma(context, (token as TextToken).Text, nodes))
{
ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Error:
if (!Error(context, (token as TextToken).Text, nodes))
{
throw new Exception((token as TextToken).Text.ToString());
}
break;
case TokenTypes.Text: Any(context, (token as TextToken).Text, nodes); break;
}
token = token.Next as PreprocessorToken;
}
}
public static void GetResult(CodeProcessorContext context, PreprocessorToken token, List <FileDescriptor> referenceFiles, List <string> fileLookups, TokenStream nodes)
{
while (token != null)
{
switch ((TokenTypes)token.Type)
{
case TokenTypes.IfDefined: Ifdef(context, token as ConditionToken, referenceFiles, fileLookups, nodes); break;
case TokenTypes.IfNotDefined: Ifndef(context, token as ConditionToken, referenceFiles, fileLookups, nodes); break;
case TokenTypes.If: If(context, token as ConditionToken, referenceFiles, fileLookups, nodes, false); break;
case TokenTypes.ElseIf: If(context, token as ConditionToken, referenceFiles, fileLookups, nodes, true); break;
case TokenTypes.Else: Else(context, referenceFiles, fileLookups, token, nodes); break;
case TokenTypes.Define:
if (!Preprocessor.Define(context, (token as TextToken).Text, nodes))
{
Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Undefine:
if (!Preprocessor.Undefine(context, (token as TextToken).Text, nodes))
{
Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Include:
if (!Include(context, (token as TextToken).Text, referenceFiles, fileLookups, nodes))
{
Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken);
}
break;
case TokenTypes.Error:
if (ProjectLookup.IgnoreErrors)
{
Application.Warning(SeverityFlags.Full, "#error directive found: '{0}'", (token as TextToken).Text.ToString());
break;
}
else
{
throw new Exception((token as TextToken).Text.ToString());
}
case TokenTypes.Text: Preprocessor.Any(context, (token as TextToken).Text, nodes); break;
}
token = token.Next as PreprocessorToken;
}
}
private int Preprocessor()
{
char ch = data[chBaseIndex];
if (ch == '#')
{
// Just scan for a new line or eof, then add this in.
int chScanningIndex = chBaseIndex;
while (++chScanningIndex <= data.Length)
{
bool eof = chScanningIndex >= data.Length - 1;
bool proceed = eof;
if (!proceed)
{
char chScanning = data[chScanningIndex];
proceed = SeparatorToken.Map.ContainsKey(chScanning) &&
SeparatorToken.Map[chScanning] == NssSeparators.NewLine;
}
if (proceed)
{
PreprocessorToken preprocessor = new PreprocessorToken();
preprocessor.PreprocessorType = NssPreprocessorType.Unknown;
int chStartIndex = chBaseIndex;
int chEndIndex = eof ? data.Length : chScanningIndex;
if (chStartIndex == chEndIndex)
{
preprocessor.Data = "";
}
else
{
preprocessor.Data = data.Substring(chStartIndex, chEndIndex - chStartIndex);
}
int chNewBaseIndex = chEndIndex;
AttachDebugData(preprocessor, DebugRanges, chBaseIndex, chNewBaseIndex - 1);
Tokens.Add(preprocessor);
chBaseIndex = chNewBaseIndex;
break;
}
}
}
return(chBaseIndex);
}
private static void Else(CodeProcessorContext context, PreprocessorToken token, TokenStream nodes)
{
int result = context.States.Pop();
if (result >= 4)
{
result = 2;
}
CheckState(context, nodes, result);
if (nodes.CanWrite && token.Child != null)
{
GetResult(context, token.Child as PreprocessorToken, nodes);
}
EndCondition(context, null, nodes);
}
/*
* Method: FindNextToken
*
* Find the next token. Return 'true' if one was found. False, otherwise.
*/
internal override bool FindNextToken()
{
int startPosition = _reader.Position;
// VB docs claim whitespace is Unicode category Zs. However,
// this category does not contain tabs. Assuming a less restrictive
// definition for whitespace...
if (_reader.SinkWhiteSpace())
{
while (_reader.SinkWhiteSpace())
{
}
// Now, we need to check for the line continuation character.
if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_'
{
// Save the current position because we may need to come back here.
int savePosition = _reader.Position - 1;
// Skip all whitespace after the '_'
while (_reader.SinkWhiteSpace())
{
}
// Now, skip all the newlines.
// Need at least one newline for this to count as line continuation.
int count = 0;
while (_reader.SinkNewLine())
{
++count;
}
if (count > 0)
{
current = new VisualBasicTokenizer.LineContinuationToken();
return true;
}
// Otherwise, fall back to plain old whitespace.
_reader.Position = savePosition;
}
current = new WhitespaceToken();
return true;
}
// Line terminators are separate from whitespace and are significant.
else if (_reader.SinkNewLine())
{
// We want one token per line terminator.
current = new VisualBasicTokenizer.LineTerminatorToken();
return true;
}
// Check for a comment--either those that start with ' or rem.
else if (_reader.SinkLineCommentStart())
{
// Skip to the first EOL.
_reader.SinkToEndOfLine();
current = new CommentToken();
return true;
}
// Identifier or keyword?
else if
(
// VB allows escaping of identifiers by surrounding them with []
// In other words,
// Date is a keyword but,
// [Date] is an identifier.
_reader.CurrentCharacter == '[' ||
_reader.MatchNextIdentifierStart()
)
{
bool escapedIdentifier = false;
if (_reader.CurrentCharacter == '[')
{
escapedIdentifier = true;
_reader.SinkCharacter();
// Now, the next character must be an identifier start.
if (!_reader.SinkIdentifierStart())
{
current = new ExpectedIdentifierToken();
return true;
}
}
// Sink the rest of the identifier.
while (_reader.SinkIdentifierPart())
{
}
// If this was an escaped identifier the we need to get the terminating ']'.
if (escapedIdentifier)
{
if (!_reader.Sink("]"))
{
current = new ExpectedIdentifierToken();
return true;
}
}
else
{
// Escaped identifiers are not allowed to have trailing type character.
_reader.SinkTypeCharacter(); // Type character is optional.
}
// An identifier that is only a '_' is illegal because it is
// ambiguous with line continuation
string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]")
{
current = new ExpectedIdentifierToken();
return true;
}
// Make an upper-case version in order to check whether this may be a keyword.
string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture);
switch (upper)
{
default:
if (Array.IndexOf(s_keywordList, upper) >= 0)
{
current = new KeywordToken();
return true;
}
// Create the token.
current = new IdentifierToken();
// Trim off the [] if this is an escaped identifier.
if (escapedIdentifier)
{
current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2);
}
return true;
case "FALSE":
case "TRUE":
current = new BooleanLiteralToken();
return true;
}
}
// Is it a hex integer?
else if (_reader.SinkHexIntegerPrefix())
{
if (!_reader.SinkMultipleHexDigits())
{
current = new ExpectedValidHexDigitToken();
return true;
}
// Sink a suffix if there is one.
_reader.SinkIntegerSuffix();
current = new HexIntegerLiteralToken();
return true;
}
// Is it an octal integer?
else if (_reader.SinkOctalIntegerPrefix())
{
if (!_reader.SinkMultipleOctalDigits())
{
current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken();
return true;
}
// Sink a suffix if there is one.
_reader.SinkIntegerSuffix();
current = new VisualBasicTokenizer.OctalIntegerLiteralToken();
return true;
}
// Is it a decimal integer?
else if (_reader.SinkMultipleDecimalDigits())
{
// Sink a suffix if there is one.
_reader.SinkDecimalIntegerSuffix();
current = new DecimalIntegerLiteralToken();
return true;
}
// Preprocessor line
else if (_reader.CurrentCharacter == '#')
{
if (_reader.SinkIgnoreCase("#if"))
{
current = new OpenConditionalDirectiveToken();
}
else if (_reader.SinkIgnoreCase("#end if"))
{
current = new CloseConditionalDirectiveToken();
}
else
{
current = new PreprocessorToken();
}
_reader.SinkToEndOfLine();
return true;
}
// Is it a separator?
else if (_reader.SinkSeparatorCharacter())
{
current = new VisualBasicTokenizer.SeparatorToken();
return true;
}
// Is it an operator?
else if (_reader.SinkOperator())
{
current = new OperatorToken();
return true;
}
// A string?
else if (_reader.Sink("\""))
{
do
{
// Inside a verbatim string "" is treated as a special character
while (_reader.Sink("\"\""))
{
}
}
while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');
// Can't end a file inside a string
if (_reader.EndOfLines)
{
current = new EndOfFileInsideStringToken();
return true;
}
current = new StringLiteralToken();
return true;
}
// We didn't recognize the token, so this is a syntax error.
_reader.SinkCharacter();
current = new UnrecognizedToken();
return true;
}
/*
* Method: FindNextToken
*
* Find the next token. Return 'true' if one was found. False, otherwise.
*/
override internal bool FindNextToken()
{
int startPosition = _reader.Position;
// VB docs claim whitespace is Unicode category Zs. However,
// this category does not contain tabs. Assuming a less restrictive
// definition for whitespace...
if (_reader.SinkWhiteSpace())
{
while (_reader.SinkWhiteSpace())
{
}
// Now, we need to check for the line continuation character.
if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_'
{
// Save the current position because we may need to come back here.
int savePosition = _reader.Position - 1;
// Skip all whitespace after the '_'
while (_reader.SinkWhiteSpace())
{
}
// Now, skip all the newlines.
// Need at least one newline for this to count as line continuation.
int count = 0;
while (_reader.SinkNewLine())
{
++count;
}
if (count > 0)
{
current = new VisualBasicTokenizer.LineContinuationToken();
return(true);
}
// Otherwise, fall back to plain old whitespace.
_reader.Position = savePosition;
}
current = new WhitespaceToken();
return(true);
}
// Line terminators are separate from whitespace and are significant.
else if (_reader.SinkNewLine())
{
// We want one token per line terminator.
current = new VisualBasicTokenizer.LineTerminatorToken();
return(true);
}
// Check for a comment--either those that start with ' or rem.
else if (_reader.SinkLineCommentStart())
{
// Skip to the first EOL.
_reader.SinkToEndOfLine();
current = new CommentToken();
return(true);
}
// Identifier or keyword?
else if
(
// VB allows escaping of identifiers by surrounding them with []
// In other words,
// Date is a keyword but,
// [Date] is an identifier.
_reader.CurrentCharacter == '[' ||
_reader.MatchNextIdentifierStart()
)
{
bool escapedIdentifier = false;
if (_reader.CurrentCharacter == '[')
{
escapedIdentifier = true;
_reader.SinkCharacter();
// Now, the next character must be an identifier start.
if (!_reader.SinkIdentifierStart())
{
current = new ExpectedIdentifierToken();
return(true);
}
}
// Sink the rest of the identifier.
while (_reader.SinkIdentifierPart())
{
}
// If this was an escaped identifier the we need to get the terminating ']'.
if (escapedIdentifier)
{
if (!_reader.Sink("]"))
{
current = new ExpectedIdentifierToken();
return(true);
}
}
else
{
// Escaped identifiers are not allowed to have trailing type character.
_reader.SinkTypeCharacter(); // Type character is optional.
}
// An identifier that is only a '_' is illegal because it is
// ambiguous with line continuation
string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]")
{
current = new ExpectedIdentifierToken();
return(true);
}
// Make an upper-case version in order to check whether this may be a keyword.
string upper = identifierOrKeyword.ToUpperInvariant();
switch (upper)
{
default:
if (Array.IndexOf(s_keywordList, upper) >= 0)
{
current = new KeywordToken();
return(true);
}
// Create the token.
current = new IdentifierToken();
// Trim off the [] if this is an escaped identifier.
if (escapedIdentifier)
{
current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2);
}
return(true);
case "FALSE":
case "TRUE":
current = new BooleanLiteralToken();
return(true);
}
}
// Is it a hex integer?
else if (_reader.SinkHexIntegerPrefix())
{
if (!_reader.SinkMultipleHexDigits())
{
current = new ExpectedValidHexDigitToken();
return(true);
}
// Sink a suffix if there is one.
_reader.SinkIntegerSuffix();
current = new HexIntegerLiteralToken();
return(true);
}
// Is it an octal integer?
else if (_reader.SinkOctalIntegerPrefix())
{
if (!_reader.SinkMultipleOctalDigits())
{
current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken();
return(true);
}
// Sink a suffix if there is one.
_reader.SinkIntegerSuffix();
current = new VisualBasicTokenizer.OctalIntegerLiteralToken();
return(true);
}
// Is it a decimal integer?
else if (_reader.SinkMultipleDecimalDigits())
{
// Sink a suffix if there is one.
_reader.SinkDecimalIntegerSuffix();
current = new DecimalIntegerLiteralToken();
return(true);
}
// Preprocessor line
else if (_reader.CurrentCharacter == '#')
{
if (_reader.SinkIgnoreCase("#if"))
{
current = new OpenConditionalDirectiveToken();
}
else if (_reader.SinkIgnoreCase("#end if"))
{
current = new CloseConditionalDirectiveToken();
}
else
{
current = new PreprocessorToken();
}
_reader.SinkToEndOfLine();
return(true);
}
// Is it a separator?
else if (_reader.SinkSeparatorCharacter())
{
current = new VisualBasicTokenizer.SeparatorToken();
return(true);
}
// Is it an operator?
else if (_reader.SinkOperator())
{
current = new OperatorToken();
return(true);
}
// A string?
else if (_reader.Sink("\""))
{
do
{
// Inside a verbatim string "" is treated as a special character
while (_reader.Sink("\"\""))
{
}
}while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');
// Can't end a file inside a string
if (_reader.EndOfLines)
{
current = new EndOfFileInsideStringToken();
return(true);
}
current = new StringLiteralToken();
return(true);
}
// We didn't recognize the token, so this is a syntax error.
_reader.SinkCharacter();
current = new UnrecognizedToken();
return(true);
}
/*
* Method: FindNextToken
*
* Find the next token. Return 'true' if one was found. False, otherwise.
*/
override internal bool FindNextToken()
{
int startPosition = _reader.Position;
// Dealing with whitespace?
if (_reader.SinkMultipleWhiteSpace())
{
current = new WhitespaceToken();
return(true);
}
// Check for one-line comment
else if (_reader.Sink("//"))
{
// Looks like a one-line comment. Follow it to the End-of-line
_reader.SinkToEndOfLine();
current = new CommentToken();
return(true);
}
// Check for multi-line comment
else if (_reader.Sink("/*"))
{
_reader.SinkUntil("*/");
// Was the ending */ found?
if (_reader.EndOfLines)
{
// No. There was a /* without a */. Return this a syntax error token.
current = new CSharpTokenizer.EndOfFileInsideCommentToken();
return(true);
}
current = new CommentToken();
return(true);
}
// Handle chars
else if (_reader.Sink("\'"))
{
while (_reader.CurrentCharacter != '\'')
{
if (_reader.Sink("\\"))
{
/* reader.Skip the escape sequence.
* This isn't exactly right. We should detect:
*
* simple-escape-sequence: one of
\' \" \\ \0 \a \b \f \n \r \t \v
*
* hexadecimal-escape-sequence:
* \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt]
*/
}
_reader.SinkCharacter();
}
if (_reader.SinkCharacter() != '\'')
{
Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick.");
}
current = new CSharpTokenizer.CharLiteralToken();
return(true);
}
// Check for verbatim string
else if (_reader.Sink("@\""))
{
do
{
// Inside a verbatim string "" is treated as a special character
while (_reader.Sink("\"\""))
{
}
}while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');
// Can't end a file inside a string
if (_reader.EndOfLines)
{
current = new EndOfFileInsideStringToken();
return(true);
}
// reader.Skip the ending quote.
current = new StringLiteralToken();
current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1);
return(true);
}
// Check for a quoted string.
else if (_reader.Sink("\""))
{
while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral())
{
// See if we have an escape sequence.
if (_reader.SinkCharacter() == '\\')
{
// This is probably an escape character.
if (_reader.SinkStringEscape())
{
// This isn't nearly right. We just do barely enough to make a string
// with an embedded escape sequence return _some_ string whose start and
// end match the real bounds of the string.
}
else
{
// This is a compiler error.
_reader.SinkCharacter();
current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
return(true);
}
}
}
// Is it a newline?
if (TokenChar.IsNewLine(_reader.CurrentCharacter))
{
current = new CSharpTokenizer.NewlineInsideStringToken();
return(true);
}
// Create the token.
if (_reader.SinkCharacter() != '\"')
{
Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote.");
}
current = new StringLiteralToken();
return(true);
}
// Identifier or keyword?
else if
(
// From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier.
_reader.CurrentCharacter == '@' ||
_reader.MatchNextIdentifierStart()
)
{
if (_reader.CurrentCharacter == '@')
{
_reader.SinkCharacter();
}
// Now, the next character must be an identifier start.
if (!_reader.SinkIdentifierStart())
{
current = new ExpectedIdentifierToken();
return(true);
}
// Sink the rest of the identifier.
while (_reader.SinkIdentifierPart())
{
}
string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
switch (identifierOrKeyword)
{
default:
if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0)
{
current = new KeywordToken();
return(true);
}
// If the identifier starts with '@' then we need to strip it off.
// The '@' is for escaping so that we can have an identifier called
// the same thing as a reserved keyword (i.e. class, if, foreach, etc)
string identifier = _reader.GetCurrentMatchedString(startPosition);
if (identifier.StartsWith("@", StringComparison.Ordinal))
{
identifier = identifier.Substring(1);
}
// Create the token.
current = new IdentifierToken();
current.InnerText = identifier;
return(true);
case "false":
case "true":
current = new BooleanLiteralToken();
return(true);
case "null":
current = new CSharpTokenizer.NullLiteralToken();
return(true);
}
}
// Open scope
else if (_reader.Sink("{"))
{
current = new CSharpTokenizer.OpenScopeToken();
return(true);
}
// Close scope
else if (_reader.Sink("}"))
{
current = new CSharpTokenizer.CloseScopeToken();
return(true);
}
// Hexidecimal integer literal
else if (_reader.SinkIgnoreCase("0x"))
{
// Sink the hex digits.
if (!_reader.SinkMultipleHexDigits())
{
current = new ExpectedValidHexDigitToken();
return(true);
}
// Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new HexIntegerLiteralToken();
return(true);
}
// Decimal integer literal
else if (_reader.SinkMultipleDecimalDigits())
{
// reader.Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new DecimalIntegerLiteralToken();
return(true);
}
// Check for single-digit operators and punctuators
else if (_reader.SinkOperatorOrPunctuator())
{
current = new OperatorOrPunctuatorToken();
return(true);
}
// Preprocessor line
else if (_reader.CurrentCharacter == '#')
{
if (_reader.Sink("#if"))
{
current = new OpenConditionalDirectiveToken();
}
else if (_reader.Sink("#endif"))
{
current = new CloseConditionalDirectiveToken();
}
else
{
current = new PreprocessorToken();
}
_reader.SinkToEndOfLine();
return(true);
}
// We didn't recognize the token, so this is a syntax error.
_reader.SinkCharacter();
current = new UnrecognizedToken();
return(true);
}
public static bool GetSyntaxTree(Stream data, SyntaxTree <PreprocessorToken> tree, PreprocessorToken root = null)
{
while (!data.Eof())
{
char c = data.Peek();
switch ((TokenTypes)c)
{
case TokenTypes.IfDefined:
case TokenTypes.IfNotDefined:
case TokenTypes.If:
case TokenTypes.ElseIf:
{
ConditionToken token = new ConditionToken((TokenTypes)c);
token.Deserialize(data);
if (root == null)
{
tree.Add(token);
}
else
{
tree.AddAppend(root, token);
}
if (!GetSyntaxTree(data, tree, token))
{
return(false);
}
}
break;
case TokenTypes.Else:
{
PreprocessorToken token = new PreprocessorToken(TokenTypes.Else);
token.Deserialize(data);
if (root == null)
{
tree.Add(token);
}
else
{
tree.AddAppend(root, token);
}
if (!GetSyntaxTree(data, tree, token))
{
return(false);
}
}
break;
case TokenTypes.Define:
case TokenTypes.Undefine:
case TokenTypes.Include:
case TokenTypes.Error:
case TokenTypes.Pragma:
{
TextToken token = new TextToken((TokenTypes)c);
token.Deserialize(data);
if (root == null)
{
tree.Add(token);
}
else
{
tree.AddAppend(root, token);
}
}
break;
case TokenTypes.Text:
{
TextToken token = new TextToken();
token.Deserialize(data);
if (root == null)
{
tree.Add(token);
}
else
{
tree.AddAppend(root, token);
}
}
break;
default:
{
if (root != null && c == 0)
{
data.Get();
return(true);
}
else
{
return(ThrowError(new CodeProcessorContext(new CodeProcessorConfig()), PreprocessorCodes.UnexpectedCharacter));
}
}
}
}
return(true);
}
internal abstract int scan(ref PreprocessorToken ppToken);
private int DoHexidecimal(StringInputBuffer buffer, PreprocessorToken ppToken, int ch, ref bool alreadyComplained, ref int len, int maxTokenLength)
{
// can't be hexidecimal or octal, is either decimal or floating point
do
{
if (len < maxTokenLength)
{
buffer.name[len++] = (char)ch;
}
else if (!alreadyComplained)
{
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
alreadyComplained = true;
}
ch = pp.getChar();
} while (ch >= '0' && ch <= '9');
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L')
{
return(pp.lFloatConst(pp.buffer, len, ch, ppToken));
}
else
{
// Finish handling signed and unsigned integers
int numericLen = len;
bool uintSign = false;
if (ch == 'u' || ch == 'U')
{
if (len < maxTokenLength)
{
buffer.name[len++] = (char)ch;
}
uintSign = true;
}
else
{
pp.ungetChar();
}
// NOT NEEDED
//ppToken.name[len] = '\0';
uint ival = 0;
const uint ONETENTHMAXINT = 0xFFFFFFFFu / 10;
const uint REMAINDERMAXINT = 0xFFFFFFFFu - 10 * ONETENTHMAXINT;
for (int i = 0; i < numericLen; i++)
{
ch = buffer.name[i] - '0';
if ((ival > ONETENTHMAXINT) || (ival == ONETENTHMAXINT && ch > REMAINDERMAXINT))
{
pp.parseContext.Error(ppToken.loc, "numeric literal too big", "", "");
ival = 0xFFFFFFFFu;
break;
}
else
{
ival = (uint)(ival * 10 + ch);
}
}
ppToken.ival = (int)ival;
ppToken.name = new string (buffer.name, 0, len);
return(uintSign ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT);
}
}
internal override int scan(ref PreprocessorToken ppToken)
{
//
// Scanner used to tokenize source stream.
//
bool AlreadyComplained = false;
int len = 0;
int ch = 0;
int ii = 0;
uint ival = 0;
ppToken.ival = 0;
ppToken.space = false;
ch = pp.getChar();
for (;;)
{
while (ch == ' ' || ch == '\t')
{
ppToken.space = true;
ch = pp.getChar();
}
ppToken.loc = pp.parseContext.getCurrentLoc();
len = 0;
switch (ch)
{
default:
return(ch); // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
// TODO : EOF for stream
// case EOF:
// return endOfInput;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
do
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.tokenText [len++] = (char)ch;
ch = pp.getChar();
}
else
{
if (!AlreadyComplained)
{
pp.parseContext.Error(ppToken.loc, "name too long", "", "");
AlreadyComplained = true;
}
ch = pp.getChar();
}
} while ((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_');
// line continuation with no token before or after makes len == 0, and need to start over skipping white space, etc.
if (len == 0)
{
continue;
}
pp.buffer.tokenText [len] = '\0';
pp.ungetChar();
ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string (pp.buffer.tokenText, 0, len));
return((int)CppEnums.IDENTIFIER);
case '0':
pp.buffer.name[len++] = (char)ch;
ch = pp.getChar();
if (ch == 'x' || ch == 'X')
{
// must be hexidecimal
bool isUnsigned = false;
pp.buffer.name[len++] = (char)ch;
ch = pp.getChar();
if ((ch >= '0' && ch <= '9') ||
(ch >= 'A' && ch <= 'F') ||
(ch >= 'a' && ch <= 'f'))
{
ival = 0;
do
{
if (ival <= 0x0fffffff)
{
pp.buffer.name[len++] = (char)ch;
if (ch >= '0' && ch <= '9')
{
ii = ch - '0';
}
else if (ch >= 'A' && ch <= 'F')
{
ii = ch - 'A' + 10;
}
else if (ch >= 'a' && ch <= 'f')
{
ii = ch - 'a' + 10;
}
else
{
pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", "");
}
ival = (uint)((ival << 4) | ii);
}
else
{
if (!AlreadyComplained)
{
pp.parseContext.Error(ppToken.loc, "hexidecimal literal too big", "", "");
AlreadyComplained = true;
}
ival = 0xffffffff;
}
ch = pp.getChar();
} while ((ch >= '0' && ch <= '9') ||
(ch >= 'A' && ch <= 'F') ||
(ch >= 'a' && ch <= 'f'));
}
else
{
pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", "");
}
if (ch == 'u' || ch == 'U')
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.name[len++] = (char)ch;
}
isUnsigned = true;
}
else
{
pp.ungetChar();
}
//NOT NEEDED
//ppToken.name[len] = '\0';
ppToken.ival = (int)ival;
if (isUnsigned)
{
return((int)CppEnums.UINTCONSTANT);
}
else
{
return((int)CppEnums.INTCONSTANT);
}
}
else
{
// could be octal integer or floating point, speculative pursue octal until it must be floating point
bool isUnsigned = false;
bool octalOverflow = false;
bool nonOctal = false;
ival = 0;
// see how much octal-like stuff we can read
while (ch >= '0' && ch <= '7')
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.name[len++] = (char)ch;
}
else if (!AlreadyComplained)
{
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
AlreadyComplained = true;
}
if (ival <= 0x1fffffff)
{
ii = ch - '0';
ival = (uint)((ival << 3) | ii);
}
else
{
octalOverflow = true;
}
ch = pp.getChar();
}
// could be part of a float...
if (ch == '8' || ch == '9')
{
nonOctal = true;
do
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.name[len++] = (char)ch;
}
else if (!AlreadyComplained)
{
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
AlreadyComplained = true;
}
ch = pp.getChar();
} while (ch >= '0' && ch <= '9');
}
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L')
{
//ppToken.name = new string(pp.buffer.name, 0, len);
return(pp.lFloatConst(pp.buffer, len, ch, ppToken));
}
// wasn't a float, so must be octal...
if (nonOctal)
{
pp.parseContext.Error(ppToken.loc, "octal literal digit too large", "", "");
}
if (ch == 'u' || ch == 'U')
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.name[len++] = (char)ch;
}
isUnsigned = true;
}
else
{
pp.ungetChar();
}
// NOT NEEDED
//ppToken.name[len] = '\0';
if (octalOverflow)
{
pp.parseContext.Error(ppToken.loc, "octal literal too big", "", "");
}
ppToken.ival = (int)ival;
ppToken.name = new string(pp.buffer.name, 0, len);
return(isUnsigned ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT);
}
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return(DoHexidecimal(pp.buffer, ppToken, ch, ref AlreadyComplained, ref len, StringInputBuffer.MAX_TOKEN_LENGTH));
case '-':
ch = pp.getChar();
if (ch == '-')
{
return((int)CppEnums.DEC_OP);
}
else if (ch == '=')
{
return((int)CppEnums.SUB_ASSIGN);
}
else
{
pp.ungetChar();
return('-');
}
case '+':
ch = pp.getChar();
if (ch == '+')
{
return((int)CppEnums.INC_OP);
}
else if (ch == '=')
{
return((int)CppEnums.ADD_ASSIGN);
}
else
{
pp.ungetChar();
return('+');
}
case '*':
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.MUL_ASSIGN);
}
else
{
pp.ungetChar();
return('*');
}
case '%':
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.MOD_ASSIGN);
}
else if (ch == '>')
{
return((int)CppEnums.RIGHT_BRACE);
}
else
{
pp.ungetChar();
return('%');
}
case ':':
ch = pp.getChar();
if (ch == '>')
{
return((int)CppEnums.RIGHT_BRACKET);
}
else
{
pp.ungetChar();
return(':');
}
case '^':
ch = pp.getChar();
if (ch == '^')
{
return((int)CppEnums.XOR_OP);
}
else
{
if (ch == '=')
{
return((int)CppEnums.XOR_ASSIGN);
}
else
{
pp.ungetChar();
return('^');
}
}
case '=':
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.EQ_OP);
}
else
{
pp.ungetChar();
return('=');
}
case '!':
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.NE_OP);
}
else
{
pp.ungetChar();
return('!');
}
case '|':
ch = pp.getChar();
if (ch == '|')
{
return((int)CppEnums.OR_OP);
}
else
{
if (ch == '=')
{
return((int)CppEnums.OR_ASSIGN);
}
else
{
pp.ungetChar();
return('|');
}
}
case '&':
ch = pp.getChar();
if (ch == '&')
{
return((int)CppEnums.AND_OP);
}
else
{
if (ch == '=')
{
return((int)CppEnums.AND_ASSIGN);
}
else
{
pp.ungetChar();
return('&');
}
}
case '<':
ch = pp.getChar();
if (ch == '<')
{
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.LEFT_ASSIGN);
}
else
{
pp.ungetChar();
return((int)CppEnums.LEFT_OP);
}
}
else
{
if (ch == '=')
{
return((int)CppEnums.LE_OP);
}
else
{
if (ch == '%')
{
return((int)CppEnums.LEFT_BRACE);
}
else if (ch == ':')
{
return((int)CppEnums.LEFT_BRACKET);
}
else
{
pp.ungetChar();
return('<');
}
}
}
case '>':
ch = pp.getChar();
if (ch == '>')
{
ch = pp.getChar();
if (ch == '=')
{
return((int)CppEnums.RIGHT_ASSIGN);
}
else
{
pp.ungetChar();
return((int)CppEnums.RIGHT_OP);
}
}
else
{
if (ch == '=')
{
return((int)CppEnums.GE_OP);
}
else
{
pp.ungetChar();
return('>');
}
}
case '.':
ch = pp.getChar();
if (ch >= '0' && ch <= '9')
{
pp.ungetChar();
return(pp.lFloatConst(pp.buffer, 0, '.', ppToken));
}
else
{
pp.ungetChar();
return('.');
}
case '/':
ch = pp.getChar();
if (ch == '/')
{
pp.inComment = true;
do
{
ch = pp.getChar();
} while (ch != '\n' && !this.IsEOF(ch));
ppToken.space = true;
pp.inComment = false;
if (this.IsEOF(ch))
{
return(END_OF_INPUT);
}
return(ch);
}
else if (ch == '*')
{
ch = pp.getChar();
do
{
while (ch != '*')
{
if (this.IsEOF(ch))
{
pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", "");
return(END_OF_INPUT);
}
ch = pp.getChar();
}
ch = pp.getChar();
if (this.IsEOF(ch))
{
pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", "");
return(END_OF_INPUT);
}
} while (ch != '/');
ppToken.space = true;
// loop again to get the next token...
break;
}
else if (ch == '=')
{
return((int)CppEnums.DIV_ASSIGN);
}
else
{
pp.ungetChar();
return('/');
}
case '"':
ch = pp.getChar();
while (ch != '"' && ch != '\n' && this.IsEOF(ch))
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.tokenText[len] = (char)ch;
len++;
ch = pp.getChar();
}
else
{
break;
}
}
;
//pp.buffer.tokenText[len] = '\0';
if (ch == '"')
{
ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string(pp.buffer.tokenText, 0, len));
ppToken.name = new string (pp.buffer.name, 0, len);
return((int)CppEnums.STRCONSTANT);
}
else
{
pp.parseContext.Error(ppToken.loc, "end of line in string", "string", "");
ppToken.name = new string (pp.buffer.name, 0, len);
return((int)CppEnums.ERROR_SY);
}
}
ch = pp.getChar();
}
}
private static void Else(CodeProcessorContext context, List <FileDescriptor> referenceFiles, List <string> fileLookups, PreprocessorToken token, TokenStream nodes)
{
int result = context.States.Pop();
if (result >= 4)
{
result = 2;
}
Preprocessor.CheckState(context, nodes, result);
if (nodes.CanWrite && token.Child != null)
{
GetResult(context, token.Child as PreprocessorToken, referenceFiles, fileLookups, nodes);
}
EndCondition(context, null, nodes);
}
internal abstract int scan(ref PreprocessorToken ppToken);
internal override int scan(ref PreprocessorToken ppToken)
{
//
// Scanner used to tokenize source stream.
//
bool AlreadyComplained = false;
int len = 0;
int ch = 0;
int ii = 0;
uint ival = 0;
ppToken.ival = 0;
ppToken.space = false;
ch = pp.getChar();
for (;;) {
while (ch == ' ' || ch == '\t') {
ppToken.space = true;
ch = pp.getChar();
}
ppToken.loc = pp.parseContext.getCurrentLoc();
len = 0;
switch (ch) {
default:
return ch; // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token)
// TODO : EOF for stream
// case EOF:
// return endOfInput;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
do
{
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
{
pp.buffer.tokenText [len++] = (char)ch;
ch = pp.getChar ();
} else
{
if (!AlreadyComplained)
{
pp.parseContext.Error (ppToken.loc, "name too long", "", "");
AlreadyComplained = true;
}
ch = pp.getChar ();
}
} while ((ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_');
// line continuation with no token before or after makes len == 0, and need to start over skipping white space, etc.
if (len == 0)
continue;
pp.buffer.tokenText [len] = '\0';
pp.ungetChar ();
ppToken.atom = pp.Symbols.Atoms.LookUpAddString (new string (pp.buffer.tokenText, 0, len));
return (int) CppEnums.IDENTIFIER;
case '0':
pp.buffer.name[len++] = (char)ch;
ch = pp.getChar();
if (ch == 'x' || ch == 'X') {
// must be hexidecimal
bool isUnsigned = false;
pp.buffer.name[len++] = (char)ch;
ch = pp.getChar();
if ((ch >= '0' && ch <= '9') ||
(ch >= 'A' && ch <= 'F') ||
(ch >= 'a' && ch <= 'f')) {
ival = 0;
do {
if (ival <= 0x0fffffff) {
pp.buffer.name[len++] = (char)ch;
if (ch >= '0' && ch <= '9') {
ii = ch - '0';
} else if (ch >= 'A' && ch <= 'F') {
ii = ch - 'A' + 10;
} else if (ch >= 'a' && ch <= 'f') {
ii = ch - 'a' + 10;
} else
pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", "");
ival = (uint)((ival << 4) | ii);
} else {
if (!AlreadyComplained) {
pp.parseContext.Error(ppToken.loc, "hexidecimal literal too big", "", "");
AlreadyComplained = true;
}
ival = 0xffffffff;
}
ch = pp.getChar();
} while ((ch >= '0' && ch <= '9') ||
(ch >= 'A' && ch <= 'F') ||
(ch >= 'a' && ch <= 'f'));
} else {
pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", "");
}
if (ch == 'u' || ch == 'U') {
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
pp.buffer.name[len++] = (char)ch;
isUnsigned = true;
} else
pp.ungetChar();
//NOT NEEDED
//ppToken.name[len] = '\0';
ppToken.ival = (int)ival;
if (isUnsigned)
{
return (int)CppEnums.UINTCONSTANT;
}
else
{
return (int)CppEnums.INTCONSTANT;
}
} else {
// could be octal integer or floating point, speculative pursue octal until it must be floating point
bool isUnsigned = false;
bool octalOverflow = false;
bool nonOctal = false;
ival = 0;
// see how much octal-like stuff we can read
while (ch >= '0' && ch <= '7') {
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
pp.buffer.name[len++] = (char)ch;
else if (! AlreadyComplained) {
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
AlreadyComplained = true;
}
if (ival <= 0x1fffffff) {
ii = ch - '0';
ival = (uint)((ival << 3) | ii);
} else
octalOverflow = true;
ch = pp.getChar();
}
// could be part of a float...
if (ch == '8' || ch == '9') {
nonOctal = true;
do {
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
pp.buffer.name[len++] = (char)ch;
else if (! AlreadyComplained) {
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
AlreadyComplained = true;
}
ch = pp.getChar();
} while (ch >= '0' && ch <= '9');
}
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L')
{
//ppToken.name = new string(pp.buffer.name, 0, len);
return pp.lFloatConst (pp.buffer, len, ch, ppToken);
}
// wasn't a float, so must be octal...
if (nonOctal)
pp.parseContext.Error(ppToken.loc, "octal literal digit too large", "", "");
if (ch == 'u' || ch == 'U') {
if (len < StringInputBuffer.MAX_TOKEN_LENGTH)
pp.buffer.name[len++] = (char)ch;
isUnsigned = true;
} else
pp.ungetChar();
// NOT NEEDED
//ppToken.name[len] = '\0';
if (octalOverflow)
pp.parseContext.Error(ppToken.loc, "octal literal too big", "", "");
ppToken.ival = (int)ival;
ppToken.name = new string(pp.buffer.name, 0, len);
return isUnsigned ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT;
}
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return DoHexidecimal (pp.buffer, ppToken, ch, ref AlreadyComplained, ref len, StringInputBuffer.MAX_TOKEN_LENGTH);
case '-':
ch = pp.getChar();
if (ch == '-') {
return (int) CppEnums.DEC_OP;
} else if (ch == '=') {
return (int) CppEnums.SUB_ASSIGN;
} else {
pp.ungetChar();
return '-';
}
case '+':
ch = pp.getChar();
if (ch == '+') {
return (int) CppEnums.INC_OP;
} else if (ch == '=') {
return (int) CppEnums.ADD_ASSIGN;
} else {
pp.ungetChar();
return '+';
}
case '*':
ch = pp.getChar();
if (ch == '=') {
return (int) CppEnums.MUL_ASSIGN;
} else {
pp.ungetChar();
return '*';
}
case '%':
ch = pp.getChar();
if (ch == '=') {
return (int) CppEnums.MOD_ASSIGN;
} else if (ch == '>'){
return (int) CppEnums.RIGHT_BRACE;
} else {
pp.ungetChar();
return '%';
}
case ':':
ch = pp.getChar();
if (ch == '>') {
return (int) CppEnums.RIGHT_BRACKET;
} else {
pp.ungetChar();
return ':';
}
case '^':
ch = pp.getChar();
if (ch == '^') {
return (int) CppEnums.XOR_OP;
} else {
if (ch == '=')
{
return (int)CppEnums.XOR_ASSIGN;
}
else
{
pp.ungetChar();
return '^';
}
}
case '=':
ch = pp.getChar();
if (ch == '=') {
return (int) CppEnums.EQ_OP;
} else {
pp.ungetChar();
return '=';
}
case '!':
ch = pp.getChar();
if (ch == '=') {
return (int) CppEnums.NE_OP;
} else {
pp.ungetChar();
return '!';
}
case '|':
ch = pp.getChar();
if (ch == '|') {
return (int) CppEnums.OR_OP;
} else {
if (ch == '=')
{
return (int)CppEnums.OR_ASSIGN;
}
else{
pp.ungetChar();
return '|';
}
}
case '&':
ch = pp.getChar();
if (ch == '&') {
return (int) CppEnums.AND_OP;
} else {
if (ch == '=')
{
return (int)CppEnums.AND_ASSIGN;
}
else{
pp.ungetChar();
return '&';
}
}
case '<':
ch = pp.getChar();
if (ch == '<') {
ch = pp.getChar();
if (ch == '=')
{
return (int)CppEnums.LEFT_ASSIGN;
}
else
{
pp.ungetChar();
return (int) CppEnums.LEFT_OP;
}
} else {
if (ch == '=')
{
return (int) CppEnums.LE_OP;
}
else
{
if (ch == '%')
{
return (int)CppEnums.LEFT_BRACE;
} else if (ch == ':')
{
return (int)CppEnums.LEFT_BRACKET;
}
else
{
pp.ungetChar();
return '<';
}
}
}
case '>':
ch = pp.getChar();
if (ch == '>') {
ch = pp.getChar();
if (ch == '=')
{
return (int)CppEnums.RIGHT_ASSIGN;
}
else{
pp.ungetChar();
return (int) CppEnums.RIGHT_OP;
}
} else {
if (ch == '=') {
return (int) CppEnums.GE_OP;
} else {
pp.ungetChar();
return '>';
}
}
case '.':
ch = pp.getChar();
if (ch >= '0' && ch <= '9')
{
pp.ungetChar();
return pp.lFloatConst(pp.buffer, 0, '.', ppToken);
}
else
{
pp.ungetChar();
return '.';
}
case '/':
ch = pp.getChar();
if (ch == '/') {
pp.inComment = true;
do {
ch = pp.getChar();
} while (ch != '\n' && ! this.IsEOF(ch));
ppToken.space = true;
pp.inComment = false;
if (this.IsEOF (ch))
{
return END_OF_INPUT;
}
return ch;
} else if (ch == '*') {
ch = pp.getChar();
do {
while (ch != '*') {
if (this.IsEOF(ch))
{
pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", "");
return END_OF_INPUT;
}
ch = pp.getChar();
}
ch = pp.getChar();
if (this.IsEOF(ch))
{
pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", "");
return END_OF_INPUT;
}
} while (ch != '/');
ppToken.space = true;
// loop again to get the next token...
break;
} else if (ch == '=') {
return (int) CppEnums.DIV_ASSIGN;
} else {
pp.ungetChar();
return '/';
}
case '"':
ch = pp.getChar();
while (ch != '"' && ch != '\n' && this.IsEOF(ch)) {
if (len < StringInputBuffer.MAX_TOKEN_LENGTH) {
pp.buffer.tokenText[len] = (char)ch;
len++;
ch = pp.getChar();
} else
break;
};
//pp.buffer.tokenText[len] = '\0';
if (ch == '"') {
ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string(pp.buffer.tokenText, 0, len));
ppToken.name = new string (pp.buffer.name, 0, len);
return (int) CppEnums.STRCONSTANT;
} else {
pp.parseContext.Error(ppToken.loc, "end of line in string", "string", "");
ppToken.name = new string (pp.buffer.name, 0, len);
return (int) CppEnums.ERROR_SY;
}
}
ch = pp.getChar();
}
}
private int DoHexidecimal(StringInputBuffer buffer, PreprocessorToken ppToken, int ch, ref bool alreadyComplained, ref int len, int maxTokenLength)
{
// can't be hexidecimal or octal, is either decimal or floating point
do {
if (len < maxTokenLength)
buffer.name[len++] = (char)ch;
else if (! alreadyComplained) {
pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", "");
alreadyComplained = true;
}
ch = pp.getChar();
} while (ch >= '0' && ch <= '9');
if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L')
{
return pp.lFloatConst(pp.buffer, len, ch, ppToken);
}
else
{
// Finish handling signed and unsigned integers
int numericLen = len;
bool uintSign = false;
if (ch == 'u' || ch == 'U')
{
if (len < maxTokenLength)
buffer.name[len++] = (char)ch;
uintSign = true;
} else
pp.ungetChar();
// NOT NEEDED
//ppToken.name[len] = '\0';
uint ival = 0;
const uint ONETENTHMAXINT = 0xFFFFFFFFu / 10;
const uint REMAINDERMAXINT = 0xFFFFFFFFu - 10 * ONETENTHMAXINT;
for (int i = 0; i < numericLen; i++)
{
ch = buffer.name[i] - '0';
if ((ival > ONETENTHMAXINT) || (ival == ONETENTHMAXINT && ch > REMAINDERMAXINT))
{
pp.parseContext.Error(ppToken.loc, "numeric literal too big", "", "");
ival = 0xFFFFFFFFu;
break;
}
else
ival = (uint)(ival * 10 + ch);
}
ppToken.ival = (int)ival;
ppToken.name = new string (buffer.name, 0, len);
return uintSign ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT;
}
}
/*
* Method: FindNextToken
*
* Find the next token. Return 'true' if one was found. False, otherwise.
*/
override internal bool FindNextToken()
{
int startPosition = _reader.Position;
// Dealing with whitespace?
if (_reader.SinkMultipleWhiteSpace())
{
current = new WhitespaceToken();
return true;
}
// Check for one-line comment
else if (_reader.Sink("//"))
{
// Looks like a one-line comment. Follow it to the End-of-line
_reader.SinkToEndOfLine();
current = new CommentToken();
return true;
}
// Check for multi-line comment
else if (_reader.Sink("/*"))
{
_reader.SinkUntil("*/");
// Was the ending */ found?
if (_reader.EndOfLines)
{
// No. There was a /* without a */. Return this a syntax error token.
current = new CSharpTokenizer.EndOfFileInsideCommentToken();
return true;
}
current = new CommentToken();
return true;
}
// Handle chars
else if (_reader.Sink("\'"))
{
while (_reader.CurrentCharacter != '\'')
{
if (_reader.Sink("\\"))
{
/* reader.Skip the escape sequence.
This isn't exactly right. We should detect:
simple-escape-sequence: one of
\' \" \\ \0 \a \b \f \n \r \t \v
hexadecimal-escape-sequence:
\x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt]
*/
}
_reader.SinkCharacter();
}
if (_reader.SinkCharacter() != '\'')
{
Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick.");
}
current = new CSharpTokenizer.CharLiteralToken();
return true;
}
// Check for verbatim string
else if (_reader.Sink("@\""))
{
do
{
// Inside a verbatim string "" is treated as a special character
while (_reader.Sink("\"\""))
{
}
}
while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"');
// Can't end a file inside a string
if (_reader.EndOfLines)
{
current = new EndOfFileInsideStringToken();
return true;
}
// reader.Skip the ending quote.
current = new StringLiteralToken();
current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1);
return true;
}
// Check for a quoted string.
else if (_reader.Sink("\""))
{
while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral())
{
// See if we have an escape sequence.
if (_reader.SinkCharacter() == '\\')
{
// This is probably an escape character.
if (_reader.SinkStringEscape())
{
// This isn't nearly right. We just do barely enough to make a string
// with an embedded escape sequence return _some_ string whose start and
// end match the real bounds of the string.
}
else
{
// This is a compiler error.
_reader.SinkCharacter();
current = new CSharpTokenizer.UnrecognizedStringEscapeToken();
return true;
}
}
}
// Is it a newline?
if (TokenChar.IsNewLine(_reader.CurrentCharacter))
{
current = new CSharpTokenizer.NewlineInsideStringToken();
return true;
}
// Create the token.
if (_reader.SinkCharacter() != '\"')
{
Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote.");
}
current = new StringLiteralToken();
return true;
}
// Identifier or keyword?
else if
(
// From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier.
_reader.CurrentCharacter == '@' ||
_reader.MatchNextIdentifierStart()
)
{
if (_reader.CurrentCharacter == '@')
{
_reader.SinkCharacter();
}
// Now, the next character must be an identifier start.
if (!_reader.SinkIdentifierStart())
{
current = new ExpectedIdentifierToken();
return true;
}
// Sink the rest of the identifier.
while (_reader.SinkIdentifierPart())
{
}
string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition);
switch (identifierOrKeyword)
{
default:
if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0)
{
current = new KeywordToken();
return true;
}
// If the identifier starts with '@' then we need to strip it off.
// The '@' is for escaping so that we can have an identifier called
// the same thing as a reserved keyword (i.e. class, if, foreach, etc)
string identifier = _reader.GetCurrentMatchedString(startPosition);
if (identifier.StartsWith("@", StringComparison.Ordinal))
{
identifier = identifier.Substring(1);
}
// Create the token.
current = new IdentifierToken();
current.InnerText = identifier;
return true;
case "false":
case "true":
current = new BooleanLiteralToken();
return true;
case "null":
current = new CSharpTokenizer.NullLiteralToken();
return true;
}
}
// Open scope
else if (_reader.Sink("{"))
{
current = new CSharpTokenizer.OpenScopeToken();
return true;
}
// Close scope
else if (_reader.Sink("}"))
{
current = new CSharpTokenizer.CloseScopeToken();
return true;
}
// Hexidecimal integer literal
else if (_reader.SinkIgnoreCase("0x"))
{
// Sink the hex digits.
if (!_reader.SinkMultipleHexDigits())
{
current = new ExpectedValidHexDigitToken();
return true;
}
// Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new HexIntegerLiteralToken();
return true;
}
// Decimal integer literal
else if (_reader.SinkMultipleDecimalDigits())
{
// reader.Skip the L, U, l, u, ul, etc.
_reader.SinkLongIntegerSuffix();
current = new DecimalIntegerLiteralToken();
return true;
}
// Check for single-digit operators and punctuators
else if (_reader.SinkOperatorOrPunctuator())
{
current = new OperatorOrPunctuatorToken();
return true;
}
// Preprocessor line
else if (_reader.CurrentCharacter == '#')
{
if (_reader.Sink("#if"))
{
current = new OpenConditionalDirectiveToken();
}
else if (_reader.Sink("#endif"))
{
current = new CloseConditionalDirectiveToken();
}
else
{
current = new PreprocessorToken();
}
_reader.SinkToEndOfLine();
return true;
}
// We didn't recognize the token, so this is a syntax error.
_reader.SinkCharacter();
current = new UnrecognizedToken();
return true;
}
