private static void GetResult(CodeProcessorContext context, PreprocessorToken token, TokenStream nodes) { while (token != null) { switch ((TokenTypes)token.Type) { case TokenTypes.IfDefined: Ifdef(context, token as ConditionToken, nodes); break; case TokenTypes.IfNotDefined: Ifndef(context, token as ConditionToken, nodes); break; case TokenTypes.If: If(context, token as ConditionToken, nodes, false); break; case TokenTypes.ElseIf: If(context, token as ConditionToken, nodes, true); break; case TokenTypes.Else: Else(context, token, nodes); break; case TokenTypes.Define: if (!Define(context, (token as TextToken).Text, nodes)) { ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Undefine: if (!Undefine(context, (token as TextToken).Text, nodes)) { ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Include: if (!Include(context, (token as TextToken).Text, nodes)) { ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Pragma: if (!Pragma(context, (token as TextToken).Text, nodes)) { ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Error: if (!Error(context, (token as TextToken).Text, nodes)) { throw new Exception((token as TextToken).Text.ToString()); } break; case TokenTypes.Text: Any(context, (token as TextToken).Text, nodes); break; } token = token.Next as PreprocessorToken; } }
public static void GetResult(CodeProcessorContext context, PreprocessorToken token, List <FileDescriptor> referenceFiles, List <string> fileLookups, TokenStream nodes) { while (token != null) { switch ((TokenTypes)token.Type) { case TokenTypes.IfDefined: Ifdef(context, token as ConditionToken, referenceFiles, fileLookups, nodes); break; case TokenTypes.IfNotDefined: Ifndef(context, token as ConditionToken, referenceFiles, fileLookups, nodes); break; case TokenTypes.If: If(context, token as ConditionToken, referenceFiles, fileLookups, nodes, false); break; case TokenTypes.ElseIf: If(context, token as ConditionToken, referenceFiles, fileLookups, nodes, true); break; case TokenTypes.Else: Else(context, referenceFiles, fileLookups, token, nodes); break; case TokenTypes.Define: if (!Preprocessor.Define(context, (token as TextToken).Text, nodes)) { Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Undefine: if (!Preprocessor.Undefine(context, (token as TextToken).Text, nodes)) { Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Include: if (!Include(context, (token as TextToken).Text, referenceFiles, fileLookups, nodes)) { Preprocessor.ThrowError(context, PreprocessorCodes.InvalidToken); } break; case TokenTypes.Error: if (ProjectLookup.IgnoreErrors) { Application.Warning(SeverityFlags.Full, "#error directive found: '{0}'", (token as TextToken).Text.ToString()); break; } else { throw new Exception((token as TextToken).Text.ToString()); } case TokenTypes.Text: Preprocessor.Any(context, (token as TextToken).Text, nodes); break; } token = token.Next as PreprocessorToken; } }
private int Preprocessor() { char ch = data[chBaseIndex]; if (ch == '#') { // Just scan for a new line or eof, then add this in. int chScanningIndex = chBaseIndex; while (++chScanningIndex <= data.Length) { bool eof = chScanningIndex >= data.Length - 1; bool proceed = eof; if (!proceed) { char chScanning = data[chScanningIndex]; proceed = SeparatorToken.Map.ContainsKey(chScanning) && SeparatorToken.Map[chScanning] == NssSeparators.NewLine; } if (proceed) { PreprocessorToken preprocessor = new PreprocessorToken(); preprocessor.PreprocessorType = NssPreprocessorType.Unknown; int chStartIndex = chBaseIndex; int chEndIndex = eof ? data.Length : chScanningIndex; if (chStartIndex == chEndIndex) { preprocessor.Data = ""; } else { preprocessor.Data = data.Substring(chStartIndex, chEndIndex - chStartIndex); } int chNewBaseIndex = chEndIndex; AttachDebugData(preprocessor, DebugRanges, chBaseIndex, chNewBaseIndex - 1); Tokens.Add(preprocessor); chBaseIndex = chNewBaseIndex; break; } } } return(chBaseIndex); }
private static void Else(CodeProcessorContext context, PreprocessorToken token, TokenStream nodes) { int result = context.States.Pop(); if (result >= 4) { result = 2; } CheckState(context, nodes, result); if (nodes.CanWrite && token.Child != null) { GetResult(context, token.Child as PreprocessorToken, nodes); } EndCondition(context, null, nodes); }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ internal override bool FindNextToken() { int startPosition = _reader.Position; // VB docs claim whitespace is Unicode category Zs. However, // this category does not contain tabs. Assuming a less restrictive // definition for whitespace... if (_reader.SinkWhiteSpace()) { while (_reader.SinkWhiteSpace()) { } // Now, we need to check for the line continuation character. if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_' { // Save the current position because we may need to come back here. int savePosition = _reader.Position - 1; // Skip all whitespace after the '_' while (_reader.SinkWhiteSpace()) { } // Now, skip all the newlines. // Need at least one newline for this to count as line continuation. int count = 0; while (_reader.SinkNewLine()) { ++count; } if (count > 0) { current = new VisualBasicTokenizer.LineContinuationToken(); return true; } // Otherwise, fall back to plain old whitespace. _reader.Position = savePosition; } current = new WhitespaceToken(); return true; } // Line terminators are separate from whitespace and are significant. else if (_reader.SinkNewLine()) { // We want one token per line terminator. current = new VisualBasicTokenizer.LineTerminatorToken(); return true; } // Check for a comment--either those that start with ' or rem. else if (_reader.SinkLineCommentStart()) { // Skip to the first EOL. _reader.SinkToEndOfLine(); current = new CommentToken(); return true; } // Identifier or keyword? else if ( // VB allows escaping of identifiers by surrounding them with [] // In other words, // Date is a keyword but, // [Date] is an identifier. _reader.CurrentCharacter == '[' || _reader.MatchNextIdentifierStart() ) { bool escapedIdentifier = false; if (_reader.CurrentCharacter == '[') { escapedIdentifier = true; _reader.SinkCharacter(); // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return true; } } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } // If this was an escaped identifier the we need to get the terminating ']'. if (escapedIdentifier) { if (!_reader.Sink("]")) { current = new ExpectedIdentifierToken(); return true; } } else { // Escaped identifiers are not allowed to have trailing type character. _reader.SinkTypeCharacter(); // Type character is optional. } // An identifier that is only a '_' is illegal because it is // ambiguous with line continuation string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]") { current = new ExpectedIdentifierToken(); return true; } // Make an upper-case version in order to check whether this may be a keyword. string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture); switch (upper) { default: if (Array.IndexOf(s_keywordList, upper) >= 0) { current = new KeywordToken(); return true; } // Create the token. current = new IdentifierToken(); // Trim off the [] if this is an escaped identifier. if (escapedIdentifier) { current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2); } return true; case "FALSE": case "TRUE": current = new BooleanLiteralToken(); return true; } } // Is it a hex integer? else if (_reader.SinkHexIntegerPrefix()) { if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return true; } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new HexIntegerLiteralToken(); return true; } // Is it an octal integer? else if (_reader.SinkOctalIntegerPrefix()) { if (!_reader.SinkMultipleOctalDigits()) { current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken(); return true; } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new VisualBasicTokenizer.OctalIntegerLiteralToken(); return true; } // Is it a decimal integer? else if (_reader.SinkMultipleDecimalDigits()) { // Sink a suffix if there is one. _reader.SinkDecimalIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return true; } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.SinkIgnoreCase("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.SinkIgnoreCase("#end if")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return true; } // Is it a separator? else if (_reader.SinkSeparatorCharacter()) { current = new VisualBasicTokenizer.SeparatorToken(); return true; } // Is it an operator? else if (_reader.SinkOperator()) { current = new OperatorToken(); return true; } // A string? else if (_reader.Sink("\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } } while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return true; } current = new StringLiteralToken(); return true; } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return true; }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // VB docs claim whitespace is Unicode category Zs. However, // this category does not contain tabs. Assuming a less restrictive // definition for whitespace... if (_reader.SinkWhiteSpace()) { while (_reader.SinkWhiteSpace()) { } // Now, we need to check for the line continuation character. if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_' { // Save the current position because we may need to come back here. int savePosition = _reader.Position - 1; // Skip all whitespace after the '_' while (_reader.SinkWhiteSpace()) { } // Now, skip all the newlines. // Need at least one newline for this to count as line continuation. int count = 0; while (_reader.SinkNewLine()) { ++count; } if (count > 0) { current = new VisualBasicTokenizer.LineContinuationToken(); return(true); } // Otherwise, fall back to plain old whitespace. _reader.Position = savePosition; } current = new WhitespaceToken(); return(true); } // Line terminators are separate from whitespace and are significant. else if (_reader.SinkNewLine()) { // We want one token per line terminator. current = new VisualBasicTokenizer.LineTerminatorToken(); return(true); } // Check for a comment--either those that start with ' or rem. else if (_reader.SinkLineCommentStart()) { // Skip to the first EOL. _reader.SinkToEndOfLine(); current = new CommentToken(); return(true); } // Identifier or keyword? else if ( // VB allows escaping of identifiers by surrounding them with [] // In other words, // Date is a keyword but, // [Date] is an identifier. _reader.CurrentCharacter == '[' || _reader.MatchNextIdentifierStart() ) { bool escapedIdentifier = false; if (_reader.CurrentCharacter == '[') { escapedIdentifier = true; _reader.SinkCharacter(); // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return(true); } } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } // If this was an escaped identifier the we need to get the terminating ']'. if (escapedIdentifier) { if (!_reader.Sink("]")) { current = new ExpectedIdentifierToken(); return(true); } } else { // Escaped identifiers are not allowed to have trailing type character. _reader.SinkTypeCharacter(); // Type character is optional. } // An identifier that is only a '_' is illegal because it is // ambiguous with line continuation string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]") { current = new ExpectedIdentifierToken(); return(true); } // Make an upper-case version in order to check whether this may be a keyword. string upper = identifierOrKeyword.ToUpperInvariant(); switch (upper) { default: if (Array.IndexOf(s_keywordList, upper) >= 0) { current = new KeywordToken(); return(true); } // Create the token. current = new IdentifierToken(); // Trim off the [] if this is an escaped identifier. if (escapedIdentifier) { current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2); } return(true); case "FALSE": case "TRUE": current = new BooleanLiteralToken(); return(true); } } // Is it a hex integer? else if (_reader.SinkHexIntegerPrefix()) { if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return(true); } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new HexIntegerLiteralToken(); return(true); } // Is it an octal integer? else if (_reader.SinkOctalIntegerPrefix()) { if (!_reader.SinkMultipleOctalDigits()) { current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken(); return(true); } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new VisualBasicTokenizer.OctalIntegerLiteralToken(); return(true); } // Is it a decimal integer? else if (_reader.SinkMultipleDecimalDigits()) { // Sink a suffix if there is one. _reader.SinkDecimalIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return(true); } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.SinkIgnoreCase("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.SinkIgnoreCase("#end if")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return(true); } // Is it a separator? else if (_reader.SinkSeparatorCharacter()) { current = new VisualBasicTokenizer.SeparatorToken(); return(true); } // Is it an operator? else if (_reader.SinkOperator()) { current = new OperatorToken(); return(true); } // A string? else if (_reader.Sink("\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return(true); } current = new StringLiteralToken(); return(true); } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return(true); }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // Dealing with whitespace? if (_reader.SinkMultipleWhiteSpace()) { current = new WhitespaceToken(); return(true); } // Check for one-line comment else if (_reader.Sink("//")) { // Looks like a one-line comment. Follow it to the End-of-line _reader.SinkToEndOfLine(); current = new CommentToken(); return(true); } // Check for multi-line comment else if (_reader.Sink("/*")) { _reader.SinkUntil("*/"); // Was the ending */ found? if (_reader.EndOfLines) { // No. There was a /* without a */. Return this a syntax error token. current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return(true); } current = new CommentToken(); return(true); } // Handle chars else if (_reader.Sink("\'")) { while (_reader.CurrentCharacter != '\'') { if (_reader.Sink("\\")) { /* reader.Skip the escape sequence. * This isn't exactly right. We should detect: * * simple-escape-sequence: one of \' \" \\ \0 \a \b \f \n \r \t \v * * hexadecimal-escape-sequence: * \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt] */ } _reader.SinkCharacter(); } if (_reader.SinkCharacter() != '\'') { Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick."); } current = new CSharpTokenizer.CharLiteralToken(); return(true); } // Check for verbatim string else if (_reader.Sink("@\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return(true); } // reader.Skip the ending quote. current = new StringLiteralToken(); current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1); return(true); } // Check for a quoted string. else if (_reader.Sink("\"")) { while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral()) { // See if we have an escape sequence. if (_reader.SinkCharacter() == '\\') { // This is probably an escape character. if (_reader.SinkStringEscape()) { // This isn't nearly right. We just do barely enough to make a string // with an embedded escape sequence return _some_ string whose start and // end match the real bounds of the string. } else { // This is a compiler error. _reader.SinkCharacter(); current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return(true); } } } // Is it a newline? if (TokenChar.IsNewLine(_reader.CurrentCharacter)) { current = new CSharpTokenizer.NewlineInsideStringToken(); return(true); } // Create the token. if (_reader.SinkCharacter() != '\"') { Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote."); } current = new StringLiteralToken(); return(true); } // Identifier or keyword? else if ( // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier. _reader.CurrentCharacter == '@' || _reader.MatchNextIdentifierStart() ) { if (_reader.CurrentCharacter == '@') { _reader.SinkCharacter(); } // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return(true); } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); switch (identifierOrKeyword) { default: if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0) { current = new KeywordToken(); return(true); } // If the identifier starts with '@' then we need to strip it off. // The '@' is for escaping so that we can have an identifier called // the same thing as a reserved keyword (i.e. class, if, foreach, etc) string identifier = _reader.GetCurrentMatchedString(startPosition); if (identifier.StartsWith("@", StringComparison.Ordinal)) { identifier = identifier.Substring(1); } // Create the token. current = new IdentifierToken(); current.InnerText = identifier; return(true); case "false": case "true": current = new BooleanLiteralToken(); return(true); case "null": current = new CSharpTokenizer.NullLiteralToken(); return(true); } } // Open scope else if (_reader.Sink("{")) { current = new CSharpTokenizer.OpenScopeToken(); return(true); } // Close scope else if (_reader.Sink("}")) { current = new CSharpTokenizer.CloseScopeToken(); return(true); } // Hexidecimal integer literal else if (_reader.SinkIgnoreCase("0x")) { // Sink the hex digits. if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return(true); } // Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new HexIntegerLiteralToken(); return(true); } // Decimal integer literal else if (_reader.SinkMultipleDecimalDigits()) { // reader.Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return(true); } // Check for single-digit operators and punctuators else if (_reader.SinkOperatorOrPunctuator()) { current = new OperatorOrPunctuatorToken(); return(true); } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.Sink("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.Sink("#endif")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return(true); } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return(true); }
public static bool GetSyntaxTree(Stream data, SyntaxTree <PreprocessorToken> tree, PreprocessorToken root = null) { while (!data.Eof()) { char c = data.Peek(); switch ((TokenTypes)c) { case TokenTypes.IfDefined: case TokenTypes.IfNotDefined: case TokenTypes.If: case TokenTypes.ElseIf: { ConditionToken token = new ConditionToken((TokenTypes)c); token.Deserialize(data); if (root == null) { tree.Add(token); } else { tree.AddAppend(root, token); } if (!GetSyntaxTree(data, tree, token)) { return(false); } } break; case TokenTypes.Else: { PreprocessorToken token = new PreprocessorToken(TokenTypes.Else); token.Deserialize(data); if (root == null) { tree.Add(token); } else { tree.AddAppend(root, token); } if (!GetSyntaxTree(data, tree, token)) { return(false); } } break; case TokenTypes.Define: case TokenTypes.Undefine: case TokenTypes.Include: case TokenTypes.Error: case TokenTypes.Pragma: { TextToken token = new TextToken((TokenTypes)c); token.Deserialize(data); if (root == null) { tree.Add(token); } else { tree.AddAppend(root, token); } } break; case TokenTypes.Text: { TextToken token = new TextToken(); token.Deserialize(data); if (root == null) { tree.Add(token); } else { tree.AddAppend(root, token); } } break; default: { if (root != null && c == 0) { data.Get(); return(true); } else { return(ThrowError(new CodeProcessorContext(new CodeProcessorConfig()), PreprocessorCodes.UnexpectedCharacter)); } } } } return(true); }
internal abstract int scan(ref PreprocessorToken ppToken);
private int DoHexidecimal(StringInputBuffer buffer, PreprocessorToken ppToken, int ch, ref bool alreadyComplained, ref int len, int maxTokenLength) { // can't be hexidecimal or octal, is either decimal or floating point do { if (len < maxTokenLength) { buffer.name[len++] = (char)ch; } else if (!alreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); alreadyComplained = true; } ch = pp.getChar(); } while (ch >= '0' && ch <= '9'); if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') { return(pp.lFloatConst(pp.buffer, len, ch, ppToken)); } else { // Finish handling signed and unsigned integers int numericLen = len; bool uintSign = false; if (ch == 'u' || ch == 'U') { if (len < maxTokenLength) { buffer.name[len++] = (char)ch; } uintSign = true; } else { pp.ungetChar(); } // NOT NEEDED //ppToken.name[len] = '\0'; uint ival = 0; const uint ONETENTHMAXINT = 0xFFFFFFFFu / 10; const uint REMAINDERMAXINT = 0xFFFFFFFFu - 10 * ONETENTHMAXINT; for (int i = 0; i < numericLen; i++) { ch = buffer.name[i] - '0'; if ((ival > ONETENTHMAXINT) || (ival == ONETENTHMAXINT && ch > REMAINDERMAXINT)) { pp.parseContext.Error(ppToken.loc, "numeric literal too big", "", ""); ival = 0xFFFFFFFFu; break; } else { ival = (uint)(ival * 10 + ch); } } ppToken.ival = (int)ival; ppToken.name = new string (buffer.name, 0, len); return(uintSign ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT); } }
internal override int scan(ref PreprocessorToken ppToken) { // // Scanner used to tokenize source stream. // bool AlreadyComplained = false; int len = 0; int ch = 0; int ii = 0; uint ival = 0; ppToken.ival = 0; ppToken.space = false; ch = pp.getChar(); for (;;) { while (ch == ' ' || ch == '\t') { ppToken.space = true; ch = pp.getChar(); } ppToken.loc = pp.parseContext.getCurrentLoc(); len = 0; switch (ch) { default: return(ch); // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token) // TODO : EOF for stream // case EOF: // return endOfInput; case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': do { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.tokenText [len++] = (char)ch; ch = pp.getChar(); } else { if (!AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "name too long", "", ""); AlreadyComplained = true; } ch = pp.getChar(); } } while ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_'); // line continuation with no token before or after makes len == 0, and need to start over skipping white space, etc. if (len == 0) { continue; } pp.buffer.tokenText [len] = '\0'; pp.ungetChar(); ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string (pp.buffer.tokenText, 0, len)); return((int)CppEnums.IDENTIFIER); case '0': pp.buffer.name[len++] = (char)ch; ch = pp.getChar(); if (ch == 'x' || ch == 'X') { // must be hexidecimal bool isUnsigned = false; pp.buffer.name[len++] = (char)ch; ch = pp.getChar(); if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')) { ival = 0; do { if (ival <= 0x0fffffff) { pp.buffer.name[len++] = (char)ch; if (ch >= '0' && ch <= '9') { ii = ch - '0'; } else if (ch >= 'A' && ch <= 'F') { ii = ch - 'A' + 10; } else if (ch >= 'a' && ch <= 'f') { ii = ch - 'a' + 10; } else { pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", ""); } ival = (uint)((ival << 4) | ii); } else { if (!AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "hexidecimal literal too big", "", ""); AlreadyComplained = true; } ival = 0xffffffff; } ch = pp.getChar(); } while ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')); } else { pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", ""); } if (ch == 'u' || ch == 'U') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.name[len++] = (char)ch; } isUnsigned = true; } else { pp.ungetChar(); } //NOT NEEDED //ppToken.name[len] = '\0'; ppToken.ival = (int)ival; if (isUnsigned) { return((int)CppEnums.UINTCONSTANT); } else { return((int)CppEnums.INTCONSTANT); } } else { // could be octal integer or floating point, speculative pursue octal until it must be floating point bool isUnsigned = false; bool octalOverflow = false; bool nonOctal = false; ival = 0; // see how much octal-like stuff we can read while (ch >= '0' && ch <= '7') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.name[len++] = (char)ch; } else if (!AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); AlreadyComplained = true; } if (ival <= 0x1fffffff) { ii = ch - '0'; ival = (uint)((ival << 3) | ii); } else { octalOverflow = true; } ch = pp.getChar(); } // could be part of a float... if (ch == '8' || ch == '9') { nonOctal = true; do { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.name[len++] = (char)ch; } else if (!AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); AlreadyComplained = true; } ch = pp.getChar(); } while (ch >= '0' && ch <= '9'); } if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') { //ppToken.name = new string(pp.buffer.name, 0, len); return(pp.lFloatConst(pp.buffer, len, ch, ppToken)); } // wasn't a float, so must be octal... if (nonOctal) { pp.parseContext.Error(ppToken.loc, "octal literal digit too large", "", ""); } if (ch == 'u' || ch == 'U') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.name[len++] = (char)ch; } isUnsigned = true; } else { pp.ungetChar(); } // NOT NEEDED //ppToken.name[len] = '\0'; if (octalOverflow) { pp.parseContext.Error(ppToken.loc, "octal literal too big", "", ""); } ppToken.ival = (int)ival; ppToken.name = new string(pp.buffer.name, 0, len); return(isUnsigned ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT); } case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return(DoHexidecimal(pp.buffer, ppToken, ch, ref AlreadyComplained, ref len, StringInputBuffer.MAX_TOKEN_LENGTH)); case '-': ch = pp.getChar(); if (ch == '-') { return((int)CppEnums.DEC_OP); } else if (ch == '=') { return((int)CppEnums.SUB_ASSIGN); } else { pp.ungetChar(); return('-'); } case '+': ch = pp.getChar(); if (ch == '+') { return((int)CppEnums.INC_OP); } else if (ch == '=') { return((int)CppEnums.ADD_ASSIGN); } else { pp.ungetChar(); return('+'); } case '*': ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.MUL_ASSIGN); } else { pp.ungetChar(); return('*'); } case '%': ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.MOD_ASSIGN); } else if (ch == '>') { return((int)CppEnums.RIGHT_BRACE); } else { pp.ungetChar(); return('%'); } case ':': ch = pp.getChar(); if (ch == '>') { return((int)CppEnums.RIGHT_BRACKET); } else { pp.ungetChar(); return(':'); } case '^': ch = pp.getChar(); if (ch == '^') { return((int)CppEnums.XOR_OP); } else { if (ch == '=') { return((int)CppEnums.XOR_ASSIGN); } else { pp.ungetChar(); return('^'); } } case '=': ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.EQ_OP); } else { pp.ungetChar(); return('='); } case '!': ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.NE_OP); } else { pp.ungetChar(); return('!'); } case '|': ch = pp.getChar(); if (ch == '|') { return((int)CppEnums.OR_OP); } else { if (ch == '=') { return((int)CppEnums.OR_ASSIGN); } else { pp.ungetChar(); return('|'); } } case '&': ch = pp.getChar(); if (ch == '&') { return((int)CppEnums.AND_OP); } else { if (ch == '=') { return((int)CppEnums.AND_ASSIGN); } else { pp.ungetChar(); return('&'); } } case '<': ch = pp.getChar(); if (ch == '<') { ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.LEFT_ASSIGN); } else { pp.ungetChar(); return((int)CppEnums.LEFT_OP); } } else { if (ch == '=') { return((int)CppEnums.LE_OP); } else { if (ch == '%') { return((int)CppEnums.LEFT_BRACE); } else if (ch == ':') { return((int)CppEnums.LEFT_BRACKET); } else { pp.ungetChar(); return('<'); } } } case '>': ch = pp.getChar(); if (ch == '>') { ch = pp.getChar(); if (ch == '=') { return((int)CppEnums.RIGHT_ASSIGN); } else { pp.ungetChar(); return((int)CppEnums.RIGHT_OP); } } else { if (ch == '=') { return((int)CppEnums.GE_OP); } else { pp.ungetChar(); return('>'); } } case '.': ch = pp.getChar(); if (ch >= '0' && ch <= '9') { pp.ungetChar(); return(pp.lFloatConst(pp.buffer, 0, '.', ppToken)); } else { pp.ungetChar(); return('.'); } case '/': ch = pp.getChar(); if (ch == '/') { pp.inComment = true; do { ch = pp.getChar(); } while (ch != '\n' && !this.IsEOF(ch)); ppToken.space = true; pp.inComment = false; if (this.IsEOF(ch)) { return(END_OF_INPUT); } return(ch); } else if (ch == '*') { ch = pp.getChar(); do { while (ch != '*') { if (this.IsEOF(ch)) { pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", ""); return(END_OF_INPUT); } ch = pp.getChar(); } ch = pp.getChar(); if (this.IsEOF(ch)) { pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", ""); return(END_OF_INPUT); } } while (ch != '/'); ppToken.space = true; // loop again to get the next token... break; } else if (ch == '=') { return((int)CppEnums.DIV_ASSIGN); } else { pp.ungetChar(); return('/'); } case '"': ch = pp.getChar(); while (ch != '"' && ch != '\n' && this.IsEOF(ch)) { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.tokenText[len] = (char)ch; len++; ch = pp.getChar(); } else { break; } } ; //pp.buffer.tokenText[len] = '\0'; if (ch == '"') { ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string(pp.buffer.tokenText, 0, len)); ppToken.name = new string (pp.buffer.name, 0, len); return((int)CppEnums.STRCONSTANT); } else { pp.parseContext.Error(ppToken.loc, "end of line in string", "string", ""); ppToken.name = new string (pp.buffer.name, 0, len); return((int)CppEnums.ERROR_SY); } } ch = pp.getChar(); } }
private static void Else(CodeProcessorContext context, List <FileDescriptor> referenceFiles, List <string> fileLookups, PreprocessorToken token, TokenStream nodes) { int result = context.States.Pop(); if (result >= 4) { result = 2; } Preprocessor.CheckState(context, nodes, result); if (nodes.CanWrite && token.Child != null) { GetResult(context, token.Child as PreprocessorToken, referenceFiles, fileLookups, nodes); } EndCondition(context, null, nodes); }
internal override int scan(ref PreprocessorToken ppToken) { // // Scanner used to tokenize source stream. // bool AlreadyComplained = false; int len = 0; int ch = 0; int ii = 0; uint ival = 0; ppToken.ival = 0; ppToken.space = false; ch = pp.getChar(); for (;;) { while (ch == ' ' || ch == '\t') { ppToken.space = true; ch = pp.getChar(); } ppToken.loc = pp.parseContext.getCurrentLoc(); len = 0; switch (ch) { default: return ch; // Single character token, including '#' and '\' (escaped newlines are handled at a lower level, so this is just a '\' token) // TODO : EOF for stream // case EOF: // return endOfInput; case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': do { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.tokenText [len++] = (char)ch; ch = pp.getChar (); } else { if (!AlreadyComplained) { pp.parseContext.Error (ppToken.loc, "name too long", "", ""); AlreadyComplained = true; } ch = pp.getChar (); } } while ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_'); // line continuation with no token before or after makes len == 0, and need to start over skipping white space, etc. if (len == 0) continue; pp.buffer.tokenText [len] = '\0'; pp.ungetChar (); ppToken.atom = pp.Symbols.Atoms.LookUpAddString (new string (pp.buffer.tokenText, 0, len)); return (int) CppEnums.IDENTIFIER; case '0': pp.buffer.name[len++] = (char)ch; ch = pp.getChar(); if (ch == 'x' || ch == 'X') { // must be hexidecimal bool isUnsigned = false; pp.buffer.name[len++] = (char)ch; ch = pp.getChar(); if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')) { ival = 0; do { if (ival <= 0x0fffffff) { pp.buffer.name[len++] = (char)ch; if (ch >= '0' && ch <= '9') { ii = ch - '0'; } else if (ch >= 'A' && ch <= 'F') { ii = ch - 'A' + 10; } else if (ch >= 'a' && ch <= 'f') { ii = ch - 'a' + 10; } else pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", ""); ival = (uint)((ival << 4) | ii); } else { if (!AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "hexidecimal literal too big", "", ""); AlreadyComplained = true; } ival = 0xffffffff; } ch = pp.getChar(); } while ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')); } else { pp.parseContext.Error(ppToken.loc, "bad digit in hexidecimal literal", "", ""); } if (ch == 'u' || ch == 'U') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) pp.buffer.name[len++] = (char)ch; isUnsigned = true; } else pp.ungetChar(); //NOT NEEDED //ppToken.name[len] = '\0'; ppToken.ival = (int)ival; if (isUnsigned) { return (int)CppEnums.UINTCONSTANT; } else { return (int)CppEnums.INTCONSTANT; } } else { // could be octal integer or floating point, speculative pursue octal until it must be floating point bool isUnsigned = false; bool octalOverflow = false; bool nonOctal = false; ival = 0; // see how much octal-like stuff we can read while (ch >= '0' && ch <= '7') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) pp.buffer.name[len++] = (char)ch; else if (! AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); AlreadyComplained = true; } if (ival <= 0x1fffffff) { ii = ch - '0'; ival = (uint)((ival << 3) | ii); } else octalOverflow = true; ch = pp.getChar(); } // could be part of a float... if (ch == '8' || ch == '9') { nonOctal = true; do { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) pp.buffer.name[len++] = (char)ch; else if (! AlreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); AlreadyComplained = true; } ch = pp.getChar(); } while (ch >= '0' && ch <= '9'); } if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') { //ppToken.name = new string(pp.buffer.name, 0, len); return pp.lFloatConst (pp.buffer, len, ch, ppToken); } // wasn't a float, so must be octal... if (nonOctal) pp.parseContext.Error(ppToken.loc, "octal literal digit too large", "", ""); if (ch == 'u' || ch == 'U') { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) pp.buffer.name[len++] = (char)ch; isUnsigned = true; } else pp.ungetChar(); // NOT NEEDED //ppToken.name[len] = '\0'; if (octalOverflow) pp.parseContext.Error(ppToken.loc, "octal literal too big", "", ""); ppToken.ival = (int)ival; ppToken.name = new string(pp.buffer.name, 0, len); return isUnsigned ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT; } case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return DoHexidecimal (pp.buffer, ppToken, ch, ref AlreadyComplained, ref len, StringInputBuffer.MAX_TOKEN_LENGTH); case '-': ch = pp.getChar(); if (ch == '-') { return (int) CppEnums.DEC_OP; } else if (ch == '=') { return (int) CppEnums.SUB_ASSIGN; } else { pp.ungetChar(); return '-'; } case '+': ch = pp.getChar(); if (ch == '+') { return (int) CppEnums.INC_OP; } else if (ch == '=') { return (int) CppEnums.ADD_ASSIGN; } else { pp.ungetChar(); return '+'; } case '*': ch = pp.getChar(); if (ch == '=') { return (int) CppEnums.MUL_ASSIGN; } else { pp.ungetChar(); return '*'; } case '%': ch = pp.getChar(); if (ch == '=') { return (int) CppEnums.MOD_ASSIGN; } else if (ch == '>'){ return (int) CppEnums.RIGHT_BRACE; } else { pp.ungetChar(); return '%'; } case ':': ch = pp.getChar(); if (ch == '>') { return (int) CppEnums.RIGHT_BRACKET; } else { pp.ungetChar(); return ':'; } case '^': ch = pp.getChar(); if (ch == '^') { return (int) CppEnums.XOR_OP; } else { if (ch == '=') { return (int)CppEnums.XOR_ASSIGN; } else { pp.ungetChar(); return '^'; } } case '=': ch = pp.getChar(); if (ch == '=') { return (int) CppEnums.EQ_OP; } else { pp.ungetChar(); return '='; } case '!': ch = pp.getChar(); if (ch == '=') { return (int) CppEnums.NE_OP; } else { pp.ungetChar(); return '!'; } case '|': ch = pp.getChar(); if (ch == '|') { return (int) CppEnums.OR_OP; } else { if (ch == '=') { return (int)CppEnums.OR_ASSIGN; } else{ pp.ungetChar(); return '|'; } } case '&': ch = pp.getChar(); if (ch == '&') { return (int) CppEnums.AND_OP; } else { if (ch == '=') { return (int)CppEnums.AND_ASSIGN; } else{ pp.ungetChar(); return '&'; } } case '<': ch = pp.getChar(); if (ch == '<') { ch = pp.getChar(); if (ch == '=') { return (int)CppEnums.LEFT_ASSIGN; } else { pp.ungetChar(); return (int) CppEnums.LEFT_OP; } } else { if (ch == '=') { return (int) CppEnums.LE_OP; } else { if (ch == '%') { return (int)CppEnums.LEFT_BRACE; } else if (ch == ':') { return (int)CppEnums.LEFT_BRACKET; } else { pp.ungetChar(); return '<'; } } } case '>': ch = pp.getChar(); if (ch == '>') { ch = pp.getChar(); if (ch == '=') { return (int)CppEnums.RIGHT_ASSIGN; } else{ pp.ungetChar(); return (int) CppEnums.RIGHT_OP; } } else { if (ch == '=') { return (int) CppEnums.GE_OP; } else { pp.ungetChar(); return '>'; } } case '.': ch = pp.getChar(); if (ch >= '0' && ch <= '9') { pp.ungetChar(); return pp.lFloatConst(pp.buffer, 0, '.', ppToken); } else { pp.ungetChar(); return '.'; } case '/': ch = pp.getChar(); if (ch == '/') { pp.inComment = true; do { ch = pp.getChar(); } while (ch != '\n' && ! this.IsEOF(ch)); ppToken.space = true; pp.inComment = false; if (this.IsEOF (ch)) { return END_OF_INPUT; } return ch; } else if (ch == '*') { ch = pp.getChar(); do { while (ch != '*') { if (this.IsEOF(ch)) { pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", ""); return END_OF_INPUT; } ch = pp.getChar(); } ch = pp.getChar(); if (this.IsEOF(ch)) { pp.parseContext.Error(ppToken.loc, "EOF in comment", "comment", ""); return END_OF_INPUT; } } while (ch != '/'); ppToken.space = true; // loop again to get the next token... break; } else if (ch == '=') { return (int) CppEnums.DIV_ASSIGN; } else { pp.ungetChar(); return '/'; } case '"': ch = pp.getChar(); while (ch != '"' && ch != '\n' && this.IsEOF(ch)) { if (len < StringInputBuffer.MAX_TOKEN_LENGTH) { pp.buffer.tokenText[len] = (char)ch; len++; ch = pp.getChar(); } else break; }; //pp.buffer.tokenText[len] = '\0'; if (ch == '"') { ppToken.atom = pp.Symbols.Atoms.LookUpAddString(new string(pp.buffer.tokenText, 0, len)); ppToken.name = new string (pp.buffer.name, 0, len); return (int) CppEnums.STRCONSTANT; } else { pp.parseContext.Error(ppToken.loc, "end of line in string", "string", ""); ppToken.name = new string (pp.buffer.name, 0, len); return (int) CppEnums.ERROR_SY; } } ch = pp.getChar(); } }
private int DoHexidecimal(StringInputBuffer buffer, PreprocessorToken ppToken, int ch, ref bool alreadyComplained, ref int len, int maxTokenLength) { // can't be hexidecimal or octal, is either decimal or floating point do { if (len < maxTokenLength) buffer.name[len++] = (char)ch; else if (! alreadyComplained) { pp.parseContext.Error(ppToken.loc, "numeric literal too long", "", ""); alreadyComplained = true; } ch = pp.getChar(); } while (ch >= '0' && ch <= '9'); if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') { return pp.lFloatConst(pp.buffer, len, ch, ppToken); } else { // Finish handling signed and unsigned integers int numericLen = len; bool uintSign = false; if (ch == 'u' || ch == 'U') { if (len < maxTokenLength) buffer.name[len++] = (char)ch; uintSign = true; } else pp.ungetChar(); // NOT NEEDED //ppToken.name[len] = '\0'; uint ival = 0; const uint ONETENTHMAXINT = 0xFFFFFFFFu / 10; const uint REMAINDERMAXINT = 0xFFFFFFFFu - 10 * ONETENTHMAXINT; for (int i = 0; i < numericLen; i++) { ch = buffer.name[i] - '0'; if ((ival > ONETENTHMAXINT) || (ival == ONETENTHMAXINT && ch > REMAINDERMAXINT)) { pp.parseContext.Error(ppToken.loc, "numeric literal too big", "", ""); ival = 0xFFFFFFFFu; break; } else ival = (uint)(ival * 10 + ch); } ppToken.ival = (int)ival; ppToken.name = new string (buffer.name, 0, len); return uintSign ? (int)CppEnums.UINTCONSTANT : (int)CppEnums.INTCONSTANT; } }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // Dealing with whitespace? if (_reader.SinkMultipleWhiteSpace()) { current = new WhitespaceToken(); return true; } // Check for one-line comment else if (_reader.Sink("//")) { // Looks like a one-line comment. Follow it to the End-of-line _reader.SinkToEndOfLine(); current = new CommentToken(); return true; } // Check for multi-line comment else if (_reader.Sink("/*")) { _reader.SinkUntil("*/"); // Was the ending */ found? if (_reader.EndOfLines) { // No. There was a /* without a */. Return this a syntax error token. current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return true; } current = new CommentToken(); return true; } // Handle chars else if (_reader.Sink("\'")) { while (_reader.CurrentCharacter != '\'') { if (_reader.Sink("\\")) { /* reader.Skip the escape sequence. This isn't exactly right. We should detect: simple-escape-sequence: one of \' \" \\ \0 \a \b \f \n \r \t \v hexadecimal-escape-sequence: \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt] */ } _reader.SinkCharacter(); } if (_reader.SinkCharacter() != '\'') { Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick."); } current = new CSharpTokenizer.CharLiteralToken(); return true; } // Check for verbatim string else if (_reader.Sink("@\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } } while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return true; } // reader.Skip the ending quote. current = new StringLiteralToken(); current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1); return true; } // Check for a quoted string. else if (_reader.Sink("\"")) { while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral()) { // See if we have an escape sequence. if (_reader.SinkCharacter() == '\\') { // This is probably an escape character. if (_reader.SinkStringEscape()) { // This isn't nearly right. We just do barely enough to make a string // with an embedded escape sequence return _some_ string whose start and // end match the real bounds of the string. } else { // This is a compiler error. _reader.SinkCharacter(); current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return true; } } } // Is it a newline? if (TokenChar.IsNewLine(_reader.CurrentCharacter)) { current = new CSharpTokenizer.NewlineInsideStringToken(); return true; } // Create the token. if (_reader.SinkCharacter() != '\"') { Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote."); } current = new StringLiteralToken(); return true; } // Identifier or keyword? else if ( // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier. _reader.CurrentCharacter == '@' || _reader.MatchNextIdentifierStart() ) { if (_reader.CurrentCharacter == '@') { _reader.SinkCharacter(); } // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return true; } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); switch (identifierOrKeyword) { default: if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0) { current = new KeywordToken(); return true; } // If the identifier starts with '@' then we need to strip it off. // The '@' is for escaping so that we can have an identifier called // the same thing as a reserved keyword (i.e. class, if, foreach, etc) string identifier = _reader.GetCurrentMatchedString(startPosition); if (identifier.StartsWith("@", StringComparison.Ordinal)) { identifier = identifier.Substring(1); } // Create the token. current = new IdentifierToken(); current.InnerText = identifier; return true; case "false": case "true": current = new BooleanLiteralToken(); return true; case "null": current = new CSharpTokenizer.NullLiteralToken(); return true; } } // Open scope else if (_reader.Sink("{")) { current = new CSharpTokenizer.OpenScopeToken(); return true; } // Close scope else if (_reader.Sink("}")) { current = new CSharpTokenizer.CloseScopeToken(); return true; } // Hexidecimal integer literal else if (_reader.SinkIgnoreCase("0x")) { // Sink the hex digits. if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return true; } // Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new HexIntegerLiteralToken(); return true; } // Decimal integer literal else if (_reader.SinkMultipleDecimalDigits()) { // reader.Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return true; } // Check for single-digit operators and punctuators else if (_reader.SinkOperatorOrPunctuator()) { current = new OperatorOrPunctuatorToken(); return true; } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.Sink("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.Sink("#endif")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return true; } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return true; }