Exemple #1
0
		} // Tokenizer constructor
	
	
	// Read the next token from the stream.  Skip whitespace and comments.
	// If there are no further tokens in the stream, return null.
	protected override Token ReadNextToken()
		{
		// Wrap the entire function in a loop so we can come back around
		// after matching a comment or whitespace.
		while (true)
			{
			if (readToEnd)
				return null;
			
			if (buffer.atEnd)
				{
				readToEnd = true;
				return null;
				}
			
			SrcLoc loc;
			loc.lineNum = buffer.lineNum;
			loc.colNum  = buffer.colNum;
			loc.absPosition = buffer.pos;
			loc.len = 1; // this will typically be overridden later
			
			int tokenLen, tokenInfo;
			if (matcher.Match(buffer, out tokenLen, out tokenInfo))
				{
				switch ((TokDispatch)tokenInfo)
					{
					case TokDispatch.ident:
						return ParseIdentifier(loc);
					
					case TokDispatch.digit:
						return ParseNumber(loc);
					
					case TokDispatch.stringLit:
						return ParseStringLiteral(loc);
					
					case TokDispatch.op:
						{
						Operator token = new Operator();
						token.rawText = buffer.ConsumeString(tokenLen);
						loc.len = tokenLen;
						token.loc = loc;
						return token;
						}
					
					case TokDispatch.keyword:
						{
						Keyword token = new Keyword();
						token.rawText = buffer.ConsumeString(tokenLen);
						loc.len = tokenLen;
						token.loc = loc;
						return token;
						}
					
					case TokDispatch.reserved:
						{
						ReservedWord token = new ReservedWord();
						token.rawText = buffer.ConsumeString(tokenLen);
						loc.len = tokenLen;
						token.loc = loc;
						return token;
						}
					
					case TokDispatch.comment:
						{
						SkipComment(loc);
						
						// Now fall through to the end of the main loop so
						// we will parse the next token.
						break;
						}
					
					case TokDispatch.ws:
						{
						// This is a whitespace character, skip it.  For
						// efficiency, we suck up all subsequent whitespace
						// at the same time (up to one buffer's worth).
						SkipWhitespace();
						
						// Now fall through to the end of the main loop so
						// we will parse the next token.
						break;
						}
					
					default:
						Trace.Assert(false);
						break;
					
					} // switch (tokenInfo)
				
				}
			else
				{
				string msg = "unexpected input character '" +
							 new String(buffer.PeekOrZero(), 1) +
							 "'";
				throw new ParseError(msg, loc);
				}
		
			} // while (true)
		
		} // ReadNextToken
Exemple #2
0
        }         // Tokenizer constructor

        // Read the next token from the stream.  Skip whitespace and comments.
        // If there are no further tokens in the stream, return null.
        protected override Token ReadNextToken()
        {
            // Wrap the entire function in a loop so we can come back around
            // after matching a comment or whitespace.
            while (true)
            {
                if (readToEnd)
                {
                    return(null);
                }

                if (buffer.atEnd)
                {
                    readToEnd = true;
                    return(null);
                }

                SrcLoc loc;
                loc.lineNum     = buffer.lineNum;
                loc.colNum      = buffer.colNum;
                loc.absPosition = buffer.pos;
                loc.len         = 1; // this will typically be overridden later

                int tokenLen, tokenInfo;
                if (matcher.Match(buffer, out tokenLen, out tokenInfo))
                {
                    switch ((TokDispatch)tokenInfo)
                    {
                    case TokDispatch.ident:
                        return(ParseIdentifier(loc));

                    case TokDispatch.digit:
                        return(ParseNumber(loc));

                    case TokDispatch.stringLit:
                        return(ParseStringLiteral(loc));

                    case TokDispatch.op:
                    {
                        Operator token = new Operator();
                        token.rawText = buffer.ConsumeString(tokenLen);
                        loc.len       = tokenLen;
                        token.loc     = loc;
                        return(token);
                    }

                    case TokDispatch.keyword:
                    {
                        Keyword token = new Keyword();
                        token.rawText = buffer.ConsumeString(tokenLen);
                        loc.len       = tokenLen;
                        token.loc     = loc;
                        return(token);
                    }

                    case TokDispatch.reserved:
                    {
                        ReservedWord token = new ReservedWord();
                        token.rawText = buffer.ConsumeString(tokenLen);
                        loc.len       = tokenLen;
                        token.loc     = loc;
                        return(token);
                    }

                    case TokDispatch.comment:
                    {
                        SkipComment(loc);

                        // Now fall through to the end of the main loop so
                        // we will parse the next token.
                        break;
                    }

                    case TokDispatch.ws:
                    {
                        // This is a whitespace character, skip it.  For
                        // efficiency, we suck up all subsequent whitespace
                        // at the same time (up to one buffer's worth).
                        SkipWhitespace();

                        // Now fall through to the end of the main loop so
                        // we will parse the next token.
                        break;
                    }

                    default:
                        Trace.Assert(false);
                        break;
                    }                     // switch (tokenInfo)
                }
                else
                {
                    string msg = "unexpected input character '" +
                                 new String(buffer.PeekOrZero(), 1) +
                                 "'";
                    throw new ParseError(msg, loc);
                }
            }     // while (true)
        }         // ReadNextToken