Beispiel #1
0
 /// <summary>
 /// Lexical analysis on the input string
 /// </summary>
 /// <param name="str">The text representation of the term</param>
 /// <returns>List of tokens contained</returns>
 /// <exception cref="ReadException">Thrown if lexical error found</exception>
 static LinkedList<Token> Tokenize(string str)
 {
     LinkedList<Token> result = new LinkedList<Token>();
     Buffer b = new Buffer(str);
     b.EatWhitespace();
     while (!b.EndOfFile())
     {
         bool error;
         Token/*?*/ token = Scan(b, out error);
         if (error)
         {
             string msg = "At position " + (b.Position - 1).ToString() + ", "
                          + (b.EndOfFile() ? "end reached while scanning string: " :
                                             "lexical error in string: ");
             throw new ReadException(msg + str);
         }
         // assert token != null;
         result.AddLast(token);
         b.EatWhitespace();
     }
     result.AddLast(new Token(Token.Kind.EOF, "eof", b.Position));
     return result;
 }