Exemple #1
0
        /**
         * This is the main lexing algorithm. It consumes source file as string and puts out token list.
         * Token consists of token type and range that token spans.
         */
        public static IToken[] Parse(string s)
        {
            LinkedList <IToken> ll = new LinkedList <IToken>();
            // use five kinds of token to produce token stream.
            var index = 0;
            var next  = 0;

            while (index < s.Length)
            {
                // try to parse as number
                if (NumberToken.ParseNumber(s, index, out var tokenNumber, out next))
                {
                    ll.AddLast(tokenNumber);
                    index = next;
                    continue;
                }

                // then try to parse as string
                if (StringToken.ParseString(s, index, out var tokenString, out next))
                {
                    ll.AddLast(tokenString);
                    index = next;
                    continue;
                }

                // then try to parse as identifier
                if (IdentifierToken.ParseIdentifier(s, index, out var tokenIdentifier, out next))
                {
                    ll.AddLast(tokenIdentifier);
                    index = next;
                    continue;
                }

                // then try to parse as comment
                if (CommentToken.ParseComment(s, index, out var tokensComment, out next))
                {
                    foreach (var t in tokensComment)
                    {
                        ll.AddLast(t);
                    }
                    index = next;
                    continue;
                }

                // then try to parse as symbol token
                if (SymbolToken.ParseSymbol(s, index, out var tokenSymbol, out next))
                {
                    ll.AddLast(tokenSymbol);
                    index = next;
                    continue;
                }

                if (Char.IsWhiteSpace(s[index]))
                {
                    // skip spaces
                    index++;
                    continue;
                }

                // otherwise token is unknown
                throw new Exception("unknown token " + s[index] + " at position " + index);
            }

            // return collected tokens
            return(ll.ToArray());
        }