예제 #1
0
        /// <summary>
        /// This function will scan a secquence of characters beginning with a digit.
        /// The function will traverse the stream until it sees a character that is no longer recognized as a digit.
        /// If the non-digit character is a period (.) and it is the first period in the sequence, the function will treat the following digits as floating points.
        /// If there are no digits after the period character, the scanner will append a 0 to the value, to ensure the value is a float in the target language.
        /// <example>The following sequences are recognized as numeric:
        /// <code>
        /// 2
        /// 42
        /// 42.1
        /// 54. # Will become 54.0
        /// </code>
        /// </example>
        /// </summary>
        private void ScanNumeric()
        {
            bool   isFloat   = false;
            string subString = CurrentChar.ToString();

            while (recogniser.IsDigit(Peek()))
            {
                Pop();
                subString += CurrentChar;
            }
            // Make sure it isn't a range
            if (Peek() == '.' && Peek(2) != '.')
            {
                isFloat = !isFloat;
                Pop();
                subString += CurrentChar;
                while (recogniser.IsDigit(Peek()))
                {
                    Pop();
                    subString += CurrentChar;
                }
                if (subString.Last() == '.')
                {
                    subString += "0";
                }
            }
            ScannerToken token = Token(TokenType.NUMERIC, subString);

            token.SymbolicType         = new TypeContext(TokenType.NUMERIC);
            token.SymbolicType.IsFloat = isFloat;
            Tokens.AddLast(token);
        }
예제 #2
0
        private void ParseChar()
        {
            var key       = Key.None;
            var modifiers = ModifierKeys.None;

            var ch = CurrentChar.ToString();

            KeyPressInfo knownKeyPress = KeyboardLayout.Instance.GetKeyGestureForChar(ch[0]);

            if (knownKeyPress != null)
            {
                key       = knownKeyPress.Key;
                modifiers = knownKeyPress.Modifiers;
            }
            else
            {
                if (char.IsUpper(ch, 0))
                {
                    ch        = ch.ToLower();
                    modifiers = ModifierKeys.Shift;
                }
                key = (Key) new KeyConverter().ConvertFromInvariantString(ch);
            }

            if (key != Key.None)
            {
                Add(new KeyPressInfo(key, modifiers));
                current++;
            }
        }
예제 #3
0
 /// <summary>
 /// Calls GetNextChar() to skip over comment characters.
 /// </summary>
 public void SkipComment(Regex commentEndRegex)
 {
     while (!commentEndRegex.IsMatch(CurrentChar.ToString() + JavaFile.PeekNextChar()) && !JavaFile.EndOfFile())
     {
         JavaFile.GetNextChar();
     }
 }
예제 #4
0
        protected override void Extract()
        {
            if (CurrentChar.IsSymbol())
            {
                if (CurrentChar == '/' && PeekChar == '>')
                {
                    type = KeywordToken.ShortClose;
                    text = KeywordToken.ShortClose.GetText();
                    NextChar();
                }
                else if (CurrentChar == '<' && PeekChar == '/')
                {
                    NextChar(2);
                    var stringBuilder = new StringBuilder();
                    while (CurrentChar != '>')
                    {
                        stringBuilder.Append(CurrentChar);
                        NextChar();
                    }

                    // NextChar();
                    text = stringBuilder.ToString();
                    // force file ending when w:document closes
                    type = text == "w:document"? KeywordToken.EOF : KeywordToken.Close;
                }
                else
                {
                    type = CurrentChar.ToTokenType();
                    text = CurrentChar.ToString();
                }
            }
            NextChar();
        }
예제 #5
0
        /// <summary>
        /// This function will scan a sequence of characters, providing a comment token.
        /// The function will traverse the sequence, until it reaches the end of line or end of file character.
        /// </summary>
        private void ScanComment()
        {
            string subString = CurrentChar.ToString();

            while (!IsEOL(Peek()) && !IsEOF(Peek()))
            {
                Pop();
                subString += CurrentChar;
            }
            Tokens.AddLast(Token(TokenType.COMMENT, subString));
        }
예제 #6
0
        public Token Next()
        {
            var tokenDescription = GetTokenDescriptionFromChar(CurrentChar);
            var stringBuilder    = new StringBuilder(CurrentChar.ToString());

            while (tokenDescription.Match(NextChar()) && CurrentChar != EOF)
            {
                stringBuilder.Append(CurrentChar);
            }

            return(new Token(tokenDescription.Type, stringBuilder.ToString()));
        }
예제 #7
0
 private void HandleKeyword()
 {
     if (TestKeywords())
     {
         return;
     }
     else
     {
         EmitToken(TokenKind.IdentifierToken, CurrentChar.ToString());
         Next();
         return;
     }
 }
예제 #8
0
        /// <summary>
        /// This function will scan a sequence of characters, providing a multiline comment token.
        /// The function will traverse the sequence, until it meets a sequence representing the end of a multiline comment.
        /// <example>An example of a valid multiline comment:
        /// <c>
        /// &lt;# This is a valid multiline comment #&gt;
        /// &lt;# This is also
        /// a valid multiline comment#&gt;
        /// </c>
        /// </example>
        /// <exception cref="Lexer.Exceptions.InvalidSyntaxException">When the traversed sequence is not closed by a quotation symbol</exception>
        /// </summary>
        private void ScanMultiLineComment()
        {
            string subString = CurrentChar.ToString();

            while (!IsEOF(Peek()) && !subString.Contains(">#"))
            {
                Pop();
                subString += CurrentChar;
            }
            if (!subString.Contains(">#"))
            {
                new InvalidSyntaxException($"Multiline comments must be closed before reaching end of file. Error at line {Line}:{Offset}.");
            }
            Tokens.AddLast(Token(TokenType.MULT_COMNT, subString));
        }
예제 #9
0
        /// <summary>
        /// This function will scan a sequence of characters providing a range token to the token list.
        /// The function will traverse while the next character is a period (.) symbol.
        /// <exception cref="Lexer.Exceptions.InvalidSyntaxException">When the traversed sequence does not produce a token with the value of exavtly '..'</exception>
        /// </summary>
        private void ScanRange()
        {
            string subString = CurrentChar.ToString();

            // This should in reality only run once
            while (Peek() == '.')
            {
                Pop();
                subString += CurrentChar;
            }
            if (subString.Length != 2)
            {
                new InvalidSyntaxException($"Invalid range symbol. Range symbol must be '..' but was '{subString}'. Error at line {Line}:{Offset}.");
            }
            Tokens.AddLast(Token(TokenType.OP_RANGE));
        }
예제 #10
0
        /// <summary>
        /// Reads a literal string until the ending quote is found.
        /// Adds 3 tokens: 2 quote tokens, 1 literal token.
        /// </summary>
        /// <example>
        /// "literal" => QuoteT -> LiteralT -> QuoteT
        /// </example>
        public void ProcessLiteral()
        {
            Token  = Tokens.LiteralT;
            Lexeme = CurrentChar.ToString();

            while (JavaFile.PeekNextChar() != '\"' && JavaFile.PeekNextChar() != '\n' && !JavaFile.EndOfFile())
            {
                JavaFile.GetNextChar();
                Lexeme += CurrentChar;
            }

            if (JavaFile.PeekNextChar() == '\"')
            {
                JavaFile.GetNextChar();
                Lexeme += CurrentChar.ToString();
            }
        }
예제 #11
0
        protected string PickName()
        {
            if (End)
            {
                return(null);
            }
            Skip();
            var from = Index;

            if (!IsNameStart)
            {
                return(null);
            }
            if (CurrentChar == '-')
            {
                var nc = NextChar;
                if (nc.HasValue)
                {
                    if (!Regex.IsMatch(nc.ToString(), @"[A-Za-zÀ-ÿ\-_]"))
                    {
                        return(null);
                    }
                    _index += 2; // skip the first two
                }
                else
                {
                    return(null);
                }
            }
            char c;

            while (!End && Regex.IsMatch(CurrentChar.ToString(), @"[A-Za-zÀ-ÿ\-_0-9]"))
            {
                _index++;
            }
            if (from == Index)
            {
                return(null);
            }
            var to = Index;

            wasWhitespace = false;
            Skip();
            return(css.Substring(from, to - from));
        }
예제 #12
0
        /// <summary>
        /// This function will scan a sequence of characters, providing a string token.
        /// The function will traverse the sequence, until it meets a quotation (") symbol.
        /// <example>An example of a valid string:
        /// <c>
        /// "Hello, World!"
        /// "I am a valid string,.!-/()*?#$"
        /// </c>
        /// </example>
        /// <exception cref="Lexer.Exceptions.InvalidSyntaxException">When the traversed sequence is not closed by a quotation symbol</exception>
        /// </summary>
        private void ScanString()
        {
            string subString = CurrentChar.ToString();

            while (Peek() != '"' && !IsEOF(Peek()))
            {
                Pop();
                subString += CurrentChar;
            }
            if (NextChar == '"')
            {
                Pop();
                subString += CurrentChar;
            }
            if (!subString.EndsWith('"'))
            {
                new InvalidSyntaxException($"Strings must be closed. Error at line {Line}:{Offset}.");
            }
            Tokens.AddLast(Token(TokenType.STRING, subString));
        }
        /// <summary>
        /// Check if a particular file name is alphanumeric or has underscore
        /// </summary>
        /// <param name="checkedValue">true: valid file name; false: invalid</param>
        /// <returns></returns>
        public static bool IsValidFileName(string fileName)
        {
            // TODO: Code review issue 24/05/2005 - KB
            // All this can be done using 1-line regular expressions (^\w(\w|\s)*$)
            bool   IsValid     = false;
            string validString = "abcdefghijklmnopqrstuvwxyz0123456789_ ";

            foreach (char CurrentChar in fileName.ToLower().ToCharArray())
            {
                if (validString.IndexOf(CurrentChar.ToString(), 0) == -1)
                {
                    IsValid = false;
                    break;
                }

                //if we come here, the string is valid
                IsValid = true;
            }
            return(IsValid);
        }
예제 #14
0
        /// <summary>
        /// Parses an identifier the ECMA-55 rules.
        /// </summary>
        /// <param name="c">The first character of the parsed identifier.</param>
        private IToken ParseIdent()
        {
            var strValueSb = new StringBuilder(CurrentChar.ToString());

            NextChar();
            while (IsDigit(CurrentChar) || IsLetter(CurrentChar))
            {
                strValueSb.Append(CurrentChar);

                NextChar();
            }

            var strValue = strValueSb.ToString().ToUpperInvariant();

            if (_keyWordsMap.ContainsKey(strValue))
            {
                return(new SimpleToken(_keyWordsMap[strValue]));
            }
            else
            {
                return(new IdentifierToken(strValue));
            }
        }
예제 #15
0
        /// <summary>
        /// Reads the first token char into Lexeme, which is then
        /// used to determine which function to use for processing.
        /// </summary>
        public void ProcessToken()
        {
            JavaFile.GetNextChar();
            Lexeme = CurrentChar.ToString();

            if (commentStartRegex.IsMatch(Lexeme + JavaFile.PeekNextChar()))
            {
                // skip the comment then continue processing the token
                ProcessComment();
                GetNextToken();
            }
            else if (char.IsLetter(Lexeme[0]))
            {
                ProcessWordToken();
            }
            else if (char.IsDigit(Lexeme[0]))
            {
                ProcessNumToken();
            }
            else if (comparisonRegex.IsMatch(Lexeme) && lookAheadCharRegex.IsMatch(JavaFile.PeekNextChar().ToString()))
            {
                ProcessDoubleToken();
            }
            else if (specialCharRegex.IsMatch(Lexeme))
            {
                ProcessSingleToken();
            }
            else if (Lexeme == "\"")
            {
                ProcessLiteral();
            }
            else
            {
                Token = Tokens.UnknownT;
            }
        }
예제 #16
0
            internal override AttributeCondition DoParse()
            {
                // http://www.w3schools.com/cssref/css_selectors.asp
                if (CurrentChar != '[')
                {
                    return(null);
                }
                Index++; // skip '['

                var attr = PickName();

                if (attr == null)
                {
                    AddError(ErrorCode.ExpectingToken, "condition");
                    var cond = SkipTillEnd();
                    return(new AttributeCondition());
                }

                if (End || CurrentChar == ']')
                {
                    if (End)
                    {
                        AddError(ErrorCode.UnexpectedEnd, "]");
                    }
                    else
                    {
                        Index++; // skip ']'
                    }
                    return(new AttributeCondition
                    {
                        Attribute = attr,
                        ConditionType = AttributeCondition.Type.HavingAttribute,
                    });
                }


                string sep;

                AttributeCondition.Type type;
                switch (CurrentChar)
                {
                case '=':
                    type = AttributeCondition.Type.Equals;
                    sep  = "=";
                    Index++;     // skip the '='
                    break;

                case '~':
                case '|':
                case '^':
                case '$':
                case '*':
                    if (NextChar != '=')
                    {
                        AddError(ErrorCode.ExpectingToken, "=");
                        SkipTillEnd();
                        return(new AttributeCondition());
                    }
                    type   = (AttributeCondition.Type)CurrentChar;
                    sep    = CurrentChar + "=";
                    Index += 2;     // skip the token
                    break;

                default:
                    AddError(ErrorCode.UnexpectedToken, CurrentChar.ToString());
                    SkipTillEnd();
                    return(new AttributeCondition());
                }

                var val = PickString();

                if (val == null)
                {
                    var ret = SkipTillEnd();
                    AddError(ErrorCode.ExpectingValue, ret);
                    SkipTillEnd();
                    return(new AttributeCondition());
                }

                if (End)
                {
                    AddError(ErrorCode.ExpectingToken, "]");
                }
                else if (CurrentChar != ']')
                {
                    AddError(ErrorCode.ExpectingToken, "]");
                    SkipTillEnd();
                    return(new AttributeCondition());
                }

                Index++; // skip ']'

                return(new AttributeCondition
                {
                    Attribute = attr,
                    Value = val,
                    ConditionType = type,
                });
            }
예제 #17
0
        public static string PCase(string strInput)
        {
            int    I;
            string CurrentChar, PrevChar;
            string strOutput;

            PrevChar  = "";
            strOutput = "";

            for (I = 1; I <= strInput.Length; I++)
            {
                CurrentChar = strInput.Substring(I - 1, 1);

                switch (PrevChar)
                {
                case "":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case " ":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case ".":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case "-":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case ",":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case "\"":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                case "'":
                    strOutput = strOutput + CurrentChar.ToString().ToUpper();
                    break;

                default:
                    switch (strOutput.ToUpper().Trim())
                    {
                    case "MC":
                        strOutput = strOutput + CurrentChar.ToString().ToUpper();
                        break;

                    case "MAC":
                        strOutput = strOutput + CurrentChar.ToString();
                        break;

                    case "O'":
                        strOutput = strOutput + CurrentChar.ToString().ToUpper();
                        break;

                    default:
                        strOutput = strOutput + CurrentChar.ToString().ToLower();
                        break;
                    }
                    break;
                }

                PrevChar = CurrentChar;
            }

            return(strOutput);
        }
예제 #18
0
        public bool NextToken()
        {
            bool hasWildcard;
            bool field;

            SkipWhiteSpaces();
            SaveLineInfo();
            IsPhrase = false;
            switch (CurrentChar)
            {
            case '\0': CurrentToken = Token.Eof; StringValue = string.Empty; return(false);

            case '(': CurrentToken = Token.LParen; StringValue = CurrentChar.ToString(); NextChar(); break;

            case ')': CurrentToken = Token.RParen; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '[': CurrentToken = Token.LBracket; StringValue = CurrentChar.ToString(); NextChar(); break;

            case ']': CurrentToken = Token.RBracket; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '{': CurrentToken = Token.LBrace; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '}': CurrentToken = Token.RBrace; StringValue = CurrentChar.ToString(); NextChar(); break;

            case ',': CurrentToken = Token.Comma; StringValue = CurrentChar.ToString(); NextChar(); break;

            case ':': CurrentToken = Token.Colon; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '+': CurrentToken = Token.Plus; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '-': CurrentToken = Token.Minus; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '^': CurrentToken = Token.Circ; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '~': CurrentToken = Token.Tilde; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '!': CurrentToken = Token.Not; StringValue = CurrentChar.ToString(); NextChar(); break;

            case '"':
            case '\'':
                StringValue  = ScanQuotedString(out hasWildcard, out field, out var isPhrase);
                CurrentToken = hasWildcard ? Token.WildcardString : field ? Token.Field : Token.String;
                IsPhrase     = isPhrase;
                break;

            case '&':
                NextChar();
                if (CurrentChar != '&')
                {
                    throw new ParserException("Invalid operator: &", CreateLastLineInfo());
                }
                CurrentToken = Token.And;
                StringValue  = "&&";
                NextChar();
                SkipWhiteSpaces();
                break;

            case '|':
                NextChar();
                if (CurrentChar != '|')
                {
                    throw new ParserException("Invalid operator: |", CreateLastLineInfo());
                }
                CurrentToken = Token.Or;
                StringValue  = "||";
                NextChar();
                SkipWhiteSpaces();
                break;

            case '<':
                NextChar();
                if (CurrentChar == '=')
                {
                    CurrentToken = Token.LTE;
                    StringValue  = "<=";
                    NextChar();
                    SkipWhiteSpaces();
                }
                else if (CurrentChar == '>')
                {
                    CurrentToken = Token.NEQ;
                    StringValue  = "<>";
                    NextChar();
                    SkipWhiteSpaces();
                }
                else
                {
                    CurrentToken = Token.LT;
                    StringValue  = "<";
                }
                break;

            case '>':
                NextChar();
                if (CurrentChar == '=')
                {
                    CurrentToken = Token.GTE;
                    StringValue  = ">=";
                    NextChar();
                    SkipWhiteSpaces();
                }
                else
                {
                    CurrentToken = Token.GT;
                    StringValue  = ">";
                }
                break;

            // -----------------------------------

            default:
                if (CurrentCharType == CharType.Digit)
                {
                    if (ScanNumber(out var numberValue, out var stringValue, out hasWildcard, out field))
                    {
                        CurrentToken = Token.Number;
                        StringValue  = stringValue;
                        NumberValue  = numberValue;
                    }
                    else
                    {
                        CurrentToken = hasWildcard ? Token.WildcardString : Token.String;
                        StringValue  = stringValue;
                    }
                }
                else
                {
                    StringValue = ScanNonQuotedString(out hasWildcard, out field, out var keyword);
                    if (keyword)
                    {
                        CurrentToken = Keywords.ScanControl(StringValue, CreateLastLineInfo());
                    }
                    else if (hasWildcard)
                    {
                        CurrentToken = Token.WildcardString;
                    }
                    else if (field)
                    {
                        CurrentToken = Token.Field;
                    }
                    else
                    {
                        CurrentToken = Keywords.ScanKeyword(StringValue);
                    }
                    SkipWhiteSpaces();
                }
                break;
            }
            return(true);
        }
예제 #19
0
            internal override Selector DoParse()
            {
                if (End)
                {
                    return(null);
                }
                var lst = new SelectorList();

                bool expectingNext = false;

                while (true)
                {
                    bool end = false;
                    switch (CurrentChar)
                    {
                    default:
                        var sel = DoParseOne();
                        if (sel == null)
                        {
                            // that's a warning
                            end = true;
                        }
                        else
                        {
                            lst.Selectors.Add(sel);
                        }
                        break;

                    case ';':
                    case '{':
                    case '}':
                    case ')':
                        // stoop
                        end = true;
                        break;
                    }

                    if (expectingNext && end)
                    {
                        // we've got a token after a ','
                        if (!End)
                        {
                            AddError(ErrorCode.UnexpectedToken, CurrentChar.ToString());
                        }
                    }

                    expectingNext = !End && CurrentChar == ',';
                    if (expectingNext)
                    {
                        Index++;
                    }

                    if (End || !expectingNext)
                    {
                        end = true;
                    }

                    if (end)
                    {
                        if (lst.Selectors.Count == 0)
                        {
                            return(null);
                        }
                        if (lst.Selectors.Count == 1)
                        {
                            return(lst.Selectors[0]);
                        }
                        return(lst);
                    }
                }
            }
예제 #20
0
 protected virtual void Extract()
 {
     text  = CurrentChar.ToString();
     value = null;
     NextChar();
 }
예제 #21
0
        private bool MultiCharacterOperatorToken()
        {
            var lexeme = CurrentChar.ToString() + PeekChar.ToString();

            return(OperatorToken(_multiCharacterOperators, lexeme));
        }
예제 #22
0
 private bool SingleCharacterOperatorToken()
 {
     return(OperatorToken(_singleCharacterOperators, CurrentChar.ToString()));
 }
예제 #23
0
        private void ScanSeparator()
        {
            AddToken(CurrentChar.ToString(), TokenType.SEPARATOR);

            Advance();
        }
예제 #24
0
        /// <summary>
        /// This function will scan a sequence of characters providing either a VAR token, a KEYWORD token, an INDEX token, a QUESTIONMARK token, or any ARRAY token.
        /// This is done by traversing the sequence of characters until it meets either a newline or any of the symbols representing any of the rest of the tokens.
        /// The time to look up a keyword is O(1) as this is done by using a hashed value in a dictionary.
        /// This function can produce multiple tokens within itself.
        /// <example> An example of possible accepted tokens:
        /// <c>
        /// a # Variable name
        /// is # Keyword
        /// for # Keyword
        /// a@2 # While the 2 is not recognised in this function this will result in tokens "VAR ARRAYINDEX NUMERIC"
        /// a[4] # This will result in the tokens "VAR ARRAYLEFT NUMERIC ARRAYRIGHT"
        /// </c>
        /// </example>
        /// </summary>
        private void ScanWord()
        {
            string       subString = CurrentChar.ToString();
            ScannerToken token;

            while (recogniser.IsAcceptedCharacter(Peek()) || recogniser.IsDigit(Peek()))
            {
                subString += Pop();
            }
            if (Regex.Match(subString.ToLower(), "(a|d)pin\\d+").Success)
            {
                if (subString.StartsWith("a"))
                {
                    token = Token(TokenType.APIN, "A" + subString.Substring(4));
                    token.SymbolicType = new TypeContext(TokenType.APIN);
                    Tokens.AddLast(token);
                }
                else
                {
                    token = Token(TokenType.DPIN, subString.Substring(4));
                    token.SymbolicType = new TypeContext(TokenType.DPIN);
                    Tokens.AddLast(token);
                }
                return;
            }
            if (Keywords.Keys.TryGetValue(subString, out TokenType tokenType))
            {
                if (tokenType == TokenType.BOOL)
                {
                    if (subString.ToLower() == "on")
                    {
                        token = Token(tokenType, "true");
                        token.SymbolicType = new TypeContext(TokenType.BOOL);
                        Tokens.AddLast(token);
                    }
                    else if (subString.ToLower() == "off")
                    {
                        token = Token(tokenType, "false");
                        token.SymbolicType = new TypeContext(TokenType.BOOL);
                        Tokens.AddLast(token);
                    }
                    else
                    {
                        token = Token(tokenType, subString);
                        token.SymbolicType = new TypeContext(TokenType.BOOL);
                        Tokens.AddLast(token);
                    }
                }
                else
                {
                    token = Token(tokenType);
                    if (TokenTypeExpressions.IsOperator(tokenType))
                    {
                        token.SymbolicType = new TypeContext(tokenType);
                    }

                    Tokens.AddLast(token);
                }
                return;
            }

            token = Token(TokenType.VAR, subString);
            if (Tokens.Any() && (Tokens.Last().Type == TokenType.FUNC || Tokens.Last().Type == TokenType.CALL))
            {
                token.SymbolicType = new TypeContext(TokenType.FUNC);
            }
            else
            {
                token.SymbolicType = new TypeContext(TokenType.VAR);
            }
            if (Peek() == '@')
            {
                Pop();
                token.SymbolicType = new TypeContext(TokenType.ARRAYINDEX);
                token.Type         = TokenType.ARRAYINDEX;
            }
            Tokens.AddLast(token);
            subString = "";
        }