Ejemplo n.º 1
0
        public Token GetNextToken()
        {
            CurStates.Clear();
            NowFinal.Clear();
            if (Machines == null) return null;
            foreach (var key in Machines.Keys) CurStates.Add(key, Machines[key].First);

            StringBuilder sb = new StringBuilder();
            while (!IsFinal() && iterator < Text.Length && CurStates.Count > 0)
            {
                if (MoveForward(Text[iterator]))
                    sb.Append(Text[iterator++]);
                else break;
            }
            Token res = new Token() { Value = sb.ToString() };
            String type = Type();
            if (type == String.Empty)
            {
                res.Type = TokenType.UNKNOWN;
                if (iterator < Text.Length)
                    sb.Append(Text[iterator]);
                res.Value = sb.ToString();
                iterator++;
            }
            else if (type == "delimiter")
                res.Type = TokenType.DELIMITER;
            else if (type == "multop")
                res.Type = TokenType.MULTOP;
            else if (type == "plusop")
                res.Type = TokenType.PLUSOP;
            else if (type == "assignop")
                res.Type = TokenType.ASSIGNOP;
            else if (type == "boolop")
                res.Type = TokenType.BOOLOP;
            else if (type == "boolop2")
                res.Type = TokenType.BOOLOP2;
            else if (type == "punctuator")
                res.Type = TokenType.PUNCTUATOR;
            else if (type == "decimal")
                res.Type = TokenType.NUMBER;
            else if (type == "literal")
                if (NowFinal.Keys.Contains("keyword") ||
                    (CurStates.ContainsKey("keyword") && CurStates["keyword"].IsFinal))
                    res.Type = TokenType.KEYWORD;
                else
                    res.Type = TokenType.LITERAL;
            else if (type == "keyword")
                res.Type = TokenType.KEYWORD;
            if (res.Type != TokenType.UNKNOWN && res.Type != TokenType.DELIMITER)
                tm.AddToken(res);
            return res;
        }
Ejemplo n.º 2
0
 public List<int> GetAllSelections(Token t)
 {
     int idx = Tokens.FindIndex(tok => tok.Type == t.Type && tok.Value == t.Value);
     return AllTokens.Where(tok => tok.Value == idx).Select(tok => tok.Key).ToList();
 }
Ejemplo n.º 3
0
 public void AddToken(Token t)
 {
     if (Tokens == null) Tokens = new List<Token>();
     if (Tokens.Count(tok => tok.Type == t.Type && tok.Value == t.Value) == 0)
         Tokens.Add(t);
 }
Ejemplo n.º 4
0
 public void AddToAll(int i, Token t)
 {
     int idx = Tokens.FindIndex(tok => tok.Type == t.Type && tok.Value == t.Value);
     if (idx != -1)
         AllTokens.Add(i, idx);
 }