public Token GetNextToken()
{
CurStates.Clear();
NowFinal.Clear();
if (Machines == null) return null;
foreach (var key in Machines.Keys) CurStates.Add(key, Machines[key].First);
StringBuilder sb = new StringBuilder();
while (!IsFinal() && iterator < Text.Length && CurStates.Count > 0)
{
if (MoveForward(Text[iterator]))
sb.Append(Text[iterator++]);
else break;
}
Token res = new Token() { Value = sb.ToString() };
String type = Type();
if (type == String.Empty)
{
res.Type = TokenType.UNKNOWN;
if (iterator < Text.Length)
sb.Append(Text[iterator]);
res.Value = sb.ToString();
iterator++;
}
else if (type == "delimiter")
res.Type = TokenType.DELIMITER;
else if (type == "multop")
res.Type = TokenType.MULTOP;
else if (type == "plusop")
res.Type = TokenType.PLUSOP;
else if (type == "assignop")
res.Type = TokenType.ASSIGNOP;
else if (type == "boolop")
res.Type = TokenType.BOOLOP;
else if (type == "boolop2")
res.Type = TokenType.BOOLOP2;
else if (type == "punctuator")
res.Type = TokenType.PUNCTUATOR;
else if (type == "decimal")
res.Type = TokenType.NUMBER;
else if (type == "literal")
if (NowFinal.Keys.Contains("keyword") ||
(CurStates.ContainsKey("keyword") && CurStates["keyword"].IsFinal))
res.Type = TokenType.KEYWORD;
else
res.Type = TokenType.LITERAL;
else if (type == "keyword")
res.Type = TokenType.KEYWORD;
if (res.Type != TokenType.UNKNOWN && res.Type != TokenType.DELIMITER)
tm.AddToken(res);
return res;
}
public List<int> GetAllSelections(Token t)
{
int idx = Tokens.FindIndex(tok => tok.Type == t.Type && tok.Value == t.Value);
return AllTokens.Where(tok => tok.Value == idx).Select(tok => tok.Key).ToList();
}
public void AddToken(Token t)
{
if (Tokens == null) Tokens = new List<Token>();
if (Tokens.Count(tok => tok.Type == t.Type && tok.Value == t.Value) == 0)
Tokens.Add(t);
}
public void AddToAll(int i, Token t)
{
int idx = Tokens.FindIndex(tok => tok.Type == t.Type && tok.Value == t.Value);
if (idx != -1)
AllTokens.Add(i, idx);
}