private IList GetTokens(string code)
{
ArrayList tokens = new ArrayList();
MatchResult match;
int currentIndex = 0;
while (true)
{
match = tokenTree.Match(code, currentIndex);
Token token = new Token();
if (match.Found)
{
//get the test from the end of the last match , to the begining of the new match
//if
string dumbText = code.Substring(currentIndex, match.Index - currentIndex);
ThrowUnknownTokenException(dumbText, tokens, currentIndex);
}
else
{
string dumbText = code.Substring(currentIndex);
ThrowUnknownTokenException(dumbText, tokens, currentIndex);
}
token.Index = match.Index;
token.Text = match.GetText();
if (match.Tag is string)
{
token.Types = new string[] {(string) match.Tag};
}
else if (match.Tag is string[])
{
token.Types = (string[]) match.Tag;
}
tokens.Add(token);
currentIndex = match.Index + match.Length;
if (!match.Found)
break;
}
return tokens;
}
private void EnsureType(Token token, string type, string near, string expected)
{
if (!token.IsType(type))
throw new UnexpectedTokenException(token.Text, token.Index, near, expected);
}
