/// <summary>
/// Breaks the input stream onto the tokens list and returns an ArrayList of strings.
/// </summary>
/// <returns>The array of items broken out from the string</returns>
internal ArrayList Lex()
{
Token token = GetNextToken();
while (true)
{
if (token != null)
{
tokens.Add(token);
}
else
{
tokens.Add(new Token(null, reader.Line, reader.Column, reader.Line, reader.Column, TokenTypes.EOF));
break;
}
token = GetNextToken();
}
// now just do a list of strings
ArrayList ar = new ArrayList(tokens.Count);
foreach (Token t in tokens)
{
if (t.Type == TokenTypes.QUOTE)
{
ar.Add(t.Value);
}
}
return(ar);
}
/// <summary>
/// Breaks the input stream onto the tokens list and returns it.
/// </summary>
/// <returns>The tokens list.</returns>
internal TokenList Lex()
{
Token token = GetNextToken();
while (true)
{
if (token != null)
{
tokens.Add(token);
}
else
{
tokens.Add(new Token(null, reader.Line, reader.Column, reader.Line, reader.Column, TokenTypes.EOF));
return(tokens);
}
token = GetNextToken();
}
}
