private LinkedList <Token> GenerateTokens()
{
string[] lines = sourceString.SplitAndKeep(new[] { '\r', '\n' }).ToArray();
string[] temp;
List <string> list = new List <string>();
foreach (string line in lines)
{
list.AddRange(line.Split('\t', ' '));
}
temp = list.ToArray();
temp = temp.Where(x => !String.IsNullOrEmpty(x)).ToArray();
tokenStrings = new LinkedList <string>(temp);
currentTokenString = tokenStrings.First;
nextTokenString = currentTokenString.Next;
previousTokenString = currentTokenString.Previous;
while (currentTokenString != null)
{
Token token = new GenericToken(currentTokenString.Value);
if (token.isOperator())
{
Operator op = new Operator(currentTokenString.Value);
op.Type = op.DetectType();
tokens.AddLast(new LinkedListNode <Token>(op));
}
else if (token.isKeyword())
{
Keyword keyword = new Keyword(currentTokenString.Value);
keyword.Type = keyword.DetectType();
tokens.AddLast(new LinkedListNode <Token>(keyword));
}
else if (token.isType())
{
Typename typename = new Typename(currentTokenString.Value);
typename.Type = typename.DetectType();
tokens.AddLast(new LinkedListNode <Token>(typename));
}
else
{
token.Type = token.DetectType();
if ((EnumTokenType)token.Type == EnumTokenType.STRING)
{
StringLiteral literal = new StringLiteral(currentTokenString.Value);
literal.Type = EnumTokenType.STRING_LITERAL;
while (!literal.Value.EndsWith("\""))
{
currentTokenString = nextTokenString;
nextTokenString = currentTokenString.Next;
if (currentTokenString.Value == null)
{
ThrowError(EnumErrorCodes.STRING_NOT_TETMINATED, "\"");
return(null);
}
literal.Value += " " + currentTokenString.Value;
}
tokens.AddLast(new LinkedListNode <Token>(literal));
}
else if ((EnumTokenType)token.Type == EnumTokenType.NUMBER)
{
NumericLiteral literal = new NumericLiteral(currentTokenString.Value);
literal.Type = EnumTokenType.NUMERIC_LITERAL;
tokens.AddLast(new LinkedListNode <Token>(literal));
}
else
{
tokens.AddLast(new LinkedListNode <Token>(token));
}
}
currentTokenString = nextTokenString;
if (nextTokenString != null)
{
nextTokenString = currentTokenString.Next;
}
if (previousTokenString != null)
{
previousTokenString = currentTokenString.Previous;
}
}
return(tokens);
}