private IToken CreateTokenFromTextToken(TextToken token)
{
switch (token.Type)
{
case TextTokenType.Number:
return(new NumberToken(token, this));
case TextTokenType.Word:
return(new VariableToken(token, this));
case TextTokenType.Operator:
return(new OperatorToken(token, this));
case TextTokenType.ParentheseStart:
return(new StartParentheseToken(token, this));
case TextTokenType.ParentheseStop:
return(new StopParentheseToken(token, this));
case TextTokenType.TernaryEnd:
return(new EndTernaryToken(token, this));
case TextTokenType.EndOfSentence:
return(new EndToken(token, this));
}
throw new NotSupportedException("Unsupported token type");
}
public static List <TextToken> TextTokenize(String text)
{
List <TextToken> tokens = new List <TextToken>();
int textLength = text.Length;
int index = 0;
//reading character by character until the end of input
while (index < textLength)
{
//if space or tab, just skip
if (char.IsWhiteSpace(text[index]))
{
index++;
}
// if starts with ":" then its ternary ending
else if (text[index] == ':')
{
TextToken token = new TextToken();
token.Type = TextTokenType.TernaryEnd;
token.RawData.Add(':');
tokens.Add(token);
index++;
}
// if starts with "(" then pick that up and continue
else if (text[index] == '(')
{
TextToken token = new TextToken();
token.Type = TextTokenType.ParentheseStart;
token.RawData.Add('(');
tokens.Add(token);
index++;
}
// if starts with "(" then pick that up and continue
else if (text[index] == ')')
{
TextToken token = new TextToken();
token.Type = TextTokenType.ParentheseStop;
token.RawData.Add(')');
tokens.Add(token);
index++;
}
//if token starts with letter, and continues
//with letter or digit, then its a word
else if (char.IsLetter(text[index]))
{
TextToken token = new TextToken();
token.Type = TextTokenType.Word;
do
{
token.RawData.Add(text[index]);
index++;
} while (index < textLength && char.IsLetterOrDigit(text[index]));
tokens.Add(token);
}
//if token starts with number, then its float or int
else if (char.IsDigit(text[index]))
{
TextToken token = new TextToken();
token.Type = TextTokenType.Number;
do
{
token.RawData.Add(text[index]);
index++;
} while (index < textLength && char.IsDigit(text[index]));
//if next is "." then its float
if (index < textLength && text[index] == '.')
{
token.RawData.Add('.');
index++;
do
{
token.RawData.Add(text[index]);
index++;
} while (index < textLength && char.IsDigit(text[index]));
}
tokens.Add(token);
}
//if dual operator ==, <=, >=, &&, ||
else if (IsDualOperator(text, index))
{
TextToken token = new TextToken();
token.Type = TextTokenType.Operator;
token.RawData.Add(text[index]);
index++;
token.RawData.Add(text[index]);
tokens.Add(token);
index++;
}
//if is operator such as "+-/*" then pick it up
else if (IsOperator(text[index]))
{
TextToken token = new TextToken();
token.Type = TextTokenType.Operator;
token.RawData.Add(text[index]);
tokens.Add(token);
index++;
}
//picking end of sentence
else if (text[index] == ';')
{
TextToken token = new TextToken();
token.Type = TextTokenType.EndOfSentence;
token.RawData.Add(';');
tokens.Add(token);
index++;
}
else
{
throw new Exception(string.Format("Not valid token {0} at {1}", text[index], index));
}
}
return(tokens);
}
public NumberToken(TextToken token, Parser parser)
: base(token, parser)
{
}
public EndToken(TextToken token, Parser parser) : base(token, parser)
{
}
public Token(TextToken token, Parser parser)
{
this.OriginalToken = token;
this.Parser = parser;
}
public OperatorToken(TextToken token, Parser parser)
: base(token, parser)
{
this.Operator = (TokenOperator)token.Data;
this.LeftBindingPower = this.OperatorPrecedence[this.Operator];
}
public StartParentheseToken(TextToken token, Parser parser)
: base(token, parser)
{
}
public VariableToken(TextToken token, Parser parser)
: base(token, parser)
{
}