//---------------------------------------------------------------------
public TokenList Tokenize(System.IO.TextReader Text)
{
TokenList tokens = new TokenList();
Token token = new Token();
int nChar = 0;
while (true)
{
int CharCode = Text.Peek();
if (CharCode == -1)
{
break;
}
if (this.WhitespaceChars.Contains("" + (char)CharCode))
{
// Accumulate a Whitespace character.
if (token.Type != TokenType.Whitespace)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Whitespace, "", nChar);
}
token.Text += (char)Text.Read();
nChar++;
}
else if (this.SymbolChars.Contains("" + (char)CharCode))
{
// Accumulate a single Symbol character.
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Symbol, ("" + (char)Text.Read()), nChar);
this.CollectToken_(tokens, token);
nChar++;
token = this.NewToken_(TokenType.Unknown, "", nChar);
}
else if (this.LiteralDelimiters.Contains("" + (char)CharCode))
{
// Accumulate a Literal character.
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Literal, ("" + (char)Text.Read()), nChar);
nChar++;
// Read until the closing delimiter.
while (true)
{
int NextCharCode = Text.Peek();
if (NextCharCode == -1)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Unknown, "", nChar);
break;
}
else if (NextCharCode == CharCode)
{
token.Text += (char)Text.Read();
nChar++;
if (!token.Text.EndsWith(this.LiteralEscapeChar + (char)CharCode))
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Unknown, "", nChar);
break;
}
}
else
{
token.Text += (char)Text.Read();
nChar++;
}
}
}
else
{
// Accumulate an identifier.
if (token.Type != TokenType.Identifier)
{
this.CollectToken_(tokens, token);
token = this.NewToken_(TokenType.Identifier, "", nChar);
}
token.Text += (char)Text.Read();
nChar++;
}
}
this.CollectToken_(tokens, token);
// Return the tokens.
return(tokens);
}
//---------------------------------------------------------------------
private void CollectToken_(TokenList Tokens, Token CurrentToken)
{
// Do some post-processing on the token and add to list of tokens.
if (CurrentToken != null)
{
switch (CurrentToken.Type)
{
case TokenType.Unknown:
// Do nothing, don't add to list.
break;
case TokenType.Whitespace:
// No post-processing, add to list.
if (this.KeepWhitespace)
{
Tokens.Add(CurrentToken);
}
break;
case TokenType.Symbol:
// No post-processing, add to list.
Tokens.Add(CurrentToken);
break;
case TokenType.Literal:
// No post-processing, add to list.
Tokens.Add(CurrentToken);
break;
case TokenType.Identifier:
// Check for Numeric.
if (CurrentToken.Type == TokenType.Identifier)
{
double dValue = 0.0;
if (double.TryParse(CurrentToken.Text, out dValue))
{
CurrentToken.Type = TokenType.Numeric;
}
}
// Check for Keyword.
if (CurrentToken.Type == TokenType.Identifier)
{
foreach (string keyword in this.Keywords)
{
if (string.Equals(CurrentToken.Text, keyword, System.StringComparison.InvariantCultureIgnoreCase))
{
CurrentToken.Type = TokenType.Keyword;
break;
}
}
}
// Add to list.
Tokens.Add(CurrentToken);
break;
}
}
return;
}
