/// <summary>
/// checks whether c is a symbol character.
/// </summary>
protected bool IsSymbol(char c)
{
return(SymbolChars.Any(t => t == c));
}
public static bool IsPersianChar(char keyChar)
{
return(((char.IsLetterOrDigit(keyChar) && PersianChars.Contains(keyChar)) || char.IsNumber(keyChar) || keyChar == ' ' || SymbolChars.Contains(keyChar)));
}
public List <Token> Tokenize(string text)
{
tokens = new List <Token>();
Token currentToken = null;
char currentChar;
for (int i = 0; i < text.Length; i++)
{
currentChar = text[i];
if (WhitespaceChars.Contains(currentChar)) // Whitespace
{
CheckIdentifier(currentToken);
if (KeepWhitespace)
{
currentToken = new Token(TokenType.Whitespace, currentChar.ToString(), i);
tokens.Add(currentToken);
}
else
{
currentToken = new Token();
}
}
else if (SymbolChars.Contains(currentChar)) // Symbol
{
CheckIdentifier(currentToken);
currentToken = new Token(TokenType.Symbol, currentChar.ToString(), i);
tokens.Add(currentToken);
}
else if (LiteralDelimiters.Contains(currentChar)) // Literal
{
CheckIdentifier(currentToken);
currentToken = new Token(TokenType.Literal, currentChar.ToString(), i);
tokens.Add(currentToken);
char delimeter = currentChar;
for (i++; i < text.Length; i++)
{
currentChar = text[i];
currentToken.Text += currentChar;
if (currentChar == delimeter && !IsEscaped(currentToken.Text))
{
if (AutoParseLiteral)
{
currentToken.Text = ParseString(currentToken.Text);
}
break;
}
}
}
else // Identifier, Numeric, Keyword
{
if (currentToken != null && currentToken.Type == TokenType.Identifier)
{
currentToken.Text += currentChar;
}
else
{
currentToken = new Token(TokenType.Identifier, currentChar.ToString(), i);
tokens.Add(currentToken);
}
if (i + 1 >= text.Length) // EOF
{
CheckIdentifier(currentToken);
}
}
}
return(tokens);
}
public Token Next()
{
ReadToken:
char ch = La(0);
switch (ch)
{
case EOF:
return(CreateToken(TokenKind.Eof, string.Empty));
case ' ':
case '\t':
{
if (_ignoreWhiteSpace)
{
Consume();
goto ReadToken;
}
return(ReadWhitespace());
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return(ReadNumber());
case '\r':
{
StartRead();
Consume();
if (La(0) == '\n')
{
Consume(); // on DOS/Windows we have \r\n for new line
}
_line++;
_column = 1;
return(CreateToken(TokenKind.Eol));
}
case '\n':
{
StartRead();
Consume();
_line++;
_column = 1;
return(CreateToken(TokenKind.Eol));
}
case '"':
{
return(ReadDoubleQuotedString());
}
case '\'':
{
return(ReadSingleQuotedString());
}
default:
{
if (Char.IsLetter(ch))
{
return(ReadWord());
}
if (SymbolChars.ContainsKey(ch))
{
StartRead();
Consume();
return(CreateToken(SymbolChars[ch]));
}
StartRead();
Consume();
return(CreateToken(TokenKind.Unknown));
}
}
}