| public MaybeCommentToken ( string str, int cmpn ) : bool | ||
| str | string | |
| cmpn | int | |
| return | bool | |
public Token(Tokenizer o, char c, int offset, int line, int ch)
{
Owner = o;
Text = c.ToString();
Offset = offset;
Line = line;
Column = ch;
Type = TokenType.None;
if (o.PrimitiveDecider != null)
{
Type = o.PrimitiveDecider(c);
}
if (Type == TokenType.None)
{
if (char.IsLetterOrDigit(c) || c == '_')
{
Type = TokenType.Alphanumeric;
}
else
{
if (Owner.IsQuoteToken(c.ToString(), 0))
{
Type = TokenType.StringQuoteStart;
}
else if (Owner.IsCommentToken(c.ToString(), 0))
{
Type = TokenType.CommentStart;
}
else if (!Owner.IgnoreSpecial &&
(Owner.MaybeQuoteToken(c.ToString(), 0) ||
Owner.MaybeCommentToken(c.ToString(), 0)))
{
Type = TokenType.MaybeSpecial;
}
if (Type == TokenType.None)
{
Type = TokenType.Symbolic;
}
}
foreach (char t in Owner.WhitespaceChars)
{
if (c == t)
{
Type = TokenType.Whitespace;
}
}
}
}
public Token (Tokenizer o, char c, int offset, int line, int ch)
{
Owner = o;
Text = c.ToString ();
Offset = offset;
Line = line;
Column = ch;
Type = TokenType.None;
if (o.PrimitiveDecider != null) {
Type = o.PrimitiveDecider (c);
}
if (Type == TokenType.None) {
if (char.IsLetterOrDigit (c) || c == '_')
Type = TokenType.Alphanumeric;
else {
if (Owner.IsQuoteToken (c.ToString (), 0)) {
Type = TokenType.StringQuoteStart;
} else if (Owner.IsCommentToken (c.ToString (), 0)) {
Type = TokenType.CommentStart;
} else if (!Owner.IgnoreSpecial &&
(Owner.MaybeQuoteToken (c.ToString (), 0) ||
Owner.MaybeCommentToken (c.ToString (), 0))) {
Type = TokenType.MaybeSpecial;
}
if (Type == TokenType.None)
Type = TokenType.Symbolic;
}
foreach (char t in Owner.WhitespaceChars)
if (c == t)
Type = TokenType.Whitespace;
}
}