public void UpdateRange(Token token)
{
if (token.StartPos < StartPos) StartPos = token.StartPos;
if (token.EndPos > EndPos) EndPos = token.EndPos;
}
/// <summary>
/// inserts the RTF codes to highlight text blocks
/// </summary>
/// <param name="token">the token to highlight, will be appended to sb</param>
/// <param name="sb">the final output string</param>
private void HighlightToken(Token token, StringBuilder sb)
{
switch (token.Type)
{
case TokenType.VARIABLE:
sb.Append(@"{{\cf1 ");
break;
case TokenType.STRING:
sb.Append(@"{{\cf2 ");
break;
case TokenType.NUMBER:
sb.Append(@"{{\cf3 ");
break;
case TokenType.BOOLEAN:
sb.Append(@"{{\cf4 ");
break;
case TokenType.SEPARATOR:
sb.Append(@"{{\cf5 ");
break;
case TokenType.OR:
sb.Append(@"{{\cf6 ");
break;
case TokenType.AND:
sb.Append(@"{{\cf7 ");
break;
case TokenType.EQUAL:
sb.Append(@"{{\cf8 ");
break;
case TokenType.NOTEQUAL:
sb.Append(@"{{\cf9 ");
break;
case TokenType.SMALLEQ:
sb.Append(@"{{\cf10 ");
break;
case TokenType.GREATEQ:
sb.Append(@"{{\cf11 ");
break;
case TokenType.SMALLTH:
sb.Append(@"{{\cf12 ");
break;
case TokenType.GREATTH:
sb.Append(@"{{\cf13 ");
break;
case TokenType.PLUS:
sb.Append(@"{{\cf14 ");
break;
case TokenType.MINUS:
sb.Append(@"{{\cf15 ");
break;
case TokenType.MULT:
sb.Append(@"{{\cf16 ");
break;
case TokenType.DIV:
sb.Append(@"{{\cf17 ");
break;
case TokenType.BROPEN:
sb.Append(@"{{\cf18 ");
break;
case TokenType.BRCLOSE:
sb.Append(@"{{\cf19 ");
break;
default:
sb.Append(@"{{\cf0 ");
break;
}
}
private List<TokenType> SkipList; // tokens to be skipped
public Scanner()
{
Regex regex;
Patterns = new Dictionary<TokenType, Regex>();
Tokens = new List<TokenType>();
LookAheadToken = null;
Skipped = new List<Token>();
SkipList = new List<TokenType>();
SkipList.Add(TokenType.WHITESPACE);
regex = new Regex(@"^\s*$", RegexOptions.Compiled);
Patterns.Add(TokenType.EOF, regex);
Tokens.Add(TokenType.EOF);
regex = new Regex(@"(\[@?[A-Z]+[%]?\])(\.[A-Za-z]+)?", RegexOptions.Compiled);
Patterns.Add(TokenType.VARIABLE, regex);
Tokens.Add(TokenType.VARIABLE);
//regex = new Regex(@"\""[0-9A-Za-z' :\-]*\""", RegexOptions.Compiled);
// fix for all language (chinese,russian, etc.)
regex = new Regex(@"\""([^\""]*)\""", RegexOptions.Compiled);
Patterns.Add(TokenType.STRING, regex);
Tokens.Add(TokenType.STRING);
regex = new Regex(@"[0-9]*(\.[0-9]+)?", RegexOptions.Compiled);
Patterns.Add(TokenType.NUMBER, regex);
Tokens.Add(TokenType.NUMBER);
regex = new Regex(@"[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]", RegexOptions.Compiled);
Patterns.Add(TokenType.BOOLEAN, regex);
Tokens.Add(TokenType.BOOLEAN);
regex = new Regex(@"#", RegexOptions.Compiled);
Patterns.Add(TokenType.SEPARATOR, regex);
Tokens.Add(TokenType.SEPARATOR);
regex = new Regex(@"\|\|", RegexOptions.Compiled);
Patterns.Add(TokenType.OR, regex);
Tokens.Add(TokenType.OR);
regex = new Regex(@"&&", RegexOptions.Compiled);
Patterns.Add(TokenType.AND, regex);
Tokens.Add(TokenType.AND);
regex = new Regex(@"==", RegexOptions.Compiled);
Patterns.Add(TokenType.EQUAL, regex);
Tokens.Add(TokenType.EQUAL);
regex = new Regex(@"!=", RegexOptions.Compiled);
Patterns.Add(TokenType.NOTEQUAL, regex);
Tokens.Add(TokenType.NOTEQUAL);
regex = new Regex(@"<=", RegexOptions.Compiled);
Patterns.Add(TokenType.SMALLEQ, regex);
Tokens.Add(TokenType.SMALLEQ);
regex = new Regex(@">=", RegexOptions.Compiled);
Patterns.Add(TokenType.GREATEQ, regex);
Tokens.Add(TokenType.GREATEQ);
regex = new Regex(@"<", RegexOptions.Compiled);
Patterns.Add(TokenType.SMALLTH, regex);
Tokens.Add(TokenType.SMALLTH);
regex = new Regex(@">", RegexOptions.Compiled);
Patterns.Add(TokenType.GREATTH, regex);
Tokens.Add(TokenType.GREATTH);
regex = new Regex(@"\+", RegexOptions.Compiled);
Patterns.Add(TokenType.PLUS, regex);
Tokens.Add(TokenType.PLUS);
regex = new Regex(@"-", RegexOptions.Compiled);
Patterns.Add(TokenType.MINUS, regex);
Tokens.Add(TokenType.MINUS);
regex = new Regex(@"\*", RegexOptions.Compiled);
Patterns.Add(TokenType.MULT, regex);
Tokens.Add(TokenType.MULT);
regex = new Regex(@"\/", RegexOptions.Compiled);
Patterns.Add(TokenType.DIV, regex);
Tokens.Add(TokenType.DIV);
regex = new Regex(@"\(", RegexOptions.Compiled);
Patterns.Add(TokenType.BROPEN, regex);
Tokens.Add(TokenType.BROPEN);
regex = new Regex(@"\)", RegexOptions.Compiled);
Patterns.Add(TokenType.BRCLOSE, regex);
Tokens.Add(TokenType.BRCLOSE);
regex = new Regex(@"\s+", RegexOptions.Compiled);
Patterns.Add(TokenType.WHITESPACE, regex);
Tokens.Add(TokenType.WHITESPACE);
}
/// <summary>
/// returns token with longest best match
/// </summary>
/// <returns></returns>
public Token LookAhead(params TokenType[] expectedtokens)
{
int i;
int startpos = StartPos;
Token tok = null;
List<TokenType> scantokens;
// this prevents double scanning and matching
// increased performance
if (LookAheadToken != null
&& LookAheadToken.Type != TokenType._UNDETERMINED_
&& LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;
// if no scantokens specified, then scan for all of them (= backward compatible)
if (expectedtokens.Length == 0)
scantokens = Tokens;
else
{
scantokens = new List<TokenType>(expectedtokens);
scantokens.AddRange(SkipList);
}
do
{
int len = -1;
TokenType index = (TokenType)int.MaxValue;
string input = Input.Substring(startpos);
tok = new Token(startpos, EndPos);
for (i = 0; i < scantokens.Count; i++)
{
Regex r = Patterns[scantokens[i]];
Match m = r.Match(input);
if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len)))
{
len = m.Length;
index = scantokens[i];
}
}
if (index >= 0 && len >= 0)
{
tok.EndPos = startpos + len;
tok.Text = Input.Substring(tok.StartPos, len);
tok.Type = index;
}
else if (tok.StartPos < tok.EndPos - 1)
{
tok.Text = Input.Substring(tok.StartPos, 1);
}
if (SkipList.Contains(tok.Type))
{
startpos = tok.EndPos;
Skipped.Add(tok);
}
else
{
// only assign to non-skipped tokens
tok.Skipped = Skipped; // assign prior skips to this token
Skipped = new List<Token>(); //reset skips
}
}
while (SkipList.Contains(tok.Type));
LookAheadToken = tok;
return tok;
}
/// <summary>
/// executes a lookahead of the next token
/// and will advance the scan on the input string
/// </summary>
/// <returns></returns>
public Token Scan(params TokenType[] expectedtokens)
{
Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
LookAheadToken = null; // reset lookahead token, so scanning will continue
StartPos = tok.EndPos;
EndPos = tok.EndPos; // set the tokenizer to the new scan position
return tok;
}
public Token GetToken(TokenType type)
{
Token t = new Token(this.StartPos, this.EndPos);
t.Type = type;
return t;
}
public void Init(string input)
{
Input = input;
StartPos = 0;
EndPos = 0;
CurrentLine = 0;
CurrentColumn = 0;
CurrentPosition = 0;
LookAheadToken = null;
}