/// <summary>
/// required to override this function from base otherwise the parsetree will consist of incorrect types of nodes
/// </summary>
/// <param name="token"></param>
/// <param name="text"></param>
/// <returns></returns>
public override ParseNode CreateNode(Token token, string text)
{
ParseTreeEvaluator node = new ParseTreeEvaluator();
node.Token = token;
node.text = text;
node.Parent = this;
return node;
}
public void UpdateRange(Token token)
{
if (token.StartPos < startpos)
startpos = token.StartPos;
if (token.EndPos > endpos)
endpos = token.EndPos;
}
/// <summary>
/// executes a lookahead of the next token
/// and will advance the scan on the input string
/// </summary>
/// <returns></returns>
public Token Scan(params TokenType[] expectedtokens)
{
Token tok = LookAhead (expectedtokens); // temporarely retrieve the lookahead
LookAheadToken = null; // reset lookahead token, so scanning will continue
StartPos = tok.EndPos;
EndPos = tok.EndPos; // set the tokenizer to the new scan position
return tok;
}
/// <summary>
/// returns token with longest best match
/// </summary>
/// <returns></returns>
public Token LookAhead(params TokenType[] expectedtokens)
{
int i;
int startpos = StartPos;
Token tok = null;
List<TokenType> scantokens;
// this prevents double scanning and matching
// increased performance
if (LookAheadToken != null
&& LookAheadToken.Type != TokenType._UNDETERMINED_
&& LookAheadToken.Type != TokenType._NONE_)
return LookAheadToken;
// if no scantokens specified, then scan for all of them (= backward compatible)
if (expectedtokens.Length == 0)
scantokens = Tokens;
else {
scantokens = new List<TokenType> (expectedtokens);
scantokens.AddRange (SkipList);
}
do {
int len = -1;
TokenType index = (TokenType)int.MaxValue;
string input = Input.Substring (startpos);
tok = new Token (startpos, EndPos);
for (i = 0; i < scantokens.Count; i++) {
Regex r = Patterns [scantokens [i]];
Match m = r.Match (input);
if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens [i] < index && m.Length == len))) {
len = m.Length;
index = scantokens [i];
}
}
if (index >= 0 && len >= 0) {
tok.EndPos = startpos + len;
tok.Text = Input.Substring (tok.StartPos, len);
tok.Type = index;
} else if (tok.StartPos < tok.EndPos - 1) {
tok.Text = Input.Substring (tok.StartPos, 1);
}
if (SkipList.Contains (tok.Type)) {
startpos = tok.EndPos;
Skipped.Add (tok);
} else {
// only assign to non-skipped tokens
tok.Skipped = Skipped; // assign prior skips to this token
Skipped = new List<Token> (); //reset skips
}
} while (SkipList.Contains(tok.Type));
LookAheadToken = tok;
return tok;
}
public void Init(string input)
{
this.Input = input;
StartPos = 0;
EndPos = 0;
CurrentLine = 0;
CurrentColumn = 0;
CurrentPosition = 0;
LookAheadToken = null;
}
public Token GetToken(TokenType type)
{
Token t = new Token (this.StartPos, this.EndPos);
t.Type = type;
return t;
}
public Scanner()
{
Regex regex;
Patterns = new Dictionary<TokenType, Regex> ();
Tokens = new List<TokenType> ();
LookAheadToken = null;
Skipped = new List<Token> ();
SkipList = new List<TokenType> ();
SkipList.Add (TokenType.WHITESPACE);
regex = new Regex (@"true|false");
Patterns.Add (TokenType.BOOLEANLITERAL, regex);
Tokens.Add (TokenType.BOOLEANLITERAL);
regex = new Regex (@"[0-9]+(UL|Ul|uL|ul|LU|Lu|lU|lu|U|u|L|l)?");
Patterns.Add (TokenType.DECIMALINTEGERLITERAL, regex);
Tokens.Add (TokenType.DECIMALINTEGERLITERAL);
regex = new Regex (@"([0-9]+\.[0-9]+([eE][+-]?[0-9]+)?([fFdDMm]?)?)|(\.[0-9]+([eE][+-]?[0-9]+)?([fFdDMm]?)?)|([0-9]+([eE][+-]?[0-9]+)([fFdDMm]?)?)|([0-9]+([fFdDMm]?))");
Patterns.Add (TokenType.REALLITERAL, regex);
Tokens.Add (TokenType.REALLITERAL);
regex = new Regex (@"0(x|X)[0-9a-fA-F]+");
Patterns.Add (TokenType.HEXINTEGERLITERAL, regex);
Tokens.Add (TokenType.HEXINTEGERLITERAL);
regex = new Regex (@"\""(\""\""|[^\""])*\""");
Patterns.Add (TokenType.STRINGLITERAL, regex);
Tokens.Add (TokenType.STRINGLITERAL);
regex = new Regex (@"[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()");
Patterns.Add (TokenType.FUNCTION, regex);
Tokens.Add (TokenType.FUNCTION);
regex = new Regex (@"[a-zA-Z_][a-zA-Z0-9_]*(?!\s*\()");
Patterns.Add (TokenType.VARIABLE, regex);
Tokens.Add (TokenType.VARIABLE);
regex = new Regex (@"(?i)pi|e");
Patterns.Add (TokenType.CONSTANT, regex);
Tokens.Add (TokenType.CONSTANT);
regex = new Regex (@"{\s*");
Patterns.Add (TokenType.BRACEOPEN, regex);
Tokens.Add (TokenType.BRACEOPEN);
regex = new Regex (@"\s*}");
Patterns.Add (TokenType.BRACECLOSE, regex);
Tokens.Add (TokenType.BRACECLOSE);
regex = new Regex (@"\(\s*");
Patterns.Add (TokenType.BRACKETOPEN, regex);
Tokens.Add (TokenType.BRACKETOPEN);
regex = new Regex (@"\s*\)");
Patterns.Add (TokenType.BRACKETCLOSE, regex);
Tokens.Add (TokenType.BRACKETCLOSE);
regex = new Regex (@";");
Patterns.Add (TokenType.SEMICOLON, regex);
Tokens.Add (TokenType.SEMICOLON);
regex = new Regex (@"\+\+");
Patterns.Add (TokenType.PLUSPLUS, regex);
Tokens.Add (TokenType.PLUSPLUS);
regex = new Regex (@"--");
Patterns.Add (TokenType.MINUSMINUS, regex);
Tokens.Add (TokenType.MINUSMINUS);
regex = new Regex (@"\|\|");
Patterns.Add (TokenType.PIPEPIPE, regex);
Tokens.Add (TokenType.PIPEPIPE);
regex = new Regex (@"&&");
Patterns.Add (TokenType.AMPAMP, regex);
Tokens.Add (TokenType.AMPAMP);
regex = new Regex (@"&(?!&)");
Patterns.Add (TokenType.AMP, regex);
Tokens.Add (TokenType.AMP);
regex = new Regex (@"\^");
Patterns.Add (TokenType.POWER, regex);
Tokens.Add (TokenType.POWER);
regex = new Regex (@"\+");
Patterns.Add (TokenType.PLUS, regex);
Tokens.Add (TokenType.PLUS);
regex = new Regex (@"-");
Patterns.Add (TokenType.MINUS, regex);
Tokens.Add (TokenType.MINUS);
regex = new Regex (@"=");
Patterns.Add (TokenType.EQUAL, regex);
Tokens.Add (TokenType.EQUAL);
regex = new Regex (@":=");
Patterns.Add (TokenType.ASSIGN, regex);
Tokens.Add (TokenType.ASSIGN);
regex = new Regex (@"!=|<>");
Patterns.Add (TokenType.NOTEQUAL, regex);
Tokens.Add (TokenType.NOTEQUAL);
regex = new Regex (@"!");
Patterns.Add (TokenType.NOT, regex);
Tokens.Add (TokenType.NOT);
regex = new Regex (@"\*");
Patterns.Add (TokenType.ASTERIKS, regex);
Tokens.Add (TokenType.ASTERIKS);
regex = new Regex (@"/");
Patterns.Add (TokenType.SLASH, regex);
Tokens.Add (TokenType.SLASH);
regex = new Regex (@"%");
Patterns.Add (TokenType.PERCENT, regex);
Tokens.Add (TokenType.PERCENT);
regex = new Regex (@"\?");
Patterns.Add (TokenType.QUESTIONMARK, regex);
Tokens.Add (TokenType.QUESTIONMARK);
regex = new Regex (@",");
Patterns.Add (TokenType.COMMA, regex);
Tokens.Add (TokenType.COMMA);
regex = new Regex (@"<=");
Patterns.Add (TokenType.LESSEQUAL, regex);
Tokens.Add (TokenType.LESSEQUAL);
regex = new Regex (@">=");
Patterns.Add (TokenType.GREATEREQUAL, regex);
Tokens.Add (TokenType.GREATEREQUAL);
regex = new Regex (@"<(?!>)");
Patterns.Add (TokenType.LESSTHAN, regex);
Tokens.Add (TokenType.LESSTHAN);
regex = new Regex (@">");
Patterns.Add (TokenType.GREATERTHAN, regex);
Tokens.Add (TokenType.GREATERTHAN);
regex = new Regex (@":");
Patterns.Add (TokenType.COLON, regex);
Tokens.Add (TokenType.COLON);
regex = new Regex (@"^$");
Patterns.Add (TokenType.EOF, regex);
Tokens.Add (TokenType.EOF);
regex = new Regex (@"\s+");
Patterns.Add (TokenType.WHITESPACE, regex);
Tokens.Add (TokenType.WHITESPACE);
}
