public Entry(Regex pattern, string type, bool ignore, CreateTokenDelegate createToken)
{
m_Pattern = pattern;
m_Type = type;
m_Ignore = ignore;
m_CreateToken = createToken;
}
public static Tokenizer CreateFrom(string code)
{
Token[] tokens = m_Tokenizer.Tokenize(code);
var tokenizer = new Tokenizer();
var createToken = new CreateTokenDelegate(CreateToken);
for (int i = 0; i < tokens.Length;)
{
string name = tokens[i++].Content;
bool ignore = tokens[i].Type == "OPERATOR";
if (ignore)
{
++i;
}
string pattern = tokens[i++].Content;
pattern = pattern.Substring(1, pattern.Length - 2);
var weight = (int)tokens[i++].State;
tokenizer.Add(name, pattern, weight, ignore, createToken);
}
return(tokenizer);
}
public void Add(string type, string regex, int weight, bool ignore, CreateTokenDelegate createToken)
{
var pattern = new Regex("^(" + regex + ")", RegexOptions.Compiled | RegexOptions.Singleline);
if (pattern.IsMatch(""))
{
throw new ArgumentException("Invalid regex! Must not match an empty string!");
}
// Inversed sorting - ensure that entries with highest weight are evaluated first instead of last!
m_Patterns.Add(-weight, new Entry(pattern, type, ignore, createToken));
}
public void Add(string type, string regex, int weight, CreateTokenDelegate createToken)
{
Add(type, regex, weight, false, createToken);
}