public void ShouldBuildInputPredicate()
{
ISituationPredicate <char> predicate;
ISituationGraph <char> graph;
LexicalRule rule;
SituationGraphFactory <char> situationGraphFactory;
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
predicate = new Terminal('a');
rule = new LexicalRule()
{
Name = "A", Predicate = (LexicalPredicate)predicate
};
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
Assert.IsTrue(graph.Contains(predicate));
predicate = new NonTerminal("A");
rule = new LexicalRule()
{
Name = "A", Predicate = (LexicalPredicate)predicate
};
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
Assert.IsTrue(graph.Contains(predicate));
predicate = new AnyTerminal();
rule = new LexicalRule()
{
Name = "A", Predicate = (LexicalPredicate)predicate
};
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
Assert.IsTrue(graph.Contains(predicate));
}
public void ShouldBuildSequenceSegment()
{
SituationGraphSegmentFactory <char> factory;
ISituationGraph <char> graph;
LexicalRule rule;
MockedSituationEdge capEdge;
ISituationGraphSegment <char> segment;
capEdge = new MockedSituationEdge();
graph = new SituationGraph <char>();
factory = new SituationGraphSegmentFactory <char>();
rule = new LexicalRule();
rule.Predicate = new Sequence(new Terminal('a'), new Terminal('b'), new Terminal('c'));
segment = factory.BuildSegment(graph, rule, rule.Predicate, capEdge.AsEnumerable());
Assert.AreEqual(3, graph.Nodes.Count());
Assert.AreEqual(1, segment.InputEdges.Count());
Assert.AreEqual(1, segment.OutputNodes.Count());
Assert.AreEqual(1, segment.OutputNodes.ElementAt(0).Edges.Count());
Assert.IsTrue(segment.InputEdges.ElementAt(0).Predicate.GetInputs().First().Match('a'));
Assert.AreEqual(1, graph.Nodes.ElementAt(0).Edges.Count());
Assert.AreEqual(1, graph.Nodes.ElementAt(1).Edges.Count());
Assert.AreEqual(1, graph.Nodes.ElementAt(2).Edges.Count());
Assert.AreEqual(capEdge, segment.OutputNodes.ElementAt(0).Edges.ElementAt(0));
}
public void ShouldBeHumanReadableWhenIsAxiomWithBullet()
{
LexicalRule rule;
Sequence predicate;
Terminal item;
predicate = new Sequence();
item = new Terminal('a');
predicate.Items.Add(item);
item = new Terminal('b');
predicate.Items.Add(item);
item = new Terminal('c');
predicate.Items.Add(item);
item = new Terminal('d');
predicate.Items.Add(item);
rule = new LexicalRule()
{
Name = "A", IsAxiom = true
};
rule.Predicate = predicate;
Assert.AreEqual("A*=(abc•d)", rule.ToString(item));
//Assert.AreEqual("A=(abcd)¤", rule.ToString(Reduce.Instance));
}
public LexicalRule Add(string pat, Sym sym)
{
var rule = new LexicalRule(pat, sym);
Add(rule);
return rule;
}
public void ShouldBuildSequencePredicate()
{
Terminal a, b, c;
Sequence predicate;
ISituationGraph <char> graph;
LexicalRule rule;
SituationGraphFactory <char> situationGraphFactory;
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
a = new Terminal('a');
b = new Terminal('b');
c = new Terminal('c');
predicate = new Sequence();
predicate.Items.Add(a);
predicate.Items.Add(b);
predicate.Items.Add(c);
rule = new LexicalRule()
{
Name = "A", Predicate = predicate
};
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
Assert.IsTrue(graph.Contains(a));
Assert.IsTrue(graph.Contains(b));
Assert.IsTrue(graph.Contains(c));
}
/// <summary>
/// Opens the lexical rule editor dialog for the selected rule.
/// </summary>
/// <param name="sender">
/// A <see cref="System.Object"/>
/// </param>
/// <param name="args">
/// A <see cref="EventArgs"/>
/// </param>
private void OnEditBtnClicked(object sender, EventArgs args)
{
TreeIter selected;
rulesTV.Selection.GetSelected(out selected);
LexicalRule rule = rulesStore.GetValue(selected, 2) as LexicalRule;
LexicalRuleEditorDialog dialog =
new LexicalRuleEditorDialog(lexicalRulesManagerDialog);
dialog.Rule = rule;
ResponseType res = dialog.Show();
if (res == ResponseType.Ok)
{
// The modified rule.
rule = dialog.Rule;
rulesStore.SetValue(selected, 0, rule.Name);
rulesStore.SetValue(selected,
1,
String.Join(" | ",
rule.LexicalExpressions.ToArray()));
rulesStore.SetValue(selected, 2, rule);
}
dialog.Destroy();
}
/// <summary>
/// <see cref="SequenceBeingMatcchedArgs"/>'s contructor.
/// </summary>
/// <param name="joinedToken">
/// A <see cref="Token"/>
/// </param>
/// <param name="matchingRule">
/// A <see cref="LexicalRule"/>
/// </param>
/// <param name="found">
/// A <see cref="System.Boolean"/>
/// </param>
public SequenceBeingMatchedArgs(Token joinedToken,
LexicalRule matchingRule,
bool found)
{
this.joinedToken = joinedToken;
this.matchingRule = matchingRule;
this.found = found;
}
public void ShouldDevelopSituationWithLeftRecursiveReduction()
{
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
LexicalRule rule1, rule2, rule3;
Situation <char> situation;
ISituationCollection <char> situations;
Terminal p1;
NonTerminal p2;
Terminal p3;
Terminal p4;
Sequence sequence;
SituationCollectionFactory <char> factory;
//A*=•{S}a
//S=•{S}b
//S=•c
p1 = new Terminal('a');
p2 = new NonTerminal("S");
p3 = new Terminal('b');
p4 = new Terminal('c');
sequence = new Sequence();
sequence.Items.Add(p2);
sequence.Items.Add(p1);
rule1 = new LexicalRule()
{
Name = "A", Predicate = sequence
};
sequence = new Sequence();
sequence.Items.Add(p2);
sequence.Items.Add(p3);
rule2 = new LexicalRule()
{
Name = "S", Predicate = sequence
};
rule3 = new LexicalRule()
{
Name = "S", Predicate = p4
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(new LexicalRule[] { rule1, rule2, rule3 });
factory = new SituationCollectionFactory <char>(graph);
situation = new Situation <char>(rule1, p2, new MockedReduceInput());
situations = factory.Develop(situation.AsEnumerable());
Assert.AreEqual(5, situations.Count);
Assert.AreEqual('a', ((ITerminalInput <char>)situations[1].Input).Value);
Assert.AreEqual('b', ((ITerminalInput <char>)situations[2].Input).Value);
Assert.AreEqual('a', ((ITerminalInput <char>)situations[3].Input).Value);
Assert.AreEqual('b', ((ITerminalInput <char>)situations[4].Input).Value);
}
public CSSGrammar()
{
Rules = new List <LexicalRule>()
{
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^(('(?:[^']|'')*'|'(?:\\.|[^\\']|)*('|\\b))|(\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b)))", RegexOptions.IgnoreCase),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"@import",
"@media"
),
},
// Selectors
new LexicalRule()
{
Type = TokenType.Builtins,
RegExpression = new Regex("^([^\r\n,{}]+)(,(?=[^}]*{)|\\s*(?={))"),
},
// Attributes
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = new Regex("^[\\w\\d#:\\s,-\\.]*-?[\\w\\d#:\\s,-\\.]+(?=;)"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public void ShouldDevelopSituations()
{
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
LexicalRule rule1, rule2, rule3, rule4;
Situation <char> situation;
ISituationCollection <char> situations;
NonTerminal p1, p2;
Terminal p3, p4;
Sequence sequence;
SituationCollectionFactory <char> factory;
p1 = new NonTerminal("B");
p2 = new NonTerminal("C");
p3 = new Terminal('b');
p4 = new Terminal('c');
sequence = new Sequence();
sequence.Items.Add(p1);
sequence.Items.Add(new Terminal('d'));
rule1 = new LexicalRule()
{
Name = "A", Predicate = sequence
};
rule2 = new LexicalRule()
{
Name = "B", Predicate = p2
};
rule3 = new LexicalRule()
{
Name = "B", Predicate = p3
};
rule4 = new LexicalRule()
{
Name = "C", Predicate = p4
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(new LexicalRule[] { rule1, rule2, rule3, rule4 });
factory = new SituationCollectionFactory <char>(graph);
situation = new Situation <char>(rule1, p1, new MockedReduceInput());
situations = factory.Develop(situation.AsEnumerable());
Assert.AreEqual(4, situations.Count);
Assert.AreEqual(p1, situations[0].Predicate);
Assert.AreEqual(p2, situations[1].Predicate);
Assert.AreEqual('d', ((ITerminalInput <char>)situations[1].Input).Value);
Assert.AreEqual(p3, situations[2].Predicate);
Assert.AreEqual('d', ((ITerminalInput <char>)situations[2].Input).Value);
Assert.AreEqual(p4, situations[3].Predicate);
Assert.AreEqual('d', ((ITerminalInput <char>)situations[3].Input).Value);
}
public void ShouldCreateNextSituationsUsingLettersRange()
{
Terminal a, c;
AnyTerminal b;
Sequence predicate;
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
ISituation <char>[] items;
LexicalRule rule;
Situation <char> situation;
SituationCollectionFactory <char> factory;
a = new Terminal('a');
b = new AnyTerminal();
c = new Terminal('c');
predicate = new Sequence();
predicate.Items.Add(a);
predicate.Items.Add(b);
predicate.Items.Add(c);
predicate.Items.Add(new Reduce());
rule = new LexicalRule()
{
Name = "A", Predicate = predicate
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
factory = new SituationCollectionFactory <char>(graph);
situation = new Situation <char>(rule, a, new MockedReduceInput());
items = factory.CreateNextSituations(situation.AsEnumerable(), new TerminalInput('b')).ToArray();
Assert.AreEqual(0, items.Length);
situation = new Situation <char>(rule, a, new MockedReduceInput());
items = factory.CreateNextSituations(situation.AsEnumerable(), new TerminalInput('a')).ToArray();
Assert.AreEqual(1, items.Length);
Assert.AreEqual(b, items[0].Predicate);
situation = new Situation <char>(rule, b, new MockedReduceInput());
items = factory.CreateNextSituations(situation.AsEnumerable(), new TerminalInput('b')).ToArray();
Assert.AreEqual(1, items.Length);
Assert.AreEqual(c, items[0].Predicate);
situation = new Situation <char>(rule, b, new MockedReduceInput());
items = factory.CreateNextSituations(situation.AsEnumerable(), new TerminalRangeInput(char.MinValue, char.MaxValue)).ToArray();
Assert.AreEqual(1, items.Length);
Assert.AreEqual(c, items[0].Predicate);
situation = new Situation <char>(rule, c, new MockedReduceInput());
items = factory.CreateNextSituations(situation.AsEnumerable(), new TerminalInput('c')).ToArray();
Assert.AreEqual(1, items.Length);
Assert.IsTrue(items[0].Predicate.Equals(new Reduce()));
}
/// <summary>
/// Launches the <see cref="SequenceBeingMatched"/> event.
/// </summary>
/// <param name="joinedToken">
/// A <see cref="Token"/>
/// </param>
/// <param name="matcherRule">
/// A <see cref="LexicalRule"/>
/// </param>
/// <param name="found">
/// If the token is a valid token.
/// </param>
protected void SequenceBeingMatchedInvoker(Token joinedToken,
LexicalRule matcherRule,
bool found)
{
if (this.SequenceBeingMatched != null)
{
SequenceBeingMatchedArgs args =
new SequenceBeingMatchedArgs(joinedToken, matcherRule, found);
SequenceBeingMatched(this, args);
}
}
public JSONGrammar()
{
Rules = new List <LexicalRule>()
{
// Key marker
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = new Regex("^\"[^\"]*(?:\\.[^\"]*)*\"(?=\\:)", RegexOptions.IgnoreCase),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^\"[^\"]*(?:\\.[^\"]*)*\"", RegexOptions.IgnoreCase),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"null"
),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public void TestSortOrder()
{
var ra = new LexicalRule("x", Sym.Id);
var rb = new LexicalRule("xx", Sym.Id);
var la = new List<LexicalRule>() { ra, rb };
var lb = new List<LexicalRule>() { rb, ra };
la.Sort();
lb.Sort();
Assert.AreEqual(rb, la[0]);
Assert.AreEqual(rb, lb[0]);
}
public void ShouldDevelopSituationWithLoopReduction()
{
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
LexicalRule rule1, rule2, rule3;
Situation <char> situation;
ISituationCollection <char> situations;
Terminal p1;
NonTerminal p2;
NonTerminal p3;
SituationCollectionFactory <char> factory;
OneOrMore oneOrMore;
//"L=a"
//"N={L}+"
//"A*={N}"
p1 = new Terminal('a');
p2 = new NonTerminal("L");
p3 = new NonTerminal("N");
rule1 = new LexicalRule()
{
Name = "L", Predicate = new Sequence(p1, new Reduce())
};
oneOrMore = new OneOrMore();
oneOrMore.Item = p2;
rule2 = new LexicalRule()
{
Name = "N", Predicate = new Sequence(oneOrMore, new Reduce())
};
rule3 = new LexicalRule()
{
Name = "A", Predicate = new Sequence(p3, new Reduce())
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(new LexicalRule[] { rule1, rule2, rule3 });
factory = new SituationCollectionFactory <char>(graph);
situation = new Situation <char>(rule3, p3, new MockedReduceInput());
situations = factory.Develop(situation.AsEnumerable());
Assert.AreEqual(5, situations.Count);
Assert.AreEqual('a', ((ITerminalInput <char>)situations[2].Input).Value);
Assert.AreEqual('a', ((ITerminalInput <char>)situations[4].Input).Value);
}
/// <summary>
/// Adds a new rule to the list.
/// </summary>
/// <param name="rule">
/// The rule which will be added.
/// </param>
private void AddRule(LexicalRule rule)
{
string expression =
String.Join(" | ", rule.LexicalExpressions.ToArray());
TreeIter newIter = rulesStore.AppendValues(rule.Name,
expression,
rule);
// We select the new row and scroll to it.
rulesTV.Selection.SelectIter(newIter);
rulesTV.ScrollToCell(rulesTV.Selection.GetSelectedRows()[0],
rulesTV.Columns[0],
true, 1.0f, 0);
}
public void ParseTest()
{
LexicalRule digit = LexicalRule.Range('0', '9');
LexerBuilder lexb = new LexerBuilder();
var blank = lexb.DefineLexeme(0, true, LexicalRule.Chars(" \n\t\r").Repeat());
var number = lexb.DefineLexeme(1, digit.Repeat() + (LexicalRule.Char('.') + digit.Repeat() | LexicalRule.Empty));
var plus = lexb.DefineLexeme(2, LexicalRule.Char('+'));
var minus = lexb.DefineLexeme(2, LexicalRule.Char('-'));
var times = lexb.DefineLexeme(2, LexicalRule.Char('*'));
var divide = lexb.DefineLexeme(2, LexicalRule.Char('/'));
var bra = lexb.DefineLexeme(3, LexicalRule.Char('('));
var ket = lexb.DefineLexeme(3, LexicalRule.Char(')'));
var plu = plus.GetParsingRule();
var min = minus.GetParsingRule();
var mul = times.GetParsingRule();
var div = divide.GetParsingRule();
var br = bra.GetParsingRule();
var ke = ket.GetParsingRule();
var num = number.GetParsingRule(i => double.Parse(i.Text));
ParsingRuleContainer <double> expr = new ParsingRuleContainer <double>();
ParsingRuleContainer <double> term = new ParsingRuleContainer <double>();
ParsingRuleContainer <double> factor = new ParsingRuleContainer <double>();
// ParsingRuleContainer<int, double> bracket = new ParsingRuleContainer<int, double>();
expr.Content
= term.Concat((plu.Concat(term, (t, y) => y) | min.Concat(term, (t, y) => - y)).Repeat(i => i.Sum()), (x, y) => x + y)
| term;
term.Content
= factor.Concat((mul.Concat(term, (s, y) => y) | (div.Concat(term, (s, y) => 1 / y))).Repeat(t => t.Count() == 0 ? 1 : t.Aggregate((x, y) => x * y)), (x, y) => x * y)
| factor;
factor.Content
= br.Concat(expr, (s, x) => x).Concat(ke, (x, s) => x)
| num;
string str = "1 * 5 + 2 * 3 / 5 - 3";
BranchedLexer lexer = lexb.GetBranchedLexer(str);
double r;
expr.TryParse(lexer, out r);
Assert.AreEqual(1.0 * 5.0 + 2.0 * 3.0 / 5.0 - 3.0, r);
}
private Token TestToken(string text, LexicalRule rule)
{
TokenSequence tokens = new TokenSequence();
foreach (char c in text)
{
tokens.Append(new Token(c.ToString(), 0, 0, new FloatBitmap(2, 2)));
}
Token res;
if (!rule.Match(tokens, out res))
{
return(null);
}
return(res);
}
public void ShouldBuildAnyLetterPredicate()
{
LexicalPredicate predicate;
ISituationGraph <char> graph;
LexicalRule rule;
SituationGraphFactory <char> situationGraphFactory;
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
predicate = new AnyTerminal();
rule = new LexicalRule()
{
Name = "A", Predicate = new Sequence(predicate, new Reduce())
};
graph = situationGraphFactory.BuildSituationGraph(rule.AsEnumerable());
Assert.AreEqual(3, graph.Nodes.Count());
}
public void LexerTest()
{
LexicalRule letter = LexicalRule.Range('A', 'Z') | LexicalRule.Range('a', 'z');
LexicalRule digit = LexicalRule.Range('0', '9');
LexerBuilder lexb = new LexerBuilder();
Lexeme blank = lexb.DefineLexeme(0, true, LexicalRule.Chars(" \n\t\r").Repeat());
Lexeme id = lexb.DefineLexeme(1, letter + (letter | digit).Repeat());
Lexeme keyword = lexb.DefineLexeme(2, LexicalRule.Literal("var") | LexicalRule.Literal("function") |
LexicalRule.Literal("new") | LexicalRule.Literal("this") | LexicalRule.Literal("for") |
LexicalRule.Literal("return"));
Lexeme number = lexb.DefineLexeme(3, digit.Repeat() + (LexicalRule.Char('.') + digit.Repeat() | LexicalRule.Empty));
Lexeme inc = lexb.DefineLexeme(4, LexicalRule.Literal("++"));
Lexeme oper = lexb.DefineLexeme(4, LexicalRule.Chars("+-*/^=<>"));
Lexeme str = lexb.DefineLexeme(5, LexicalRule.Char('\'') +
(LexicalRule.NotChar('\'') | LexicalRule.Literal(@"\'")).Repeat() + LexicalRule.Char('\''));
Lexeme bracket = lexb.DefineLexeme(6, LexicalRule.Chars("()[]{}"));
Lexeme deli = lexb.DefineLexeme(7, LexicalRule.Chars(",;:"));
Lexeme comm = lexb.DefineLexeme(10, true, LexicalRule.Literal("//") +
LexicalRule.NotChars("\n\r").Repeat() + LexicalRule.Chars("\n\r"));
Lexeme commul = lexb.DefineLexeme(10, true, LexicalRule.Literal("/*") +
(LexicalRule.Char('/') | LexicalRule.Char('*').Repeat() + LexicalRule.NotChars("/*")).Repeat() +
LexicalRule.Char('*') + LexicalRule.Char('/'));
var input = System.IO.File.ReadAllText("test_data/1.input.txt");
var expected = System.IO.File.ReadAllText("test_data/1.expected.txt");
string actual;
{
var sb = new System.Text.StringBuilder();
BranchedLexer blexer = lexb.GetBranchedLexer(input);
Token t;
while ((t = blexer.Read()) != null)
{
sb.AppendLine(t.ToString());
}
actual = sb.ToString();
}
if (expected != actual)
{
System.IO.File.WriteAllText("test_data/1.actual.txt", actual);
Assert.Fail();
}
}
public void ShouldDevelopSituationEndingWithReductionOnNonTerminal()
{
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
LexicalRule rule1, rule2;
Situation <char> situation;
ISituationCollection <char> situations;
Terminal p1;
NonTerminal p2;
Terminal p3;
Sequence sequence;
SituationCollectionFactory <char> factory;
p1 = new Terminal('a');
p2 = new NonTerminal("B");
p3 = new Terminal('b');
sequence = new Sequence();
sequence.Items.Add(p1);
sequence.Items.Add(p2);
sequence.Items.Add(new Reduce());
rule1 = new LexicalRule()
{
Name = "A", Predicate = sequence
};
rule2 = new LexicalRule()
{
Name = "B", Predicate = p3
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(new LexicalRule[] { rule1, rule2 });
factory = new SituationCollectionFactory <char>(graph);
situation = new Situation <char>(rule1, p2, new MockedReduceInput());
situations = factory.Develop(situation.AsEnumerable());
Assert.AreEqual(2, situations.Count);
Assert.AreEqual(p3, situations[1].Predicate);
}
public void Add(LexicalRule rule)
{
if (!rule.StartChar.HasValue)
{
//Then this is a 'special' rule, such as an identifier
//token matcher
specialRules.Add(rule);
return;
}
char start = rule.StartChar.GetValueOrDefault();
List<LexicalRule> rulesForStartChar = null;
if (!rules.TryGetValue(start, out rulesForStartChar))
{
rulesForStartChar = new List<LexicalRule>();
rules.Add(start, rulesForStartChar);
}
rulesForStartChar.Add(rule);
rulesForStartChar.Sort();
}
public void ShouldNotBuildOneOrMoreSegment()
{
SituationGraphSegmentFactory <char> factory;
ISituationGraph <char> graph;
LexicalRule rule;
MockedSituationEdge capEdge;
OneOrMore predicate;
capEdge = new MockedSituationEdge();
graph = new SituationGraph <char>();
factory = new SituationGraphSegmentFactory <char>();
predicate = new OneOrMore();
rule = new LexicalRule()
{
Name = "A", Predicate = predicate
};
Assert.ThrowsException <ArgumentNullException>(() => factory.BuildSegment(null, rule, predicate, capEdge.AsEnumerable()));
Assert.ThrowsException <ArgumentNullException>(() => factory.BuildSegment(graph, null, predicate, capEdge.AsEnumerable()));
Assert.ThrowsException <ArgumentNullException>(() => factory.BuildSegment(graph, rule, null as OneOrMore, capEdge.AsEnumerable()));
Assert.ThrowsException <ArgumentNullException>(() => factory.BuildSegment(graph, rule, predicate, null));
}
public void ShouldCreateAxiomSituations()
{
Terminal a, b, c;
Sequence predicate;
SituationGraphFactory <char> situationGraphFactory;
ISituationGraph <char> graph;
ISituation <char>[] items;
LexicalRule rule1, rule2;
SituationCollectionFactory <char> factory;
a = new Terminal('a');
b = new Terminal('b');
c = new Terminal('c');
predicate = new Sequence();
predicate.Items.Add(a);
predicate.Items.Add(b);
predicate.Items.Add(c);
rule1 = new LexicalRule()
{
Name = "A", Predicate = predicate, IsAxiom = true
};
rule2 = new LexicalRule()
{
Name = "A", Predicate = predicate, IsAxiom = false
};
situationGraphFactory = new SituationGraphFactory <char>(new SituationGraphSegmentFactory <char>());
graph = situationGraphFactory.BuildSituationGraph(new LexicalRule[] { rule1, rule2 });
factory = new SituationCollectionFactory <char>(graph);
items = factory.CreateAxiomSituations().ToArray();
Assert.AreEqual(1, items.Length);
Assert.AreEqual(rule1, items[0].Rule);
Assert.AreEqual(a, items[0].Predicate);
}
public void ShouldBuildAutomatonTableFromExtendedSequence()
{
AutomatonTableFactory <char> factory;
IAutomatonTable <char> automatonTable;
LexicalRule rule;
Sequence predicate;
AutomatonTableParser parser;
factory = new AutomatonTableFactory <char>( );
predicate = new Sequence();
predicate.Items.Add(new Terminal('a'));
predicate.Items.Add(new Terminal('b'));
predicate.Items.Add(new Terminal('c'));
rule = new LexicalRule()
{
Name = "rule", IsAxiom = true
};
rule.Predicate = new ZeroOrMore()
{
Item = predicate
};
automatonTable = factory.BuildAutomatonTable(SituationCollectionFactoryHelper.BuildSituationCollectionFactory(rule.AsEnumerable()), new DistinctInputFactory(new RangeValueProvider()));
Assert.IsNotNull(automatonTable);
parser = new AutomatonTableParser(automatonTable);
Assert.AreEqual(1, parser.ActionCount);
Assert.IsTrue(parser.Parse(new TerminalInput('a')));
Assert.AreEqual(1, parser.ActionCount);
Assert.IsTrue(parser.Parse(new TerminalInput('b')));
Assert.AreEqual(1, parser.ActionCount);
Assert.IsTrue(parser.Parse(new TerminalInput('c')));
Assert.AreEqual(1, parser.ActionCount);
Assert.IsTrue(parser.Parse(new TerminalInput('a')));
}
public void MatchTextTest()
{
LexicalRule rule = new LexicalRule();
rule.Name = "NUMBER";
rule.LexicalExpressions.Add(@"[0-9]+");
rule.LexicalExpressions.Add(@"[0-9]+\.[0-9]+");
// Should accept
string number = "290";
Token token = TestToken(number, rule);
Assert.IsNotNull(token, "El token es nulo para {0}", number);
Assert.AreEqual("NUMBER", token.Type,
"El tipo del token no es correcto para {0}", number);
Assert.AreEqual(number, token.Text,
"El texto del token no es correcto para {0}", number);
// Should accept
number = "290.23";
token = TestToken(number, rule);
Assert.IsNotNull(token, "El token es nulo para {0}", number);
Assert.AreEqual("NUMBER", token.Type,
"El tipo del token no es correcto para {0}", number);
Assert.AreEqual(number, token.Text,
"El texto del token no es correcto para {0}", number);
// Should fail
number = "2fdsf90.23";
token = TestToken(number, rule);
Assert.IsNull(token, "El token no es nulo para {0}", number);
}
public void ShouldBeHumanReadable()
{
LexicalRule rule;
Sequence predicate;
Terminal item;
predicate = new Sequence();
item = new Terminal('a');
predicate.Items.Add(item);
item = new Terminal('b');
predicate.Items.Add(item);
item = new Terminal('c');
predicate.Items.Add(item);
item = new Terminal('d');
predicate.Items.Add(item);
rule = new LexicalRule()
{
Name = "A"
};
rule.Predicate = predicate;
Assert.AreEqual("A=(abcd)", rule.ToString());
}
public void TestLexicalMatch1()
{
var lexer = new Lexer("foo bar");
var rule = new LexicalRule("foo", Sym.KwClass);
Assert.IsNotNull(rule.Match(lexer));
}
public CPlusPlusGrammar()
{
Rules = new List <LexicalRule>()
{
// Single line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(\\/\\/\\/?[^\r\n]*)"),
},
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^(\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b))", RegexOptions.IgnoreCase),
},
// Char Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^('(\\w\\d){1}')", RegexOptions.IgnoreCase),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"NULL",
"nullptr"
),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Single Char Operator
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^[\\+\\-\\*/%&|\\^~<>!]"),
},
// Double character comparison operators
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^((==)|(!=)|(<=)|(>=)|(<<)|(>>>?)|(//)|(\\*\\*))"),
},
// Single Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^[\\(\\)\\[\\]\\{\\}@,:`=;\\.]"),
},
// Double Char Operator
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((\\+=)|(\\-=)|(\\*=)|(%=)|(/=)|(\\++)|(\\--))"),
},
// Triple Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((//=)|(>>=)|(<<=)|(\\*\\*=))"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"int",
"float",
"while",
"private",
"char",
"catch",
"import",
"module",
"export",
"virtual",
"operator",
"sizeof",
"dynamic_cast|10",
"typedef",
"const_cast|10",
"const",
"for",
"static_cast|10",
"union",
"namespace",
"unsigned",
"long",
"volatile",
"static",
"protected",
"bool",
"template",
"mutable",
"if",
"public",
"friend",
"do",
"goto",
"auto",
"void",
"enum",
"else",
"break",
"extern",
"using",
"asm",
"case",
"typeid",
"short",
"retinterpret_cast|10",
"default",
"double",
"register",
"explicit",
"signed",
"typename",
"try",
"this",
"switch",
"continue",
"inline",
"delete",
"alignof",
"constexpr",
"decltype",
"noexcept",
"static_assert",
"thread_local",
"restrict",
"_Bool",
"complex",
"_Complex",
"_Imaginary",
"atomic_bool",
"atomic_char",
"atomic_schar",
"atomic_uchar",
"atomic_short",
"atomic_ushort",
"atomic_int",
"atomic_uint",
"atomic_long",
"atomic_ulong",
"atomic_llong",
"atomic_ullong",
"new",
"throw",
"return",
"and",
"or",
"not"
),
},
// Commonly-used classes (generally part of `System` namespace)
new LexicalRule()
{
Type = TokenType.Builtins,
RegExpression = LexicalRule.WordRegex(
"std",
"string",
"cin",
"cout",
"cerr",
"clog",
"stdin",
"stdout",
"stderr",
"stringstream",
"istringstream",
"ostringstream",
"auto_ptr",
"deque",
"list",
"queue",
"stack",
"vector",
"map",
"set",
"bitset",
"multiset",
"multimap",
"unordered_set",
"unordered_map",
"unordered_multiset",
"unordered_multimap",
"array",
"shared_ptr",
"abort",
"abs",
"acos",
"asin",
"atan2",
"atan",
"calloc",
"ceil",
"cost",
"cos",
"exit",
"exp",
"fabs",
"floor",
"fmod",
"fprintf",
"fputs",
"free",
"frexp",
"fscanf",
"isalnum",
"isalpha",
"iscntrl",
"isdigit",
"isgraph",
"islower",
"isprint",
"ispunct",
"isspace",
"isupper",
"isxdigit",
"tolower",
"toupper",
"labs",
"ldexp",
"log10",
"log",
"malloc",
"realloc",
"memchr",
"memcmp",
"memcpy",
"memset",
"modf",
"pow",
"printf",
"putchar",
"puts",
"scanf",
"sinh",
"sin",
"snprintf",
"sprintf",
"sqrt",
"sscanf",
"strcat",
"strchr",
"strcmp",
"strcpy",
"strcspn",
"strlen",
"strncat",
"strncmp",
"strncpy",
"strpbrk",
"strrchr",
"strspn",
"strstr",
"tanh",
"tan",
"vfprintf",
"vprintf",
"vsprintf",
"endl",
"initializer_list",
"unique_ptr"
),
},
// Identifier
new LexicalRule()
{
Type = TokenType.Identifier,
RegExpression = new Regex("^[_A-Za-z][_A-Za-z0-9]*")
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
private void AddRule(LexicalRule rule)
{
table.Add(rule);
}
public CSharpGrammar()
{
Rules = new List <LexicalRule>()
{
// Single line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(\\/\\/\\/?[^\r\n]*)"),
},
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^([@|$]\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b))", RegexOptions.IgnoreCase),
},
// Char Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^('(\\w\\d){1}')", RegexOptions.IgnoreCase),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"null"
),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Single Char Operator
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^[\\+\\-\\*/%&|\\^~<>!]"),
},
// Double character comparison operators
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^((==)|(!=)|(<=)|(>=)|(<<)|(>>>?)|(//)|(\\*\\*))"),
},
// Single Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^[\\(\\)\\[\\]\\{\\}@,:`=;\\.]"),
},
// Double Char Operator
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((\\+=)|(\\-=)|(\\*=)|(%=)|(/=)|(\\++)|(\\--))"),
},
// Triple Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((//=)|(>>=)|(<<=)|(\\*\\*=))"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"abstract",
"as",
"base",
"bool",
"break",
"byte",
"case",
"catch",
"char",
"checked",
"class",
"const",
"continue",
"decimal",
"default",
"delegate",
"do",
"double",
"else",
"enum",
"event",
"explicit",
"extern",
"finally",
"fixed",
"float",
"for",
"foreach",
"goto",
"if",
"implicit",
"in",
"int",
"interface",
"internal",
"is",
"lock",
"long",
"namespace",
"new",
"object",
"operator",
"out",
"override",
"params",
"private",
"protected",
"public",
"readonly",
"ref",
"return",
"sbyte",
"sealed",
"short",
"sizeof",
"stackalloc",
"static",
"string",
"struct",
"switch",
"this",
"throw",
"try",
"typeof",
"uint",
"ulong",
"unchecked",
"unsafe",
"ushort",
"using",
"void",
"volatile",
"while"
),
},
// Commonly-used classes (usually part of `System` namespace)
new LexicalRule()
{
Type = TokenType.Builtins,
RegExpression = LexicalRule.WordRegex(
"Boolean",
"Byte",
"Console",
"SByte",
"Char",
"Decimal",
"Double",
"Enum",
"Single",
"Int32",
"UInt32",
"Int64",
"UInt64",
"Object",
"Int16",
"UInt16",
"String",
"StringBuilder",
"Exception",
"Guid",
"DateTime",
"DateTimeOffset",
"TimeSpan",
"Uri",
"UriKind",
"StringBuilder"
),
},
// Identifiers
new LexicalRule()
{
Type = TokenType.Identifier,
RegExpression = new Regex("^[_A-Za-z][_A-Za-z0-9]*")
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public SQLGrammar()
{
Rules = new List <LexicalRule>()
{
// Single-line Comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(--|#)[^\r\n]*"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s")
},
// Multi-line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^(('(?:[^']|'')*'|'(?:\\.|[^\\']|)*('|\\b))|(\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b)))", RegexOptions.IgnoreCase),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"null"
),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegexCaseInsensitive(
"abort",
"action",
"add",
"after",
"all",
"alter",
"analyze",
"and",
"as",
"asc",
"attach",
"autoincrement",
"before",
"begin",
"between",
"by",
"cascade",
"case",
"cast",
"check",
"collate",
"column",
"commit",
"conflict",
"constraint",
"create",
"cross",
"current_date",
"current_time",
"current_timestamp",
"database",
"default",
"deferrable",
"deferred",
"delete",
"desc",
"detach",
"distinct",
"drop",
"each",
"else",
"end",
"escape",
"except",
"exclusive",
"exists",
"explain",
"fail",
"for",
"foreign",
"from",
"full",
"glob",
"group",
"having",
"if",
"ignore",
"immediate",
"in",
"index",
"indexed",
"initially",
"inner",
"insert",
"instead",
"intersect",
"into",
"is",
"isnull",
"join",
"key",
"left",
"like",
"limit",
"match",
"natural",
"no",
"not",
"notnull",
"null",
"of",
"offset",
"on",
"or",
"order",
"outer",
"plan",
"pragma",
"primary",
"query",
"raise",
"recursive",
"references",
"regexp",
"reindex",
"release",
"rename",
"replace",
"restrict",
"right",
"rollback",
"row",
"savepoint",
"select",
"set",
"table",
"temp",
"temporary",
"then",
"to",
"transaction",
"trigger",
"union",
"unique",
"update",
"using",
"vacuum",
"values",
"view",
"virtual",
"when",
"where",
"with",
"without"
),
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public JavascriptGrammar()
{
Rules = new List <LexicalRule>()
{
// Single line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(\\/\\/[^\r\n]*)"),
},
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^((@'(?:[^']|'')*'|'(?:\\.|[^\\']|)*('|\\b))|(@\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b)))", RegexOptions.IgnoreCase),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"undefined",
"null"
),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Single Char Operator
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^[\\+\\-\\*/%&|\\^~<>!]"),
},
// Double/Triple character comparison operators
new LexicalRule()
{
Type = TokenType.Operator,
RegExpression = new Regex("^((===?)|(!==?)|(<==?)|(>==?)|(<<)|(>>>?)|(//)|(\\*\\*))"),
},
// Single Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^[\\(\\)\\[\\]\\{\\}@,:`=;\\.]"),
},
// Double Char Operator
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((\\+=)|(\\-=)|(\\*=)|(%=)|(/=)|(\\++)|(\\--))"),
},
// Triple Delimiter
new LexicalRule()
{
Type = TokenType.Delimiter,
RegExpression = new Regex("^((//=)|(>>=)|(<<=)|(\\*\\*=))"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"arguments",
"break",
"case",
"catch",
"class",
"const",
"continue",
"debugger",
"default",
"delete",
"do",
"else",
"export",
"extends",
"finally",
"for",
"function",
"if",
"import",
"in",
"instanceof",
"new",
"return",
"super",
"switch",
"this",
"throw",
"try",
"typeof",
"var",
"void",
"while",
"with",
"yield",
"enum",
"implements",
"interface",
"let",
"package",
"await",
"private",
"protected",
"public",
"static",
"console"
),
},
// Built in objects
new LexicalRule()
{
Type = TokenType.Builtins,
RegExpression = LexicalRule.WordRegex(
"Infinity",
"NaN",
"Object",
"Function",
"Boolean",
"Symbol",
"Error",
"EvalError",
"InternalError",
"RangeError",
"ReferenceError",
"SyntaxError",
"TypeError",
"URIError",
"Number",
"Math",
"Date",
"String",
"RegExp",
"Array",
"Int8Array",
"Uint8Array",
"Uint8ClampedArray",
"Int16Array",
"Uint16Array",
"Int32Array",
"Uint32Array",
"Float32Array",
"Float64Array",
"Map",
"Set",
"WeakMap",
"WeakSet",
"ArrayBuffer",
"DataView",
"JSON",
"Promise",
"Generator",
"GeneratorFunction",
"Reflect",
"Proxy",
"Intl",
"WebAssembly"
),
},
// Identifiers
new LexicalRule()
{
Type = TokenType.Identifier,
RegExpression = new Regex("^[_A-Za-z][_A-Za-z0-9]*")
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public RubyGrammar()
{
Rules = new List <LexicalRule>()
{
// Single line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(#[^\r\n]*)"),
},
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^=begin([^*])*\\=end"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^(('(?:[^']|'')*'|'(?:\\.|[^\\']|)*('|\\b))|(\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b)))", RegexOptions.IgnoreCase),
},
// Function/class signatures
new LexicalRule()
{
Type = TokenType.Builtins,
// TODO: Get this working as expected
RegExpression = new Regex(@"^(?<=(def|class)\s*)([\w\d\:]+)(?=[\s])", RegexOptions.IgnoreCase),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"true",
"false",
"nil"
),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"and",
"then",
"def",
"defined",
"module",
"in",
"return",
"redo",
"if",
"BEGIN",
"retry",
"end",
"for",
"self",
"when",
"next",
"until",
"do",
"begin",
"unless",
"END",
"rescue",
"else",
"break",
"undef",
"not",
"super",
"class",
"case",
"require",
"raise",
"yield",
"alias",
"while",
"ensure",
"elsif",
"or",
"include",
"attr_reader",
"attr_writer",
"attr_accessor"
),
},
// Identifiers
new LexicalRule()
{
Type = TokenType.Identifier,
RegExpression = new Regex("^[_A-Za-z][_A-Za-z0-9]*")
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}
public PHPGrammar()
{
Rules = new List <LexicalRule>()
{
// Single line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^(\\/\\/[^\r\n]*)"),
},
// Multi line comment
new LexicalRule()
{
Type = TokenType.Comment,
RegExpression = new Regex("^\\/\\*(\\*(?!\\/)|[^*])*\\*\\/"),
},
// String Marker
new LexicalRule()
{
Type = TokenType.String,
RegExpression = new Regex("^(('(?:[^']|'')*'|'(?:\\.|[^\\']|)*('|\\b))|(\"(?:[^\"]|\"\")*\"|\"(?:\\.|[^\\\"])*(\"|\\b)))", RegexOptions.IgnoreCase),
},
// Literals
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = LexicalRule.WordRegex(
"FALSE",
"TRUE",
"NULL"
),
},
// Numbers
new LexicalRule()
{
Type = TokenType.Number,
RegExpression = new Regex("^\\d+(((\\.)|(x))\\d*)?"),
},
// Whitespace
new LexicalRule()
{
Type = TokenType.WhiteSpace,
RegExpression = new Regex("^\\s"),
},
// Keywords
new LexicalRule()
{
Type = TokenType.Keyword,
RegExpression = LexicalRule.WordRegex(
"and",
"include_once",
"list",
"abstract",
"global",
"private",
"echo",
"interface",
"as",
"static",
"endswith",
"array",
"null",
"if",
"endwhile",
"or",
"const",
"for",
"endforeach",
"self",
"var",
"while",
"isset",
"public",
"protected",
"exit",
"foreach",
"throw",
"elseif",
"include",
"__FILE__",
"empty",
"require_once",
"do",
"xor",
"return",
"parent",
"clone",
"use",
"__CLASS__",
"__LINE__",
"else",
"break",
"print",
"eval",
"new",
"catch",
"__METHOD__",
"case",
"exception",
"default",
"die",
"require",
"__FUNCTION__",
"enddeclare",
"final",
"try",
"switch",
"continue",
"endfor",
"endif",
"declare",
"unset",
"true",
"false",
"trait",
"goto",
"instanceof",
"insteadof",
"__DIR__",
"__NAMESPACE__",
"yield",
"finally"
),
},
new LexicalRule()
{
Type = TokenType.Builtins,
RegExpression = LexicalRule.WordRegex(
"\\$GLOBALS",
"\\$_SERVER",
"\\$_REQUEST",
"\\$_POST",
"\\$_GET",
"\\$_FILES",
"\\$_ENV",
"\\$_COOKIE",
"\\$_SESSION"
),
},
// Identifiers
new LexicalRule()
{
Type = TokenType.Identifier,
RegExpression = new Regex("^[_A-Za-z][_A-Za-z0-9]*")
},
// Any
new LexicalRule()
{
Type = TokenType.Unknown,
RegExpression = new Regex("^."),
},
};
}