private void Assign(NonTerminal head, Terminal term, Production production)
{
if (predictiveParseTable[head][term].HasValue)
throw new InvalidGrammarException("Grammar is not LL(1)");
predictiveParseTable[head][term] = production;
}
public void CorrectLineAndColumnNumbers()
{
var input = "Hello World!\nSecond line\r\nFoo";
MemoryStream m = new MemoryStream();
var w = new StreamWriter(m);
w.Write(input);
w.Flush();
m.Position = 0;
var word = new Terminal("Word", "\\S+");
var whitespace = new Terminal("Whitespace", " |\n|\r");
var lexer = new Lexer(m, word, whitespace);
Token[] tokens = lexer.ToArray();
Assert.AreEqual(10, tokens.Length);
AssertToken(tokens[0], "Word", 1, 1);
AssertToken(tokens[1], "Whitespace", 1, 6);
AssertToken(tokens[2], "Word", 1, 7);
AssertToken(tokens[3], "Whitespace", 1, 13);
AssertToken(tokens[4], "Word", 2, 1);
AssertToken(tokens[5], "Whitespace", 2, 7);
AssertToken(tokens[6], "Word", 2, 8);
AssertToken(tokens[7], "Whitespace", 2, 12);
AssertToken(tokens[8], "Whitespace", 2, 13);
AssertToken(tokens[9], "Word", 3, 1);
}
public Token(Terminal terminal, string value, int line, int column)
{
this.Terminal = terminal;
this.Value = value;
this.Line = line;
this.Column = column;
}
public void CastBnfTermIntoConcatenationRule()
{
BnfTerm a = new Terminal("a");
ConcatenationRule c = a;
Assert.IsNotNull(c);
}
public void AlternateTerms()
{
BnfTerm a = new Terminal("a");
BnfTerm b = new Terminal("b");
RuleAlternation c = a | b;
Assert.IsNotNull(c);
}
public void ConcatenateTerms()
{
BnfTerm a = new Terminal("a");
BnfTerm b = new Terminal("b");
ConcatenationRule c = a + b;
Assert.IsNotNull(c);
}
public void DoNotMatchAStringNotAtIndexZero()
{
Terminal t = new Terminal("AAA", "AAA");
string aString = "BBBAAA";
string match;
Assert.IsFalse(t.Match(aString, 0, out match));
Assert.IsNull(match);
}
public void MatchAStringNotAtIndexZeroButWithOffset()
{
Terminal t = new Terminal("AAA", "AAA");
string aString = "BBBAAA";
string match;
Assert.IsTrue(t.Match(aString, 3, out match));
Assert.AreEqual("AAA", match);
}
public void MatchAString()
{
Terminal t = new Terminal("AAA", "AAA");
const string aString = "AAABBB";
string match;
Assert.IsTrue(t.Match(aString, 0, out match));
Assert.AreEqual("AAA", match);
}
public void ConcatenationOfNonTerminalAndTerminal()
{
NonTerminal a = new NonTerminal("a");
Terminal b = new Terminal("b");
var c = a | b;
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(RuleAlternation));
}
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket)
{
Terminal open = new Terminal(Regex.Escape(openBracket));
Terminal close = new Terminal(Regex.Escape(closeBracket));
NonTerminal s = new NonTerminal("S");
NonTerminal sPrime = new NonTerminal("S'");
s.Rules = open + sPrime + close;
sPrime.Rules = s | "a";
return s;
}
private static NonTerminal ConstructGrammar()
{
NonTerminal R = new NonTerminal("R");
NonTerminal S = new NonTerminal("S");
NonTerminal V = new NonTerminal("V");
Terminal id = new Terminal("id", @"([A-Za-z]+)([0-9A-Za-z]*)");
R.Rules = S;
S.Rules = V + "=" + V;
V.Rules = id;
return R;
}
public void InvalidInputStream()
{
var myString = "Hello World!";
MemoryStream m = new MemoryStream();
var w = new StreamWriter(m);
w.Write(myString);
w.Flush();
m.Position = 0;
Terminal singleCharacter = new Terminal("Match a single A", "A");
Lexer t = new Lexer(m, singleCharacter);
Token[] characters = t.ToArray();
}
private static NonTerminal ConstructGrammar(string openBracket, string closeBracket)
{
Terminal open = new Terminal(Regex.Escape(openBracket));
Terminal close = new Terminal(Regex.Escape(closeBracket));
NonTerminal start = new NonTerminal("start");
NonTerminal wbbList = new NonTerminal("wbb List");
NonTerminal openBracketPlusStuff = new NonTerminal("( + L|)");
NonTerminal stuffPlusClose = new NonTerminal("L + ) | )");
start.Rules = openBracketPlusStuff + wbbList;
wbbList.Rules = openBracketPlusStuff + wbbList | "";
openBracketPlusStuff.Rules = open + stuffPlusClose;
stuffPlusClose.Rules = wbbList + close | close;
return start;
}
private static NonTerminal ConstructGrammar()
{
NonTerminal expression = new NonTerminal("expression");
NonTerminal term = new NonTerminal("term");
NonTerminal factor = new NonTerminal("factor");
Terminal number = new Terminal("number", @"[0-9]+(\.[0-9]+)?");
Terminal multiply = new Terminal("*", Regex.Escape("*"));
Terminal divide = new Terminal("/", Regex.Escape("/"));
Terminal add = new Terminal("+", Regex.Escape("+"));
Terminal subtract = new Terminal("-", Regex.Escape("-"));
Terminal openBracket = new Terminal("(", Regex.Escape("("));
Terminal closeBracket = new Terminal(")", Regex.Escape(")"));
expression.Rules = expression + add + term | expression + subtract + term | term;
term.Rules = term + multiply + factor | term + divide + factor | factor;
factor.Rules = openBracket + expression + closeBracket | number;
return expression;
}
public void ConstructAGrammar()
{
NonTerminal op = new NonTerminal("Operator");
NonTerminal expr = new NonTerminal("Expression");
Terminal number = new Terminal("Number", @"\b\d+\b");
Terminal plus = new Terminal("Plus", @"\+");
Terminal multiply = @"\*";
op.Rules = plus | multiply;
expr.Rules = number + op + expr;
Terminal whitespace = new Terminal("Whitespace", " |\n|\r", true);
Grammar g = new Grammar(expr, whitespace);
Assert.AreEqual(3, g.Terminals.Count());
Assert.AreEqual(3, g.NonTerminals.Count()); //Although the grammar only specifies 2 Nonterminals, grammar creates 1 internally for the root
Lexer lexer = g.CreateLexer("1 + 2 * 3");
Assert.AreEqual(5, lexer.Count());
}
public void ConstructTerminal()
{
Terminal t = new Terminal("Test Terminal", ".*");
}
public void IsWhitespaceIgnored()
{
var input = "Hello World!\nSecond line\r\nFoo";
MemoryStream m = new MemoryStream();
var w = new StreamWriter(m);
w.Write(input);
w.Flush();
m.Position = 0;
var letter = new Terminal("Letter", "\\S");
var whitespace = new Terminal("Whitespace", " |\n|\r", true);
var lexer = new Lexer(m, letter, whitespace);
foreach (var match in lexer)
Assert.IsFalse(match.Terminal.IsIgnored);
}
public void CastTerminalIntoConcatenationRule()
{
ConcatenationRule a = new Terminal("aaa");
Assert.IsNotNull(a);
}
public void ReadATokenStream()
{
var myString = "Hello World!";
MemoryStream m = new MemoryStream();
var w = new StreamWriter(m);
w.Write(myString);
w.Flush();
m.Position = 0;
Terminal singleCharacter = new Terminal("Match a single character", ".");
Lexer t = new Lexer(m, singleCharacter);
Token[] characters = t.ToArray();
Assert.AreEqual(myString.Length, characters.Length);
for (int i = 0; i < myString.Length; i++)
Assert.AreEqual(myString[i].ToString(), characters[i].Value);
}
public void ConcatenationOfTypedTerminals()
{
Terminal a = new Terminal("a");
Terminal b = new Terminal("b");
var c = a + b;
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(ConcatenationRule));
}
public void ConcatenationOfStringAndTerminal()
{
Terminal b = new Terminal("b");
var c = "a" | b;
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(RuleAlternation));
}
public void TerminalPlusTerminalReturnsRule()
{
Terminal a = new Terminal("a");
Terminal b = new Terminal("b");
ConcatenationRule operatorResult = a + b;
ConcatenationRule methodResult = Terminal.Plus(a, b);
Assert.IsNotNull(operatorResult);
Assert.IsNotNull(methodResult);
Assert.AreEqual(operatorResult, methodResult);
}
public void ConcatenationOfTerminalAndString()
{
Terminal a = new Terminal("a");
var c = a | "b";
Assert.IsNotNull(c);
Assert.AreEqual(2, c.Count);
Assert.IsInstanceOfType(c, typeof(RuleAlternation));
}