public void ParseRunnerShouldEmitTokenWhenIgnoreCharacterIsEncountered()
{
const string input = "aa aa";
ProductionExpression S = "S";
S.Rule = _wordRule + S | _wordRule;
var grammar = new GrammarExpression(
S,
new[] { S },
new[] { _whitespaceRule },
null)
.ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
for (int i = 0; i < 2; i++)
{
Assert.IsTrue(parseRunner.Read());
}
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(2, chart.EarleySets.Count);
}
public void EbnfParserShouldParseComplexGrammarWithRepeat()
{
var stringBuilder = new StringBuilder()
//.AppendLine("file = ws directives ws ;")
.AppendLine("file = \"1\" { \"2\" } \"1\";");
//.AppendLine("directives = directive { ows directive };")
//.AppendLine("directive = \"0\" | \"1\"; ");
var actual = Parse(stringBuilder.ToString());
var grammar = new EbnfGrammar();
var parseEngine = new ParseEngine(grammar, new ParseEngineOptions(false));
var parseRunner = new ParseRunner(parseEngine, stringBuilder.ToString());
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse Ebnf. Error at position {parseRunner.Position}.");
}
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"Unable to parse Ebnf. Error at position {parseRunner.Position}");
}
var parseForest = parseEngine.GetParseForestRootNode();
var visitor = new LoggingForestNodeVisitor(Console.Out);
parseForest.Accept(visitor);
}
#pragma warning disable CC0091 // Use static method
public EbnfDefinition Parse(string ebnf)
{
var grammar = new EbnfGrammar();
var parseEngine = new ParseEngine(
grammar,
new ParseEngineOptions(
optimizeRightRecursion: true,
loggingEnabled: false));
var parseRunner = new ParseRunner(parseEngine, ebnf);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse Ebnf. Error at line {parseRunner.Line}, column {parseRunner.Column}.");
}
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"Ebnf parse not accepted. Error at line {parseRunner.Line}, column {parseRunner.Column}.");
}
var parseForest = parseEngine.GetParseForestRootNode();
var parseTree = new InternalTreeNode(
parseForest as IInternalForestNode,
new SelectFirstChildDisambiguationAlgorithm());
var ebnfVisitor = new EbnfVisitor();
parseTree.Accept(ebnfVisitor);
return(ebnfVisitor.Definition);
}
public IInternalTreeNode Parse(string source)
{
var grammar = MakeGrammar();
var parseEngine = new ParseEngine(
grammar,
new ParseEngineOptions(loggingEnabled: true));
var parseRunner = new ParseRunner(parseEngine, source);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse expression. Error at line {parseRunner.Line+1}, column {parseRunner.Column+1}.");
}
Console.WriteLine("-----");
Console.ReadKey(true);
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"expression parse not accepted. Error at line {parseRunner.Line+1}, column {parseRunner.Column+1}.");
}
return(parseEngine.GetParseTree());
}
public void ParseRunnerWhenNoLexemesMatchCharacterShouldCreateNewLexeme()
{
#if false
const string input = "aaaa";
ProductionExpression A = "A", S = "S";
A.Rule = (Expr)'a' + 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
S.Rule = (a + S) | a;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
for (var i = 0; i < 3; i++)
{
Assert.IsTrue(parseRunner.Read());
}
Assert.AreEqual(2, chart.Count);
#endif
}
public IEbnfDefinition Parse(string ebnf)
{
var grammar = new EbnfGrammar();
var parseEngine = new ParseEngine(
grammar,
new ParseEngineOptions());
var parseRunner = new ParseRunner(parseEngine, ebnf);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse Ebnf. Error at line {parseRunner.Line}, column {parseRunner.Column}.");
}
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"Ebnf parse not accepted. Error at line {parseRunner.Line}, column {parseRunner.Column}.");
}
var parseTree = parseEngine.GetParseTree();
var ebnfVisitor = new EbnfVisitor();
parseTree.Accept(ebnfVisitor);
return(ebnfVisitor.Definition);
}
public Regex Parse(string regularExpression)
{
var grammar = new RegexGrammar();
var parseEngine = new ParseEngine(grammar, new ParseEngineOptions(optimizeRightRecursion: true));
var parseRunner = new ParseRunner(parseEngine, regularExpression);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse regular expression. Error at position {parseRunner.Position}.");
}
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"Error parsing regular expression. Error at position {parseRunner.Position}");
}
var parseForest = parseEngine.GetParseForestRootNode();
var parseTree = new InternalTreeNode(
parseForest as IInternalForestNode,
new SelectFirstChildDisambiguationAlgorithm());
var regexVisitor = new RegexVisitor();
parseTree.Accept(regexVisitor);
return(regexVisitor.Regex);
}
public Regex Parse(string regularExpression)
{
var grammar = new RegexGrammar();
var parseEngine = new ParseEngine(grammar, new ParseEngineOptions());
var parseRunner = new ParseRunner(parseEngine, regularExpression);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
throw new Exception(
$"Unable to parse regular expression. Error at position {parseRunner.Position}.");
}
}
if (!parseEngine.IsAccepted())
{
throw new Exception(
$"Error parsing regular expression. Error at position {parseRunner.Position}");
}
var parseTree = parseEngine.GetParseTree();
var regexVisitor = new RegexVisitor();
parseTree.Accept(regexVisitor);
return(regexVisitor.Regex);
}
private void FailParseAtPosition(string input, int position)
{
var parseRunner = new ParseRunner(this.parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
if (i < position)
{
Assert.IsTrue(parseRunner.Read(),
$"Line 0, Column {this.parseEngine.Location} : Invalid Character {input[i]}");
}
else
{
Assert.IsFalse(parseRunner.Read());
}
}
}
public void ParseRunnerShouldEmitTokenBetweenLexerRulesAndEndOfFile()
{
const string input = "aa";
ProductionExpression S = "S";
S.Rule = 'a' + S | 'a';
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(1, chart.EarleySets.Count);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(3, chart.EarleySets.Count);
}
private static void RunParse(ParseEngine parseEngine, string input)
{
var parseRunner = new ParseRunner(parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(parseRunner.Read(), $"Error parsing at position {i}");
}
Assert.IsTrue(parseRunner.ParseEngine.IsAccepted());
}
private IForestNode ParseInput(string input)
{
var parseRunner = new ParseRunner(_parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(parseRunner.Read(), $"Error found in position {parseRunner.Position}");
}
Assert.IsTrue(parseRunner.ParseEngine.IsAccepted());
return(parseRunner.ParseEngine.GetParseForestRootNode());
}
public bool Parse()
{
var grammar = new JsonGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, json);
while (!parseRunner.EndOfStream() && parseRunner.Read())
{
}
return(parseRunner.ParseEngine.IsAccepted());
}
private void ParseInput(string input)
{
var parseRunner = new ParseRunner(this.parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
if (!parseRunner.Read())
{
Assert.Fail($"Line 0, Column {this.parseEngine.Location} : Invalid Character {input[i]}");
}
}
}
protected void ParseInput(string input)
{
var parseRunner = new ParseRunner(_parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
if (!parseRunner.Read())
{
Assert.Fail($"Line {parseRunner.Line + 1}, Column {parseRunner.Column + 1} : Invalid Character '{input[i]}'");
}
}
}
public static void Main(string[] args)
{
string grammarText = LoadFromResource(nameof(DefaultNamespaceName), "Grammar", "syntax5.ebnf");
//string input = File.ReadAllText("/etc/apache2/httpd.conf", Encoding.UTF8);
string input = "1 1";
var definition = new EbnfParser().Parse(grammarText);
var grammar = new EbnfGrammarGenerator().Generate(definition);
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var recognized = false;
var errorPosition = 0;
while (!parseRunner.EndOfStream())
{
recognized = parseRunner.Read();
if (!recognized)
{
errorPosition = parseRunner.Position;
break;
}
}
var accepted = false;
if (recognized)
{
accepted = parseRunner.ParseEngine.IsAccepted();
if (!accepted)
{
errorPosition = parseRunner.Position;
}
}
Console.WriteLine($"Recognized: {recognized}, Accepted: {accepted}");
if (!recognized || !accepted)
{
Console.Error.WriteLine($"Error at position {errorPosition}");
}
// get the parse forest root from the parse engine
var parseForestRoot = parseEngine.GetParseForestRootNode();
// create a internal tree node and supply the disambiguation algorithm for tree traversal.
var parseTree = new InternalTreeNode(
parseForestRoot,
new SelectFirstChildDisambiguationAlgorithm());
Console.WriteLine(parseTree.ToString());
}
public void ParseRunnerShouldUseExistingMatchingLexemesToPerformMatch()
{
#if false
const string input = "aaaa";
ProductionExpression A = "A";
A.Rule = (Expr)'a' + 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
ProductionExpression S = "S";
S.Rule = (a + S) | a;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(1, chart.Count);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(1, chart.Count);
#endif
}
public void ParseRunnerShouldEmitTokenWhenCharacterMatchesNextProduction()
{
const string input = "aabb";
ProductionExpression A = "A";
A.Rule =
'a' + A
| 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
ProductionExpression B = "B";
B.Rule =
'b' + B
| 'b';
var bGrammar = new GrammarExpression(B, new[] { B }).ToGrammar();
var b = new GrammarLexerRule("b", bGrammar);
ProductionExpression S = "S";
S.Rule = (Expr)
a + b;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var chart = GetParseEngineChart(parseEngine);
var parseRunner = new ParseRunner(parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(parseRunner.Read());
if (i < 2)
{
Assert.AreEqual(1, chart.Count);
}
else if (i < 3)
{
Assert.AreEqual(2, chart.Count);
}
else
{
Assert.AreEqual(3, chart.Count);
}
}
}
public void BnfShouldProduceParseChartForTextGrammar()
{
var grammar = new BnfGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, _bnfText);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
var position = parseRunner.Position;
var startIndex = 0;
for (int i = position; i >= 0; i--)
{
if (_bnfText[i] == '\n' && i > 0)
{
if (_bnfText[i - 1] == '\r')
{
startIndex = i;
break;
}
}
}
var endIndex = _bnfText.IndexOf(
System.Environment.NewLine,
position,
System.StringComparison.CurrentCulture);
endIndex = endIndex < 0 ? _bnfText.Length : endIndex;
var length = endIndex - startIndex;
var stringBuilder = new StringBuilder();
stringBuilder
.Append($"Error parsing input string at position {parseRunner.Position}.")
.AppendLine()
.Append($"start: {startIndex}")
.AppendLine()
.AppendLine(_bnfText.Substring(startIndex, length));
Assert.Fail(stringBuilder.ToString());
}
}
Assert.IsTrue(
parseRunner.ParseEngine.IsAccepted(),
$"error at position {parseRunner.Position}");
}
public void BnfShouldParseLargeGrammarInFile()
{
var bnf = File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(), "Bnf", "AnsiC.bnf"));
Assert.IsFalse(string.IsNullOrEmpty(bnf));
var grammar = new BnfGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, bnf);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
Assert.Fail($"Error Parsing At Position {parseRunner.Position}");
}
}
Assert.IsTrue(parseEngine.IsAccepted());
}
private IForestNode ParseInput(string input)
{
var parseRunner = new ParseRunner(_parseEngine, input);
while (!parseRunner.EndOfStream())
{
if (!parseRunner.Read())
{
Assert.Fail($"Error found in on line {parseRunner.Line}, column {parseRunner.Column}");
}
}
if (!parseRunner.ParseEngine.IsAccepted())
{
Assert.Fail("Parse was not accepted.");
}
return(parseRunner.ParseEngine.GetParseForestRootNode());
}
public void TestProtcolBuffersV3Grammar()
{
var ebnfPath = Path.Combine(TestContext.TestDeploymentDir, "Runtime", GrammarFile);
var ebnf = File.ReadAllText(ebnfPath);
var ebnfGenerator = new EbnfGrammarGenerator();
var ebnfParser = new EbnfParser();
var ebnfDefintion = ebnfParser.Parse(ebnf);
var parseEngine = new ParseEngine(ebnfGenerator.Generate(ebnfDefintion));
var inputPath = Path.Combine(TestContext.TestDeploymentDir, "Runtime", ProtoFile);
var input = File.ReadAllText(inputPath);
var parseRunner = new ParseRunner(parseEngine, input);
while (!parseRunner.EndOfStream())
{
Assert.IsTrue(parseRunner.Read());
}
Assert.IsTrue(parseRunner.ParseEngine.IsAccepted());
}
private static InternalTreeNode GetTreeNode(Grammar grammar, string input)
{
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
while (!parseRunner.EndOfStream())
{
Assert.IsTrue(parseRunner.Read());
}
Assert.IsTrue(parseEngine.IsAccepted());
var parseForest = parseEngine.GetParseForestRootNode();
Assert.IsNotNull(parseForest);
var internalNode = parseForest;
var treeNode = new InternalTreeNode(internalNode);
return(treeNode);
}
private static InternalTreeNode GetTreeNode(IGrammar grammar, string input)
{
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
while (!parseRunner.EndOfStream())
{
Assert.IsTrue(parseRunner.Read());
}
Assert.IsTrue(parseEngine.IsAccepted());
var parseForest = parseEngine.GetParseForestRootNode();
Assert.IsTrue(parseForest is IInternalForestNode);
var internalNode = parseForest as IInternalForestNode;
var disambiguationAlgorithm = new SelectFirstChildDisambiguationAlgorithm();
var treeNode = new InternalTreeNode(internalNode, disambiguationAlgorithm);
return(treeNode);
}
private static void Main(string[] args)
{
var sampleBnf = File.ReadAllText(
Path.Combine(Environment.CurrentDirectory, "10000.json"));
var grammar = new JsonGrammar();
var loopCount = 100;
for (long i = 0; i < loopCount; i++)
{
Console.WriteLine($"Iteration {i} of {loopCount}");
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, sampleBnf);
while (!parseRunner.EndOfStream() && parseRunner.Read())
{
}
var result = parseRunner.ParseEngine.IsAccepted();
}
}
public void NodeVisitorShouldWalkSimpleRegex()
{
var regexGrammar = new RegexGrammar();
var regexParseEngine = new ParseEngine(regexGrammar);
var regexLexer = new ParseRunner(regexParseEngine, @"[(]\d[)]");
while (!regexLexer.EndOfStream())
{
if (!regexLexer.Read())
{
Assert.Fail($"error parsing input at position {regexLexer.Position}");
}
}
Assert.IsTrue(regexParseEngine.IsAccepted());
var nodeVisitor = new LoggingNodeVisitor(
new SelectFirstChildDisambiguationAlgorithm());
var root = regexParseEngine.GetParseForestRootNode();
root.Accept(nodeVisitor);
Assert.AreEqual(31, nodeVisitor.VisitLog.Count);
}
public void NodeVisitorShouldEnumerateAllParseTrees()
{
ProductionExpression And = "AND",
Panda = "Panda",
AAn = "AAn",
ShootsLeaves = "ShootsAndLeaves",
EatsShootsLeaves = "EatsShootsLeaves"
;
And.Rule = (Expr)'a' + 'n' + 'd';
var and = new GrammarExpression(And, new[] { And }).ToGrammar();
Panda.Rule = (Expr)'p' + 'a' + 'n' + 'd' + 'a';
var panda = new GrammarExpression(Panda, new[] { Panda }).ToGrammar();
AAn.Rule = (Expr)'a' | (Expr)'a' + 'n';
var aAn = new GrammarExpression(AAn, new[] { AAn }).ToGrammar();
ShootsLeaves.Rule =
(Expr)"shoots"
| (Expr)"leaves";
var shootsLeaves = new GrammarExpression(ShootsLeaves, new[] { ShootsLeaves }).ToGrammar();
EatsShootsLeaves.Rule =
(Expr)'e' + 'a' + 't' + 's'
| (Expr)'s' + 'h' + 'o' + 'o' + 't' + 's'
| (Expr)'l' + 'e' + 'a' + 'v' + 'e' + 's';
var eatsShootsLeaves = new GrammarExpression(EatsShootsLeaves, new[] { EatsShootsLeaves }).ToGrammar();
ProductionExpression
S = "S", NP = "NP", VP = "VP", NN = "NN",
NNS = "NNS", DT = "DT", CC = "CC", VBZ = "VBZ";
S.Rule =
NP + VP + '.';
NP.Rule =
NN
| NNS
| DT + NN
| NN + NNS
| NNS + CC + NNS;
VP.Rule = VBZ + NP
| VP + VBZ + NNS
| VP + CC + VP
| VP + VP + CC + VP
| VBZ;
CC.Rule = new GrammarLexerRule(nameof(CC), and);
DT.Rule = new GrammarLexerRule(nameof(DT), aAn);
NN.Rule = new GrammarLexerRule(nameof(NN), panda);
NNS.Rule = new GrammarLexerRule(nameof(NNS), shootsLeaves);
VBZ.Rule = new GrammarLexerRule(nameof(VBZ), eatsShootsLeaves);
var grammar = new GrammarExpression(
S,
new[] { S, NP, VP, CC, DT, NN, NNS, VBZ },
new[] { new LexerRuleModel(_whitespace) })
.ToGrammar();
var sentence = "a panda eats shoots and leaves.";
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, sentence);
while (!parseRunner.EndOfStream())
{
Assert.IsTrue(parseRunner.Read(),
$"Error parsing position: {parseRunner.Position}");
}
Assert.IsTrue(parseRunner.ParseEngine.IsAccepted());
}