public static void Do() {
// S -> aSa | bSb | ε
var productions = new List<Production> {
// construct productions by passing arguments...
new Production(
lhs: Nonterminal.Of("S"),
rhs: new Sentence { Terminal.Of("a"), Nonterminal.Of("S"), Terminal.Of("a") },
weight: 20
),
// or from a string...
CFGParser.Production(@"<S> -> 'b' <S> 'b' [10]"),
CFGParser.Production(@"<S> -> ε [5]"),
};
var cfg = new Grammar(productions, Nonterminal.Of("S"));
// var cnf = cfg.ToCNF();
//var probs = cfg.EstimateProbabilities(1000000);
//foreach (var entry in probs) {
// var key = entry.Key;
// var value = entry.Value;
// if (key.Length <= 4) {
// Console.WriteLine("{0}: {1}", key, value);
// }
//}
// Print out the new CNF grammar
// Console.WriteLine(cnf);
var ep = new EarleyParser(cfg);
// var cp = new CykParser(cnf);
// Does this grammar accept the string "aabb"?
Console.WriteLine("aabb: {0}", ep.ParseGetProbability(Sentence.FromLetters("aabb")));
// How about "abba"?
Console.WriteLine("abba: {0}", ep.ParseGetProbability(Sentence.FromLetters("abba")));
Console.WriteLine(ep.ParseGetForest(Sentence.FromLetters("abba")));
for (int i = 0; i < 5; i++) {
Console.WriteLine(cfg.ProduceRandom().AsTerminals());
}
var sentences = cfg.ProduceToDepth(3);
foreach (var sentence in sentences) {
Console.WriteLine(sentence.Value.AsTerminals());
}
var gg = new GrammarGenerator(1);
var terminals = new List<Terminal> { Terminal.Of("a"), Terminal.Of("b") };
var randGram = gg.NextCFG(
numNonterminals: 4,
numProductions: 10,
maxProductionLength: 4,
terminals: terminals
);
Console.WriteLine(randGram);
}
private static void ExecuteTest(Grammar g, List<Sentence> sentences) {
CNFGrammar h = g.ToCNF();
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
foreach (var sentence in sentences) {
var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
}
}
private static void ExecuteTest(Grammar g, string input)
{
var earley1 = new EarleyParser(g);
var earley2 = new EarleyParser2(g);
var sentence = Sentence.FromWords(input);
var p1 = earley1.ParseGetProbability(sentence);
var p2 = earley2.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
}
private static void ExecuteTest(Grammar g, List <Sentence> sentences)
{
CNFGrammar h = g.ToCNF();
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
foreach (var sentence in sentences)
{
var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
}
}
private static void BnfParse(Sentence sentence)
{
var g = Bnf.Grammar();
//var h = g.ToCNF(); // too much memory
var earley = new EarleyParser(g);
// var cyk = new CykParser(h);
var earley2 = new EarleyParser2(g);
// var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
var p3 = earley2.ParseGetProbability(sentence);
// Helpers.AssertNear(p1, p2);
Helpers.AssertNear(p2, p3);
Assert.IsTrue(p2 > 0.0);
}
public static void Do()
{
// S -> aSa | bSb | ε
var productions = new List <Production> {
// construct productions by passing arguments...
new Production(
lhs: Nonterminal.Of("S"),
rhs: new Sentence {
Terminal.Of("a"), Nonterminal.Of("S"), Terminal.Of("a")
},
weight: 20
),
// or from a string...
CFGParser.Production(@"<S> -> 'b' <S> 'b' [10]"),
CFGParser.Production(@"<S> -> ε [5]"),
};
var cfg = new Grammar(productions, Nonterminal.Of("S"));
// var cnf = cfg.ToCNF();
//var probs = cfg.EstimateProbabilities(1000000);
//foreach (var entry in probs) {
// var key = entry.Key;
// var value = entry.Value;
// if (key.Length <= 4) {
// Console.WriteLine("{0}: {1}", key, value);
// }
//}
// Print out the new CNF grammar
// Console.WriteLine(cnf);
var ep = new EarleyParser(cfg);
// var cp = new CykParser(cnf);
// Does this grammar accept the string "aabb"?
Console.WriteLine("aabb: {0}", ep.ParseGetProbability(Sentence.FromLetters("aabb")));
// How about "abba"?
Console.WriteLine("abba: {0}", ep.ParseGetProbability(Sentence.FromLetters("abba")));
Console.WriteLine(ep.ParseGetForest(Sentence.FromLetters("abba")));
for (int i = 0; i < 5; i++)
{
Console.WriteLine(cfg.ProduceRandom().AsTerminals());
}
var sentences = cfg.ProduceToDepth(3);
foreach (var sentence in sentences)
{
Console.WriteLine(sentence.Value.AsTerminals());
}
var gg = new GrammarGenerator(1);
var terminals = new List <Terminal> {
Terminal.Of("a"), Terminal.Of("b")
};
var randGram = gg.NextCFG(
numNonterminals: 4,
numProductions: 10,
maxProductionLength: 4,
terminals: terminals
);
Console.WriteLine(randGram);
}
public void RandomParsingTest(
int _numGrammars = 10000,
int _numNonterminals = 10,
int _numTerminals = 5,
int _numProductions = 30,
int _maxProductionLength = 8,
int _maxInputLength = 6,
int seed = 0
) {
var printStatus = true;
var range = Enumerable.Range(0, _numTerminals);
var terminals = new List<Terminal>(range.Select((x) => Terminal.Of("x" + x)));
Console.WriteLine("Preparing sentences");
var preparedSentences = new List<Sentence>();
for (int length = 0; length <= _maxInputLength; length++) {
var combinations = CFGLibTest.Helpers.CombinationsWithRepetition(terminals, length);
foreach (var target in combinations) {
var sentence = new Sentence(target);
preparedSentences.Add(sentence);
}
}
var randg = new GrammarGenerator(seed);
var preparedGrammars = new List<Grammar>(_numGrammars);
var preparedGrammarsCNF = new List<CNFGrammar>(_numGrammars);
Console.WriteLine("Preparing grammars");
for (int i = 0; i < _numGrammars; i++) {
Grammar g = null;
while (g == null) {
// g = randg.NextCNF(_numNonterminals, _numProductions, terminals);
g = randg.NextCFG(_numNonterminals, _numProductions, _maxProductionLength, terminals, true);
if (g.Productions.Count() == 0) {
g = null;
}
}
// Console.WriteLine("---------------{0}/{1}---------------", i.ToString("D5"), _numGrammars.ToString("D5"));
// Console.WriteLine(g.ToCodeString());
var h = g.ToCNF();
//return;
// Console.WriteLine(g);
// g.PrintProbabilities(2, 3);
preparedGrammars.Add(g);
preparedGrammarsCNF.Add(h);
}
Console.WriteLine("starting");
var sw = Stopwatch.StartNew();
int count = 0;
for (int grammarIndex = 0; grammarIndex < _numGrammars; grammarIndex++) {
if (printStatus) {
Console.WriteLine("---------------{0}/{1}---------------", grammarIndex.ToString("D5"), _numGrammars.ToString("D5"));
}
var g = preparedGrammars[grammarIndex];
var h = preparedGrammarsCNF[grammarIndex];
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
// Console.WriteLine(g.ToCodeString());
// Console.Write("{0}, ", count);
count++;
var accepts = 0;
foreach (var sentence in preparedSentences) {
try {
var p1 = earley.ParseGetProbability(sentence);
var p2 = cyk.ParseGetProbability(sentence);
if (!Helpers.IsNear(p2, p1)) {
throw new Exception();
}
var accepts1 = p1 > 0;
var accepts2 = p2 > 0;
if (accepts2) {
accepts++;
}
} catch (Exception) {
Report(g, sentence);
throw;
}
}
if (printStatus) {
Console.WriteLine("Accepted {0} / {1}", accepts, preparedSentences.Count);
}
}
sw.Stop();
Console.WriteLine();
Console.WriteLine("inner Elapsed: {0}s", sw.Elapsed.TotalMilliseconds / 1000.0);
// Console.WriteLine("Per CYK: {0}ms", sw.Elapsed.TotalMilliseconds / (_numGrammars * preparedSentences.Count));
}
public void RandomParsingTest(
int _numGrammars = 10000,
int _numNonterminals = 10,
int _numTerminals = 5,
int _numProductions = 30,
int _maxProductionLength = 8,
int _maxInputLength = 6,
int seed = 0
)
{
var printStatus = true;
var range = Enumerable.Range(0, _numTerminals);
var terminals = new List <Terminal>(range.Select((x) => Terminal.Of("x" + x)));
Console.WriteLine("Preparing sentences");
var preparedSentences = new List <Sentence>();
for (int length = 0; length <= _maxInputLength; length++)
{
var combinations = CFGLibTest.Helpers.CombinationsWithRepetition(terminals, length);
foreach (var target in combinations)
{
var sentence = new Sentence(target);
preparedSentences.Add(sentence);
}
}
var randg = new GrammarGenerator(seed);
var preparedGrammars = new List <Grammar>(_numGrammars);
var preparedGrammarsCNF = new List <CNFGrammar>(_numGrammars);
Console.WriteLine("Preparing grammars");
for (int i = 0; i < _numGrammars; i++)
{
Grammar g = null;
while (g == null)
{
// g = randg.NextCNF(_numNonterminals, _numProductions, terminals);
g = randg.NextCFG(_numNonterminals, _numProductions, _maxProductionLength, terminals, true);
if (g.Productions.Count() == 0)
{
g = null;
}
}
// Console.WriteLine("---------------{0}/{1}---------------", i.ToString("D5"), _numGrammars.ToString("D5"));
// Console.WriteLine(g.ToCodeString());
var h = g.ToCNF();
//return;
// Console.WriteLine(g);
// g.PrintProbabilities(2, 3);
preparedGrammars.Add(g);
preparedGrammarsCNF.Add(h);
}
Console.WriteLine("starting");
var sw = Stopwatch.StartNew();
int count = 0;
for (int grammarIndex = 0; grammarIndex < _numGrammars; grammarIndex++)
{
if (printStatus)
{
Console.WriteLine("---------------{0}/{1}---------------", grammarIndex.ToString("D5"), _numGrammars.ToString("D5"));
}
var g = preparedGrammars[grammarIndex];
var h = preparedGrammarsCNF[grammarIndex];
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
// Console.WriteLine(g.ToCodeString());
// Console.Write("{0}, ", count);
count++;
var accepts = 0;
foreach (var sentence in preparedSentences)
{
try {
var p1 = earley.ParseGetProbability(sentence);
var p2 = cyk.ParseGetProbability(sentence);
if (!Helpers.IsNear(p2, p1))
{
throw new Exception();
}
var accepts1 = p1 > 0;
var accepts2 = p2 > 0;
if (accepts2)
{
accepts++;
}
} catch (Exception) {
Report(g, sentence);
throw;
}
}
if (printStatus)
{
Console.WriteLine("Accepted {0} / {1}", accepts, preparedSentences.Count);
}
}
sw.Stop();
Console.WriteLine();
Console.WriteLine("inner Elapsed: {0}s", sw.Elapsed.TotalMilliseconds / 1000.0);
// Console.WriteLine("Per CYK: {0}ms", sw.Elapsed.TotalMilliseconds / (_numGrammars * preparedSentences.Count));
}