public void TestCFG_Overlaps()
{
var input =
@"S -> \( S \) | [abx] [bxd] [xde]
";
ContextFreeGrammar cfg = GrammarParser <BDD> .Parse(MkAutomaton, input);
Assert.IsNotNull(cfg.BuiltinTerminalAlgebra);
Assert.AreEqual("S", cfg.StartSymbol.Name);
Assert.AreEqual(1, cfg.NonterminalCount);
Assert.AreEqual(2, cfg.ProductionCount);
Assert.IsFalse(cfg.IsInGNF());
var aut = MkAutomaton(@"\((xx)+\)");
BDD[] witness;
var no = cfg.Overlaps <BDD>(aut, out witness);
Assert.IsFalse(no);
var aut2 = MkAutomaton(@"\(x+\)");
BDD[] witness2 = null;
var yes = cfg.Overlaps <BDD>(aut2, out witness2);
Assert.IsTrue(yes);
string concrete_witness = new string(Array.ConvertAll(witness2, GetChar));
Assert.AreEqual <string>("(xxx)", concrete_witness);
}
public override bool CallRescue(Coderack coderack, IParsedPhrase input, PatternTemplateSource patternTemplateSource, string reason, IContinuation skip, IContinuation succ, IFailure fail)
{
List <string> words = GroupPhrase.PhraseToTexts(input);
bool changed = false;
List <string> corrected = new List <string>();
foreach (string word in words)
{
string correct = comparer.GetCorrects(word)[0];
if (correct.ToLower() != word.ToLower())
{
changed = true;
}
corrected.Add(correct);
}
if (changed)
{
IParsedPhrase correct = parser.Parse(StringUtilities.JoinWords(corrected));
IFailure fallfail = fallback.MakeFailure(input, patternTemplateSource, succ, fail, coderack);
patternTemplateSource.Generate(coderack, correct, succ, fallfail, weight);
return(true);
}
else
{
return(fallback.CallRescue(coderack, input, patternTemplateSource, reason, skip, succ, fail));
}
}
public static void CorrectParse_WeakPrecedenceGrammar()
{
var rules = new[]
{
"S aASb",
"S d",
"A Ac",
"A c",
};
var grammar = GrammarParser.Parse(rules);
var expectedSRules = new []
{
new Rule(new NonTerminal('S'), new Terminal('a'), new NonTerminal('A'), new NonTerminal('S'), new Terminal('b')),
new Rule(new NonTerminal('S'), new Terminal('d')),
};
var expectedARules = new[]
{
new Rule(new NonTerminal('A'), new NonTerminal('A'), new Terminal('c')),
new Rule(new NonTerminal('A'), new Terminal('c')),
};
grammar[new NonTerminal('S')].Should().BeEquivalentTo(expectedSRules);
grammar[new NonTerminal('A')].Should().BeEquivalentTo(expectedARules);
}
public IParsedPhrase ToPhrase(POSTagger tagger, GrammarParser parser)
{
if (name.Contains(" "))
{
List <IParsedPhrase> phrases = new List <IParsedPhrase>();
List <string> words = StringUtilities.SplitWords(name, true);
foreach (string word in words)
{
phrases.Add(new WordPhrase(word, "??"));
}
List <KeyValuePair <string, string> > tokens = tagger.ResolveUnknowns(phrases);
return(parser.Parse(tokens));
}
else
{
if (kind == Kind.Event)
{
return(new WordPhrase(name, "VB"));
}
if (kind == Kind.Attribute)
{
return(new WordPhrase(name, "JJ"));
}
// entity
return(new WordPhrase(name, "NN"));
}
}
void DoMatching(List <PatternTemplateSource> dicta, string input)
{
if (verbose > 0)
{
Console.WriteLine("Parsing input...");
}
IParsedPhrase phrase = parser.Parse(input);
if (verbose > 0)
{
Console.WriteLine("Matching templates...");
}
// Add a codelet for each of these, to match the input
if (!serialmode)
{
foreach (PatternTemplateSource dictum in dicta)
{
IFailure fail = tryToRescueMatch.MakeFailure(phrase, dictum, this, new NopCallable(), coderack);
dictum.Generate(coderack, phrase, this, fail, 1.0);
}
}
else
{
SerialTemplateMatcher matcher = new SerialTemplateMatcher(this, this, coderack, tryToRescueMatch, phrase, dicta, 1.0);
matcher.MatchNextSentence();
}
RunToEnd();
}
public static void Main(string[] args)
{
PluginEnvironment plugenv = new PluginEnvironment(new MainClass());
string plugbase = "/Users/jrising/projects/virsona/github";
plugenv.Initialize(plugbase + "/config.xml", new NameValueCollection());
// Test 1: POS Tagging
POSTagger tagger = new POSTagger(plugenv);
List <KeyValuePair <string, string> > tagged = tagger.TagList(StringUtilities.SplitWords("This is a test.", false));
foreach (KeyValuePair <string, string> kvp in tagged)
{
Console.WriteLine(kvp.Key + ": " + kvp.Value);
}
// Test 2: Grammar parsing
GrammarParser parser = new GrammarParser(plugenv);
IParsedPhrase before = parser.Parse("This is a rug and a keyboard.");
Console.WriteLine(before.ToString());
// Test 5: Pluralize nouns and conjugate verbs
Nouns nouns = new Nouns(plugenv);
Console.WriteLine("person becomes " + nouns.Pluralize("person"));
Verbs verbs = new Verbs(plugenv);
Console.WriteLine("goes becomes " + verbs.ComposePast("goes"));
Console.WriteLine("eats becomes " + verbs.ComposePrespart(verbs.InputToBase("eats")));
// Test 3: Paraphrasing
Random randgen = new Random();
try {
IParsedPhrase after = parser.Paraphrase(before, null, null, randgen.NextDouble());
Console.WriteLine(after.Text);
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
}
// Test 4: Look up some indices
WordNetAccess wordnet = new WordNetAccess(plugenv);
List <string> synonyms = null;
try {
synonyms = wordnet.GetExactSynonyms("rug", WordNetAccess.PartOfSpeech.Noun);
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
}
if (synonyms == null)
{
Console.WriteLine("Could not find a synonym for 'rug'. Is Memcached installed?");
}
else
{
Console.WriteLine("Synonyms: " + string.Join(", ", synonyms.ToArray()));
}
}
public void InGrammarParser_20x02y00_return_xyxy()
{
string expected = "xyyxyy";
string actual = GrammarParser.Parse("2[x2[y]]");
Assert.AreEqual(expected, actual);
}
public void InGrammarParser_30xyz02za0bc_return_xyxy()
{
string expected = "xyzxyzxyzzazabc";
string actual = GrammarParser.Parse("3[xyz]2[za]bc");
Assert.AreEqual(expected, actual);
}
public void InGrammarParser_xyz_return_xzy()
{
string expected = "xyz";
string actual = GrammarParser.Parse("xyz");
Assert.AreEqual(expected, actual);
}
public static char[] ReturnCorrectLastSymbols_When(char nonTerminal, params string[] rules)
{
var grammar = GrammarParser.Parse(rules);
return(grammar.GetAllLastSymbols(new NonTerminal(nonTerminal))
.Select(c => c.Value)
.OrderBy(c => c)
.ToArray());
}
public void GrammarGradingTest1() // balanced parenthesis (wordsInGrammar) !!!could change if grading scale is changed!!!
{
String sg1 = "S -> S S|(S)|";
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg1);
var res = GrammarGrading.gradeWordsInGrammar(g, new[] { "()", "())()()(", "()((" }, new[] { "()(", "())()(", "xyz" }, 10);
Assert.IsTrue(res.Item1 == 5);
}
/// <summary>
/// 解析指令
/// </summary>
/// <returns></returns>
public Result Parse()
{
parseResult = Result.FAILED;
if (grammarParser == null)
{
grammarParser = new GrammarParser(this);
}
parseResult = grammarParser.Parse();
return(parseResult);
}
public void EqualityTest2()
{
String sg1 = "S->aSb|absjjfhghs|";
String sg2 = "S->aSb|aaSbb|";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2[0].Equals("absjjfhghs") && res.Item3.Count == 0);
}
public void TestCFG_Parser_WithDummyAutomaton()
{
var input = @"
START -> AS BS
AS -> AS (a) (a) | (a) (a) | (@)
BS -> (b) (b) (b) BS | (b) (b) (b)";
TestCFGParser_validate(input);
var input2 = GrammarParser <string> .Parse(MapTerminalToDummyAutomaton, input).ToString();
TestCFGParser_validate(input2);
}
public void EqualityTest6() // balanced parenthesis
{
String sg1 = "S -> S S|(S)|";
String sg2 = "X -> | X X | (L L->X) | X X)";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void EqualityTest5() // (a|b)^n
{
String sg1 = "S -> a|b|aa|aS|bS|bbbbS";
String sg2 = "S->X|X S X->a|b";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void EqualityTest4() // (a|b|c|d|e|f|g|h|i|j)*
{
String sg1 = "S->|a|b|c|d|e|f|g|h|i|j|S S";
String sg2 = "S->X|X S| X->a|b|c|d|e|f|g|h|i|j";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 50);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void EqualityTest3() // a^n b^n
{
String sg1 = "S->aT T->aT U|b U->b";
String sg2 = "P->aR R->abb|aRb|b";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void prefixTest1() // balanced parenthesis
{
String sg1 = "S -> S S|(S)|";
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg1);
g = GrammarUtilities.getEquivalentCNF(g);
Assert.AreEqual(-2, GrammarUtilities.longestPrefixLength(g, "()(())()()")); // full
Assert.AreEqual("()(())".Length, GrammarUtilities.longestPrefixLength(g, "()(()))()()"));
Assert.AreEqual("()(())((((".Length, GrammarUtilities.longestPrefixLength(g, "()(())(((("));
Assert.AreEqual("".Length, GrammarUtilities.longestPrefixLength(g, ")(()()())"));
}
public void GrammarDerivationTest() //tests derivation computation in combination with parsing (grammar and derivation)
{
String sg = "S->S S S | x | A | ab | A -> A A | S | a | ";
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg);
var der1 = DerivationParser <char> .Parse(f1, "S \n A \n A A \n A \n A A \n A S \n A \n S \n S S S \n S a b S \n x a b S \n x a b S S S \n x a b x S S \n x a b x x S \n x a b x x x");
Assert.IsTrue(Derivation.comparator.Equals(der1[0], new GrammarSymbol[] { g.StartSymbol })); //check start
for (int i = 1; i < der1.Count; i++)
{
Assert.IsTrue(Derivation.isValidDerivationStep(g.GetProductions(), der1[i - 1], der1[i])); //check step
}
}
public void prefixTest2() // a^n b^n
{
String sg1 = "S->aSb|S X| X->cX";
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg1);
g = GrammarUtilities.getEquivalentCNF(g);
Assert.AreEqual(-2, GrammarUtilities.longestPrefixLength(g, "aaaaaaaaaaaaaaaabbbbbbbbbbbbbbbb")); // full
Assert.AreEqual("a".Length, GrammarUtilities.longestPrefixLength(g, "a"));
Assert.AreEqual("".Length, GrammarUtilities.longestPrefixLength(g, "b"));
Assert.AreEqual("aab".Length, GrammarUtilities.longestPrefixLength(g, "aab"));
Assert.AreEqual("aabb".Length, GrammarUtilities.longestPrefixLength(g, "aabbb"));
Assert.AreEqual("aaaaaaaaaaaabbbbbbbbbbbb".Length, GrammarUtilities.longestPrefixLength(g, "aaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbababc"));
}
public void GrammarGradingTest2() // a^n b^n (grammar equality) !!!could change if grading scale is changed!!!
{
String sg1 = "S->absjjfhghs|X X->aXb|";
String sg2 = "S->aSb|aaSbb|";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarGrading.gradeGrammarEquality(g1, g2, 10, 50);
var res2 = GrammarGrading.gradeGrammarEquality(g2, g1, 10, 50);
Assert.IsTrue(res.Item1 == 9);
Assert.IsTrue(res2.Item1 == 9); //mirrored
}
public void EqualityTest7() // empty grammars and invariants
{
String sg1 = "S -> S S|(S)|";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, null, true, 25);
Assert.IsTrue(res.Item1 == 0 && res.Item2.Count > 0 && res.Item3.Count == 0);
res = GrammarUtilities.findDifferenceWithTimelimit(null, g1, true, 25);
Assert.IsTrue(res.Item1 == 0 && res.Item2.Count == 0 && res.Item3.Count > 0);
res = GrammarUtilities.findDifferenceWithTimelimit(g1, g1, true, 25);
Assert.IsTrue(res.Item1 > 0 && res.Item2.Count == 0 && res.Item3.Count == 0);
res = GrammarUtilities.findDifferenceWithTimelimit(null, null, true, 30);
Assert.IsTrue(res.Item1 == 0 && res.Item2.Count == 0 && res.Item3.Count == 0);
}
public AST Run(string input, bool print)
{
GrammarLexer lexer = new GrammarLexer(input);
GrammarParser parser = new GrammarParser(lexer);
// Executes the parsing
ParseResult result = parser.Parse();
CheckErrors(result);
// Prints the produced syntax tree
if (print)
{
Print(result.Root, new bool[] { });
}
return(_astBuilder.GetAST(result.Root));
}
public void CYK_Test3()
{
String sg1 = "S -> A B C A-> B-> C->"; // = {""} Testing epsilon
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg1);
g = GrammarUtilities.getEquivalentCNF(g);
Assert.IsTrue(GrammarUtilities.isWordInGrammar(g, ""));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "ba"));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "ca"));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "d"));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "abcc"));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "cba"));
Assert.IsFalse(GrammarUtilities.isWordInGrammar(g, "ababababababababababababababababababab"));
}
public static IParsedPhrase ProducedPhrase(Context context, POSTagger tagger, GrammarParser parser)
{
List <IParsedPhrase> phrases = new List <IParsedPhrase>();
foreach (IContent content in context.Contents)
{
if (content is Word)
{
phrases.Add(new WordPhrase(content.Name));
}
else if (content is Special && (content.Name.StartsWith("*") || content.Name.StartsWith("_")))
{
List <IContent> words = GetStarValue(context, content.Name);
foreach (IContent word in words)
{
phrases.Add(new WordPhrase(content.Name));
}
}
else if (content is Variable)
{
IParsedPhrase phrase = ((Variable)content).Produce(context, tagger, parser);
if (phrase == null)
{
return(null); // failed!
}
phrases.Add(phrase);
}
else if (content is Concept)
{
phrases.Add(new WordPhrase(((Concept)content).Name));
}
else
{
phrases.Add(new WordPhrase(content.Name));
}
}
if (phrases.Count == 0)
{
return(null);
}
List <KeyValuePair <string, string> > tokens = tagger.ResolveUnknowns(phrases);
return(parser.Parse(tokens));
}
public static void CorrectFindLessAndEqualAndGreateSets_WhenG31()
{
var grammar = GrammarParser.Parse("S aFSb", "S c", "F Fb", "F b");
var precedence = new GrammarPrecedence(grammar);
precedence.Less.Select(p => (p.Item1.Value, p.Item2.Value))
.Should()
.BeEquivalentTo(('a', 'F'), ('a', 'b'), ('F', 'a'), ('F', 'c'), ('^', 'S'), ('^', 'a'), ('^', 'c'));
precedence.Equal.Select(p => (p.Item1.Value, p.Item2.Value))
.Should()
.BeEquivalentTo(('a', 'F'), ('F', 'S'), ('S', 'b'), ('F', 'b'));
precedence.Greater.Select(p => (p.Item1.Value, p.Item2.Value))
.Should()
.BeEquivalentTo(('b', 'a'), ('b', 'c'), ('b', 'b'), ('c', 'b'), ('S', '$'), ('b', '$'), ('c', '$'));
}
public static void CorrectParse_SimplePrecedenceGrammar()
{
var rules = new[]
{
"S aSSb",
"S c"
};
var grammar = GrammarParser.Parse(rules);
var expectedSRules = new[]
{
new Rule(new NonTerminal('S'), new Terminal('a'), new NonTerminal('S'), new NonTerminal('S'), new Terminal('b')),
new Rule(new NonTerminal('S'), new Terminal('c')),
};
grammar[new NonTerminal('S')].Should().BeEquivalentTo(expectedSRules);
}
public void SkipGramarParseTest()
{
// get skip parse table
Grammar skipGrammar = GrammarParser.Parse(c_skipGrammar);
var skipParseTable = ParserTableConstructor.Construct(skipGrammar);
var skipGrammarStr = string.Join(Environment.NewLine, skipGrammar.Rules);
var skipTableStr = new ParserTableFormatter().Format(skipParseTable);
Console.Write(skipGrammarStr);
Console.WriteLine();
Console.Write(skipTableStr);
Console.WriteLine();
Console.WriteLine();
var skipParser = new Parser(skipParseTable);
skipParser.Parse("till 5 pm".Split(' '));
skipParser.Parse("begin at 9 am and run to 5 pm nonstop".Split(' '));
}
public void TestCFG_Intersect()
{
//same as above but with a regex as a single terminal because there are no spaces between the sub-terminals
//but outcome must be equivalent
var input =
@"S -> \( S \) | [abx][bxd][xde]
";
ContextFreeGrammar cfg = GrammarParser <BDD> .Parse(MkAutomaton, input);
Assert.AreEqual("S", cfg.StartSymbol.Name);
Assert.AreEqual(1, cfg.NonterminalCount);
Assert.AreEqual(2, cfg.ProductionCount);
string w;
Assert.IsFalse(cfg.IntersectsWith(@"^\((xx)+\)$", out w));
Assert.IsTrue(cfg.IntersectsWith(@"^\(x+\)$", out w));
Assert.AreEqual <string>("(xxx)", w);
}