private static void ComplexMovementTest()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "N"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "N"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));
//grammar.AddRule(new Rule(1, "CP", new[] { "IP" }, 0, 0));
//grammar.AddRule(new Rule(1, "IP", new[] { "NP" }, 0, 0));
//grammar.AddRule(new Rule(1, "VP", new[] { "V1", "NP" }, 0, 1));
//grammar.AddRule(new Rule(1, "VP", new[] { "V2", "PP" }, 0, 1));
//grammar.AddRule(new Rule(1, "PP", new[] { "P", "NP" }, 0, 1));
//grammar.AddRule(new Rule(1, "NP", new[] { "D", "N" }, 1, 0));
grammar.AddNonTerminalToLandingSites("CP");
grammar.AddNonTerminalToLandingSites("IP");
grammar.AddMoveable("NP");
grammar.AddMoveable("PP");
grammar.GenerateDerivedRulesFromSchema();
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("Who Who Who");
n.Print();
}
internal Grammar GetInitialGrammar_old()
{
var initialGrammar = new Grammar(voc);
initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
initialGrammar.POSTypes = posTypes;
foreach (var sentence in sentencesWithCounts.Keys)
{
var words = sentence.Split();
//assume unamibigous interpretation of words. take the first POS for each word.
var posList = words.Select(x => voc[x].First()).ToArray();
if (posList.Length < 2) continue;
var lastRuleName =
initialGrammar.AddRule(new Rule(0, null, new[] {posList[0], posList[1]}, 1, 0));
for (var i = 0; i < posList.Length - 2; i++)
{
lastRuleName =
initialGrammar.AddRule(new Rule(0, null, new[] {lastRuleName, posList[i + 2]}, 1, 0));
}
initialGrammar.AddRule(new Rule(0, initialGrammar.StartSymbol, new[] {lastRuleName}, 0, 0));
}
return initialGrammar;
}
internal Grammar GetInitialGrammar()
{
var initialGrammar = new Grammar(voc);
initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
initialGrammar.POSTypes = posTypes;
var posInText = voc.POSWithPossibleWords.Keys;
foreach (var pos in posInText)
{
initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos, initialGrammar.StartSymbol},
0, 1));
initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos}, 0, 0));
}
return initialGrammar;
}
private static void TestAmbiguityGrammar()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(0, "START", new[] {"VP"}, 0, 0));
grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"VP", "NP"}, 0, 0));
grammar.AddRule(new Rule(0, "NP", new[] {"NP", "NP"}, 0, 0));
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("kissed John David");
n.Print();
}
public static void CreateSimpleGrammar(Grammar grammar)
{
AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 1, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V0"}, 0, 0));
grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V3", "START"}, 0, 1));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"P", "NP"}, 0, 1));
}
public static void CreateMovementGrammar(Grammar grammar)
{
AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"C", "IP"}, 0, 1));
grammar.AddRule(new Rule(0, "IP", new[] {"NP", "VP"}, 1, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(2, "VP", new[] {"V3", "CP"}, 0, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));
grammar.AddNonTerminalToLandingSites("CP");
grammar.AddMoveable("NP");
grammar.AddMoveable("PP");
}
public static void CreatePalindromeGrammar(Grammar grammar)
{
grammar.AddAllPOSTypesToDictionary(new[] {"A", "B"});
grammar.AddRule(new Rule(2, "START", new[] {"A", "A"}, 0, 1));
grammar.AddRule(new Rule(2, "START", new[] {"B", "B"}, 0, 1));
grammar.AddRule(new Rule(2, "START", new[] {"AP", "A"}, 1, 1));
grammar.AddRule(new Rule(2, "START", new[] {"BP", "B"}, 1, 1));
grammar.AddRule(new Rule(1, "AP", new[] {"A", "START"}, 0, 0));
grammar.AddRule(new Rule(1, "BP", new[] {"B", "START"}, 0, 0));
}
private static void TestMovement2()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"VP", "VP"}, 1, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"VP", "PP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"NP", "V1"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("the man arrived to Mary"); //supposed to fail in parsing!!
n.Print();
}
public static void CreateMovementGrammarCompetitor_old(Grammar grammar)
{
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"P", "START"}, 0, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 1)); //1.58496250072116
grammar.AddRule(new Rule(0, "CP", new[] {"VP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"C"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "DP"}, 0, 1)); //2.80735493280681
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"VP", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"NP", "V3"}, 1, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"DP", "V3"}, 1, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0)); //1
grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));
grammar.AddRule(new Rule(0, "DP", new[] {"D", "N"}, 0, 1)); //0
}
public static void CreateMovementGrammarCompetitor(Grammar grammar)
{
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"C", "START"}, 0, 1));
grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"PP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V3", "START"}, 0, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));
grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));
}
private static Grammar InfiniteMovementGrammar3()
{
var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "NP", new[] {"V0P", "NP"}, 1, 0));
grammar.AddRule(new Rule(1, "V0P", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"NP", "V0"}, 1, 1));
grammar.AddRule(new Rule(1, "START", new[] {"V0", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "START", new[] {"PP", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "START", new[] {"NP", "NP"}, 1, 1));
grammar.AddRule(new Rule(1, "V0P", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V0P", "P"}, 1, 1));
grammar.AddRule(new Rule(1, "V0P", new[] {"NP", "V0P"}, 1, 1));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 1));
var does = grammar.DoesGrammarAllowInfiniteMovement();
return grammar;
}
private static Grammar InfiniteMovementGrammar1()
{
var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP"}, 0, 0));
grammar.AddRule(new Rule(1, "NP", new[] {"PP", "NP"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"NP", "PP"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
var does = grammar.DoesGrammarAllowInfiniteMovement();
return grammar;
}
//generate a new rule from random existing productions.
public bool InsertRule(Grammar grammar)
{
for (var i = 0; i < NumberOfRetries; i++)
{
var productions = new List<string>();
var randomDaughter = grammar.StartSymbol;
while (randomDaughter == grammar.StartSymbol)
randomDaughter = grammar.GetRandomNonTerminal(); //the first daughter is never the start symbol.
productions.Add(randomDaughter);
if (_rand.NextDouble() < 0.5f)
productions.Add(grammar.GetRandomNonTerminal());
var newRule = new Rule();
newRule.Occurrences = 1;
newRule.Production = productions.Select(x => new NonTerminalObject(x)).ToArray();
newRule.HeadPosition = _rand.Next(newRule.Production.Length);
newRule.ComplementPosition = _rand.Next(newRule.Production.Length);
if (newRule.HeadTerm == grammar.StartSymbol)
//never let the head be the start symbol. the start symbol can only be the second term(see above).
newRule.HeadPosition = 0;
var ruleName = grammar.StartSymbol;
if (_rand.NextDouble() < 0.9f)
//90% probability of projecting regular head stucture. 10% allow to project to the START symbol.
{
try
{
ruleName = grammar.NonTerminalsTypeDictionary[newRule.HeadTerm] + "P";
}
catch
{
throw new Exception(string.Format("rule head term not found", newRule.HeadTerm));
}
}
newRule.Name = new NonTerminalObject(ruleName);
if (grammar.AreHeadRelationsConsistent(newRule))
{
grammar.AddRule(newRule);
return true;
}
}
return false;
}
public bool InsertRuleWithANewSymbol(Grammar grammar)
{
for (var i = 0; i < NumberOfRetries; i++)
{
var existingLHSNonTerminal = grammar.GetRandomRule().Name.NonTerminal;
if (existingLHSNonTerminal == grammar.StartSymbol) continue;
var newLHSNonTerminal = grammar.GetNextAvailableProjectionName();
var typeOfExistingLHS = grammar.NonTerminalsTypeDictionary[existingLHSNonTerminal];
grammar.AddProjectionTypeToTypeDictionary(newLHSNonTerminal, typeOfExistingLHS);
//grammar.AddRule(new Rule(0, grammar.StartSymbol, new[] { newLHSNonTerminal, grammar.StartSymbol }, 0, 1));
// grammar.AddRule(new Rule(0, grammar.StartSymbol, new[] { newLHSNonTerminal }, 0, 0));
grammar.AddRule(new Rule(0, newLHSNonTerminal, new[] {existingLHSNonTerminal}, 0, 0));
return true;
}
return false;
}
public bool ChangeHeadOfRule(Grammar grammar)
{
for (var i = 0; i < NumberOfRetries; i++)
{
var rule = grammar.GetRandomRule();
if (!rule.IsInitialRule()) continue; //do not change complements of schematic rules. (push/pop)
if (rule.Production.Length > 1)
{
var newRule = new Rule(rule);
newRule.HeadPosition = (rule.HeadPosition + 1)%rule.Production.Length;
if (grammar.AreHeadRelationsConsistent(newRule))
{
grammar.DeleteRule(rule);
grammar.AddRule(newRule);
return true;
}
}
}
return false;
}