private static void ComplexMovementTest()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "N"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "N"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));
//grammar.AddRule(new Rule(1, "CP", new[] { "IP" }, 0, 0));
//grammar.AddRule(new Rule(1, "IP", new[] { "NP" }, 0, 0));
//grammar.AddRule(new Rule(1, "VP", new[] { "V1", "NP" }, 0, 1));
//grammar.AddRule(new Rule(1, "VP", new[] { "V2", "PP" }, 0, 1));
//grammar.AddRule(new Rule(1, "PP", new[] { "P", "NP" }, 0, 1));
//grammar.AddRule(new Rule(1, "NP", new[] { "D", "N" }, 1, 0));
grammar.AddNonTerminalToLandingSites("CP");
grammar.AddNonTerminalToLandingSites("IP");
grammar.AddMoveable("NP");
grammar.AddMoveable("PP");
grammar.GenerateDerivedRulesFromSchema();
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("Who Who Who");
n.Print();
}
public SimulatedAnnealing(Learner l)
{
SimulatedAnnealingRunningParameters pam;
using (var file = File.OpenText(@"SimulatedAnnealingParameters.json"))
{
var serializer = new JsonSerializer();
pam =
(SimulatedAnnealingRunningParameters)
serializer.Deserialize(file, typeof(SimulatedAnnealingRunningParameters));
}
learner = l;
threshold = pam.ThresholdTemperature;
coolingFactor = pam.CoolingFactor;
currentIteration = bestIteration = 1;
reportEveryNIterations = pam.ReportEveryNIteration;
currentHypothesis = bestHypothesis = learner.GetInitialGrammar();
currentHypothesis.GenerateDerivedRulesFromSchema();
currentEnergy = bestEnergy = learner.Energy(currentHypothesis);
currentHypothesis.GenerateInitialRulesFromDerivedRules();
currentTemp = currentEnergy.TotalEnergy*pam.InitialTemperatureTimesInitialEnegrgy;
using (var sw = File.AppendText("SessionReport.txt"))
{
sw.WriteLine(string.Format("cooling factor: {0}, initial energy: {1}, initial temperature: {2}",
coolingFactor, currentEnergy, currentTemp));
}
}
internal Grammar GetInitialGrammar_old()
{
var initialGrammar = new Grammar(voc);
initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
initialGrammar.POSTypes = posTypes;
foreach (var sentence in sentencesWithCounts.Keys)
{
var words = sentence.Split();
//assume unamibigous interpretation of words. take the first POS for each word.
var posList = words.Select(x => voc[x].First()).ToArray();
if (posList.Length < 2) continue;
var lastRuleName =
initialGrammar.AddRule(new Rule(0, null, new[] {posList[0], posList[1]}, 1, 0));
for (var i = 0; i < posList.Length - 2; i++)
{
lastRuleName =
initialGrammar.AddRule(new Rule(0, null, new[] {lastRuleName, posList[i + 2]}, 1, 0));
}
initialGrammar.AddRule(new Rule(0, initialGrammar.StartSymbol, new[] {lastRuleName}, 0, 0));
}
return initialGrammar;
}
public Grammar(Grammar otherGrammar)
{
Rules = otherGrammar.Rules.ToDictionary(x => x.Key, x => x.Value.Select(y => new Rule(y)).ToList());
var rules = Rules.Values.SelectMany(l => l).ToArray();
ruleNumberDictionary =
rules.Select(x => new KeyValuePair<int, Rule>(x.Number, x)).ToDictionary(x => x.Key, x => x.Value);
nonTerminalCounts = otherGrammar.nonTerminalCounts.ToDictionary(x => x.Key,
x => new NonTerminalCounts(x.Value));
if (otherGrammar.Moveables != null)
Moveables = new HashSet<string>(otherGrammar.Moveables);
if (otherGrammar.LandingSites != null)
LandingSites = new HashSet<string>(otherGrammar.LandingSites);
StartSymbol = otherGrammar.StartSymbol;
POSTypes = new HashSet<string>(otherGrammar.POSTypes);
Vocabulary = otherGrammar.Vocabulary;
if (otherGrammar.availableRuleNumbers != null)
availableRuleNumbers = new Stack<int>(otherGrammar.availableRuleNumbers);
numberOfRules = otherGrammar.numberOfRules;
NonTerminalsTypeDictionary = otherGrammar.NonTerminalsTypeDictionary.ToDictionary(x => x.Key, x => x.Value);
inverseRules = otherGrammar.inverseRules.ToDictionary(x => new InverseKeyType(x.Key), x => x.Value);
nullableProductions = otherGrammar.nullableProductions.ToDictionary(x => x.Key, x => x.Value);
}
public static int GetGrammarLength(Grammar grammar)
{
var rules = grammar.ruleNumberDictionary.Values.ToArray();
var productions = rules.SelectMany(r => r.Production);
var names = rules.Select(r => r.Name);
var overallTerms = productions.Concat(names);
var overallNonTerminals = overallTerms.SelectMany(r => r.ParticipatingNonTerminals()).ToArray();
var termDic = overallNonTerminals.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count());
var counts = termDic.Select(x => x.Value).ToArray();
var totalCount = counts.Sum();
//length of symbol table = length of symbol codewords + length of probability hashtable:
//length of symbol codewords is:
var symbolCodewordsLength =
counts.Select(x => RequiredBitsGivenProbability(x/(float) totalCount)).Sum();
//length of probability hashtable is:
var symbolHashTableBits = counts.Length*BitsForProbabilityEncoding;
//length of codewords contributed by symbols in rules
var overallSymbolRepresentationLength =
counts.Select(x => RequiredBitsGivenProbability(x/(float) totalCount)*x).Sum();
var ruleHashTableBits = rules.Length*(3 + BitsForProbabilityEncoding);
//see documentation. 3 bits for encoding head position, complement position, number of sons.
return symbolCodewordsLength + symbolHashTableBits + overallSymbolRepresentationLength + ruleHashTableBits;
}
public Tuple<EnergyData, Grammar> Run()
{
var rand = new Random();
while (currentTemp > threshold)
{
try
{
var newHypothesis = learner.GetNeighbor(currentHypothesis);
EnergyData newEnergy = null;
if (newHypothesis != null)
{
newHypothesis.GenerateDerivedRulesFromSchema();
newEnergy = learner.Energy(newHypothesis);
newHypothesis.GenerateInitialRulesFromDerivedRules();
}
if (newEnergy != null)
{
if (newEnergy < bestEnergy)
{
bestEnergy = newEnergy;
bestHypothesis = newHypothesis;
bestIteration = currentIteration;
}
var prob = P(currentEnergy, newEnergy, currentTemp);
if (rand.NextDouble() < prob)
{
// moved to new hypothesis
currentHypothesis = newHypothesis;
currentEnergy = newEnergy;
}
}
currentIteration++;
if (currentIteration%reportEveryNIterations == 0)
Console.WriteLine("Iteration {0}", currentIteration);
currentTemp *= coolingFactor;
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
var actualRuleDistributions = learner.CollectUsages(bestHypothesis);
var s = string.Format("{0}. ({1}) \nBest Hypothesis: \n{2}Best so far: #{3} with energy: {4}\n",
currentIteration, currentTemp, bestHypothesis.GrammarWithRuleUsages(actualRuleDistributions),
bestIteration, bestEnergy);
Console.WriteLine(s);
using (var sw = File.AppendText("SessionReport.txt"))
{
sw.WriteLine(s);
}
return new Tuple<EnergyData, Grammar>(bestEnergy, bestHypothesis);
}
public static EnergyData TotalLength(Grammar grammar, Tuple<Node, int>[] parseTreesWithCounts)
{
var e = new EnergyData();
//var rules = grammar.ruleNumberDictionary.Values;
e.GrammarEnergy = GetGrammarLength(grammar);
e.DataEnergy = GetDataLength(grammar, parseTreesWithCounts);
return e;
}
public Learner(Vocabulary v, Dictionary<string, string> n, HashSet<string> p,
string[] s, Grammar o)
{
voc = v;
nonTerminalTypeDic = n;
posTypes = p;
originalGrammar = o;
sentencesWithCounts = s.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count());
}
private static void Main(string[] args)
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
ProgramParams programParams;
using (var file = File.OpenText(@"ProgramParameters.json"))
{
var serializer = new JsonSerializer();
programParams = (ProgramParams) serializer.Deserialize(file, typeof(ProgramParams));
}
if (programParams.DataWithMovement)
CreateMovementGrammar(grammar);
else
CreateSimpleGrammar(grammar);
grammar.GenerateDerivedRulesFromSchema();
var p = new Parser(grammar);
var data = p.GenerateSentences(programParams.NumberOfDataSentences);
using (var sw = File.AppendText("SessionReport.txt"))
{
sw.WriteLine("-------------------");
sw.WriteLine("Session {0} ", DateTime.Now.ToString("MM/dd/yyyy h:mm tt"));
sw.WriteLine("sentences: {0}, runs: {1}, movement: {2}", programParams.NumberOfDataSentences,
programParams.NumberOfRuns, programParams.DataWithMovement);
}
var stopWatch = StartWatch();
var learner = new Learner(voc, grammar.NonTerminalsTypeDictionary, grammar.POSTypes, data, grammar);
learner.originalGrammar.GenerateDerivedRulesFromSchema();
var targetGrammarEnergy = learner.Energy(learner.originalGrammar);
learner.originalGrammar.GenerateInitialRulesFromDerivedRules();
var s = string.Format("Target Hypothesis:\n{0} with energy: {1}\n", learner.originalGrammar,
targetGrammarEnergy);
Console.WriteLine(s);
using (var sw = File.AppendText("SessionReport.txt"))
{
sw.WriteLine(s);
}
for (var i = 0; i < programParams.NumberOfRuns; i++)
{
var sa = new SimulatedAnnealing(learner);
sa.Run();
}
StopWatch(stopWatch);
}
public static void CreatePalindromeGrammar(Grammar grammar)
{
grammar.AddAllPOSTypesToDictionary(new[] {"A", "B"});
grammar.AddRule(new Rule(2, "START", new[] {"A", "A"}, 0, 1));
grammar.AddRule(new Rule(2, "START", new[] {"B", "B"}, 0, 1));
grammar.AddRule(new Rule(2, "START", new[] {"AP", "A"}, 1, 1));
grammar.AddRule(new Rule(2, "START", new[] {"BP", "B"}, 1, 1));
grammar.AddRule(new Rule(1, "AP", new[] {"A", "START"}, 0, 0));
grammar.AddRule(new Rule(1, "BP", new[] {"B", "START"}, 0, 0));
}
public static void CreateSimpleGrammar(Grammar grammar)
{
AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 1, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V0"}, 0, 0));
grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"V3", "START"}, 0, 1));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"P", "NP"}, 0, 1));
}
public static void AddTypesToGrammar(Grammar grammar)
{
grammar.AddAllPOSTypesToDictionary(new[] {"D", "N", "V0", "V1", "V2", "V3", "P", "C", "V", "I"});
grammar.NonTerminalsTypeDictionary["V0"] = "V";
grammar.NonTerminalsTypeDictionary["V1"] = "V";
grammar.NonTerminalsTypeDictionary["V2"] = "V";
grammar.NonTerminalsTypeDictionary["V3"] = "V";
grammar.NonTerminalsTypeDictionary["VP"] = "V";
grammar.NonTerminalsTypeDictionary["V0P"] = "V";
grammar.NonTerminalsTypeDictionary["V1P"] = "V";
grammar.NonTerminalsTypeDictionary["V2P"] = "V";
grammar.NonTerminalsTypeDictionary["V3P"] = "V";
}
public Dictionary<int, HashSet<int>> ComputeLeftCorner(Grammar grammar)
{
var rules = grammar.ruleNumberDictionary.Values.ToList();
//key - nonterminal, value - set of the numbers of reachable rules by transitive left corner.
var leftCorners = new Dictionary<int, HashSet<int>>();
foreach (var item in rules)
{
var ruleNumber = item.Number;
if (!leftCorners.ContainsKey(ruleNumber))
leftCorners[ruleNumber] = new HashSet<int>();
var ruleList = grammar[item.Production[0].NonTerminal];
if (ruleList != null)
{
foreach (var predicted in ruleList)
{
if (!leftCorners[ruleNumber].Contains(predicted.Number))
leftCorners[ruleNumber].Add(predicted.Number);
}
}
}
var changed = true;
while (changed)
{
changed = false;
foreach (var item in leftCorners)
{
var ruleNumber = item.Key;
var reachableRules = item.Value;
foreach (var reachable in reachableRules)
{
var reachablesFromReachable = leftCorners[reachable];
foreach (var reachreach in reachablesFromReachable)
{
if (!leftCorners[ruleNumber].Contains(reachreach))
{
leftCorners[ruleNumber].Add(reachreach);
changed = true;
}
}
}
}
}
return leftCorners;
}
//generate a new rule from random existing productions.
public bool InsertRule(Grammar grammar)
{
for (var i = 0; i < NumberOfRetries; i++)
{
var productions = new List<string>();
var randomDaughter = grammar.StartSymbol;
while (randomDaughter == grammar.StartSymbol)
randomDaughter = grammar.GetRandomNonTerminal(); //the first daughter is never the start symbol.
productions.Add(randomDaughter);
if (_rand.NextDouble() < 0.5f)
productions.Add(grammar.GetRandomNonTerminal());
var newRule = new Rule();
newRule.Occurrences = 1;
newRule.Production = productions.Select(x => new NonTerminalObject(x)).ToArray();
newRule.HeadPosition = _rand.Next(newRule.Production.Length);
newRule.ComplementPosition = _rand.Next(newRule.Production.Length);
if (newRule.HeadTerm == grammar.StartSymbol)
//never let the head be the start symbol. the start symbol can only be the second term(see above).
newRule.HeadPosition = 0;
var ruleName = grammar.StartSymbol;
if (_rand.NextDouble() < 0.9f)
//90% probability of projecting regular head stucture. 10% allow to project to the START symbol.
{
try
{
ruleName = grammar.NonTerminalsTypeDictionary[newRule.HeadTerm] + "P";
}
catch
{
throw new Exception(string.Format("rule head term not found", newRule.HeadTerm));
}
}
newRule.Name = new NonTerminalObject(ruleName);
if (grammar.AreHeadRelationsConsistent(newRule))
{
grammar.AddRule(newRule);
return true;
}
}
return false;
}
internal Grammar GetInitialGrammar()
{
var initialGrammar = new Grammar(voc);
initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
initialGrammar.POSTypes = posTypes;
var posInText = voc.POSWithPossibleWords.Keys;
foreach (var pos in posInText)
{
initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos, initialGrammar.StartSymbol},
0, 1));
initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos}, 0, 0));
}
return initialGrammar;
}
private static void TestAmbiguityGrammar()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(0, "START", new[] {"VP"}, 0, 0));
grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"VP", "NP"}, 0, 0));
grammar.AddRule(new Rule(0, "NP", new[] {"NP", "NP"}, 0, 0));
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("kissed John David");
n.Print();
}
private Tuple<Node, int>[] ParseAllSentences(Grammar currentHypothesis)
{
var allParses = new Tuple<Node, int>[sentencesWithCounts.Count];
try
{
Parallel.ForEach(sentencesWithCounts, (sentenceItem, state, i) =>
{
var parser = new Parser(currentHypothesis);
var n = parser.ParseSentence(sentenceItem.Key);
allParses[i] = Tuple.Create(n, sentenceItem.Value);
});
return allParses;
}
catch (Exception)
{
return null; //parsing failed.
}
}
public static void CreateMovementGrammar(Grammar grammar)
{
AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"C", "IP"}, 0, 1));
grammar.AddRule(new Rule(0, "IP", new[] {"NP", "VP"}, 1, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(2, "VP", new[] {"V3", "CP"}, 0, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));
grammar.AddNonTerminalToLandingSites("CP");
grammar.AddMoveable("NP");
grammar.AddMoveable("PP");
}
public bool InsertRuleWithANewSymbol_old(Grammar grammar)
{
//grammar.CleanOrphanedRules();
//for (var i = 0; i < NUMBER_OF_RETRIES; i++)
//{
// var fatherRule = grammar.GetRandomRule();
// var daughterPos = rand.Next(fatherRule.Production.Length);
// var daughter = fatherRule.Production[daughterPos].NonTerminal;
// if (grammar.isPOS(daughter)) continue; //needs to be a projection
// var newLHSSymbol = grammar.GetNextAvailableProjectionName();
// var daughterRule = grammar.GetRandomRuleForAGivenLHS(daughter);
// if (daughterRule.Production.Length < 2) continue; //the daughter rule must have two daughters.
// //pick out (in random) a rule out of the 4 below which keep the head relations consistent,
// var arr = new Rule[4];
// var randomProjection = grammar.GetRandomRule().Name.NonTerminal;
// if (randomProjection == grammar.StartSymbol) continue; //all projections except START
// arr[0] = new Rule(1.0f, newLHSSymbol, new[] { daughter, randomProjection }, 0, 1);
// arr[1] = new Rule(1.0f, newLHSSymbol, new[] { randomProjection, daughter }, 1, 0);
// arr[2] = new Rule(1.0f, newLHSSymbol, new[] { daughter, randomProjection }, 1, 0);
// arr[3] = new Rule(1.0f, newLHSSymbol, new[] { randomProjection, daughter }, 0, 1);
// var newRule = arr[rand.Next(4)];
// if (!grammar.AreHeadRelationsConsistent(newRule)) continue;
// var oldFatherProductions = new string[fatherRule.Production.Length];
// fatherRule.Production.CopyTo(oldFatherProductions, 0);
// oldFatherProductions[daughterPos] = newLHSSymbol;
// var newFatherRule = new Rule(fatherRule);
// newFatherRule.Production = oldFatherProductions.Select(x => new NonTerminalObject(x));
// if (!grammar.AreHeadRelationsConsistent(newFatherRule)) continue;
// var ruleName = grammar.AddRule(newRule);
// if (ruleName != newRule.Name.NonTerminal) continue; // in case that the new rule that was added already exists in the grammar.
// grammar.AddRule(newFatherRule);
// return true;
//}
return false;
}
public static int GetDataLength(Grammar grammar, Tuple<Node, int>[] parseTreesWithCounts)
{
var sum = 0;
foreach (var parseTree in parseTreesWithCounts)
{
var root = parseTree.Item1; //the node.
sum += root.Bits*parseTree.Item2;
//Item2 = number of occurrences of that sentence in the corpus
}
return sum;
}
private static Grammar InfiniteMovementGrammar1()
{
var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP"}, 0, 0));
grammar.AddRule(new Rule(1, "NP", new[] {"PP", "NP"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"NP", "PP"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
var does = grammar.DoesGrammarAllowInfiniteMovement();
return grammar;
}
private static Grammar InfiniteMovementGrammar3()
{
var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "NP", new[] {"V0P", "NP"}, 1, 0));
grammar.AddRule(new Rule(1, "V0P", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"NP", "V0"}, 1, 1));
grammar.AddRule(new Rule(1, "START", new[] {"V0", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "START", new[] {"PP", "NP"}, 0, 1));
grammar.AddRule(new Rule(1, "START", new[] {"NP", "NP"}, 1, 1));
grammar.AddRule(new Rule(1, "V0P", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V0P", "P"}, 1, 1));
grammar.AddRule(new Rule(1, "V0P", new[] {"NP", "V0P"}, 1, 1));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 1));
var does = grammar.DoesGrammarAllowInfiniteMovement();
return grammar;
}
public Parser(Grammar g, bool debug = false)
{
Grammar = g;
Debug = debug;
ruleLogProbabilities = Grammar.GetLogProbabilitiesOfRules();
}
public EnergyData Energy(Grammar currentHypothesis)
{
var allParses = ParseAllSentences(currentHypothesis);
if (allParses != null)
return Encoder.TotalLength(currentHypothesis, allParses);
return null;
}
public static void CreateMovementGrammarCompetitor(Grammar grammar)
{
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"C", "START"}, 0, 1));
grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"PP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"V3", "START"}, 0, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));
grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));
}
public static void CreateMovementGrammarCompetitor_old(Grammar grammar)
{
Program.AddTypesToGrammar(grammar);
grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
grammar.AddRule(new Rule(1, "START", new[] {"P", "START"}, 0, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 1)); //1.58496250072116
grammar.AddRule(new Rule(0, "CP", new[] {"VP", "C"}, 1, 0));
grammar.AddRule(new Rule(0, "CP", new[] {"C"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1", "DP"}, 0, 1)); //2.80735493280681
grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"VP", "NP"}, 0, 1));
grammar.AddRule(new Rule(0, "VP", new[] {"NP", "V3"}, 1, 0));
grammar.AddRule(new Rule(0, "VP", new[] {"DP", "V3"}, 1, 1));
grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0)); //1
grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));
grammar.AddRule(new Rule(0, "DP", new[] {"D", "N"}, 0, 1)); //0
}
private static void TestMovement2()
{
var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
var grammar = new Grammar(voc);
Program.AddTypesToGrammar(grammar);
grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
grammar.AddRule(new Rule(1, "START", new[] {"VP", "VP"}, 1, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"VP", "PP"}, 0, 1));
grammar.AddRule(new Rule(1, "VP", new[] {"NP", "V1"}, 1, 0));
grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));
var parser = new Parser(grammar, true);
var n = parser.ParseSentence("the man arrived to Mary"); //supposed to fail in parsing!!
n.Print();
}
public bool ChangeRightToLeftBranch(Grammar grammar)
{
/* for (var i = 0; i < NUMBER_OF_RETRIES; i++)
{
var outerRule = grammar.GetRandomRule();
if (outerRule.Production.Length < 2) continue;
var A = outerRule.Production[0];
var D = outerRule.Production[1];
if (!grammar.Rules.ContainsKey(D.NonTerminal)) continue;
var innerRule = grammar.GetRandomRuleForAGivenLHS(D.NonTerminal);
if (innerRule.Production.Length < 2) continue;
if (outerRule.Equals(innerRule)) continue;
var B = innerRule.Production[0];
var C = innerRule.Production[1];
var oldOuterRuleName = outerRule.Name.NonTerminal;
var newOuterRule = new Rule(outerRule);
newOuterRule.Production = new[] { D, C };
var newInnerRule = new Rule(innerRule);
innerRule.Name = D;
newInnerRule.Production = new[] { A, B };
// the new rules may be inconsistent with head-driven grammar.
// for instance, switching branches may create an inconsistent rule NP -> NP V.
if (!grammar.AreHeadRelationsConsistent(newInnerRule) ||
!grammar.AreHeadRelationsConsistent(newOuterRule)) continue;
//Console.WriteLine("found HeadRelation consistent");
var landingSiteInner = grammar.LandingSites.Contains(innerRule);
var landingSiteOuter = grammar.LandingSites.Contains(outerRule);
grammar.DeleteRule(innerRule);
grammar.DeleteRule(outerRule);
//the grammar may change the rule name according to the head-driven logic.
var newInnerRuleName = grammar.AddRule(newInnerRule, landingSiteInner);
newOuterRule.Production = new[] { new NonTerminalObject(newInnerRuleName), C };
var newOuterRuleName = grammar.AddRule(newOuterRule, landingSiteOuter);
//print("change_left_branch_to_right_branch::new outer rule (inserted): {0} new inner rule (inserted) {1}"\
// .format(new_outer_rule,new_inner_rule))
return true;
}*/
return false;
}
public Dictionary<int, int> CollectUsages(Grammar currentHypothesis)
{
var allParses = ParseAllSentences(currentHypothesis);
var usagesDic = new Dictionary<int, int>();
if (allParses != null)
{
foreach (var item in allParses)
Encoder.CollectRuleUsages(item.Item1, usagesDic, item.Item2);
return usagesDic;
}
return null;
}
internal Grammar GetNeighbor(Grammar currentHypothesis)
{
while (true)
{
var m = GrammarPermutations.GetWeightedRandomMutation();
var newGrammar = new Grammar(currentHypothesis);
var res = m(newGrammar);
if (res)
{
//Console.WriteLine("mutation accepted: {0}", m);
//Console.WriteLine("New grammar {0}",newGrammar.ToString());
}
else
{
//if there is no mutation to accept, don't bother re-parsing the current grammar.
//Console.WriteLine("mutation rejected");
return null;
}
var infinite = newGrammar.DoesGrammarAllowInfiniteMovement();
if (!infinite) return newGrammar;
}
}