Example #1
0
        private static void ComplexMovementTest()
        {
            var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
            var grammar = new Grammar(voc);
            Program.AddTypesToGrammar(grammar);

            grammar.AddProjectionTypeToTypeDictionary("CP", "N"); // no overt POS here, head of IP is V.
            grammar.AddProjectionTypeToTypeDictionary("IP", "N"); // no overt POS here, head of IP is V.
            grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));
            grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));

            //grammar.AddRule(new Rule(1, "CP", new[] { "IP" }, 0, 0));
            //grammar.AddRule(new Rule(1, "IP", new[] { "NP" }, 0, 0));
            //grammar.AddRule(new Rule(1, "VP", new[] { "V1", "NP" }, 0, 1));
            //grammar.AddRule(new Rule(1, "VP", new[] { "V2", "PP" }, 0, 1));
            //grammar.AddRule(new Rule(1, "PP", new[] { "P", "NP" }, 0, 1));
            //grammar.AddRule(new Rule(1, "NP", new[] { "D", "N" }, 1, 0));
            grammar.AddNonTerminalToLandingSites("CP");
            grammar.AddNonTerminalToLandingSites("IP");
            grammar.AddMoveable("NP");
            grammar.AddMoveable("PP");
            grammar.GenerateDerivedRulesFromSchema();

            var parser = new Parser(grammar, true);
            var n = parser.ParseSentence("Who Who Who");
            n.Print();
        }
        public SimulatedAnnealing(Learner l)
        {
            SimulatedAnnealingRunningParameters pam;
            using (var file = File.OpenText(@"SimulatedAnnealingParameters.json"))
            {
                var serializer = new JsonSerializer();
                pam =
                    (SimulatedAnnealingRunningParameters)
                        serializer.Deserialize(file, typeof(SimulatedAnnealingRunningParameters));
            }


            learner = l;

            threshold = pam.ThresholdTemperature;
            coolingFactor = pam.CoolingFactor;
            currentIteration = bestIteration = 1;
            reportEveryNIterations = pam.ReportEveryNIteration;

            currentHypothesis = bestHypothesis = learner.GetInitialGrammar();

            currentHypothesis.GenerateDerivedRulesFromSchema();
            currentEnergy = bestEnergy = learner.Energy(currentHypothesis);
            currentHypothesis.GenerateInitialRulesFromDerivedRules();

            currentTemp = currentEnergy.TotalEnergy*pam.InitialTemperatureTimesInitialEnegrgy;
            using (var sw = File.AppendText("SessionReport.txt"))
            {
                sw.WriteLine(string.Format("cooling factor: {0}, initial energy: {1}, initial temperature: {2}",
                    coolingFactor, currentEnergy, currentTemp));
            }
        }
Example #3
0
        internal Grammar GetInitialGrammar_old()
        {
            var initialGrammar = new Grammar(voc);
            initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
            initialGrammar.POSTypes = posTypes;

            foreach (var sentence in sentencesWithCounts.Keys)
            {
                var words = sentence.Split();

                //assume unamibigous interpretation of words. take the first POS for each word.
                var posList = words.Select(x => voc[x].First()).ToArray();

                if (posList.Length < 2) continue;

                var lastRuleName =
                    initialGrammar.AddRule(new Rule(0, null, new[] {posList[0], posList[1]}, 1, 0));

                for (var i = 0; i < posList.Length - 2; i++)
                {
                    lastRuleName =
                        initialGrammar.AddRule(new Rule(0, null, new[] {lastRuleName, posList[i + 2]}, 1, 0));
                }

                initialGrammar.AddRule(new Rule(0, initialGrammar.StartSymbol, new[] {lastRuleName}, 0, 0));
            }

            return initialGrammar;
        }
Example #4
0
        public Grammar(Grammar otherGrammar)
        {
            Rules = otherGrammar.Rules.ToDictionary(x => x.Key, x => x.Value.Select(y => new Rule(y)).ToList());
            var rules = Rules.Values.SelectMany(l => l).ToArray();
            ruleNumberDictionary =
                rules.Select(x => new KeyValuePair<int, Rule>(x.Number, x)).ToDictionary(x => x.Key, x => x.Value);
            nonTerminalCounts = otherGrammar.nonTerminalCounts.ToDictionary(x => x.Key,
                x => new NonTerminalCounts(x.Value));

            if (otherGrammar.Moveables != null)
                Moveables = new HashSet<string>(otherGrammar.Moveables);
            if (otherGrammar.LandingSites != null)
                LandingSites = new HashSet<string>(otherGrammar.LandingSites);
            StartSymbol = otherGrammar.StartSymbol;
            POSTypes = new HashSet<string>(otherGrammar.POSTypes);
            Vocabulary = otherGrammar.Vocabulary;
            if (otherGrammar.availableRuleNumbers != null)
                availableRuleNumbers = new Stack<int>(otherGrammar.availableRuleNumbers);
            numberOfRules = otherGrammar.numberOfRules;

            NonTerminalsTypeDictionary = otherGrammar.NonTerminalsTypeDictionary.ToDictionary(x => x.Key, x => x.Value);
            inverseRules = otherGrammar.inverseRules.ToDictionary(x => new InverseKeyType(x.Key), x => x.Value);

            nullableProductions = otherGrammar.nullableProductions.ToDictionary(x => x.Key, x => x.Value);
        }
Example #5
0
        public static int GetGrammarLength(Grammar grammar)
        {
            var rules = grammar.ruleNumberDictionary.Values.ToArray();
            var productions = rules.SelectMany(r => r.Production);
            var names = rules.Select(r => r.Name);

            var overallTerms = productions.Concat(names);
            var overallNonTerminals = overallTerms.SelectMany(r => r.ParticipatingNonTerminals()).ToArray();
            var termDic = overallNonTerminals.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count());

            var counts = termDic.Select(x => x.Value).ToArray();
            var totalCount = counts.Sum();

            //length of symbol table = length of symbol codewords + length of probability hashtable:
            //length of symbol codewords is:
            var symbolCodewordsLength =
                counts.Select(x => RequiredBitsGivenProbability(x/(float) totalCount)).Sum();
            //length of probability hashtable is:
            var symbolHashTableBits = counts.Length*BitsForProbabilityEncoding;

            //length of codewords contributed by symbols in rules 
            var overallSymbolRepresentationLength =
                counts.Select(x => RequiredBitsGivenProbability(x/(float) totalCount)*x).Sum();

            var ruleHashTableBits = rules.Length*(3 + BitsForProbabilityEncoding);
            //see documentation. 3 bits for encoding head position, complement position, number of sons.

            return symbolCodewordsLength + symbolHashTableBits + overallSymbolRepresentationLength + ruleHashTableBits;
        }
        public Tuple<EnergyData, Grammar> Run()
        {
            var rand = new Random();
            while (currentTemp > threshold)
            {
                try
                {
                    var newHypothesis = learner.GetNeighbor(currentHypothesis);

                    EnergyData newEnergy = null;
                    if (newHypothesis != null)
                    {
                        newHypothesis.GenerateDerivedRulesFromSchema();
                        newEnergy = learner.Energy(newHypothesis);
                        newHypothesis.GenerateInitialRulesFromDerivedRules();
                    }

                    if (newEnergy != null)
                    {
                        if (newEnergy < bestEnergy)
                        {
                            bestEnergy = newEnergy;
                            bestHypothesis = newHypothesis;
                            bestIteration = currentIteration;
                        }

                        var prob = P(currentEnergy, newEnergy, currentTemp);

                        if (rand.NextDouble() < prob)
                        {
                            // moved to new hypothesis
                            currentHypothesis = newHypothesis;
                            currentEnergy = newEnergy;
                        }
                    }
                    currentIteration++;
                    if (currentIteration%reportEveryNIterations == 0)
                        Console.WriteLine("Iteration {0}", currentIteration);
                    currentTemp *= coolingFactor;
                }

                catch (Exception e)
                {
                    Console.WriteLine(e.ToString());
                }
            }

            var actualRuleDistributions = learner.CollectUsages(bestHypothesis);

            var s = string.Format("{0}. ({1}) \nBest Hypothesis: \n{2}Best so far: #{3} with energy: {4}\n",
                currentIteration, currentTemp, bestHypothesis.GrammarWithRuleUsages(actualRuleDistributions),
                bestIteration, bestEnergy);
            Console.WriteLine(s);
            using (var sw = File.AppendText("SessionReport.txt"))
            {
                sw.WriteLine(s);
            }
            return new Tuple<EnergyData, Grammar>(bestEnergy, bestHypothesis);
        }
Example #7
0
        public static EnergyData TotalLength(Grammar grammar, Tuple<Node, int>[] parseTreesWithCounts)
        {
            var e = new EnergyData();
            //var rules = grammar.ruleNumberDictionary.Values;

            e.GrammarEnergy = GetGrammarLength(grammar);
            e.DataEnergy = GetDataLength(grammar, parseTreesWithCounts);

            return e;
        }
Example #8
0
        public Learner(Vocabulary v, Dictionary<string, string> n, HashSet<string> p,
            string[] s, Grammar o)
        {
            voc = v;
            nonTerminalTypeDic = n;
            posTypes = p;
            originalGrammar = o;

            sentencesWithCounts = s.GroupBy(x => x).ToDictionary(g => g.Key, g => g.Count());
        }
Example #9
0
        private static void Main(string[] args)
        {
            var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
            var grammar = new Grammar(voc);

            ProgramParams programParams;
            using (var file = File.OpenText(@"ProgramParameters.json"))
            {
                var serializer = new JsonSerializer();
                programParams = (ProgramParams) serializer.Deserialize(file, typeof(ProgramParams));
            }


            if (programParams.DataWithMovement)
                CreateMovementGrammar(grammar);
            else
                CreateSimpleGrammar(grammar);
            grammar.GenerateDerivedRulesFromSchema();

            var p = new Parser(grammar);
            var data = p.GenerateSentences(programParams.NumberOfDataSentences);

            using (var sw = File.AppendText("SessionReport.txt"))
            {
                sw.WriteLine("-------------------");
                sw.WriteLine("Session {0} ", DateTime.Now.ToString("MM/dd/yyyy h:mm tt"));
                sw.WriteLine("sentences: {0}, runs: {1}, movement: {2}", programParams.NumberOfDataSentences,
                    programParams.NumberOfRuns, programParams.DataWithMovement);
            }

            var stopWatch = StartWatch();

            var learner = new Learner(voc, grammar.NonTerminalsTypeDictionary, grammar.POSTypes, data, grammar);

            learner.originalGrammar.GenerateDerivedRulesFromSchema();
            var targetGrammarEnergy = learner.Energy(learner.originalGrammar);
            learner.originalGrammar.GenerateInitialRulesFromDerivedRules();
            var s = string.Format("Target Hypothesis:\n{0} with energy: {1}\n", learner.originalGrammar,
                targetGrammarEnergy);

            Console.WriteLine(s);
            using (var sw = File.AppendText("SessionReport.txt"))
            {
                sw.WriteLine(s);
            }

            for (var i = 0; i < programParams.NumberOfRuns; i++)
            {
                var sa = new SimulatedAnnealing(learner);
                sa.Run();
            }
            StopWatch(stopWatch);
        }
Example #10
0
        public static void CreatePalindromeGrammar(Grammar grammar)
        {
            grammar.AddAllPOSTypesToDictionary(new[] {"A", "B"});

            grammar.AddRule(new Rule(2, "START", new[] {"A", "A"}, 0, 1));
            grammar.AddRule(new Rule(2, "START", new[] {"B", "B"}, 0, 1));

            grammar.AddRule(new Rule(2, "START", new[] {"AP", "A"}, 1, 1));
            grammar.AddRule(new Rule(2, "START", new[] {"BP", "B"}, 1, 1));

            grammar.AddRule(new Rule(1, "AP", new[] {"A", "START"}, 0, 0));
            grammar.AddRule(new Rule(1, "BP", new[] {"B", "START"}, 0, 0));
        }
Example #11
0
        public static void CreateSimpleGrammar(Grammar grammar)
        {
            AddTypesToGrammar(grammar);
            grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 1, 1));

            grammar.AddRule(new Rule(1, "VP", new[] {"V0"}, 0, 0));
            grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
            grammar.AddRule(new Rule(1, "VP", new[] {"V2", "PP"}, 0, 1));
            grammar.AddRule(new Rule(1, "VP", new[] {"V3", "START"}, 0, 1));

            grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));
            grammar.AddRule(new Rule(1, "PP", new[] {"P", "NP"}, 0, 1));
        }
Example #12
0
 public static void AddTypesToGrammar(Grammar grammar)
 {
     grammar.AddAllPOSTypesToDictionary(new[] {"D", "N", "V0", "V1", "V2", "V3", "P", "C", "V", "I"});
     grammar.NonTerminalsTypeDictionary["V0"] = "V";
     grammar.NonTerminalsTypeDictionary["V1"] = "V";
     grammar.NonTerminalsTypeDictionary["V2"] = "V";
     grammar.NonTerminalsTypeDictionary["V3"] = "V";
     grammar.NonTerminalsTypeDictionary["VP"] = "V";
     grammar.NonTerminalsTypeDictionary["V0P"] = "V";
     grammar.NonTerminalsTypeDictionary["V1P"] = "V";
     grammar.NonTerminalsTypeDictionary["V2P"] = "V";
     grammar.NonTerminalsTypeDictionary["V3P"] = "V";
 }
Example #13
0
        public Dictionary<int, HashSet<int>> ComputeLeftCorner(Grammar grammar)
        {
            var rules = grammar.ruleNumberDictionary.Values.ToList();
            //key - nonterminal, value - set of the numbers of reachable rules by transitive left corner.
            var leftCorners = new Dictionary<int, HashSet<int>>();

            foreach (var item in rules)
            {
                var ruleNumber = item.Number;
                if (!leftCorners.ContainsKey(ruleNumber))
                    leftCorners[ruleNumber] = new HashSet<int>();


                var ruleList = grammar[item.Production[0].NonTerminal];
                if (ruleList != null)
                {
                    foreach (var predicted in ruleList)
                    {
                        if (!leftCorners[ruleNumber].Contains(predicted.Number))
                            leftCorners[ruleNumber].Add(predicted.Number);
                    }
                }
            }

            var changed = true;
            while (changed)
            {
                changed = false;
                foreach (var item in leftCorners)
                {
                    var ruleNumber = item.Key;
                    var reachableRules = item.Value;

                    foreach (var reachable in reachableRules)
                    {
                        var reachablesFromReachable = leftCorners[reachable];

                        foreach (var reachreach in reachablesFromReachable)
                        {
                            if (!leftCorners[ruleNumber].Contains(reachreach))
                            {
                                leftCorners[ruleNumber].Add(reachreach);
                                changed = true;
                            }
                        }
                    }
                }
            }
            return leftCorners;
        }
        //generate a new rule from random existing productions.
        public bool InsertRule(Grammar grammar)
        {
            for (var i = 0; i < NumberOfRetries; i++)
            {
                var productions = new List<string>();
                var randomDaughter = grammar.StartSymbol;
                while (randomDaughter == grammar.StartSymbol)
                    randomDaughter = grammar.GetRandomNonTerminal(); //the first daughter is never the start symbol.

                productions.Add(randomDaughter);

                if (_rand.NextDouble() < 0.5f)
                    productions.Add(grammar.GetRandomNonTerminal());

                var newRule = new Rule();
                newRule.Occurrences = 1;
                newRule.Production = productions.Select(x => new NonTerminalObject(x)).ToArray();

                newRule.HeadPosition = _rand.Next(newRule.Production.Length);
                newRule.ComplementPosition = _rand.Next(newRule.Production.Length);

                if (newRule.HeadTerm == grammar.StartSymbol)
                    //never let the head be the start symbol. the start symbol can only be the second term(see above).
                    newRule.HeadPosition = 0;


                var ruleName = grammar.StartSymbol;
                if (_rand.NextDouble() < 0.9f)
                    //90% probability of projecting regular head stucture. 10% allow to project to the START symbol.
                {
                    try
                    {
                        ruleName = grammar.NonTerminalsTypeDictionary[newRule.HeadTerm] + "P";
                    }
                    catch
                    {
                        throw new Exception(string.Format("rule head term not found", newRule.HeadTerm));
                    }
                }
                newRule.Name = new NonTerminalObject(ruleName);

                if (grammar.AreHeadRelationsConsistent(newRule))
                {
                    grammar.AddRule(newRule);
                    return true;
                }
            }
            return false;
        }
Example #15
0
        internal Grammar GetInitialGrammar()
        {
            var initialGrammar = new Grammar(voc);
            initialGrammar.NonTerminalsTypeDictionary = nonTerminalTypeDic;
            initialGrammar.POSTypes = posTypes;
            var posInText = voc.POSWithPossibleWords.Keys;

            foreach (var pos in posInText)
            {
                initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos, initialGrammar.StartSymbol},
                    0, 1));
                initialGrammar.AddRule(new Rule(1, initialGrammar.StartSymbol, new[] {pos}, 0, 0));
            }

            return initialGrammar;
        }
Example #16
0
        private static void TestAmbiguityGrammar()
        {
            var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
            var grammar = new Grammar(voc);
            Program.AddTypesToGrammar(grammar);


            grammar.AddRule(new Rule(0, "START", new[] {"VP"}, 0, 0));
            grammar.AddRule(new Rule(1, "VP", new[] {"V1", "NP"}, 0, 1));
            grammar.AddRule(new Rule(1, "VP", new[] {"VP", "NP"}, 0, 0));
            grammar.AddRule(new Rule(0, "NP", new[] {"NP", "NP"}, 0, 0));


            var parser = new Parser(grammar, true);
            var n = parser.ParseSentence("kissed John David");
            n.Print();
        }
Example #17
0
        private Tuple<Node, int>[] ParseAllSentences(Grammar currentHypothesis)
        {
            var allParses = new Tuple<Node, int>[sentencesWithCounts.Count];

            try
            {
                Parallel.ForEach(sentencesWithCounts, (sentenceItem, state, i) =>
                {
                    var parser = new Parser(currentHypothesis);
                    var n = parser.ParseSentence(sentenceItem.Key);
                    allParses[i] = Tuple.Create(n, sentenceItem.Value);
                });
                return allParses;
            }
            catch (Exception)
            {
                return null; //parsing failed.
            }
        }
Example #18
0
        public static void CreateMovementGrammar(Grammar grammar)
        {
            AddTypesToGrammar(grammar);
            grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.

            grammar.AddRule(new Rule(1, "START", new[] {"IP"}, 0, 0));
            grammar.AddRule(new Rule(1, "START", new[] {"CP"}, 0, 0));

            grammar.AddRule(new Rule(0, "CP", new[] {"C", "IP"}, 0, 1));
            grammar.AddRule(new Rule(0, "IP", new[] {"NP", "VP"}, 1, 1));

            grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
            grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
            grammar.AddRule(new Rule(2, "VP", new[] {"V3", "CP"}, 0, 1));

            grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
            grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));

            grammar.AddNonTerminalToLandingSites("CP");
            grammar.AddMoveable("NP");
            grammar.AddMoveable("PP");
        }
        public bool InsertRuleWithANewSymbol_old(Grammar grammar)
        {
            //grammar.CleanOrphanedRules();
            //for (var i = 0; i < NUMBER_OF_RETRIES; i++)
            //{
            //    var fatherRule = grammar.GetRandomRule();
            //    var daughterPos = rand.Next(fatherRule.Production.Length);
            //    var daughter = fatherRule.Production[daughterPos].NonTerminal;

            //    if (grammar.isPOS(daughter)) continue; //needs to be a projection

            //    var newLHSSymbol = grammar.GetNextAvailableProjectionName();
            //    var daughterRule = grammar.GetRandomRuleForAGivenLHS(daughter);
            //    if (daughterRule.Production.Length < 2) continue; //the daughter rule must have two daughters.

            //    //pick out (in random) a rule out of the 4 below which keep the head relations consistent,
            //    var arr = new Rule[4];
            //    var randomProjection = grammar.GetRandomRule().Name.NonTerminal;
            //    if (randomProjection == grammar.StartSymbol) continue; //all projections except START

            //    arr[0] = new Rule(1.0f, newLHSSymbol, new[] { daughter, randomProjection }, 0, 1);
            //    arr[1] = new Rule(1.0f, newLHSSymbol, new[] { randomProjection, daughter }, 1, 0);
            //    arr[2] = new Rule(1.0f, newLHSSymbol, new[] { daughter, randomProjection }, 1, 0);
            //    arr[3] = new Rule(1.0f, newLHSSymbol, new[] { randomProjection, daughter }, 0, 1);

            //    var newRule = arr[rand.Next(4)];
            //    if (!grammar.AreHeadRelationsConsistent(newRule)) continue;

            //    var oldFatherProductions = new string[fatherRule.Production.Length];
            //    fatherRule.Production.CopyTo(oldFatherProductions, 0);
            //    oldFatherProductions[daughterPos] = newLHSSymbol;

            //    var newFatherRule = new Rule(fatherRule);
            //    newFatherRule.Production = oldFatherProductions.Select(x => new NonTerminalObject(x));

            //    if (!grammar.AreHeadRelationsConsistent(newFatherRule)) continue;
            //    var ruleName = grammar.AddRule(newRule);
            //    if (ruleName != newRule.Name.NonTerminal) continue; // in case that the new rule that was added already exists in the grammar.
            //    grammar.AddRule(newFatherRule);
            //    return true;
            //}
            return false;
        }
Example #20
0
 public static int GetDataLength(Grammar grammar, Tuple<Node, int>[] parseTreesWithCounts)
 {
     var sum = 0;
     foreach (var parseTree in parseTreesWithCounts)
     {
         var root = parseTree.Item1; //the node.
         sum += root.Bits*parseTree.Item2;
         //Item2 = number of occurrences of that sentence in the corpus
     }
     return sum;
 }
Example #21
0
 private static Grammar InfiniteMovementGrammar1()
 {
     var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
     var grammar = new Grammar(voc);
     Program.AddTypesToGrammar(grammar);
     grammar.AddRule(new Rule(1, "START", new[] {"NP"}, 0, 0));
     grammar.AddRule(new Rule(1, "NP", new[] {"PP", "NP"}, 1, 0));
     grammar.AddRule(new Rule(1, "PP", new[] {"NP", "PP"}, 1, 0));
     grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
     var does = grammar.DoesGrammarAllowInfiniteMovement();
     return grammar;
 }
Example #22
0
        private static Grammar InfiniteMovementGrammar3()
        {
            var voc = Vocabulary.GetVocabularyFromFile(@"..\..\..\Input\Vocabulary.json");
            var grammar = new Grammar(voc);
            Program.AddTypesToGrammar(grammar);
            grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
            grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
            grammar.AddRule(new Rule(1, "NP", new[] {"V0P", "NP"}, 1, 0));

            grammar.AddRule(new Rule(1, "V0P", new[] {"V2"}, 0, 0));
            grammar.AddRule(new Rule(1, "START", new[] {"NP", "V0"}, 1, 1));
            grammar.AddRule(new Rule(1, "START", new[] {"V0", "NP"}, 0, 1));
            grammar.AddRule(new Rule(1, "START", new[] {"PP", "NP"}, 0, 1));
            grammar.AddRule(new Rule(1, "START", new[] {"NP", "NP"}, 1, 1));
            grammar.AddRule(new Rule(1, "V0P", new[] {"V1"}, 0, 0));
            grammar.AddRule(new Rule(1, "PP", new[] {"V0P", "P"}, 1, 1));

            grammar.AddRule(new Rule(1, "V0P", new[] {"NP", "V0P"}, 1, 1));
            grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 1));
            var does = grammar.DoesGrammarAllowInfiniteMovement();
            return grammar;
        }
Example #23
0
 public Parser(Grammar g, bool debug = false)
 {
     Grammar = g;
     Debug = debug;
     ruleLogProbabilities = Grammar.GetLogProbabilitiesOfRules();
 }
Example #24
0
        public EnergyData Energy(Grammar currentHypothesis)
        {
            var allParses = ParseAllSentences(currentHypothesis);
            if (allParses != null)
                return Encoder.TotalLength(currentHypothesis, allParses);

            return null;
        }
Example #25
0
        public static void CreateMovementGrammarCompetitor(Grammar grammar)
        {
            Program.AddTypesToGrammar(grammar);

            grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
            grammar.AddRule(new Rule(1, "START", new[] {"C", "START"}, 0, 1));

            grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 0));
            grammar.AddRule(new Rule(0, "CP", new[] {"PP", "C"}, 1, 0));

            grammar.AddRule(new Rule(0, "VP", new[] {"V1", "NP"}, 0, 1));
            grammar.AddRule(new Rule(0, "VP", new[] {"V2", "PP"}, 0, 1));
            grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
            grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));
            grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 1));
            grammar.AddRule(new Rule(0, "VP", new[] {"V3", "START"}, 0, 1));

            grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0));
            grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));


            grammar.AddRule(new Rule(0, "PP", new[] {"P", "NP"}, 0, 1));
        }
Example #26
0
        public static void CreateMovementGrammarCompetitor_old(Grammar grammar)
        {
            Program.AddTypesToGrammar(grammar);

            grammar.AddRule(new Rule(1, "START", new[] {"NP", "VP"}, 0, 0));
            grammar.AddRule(new Rule(1, "START", new[] {"P", "START"}, 0, 0));

            grammar.AddRule(new Rule(0, "CP", new[] {"NP", "C"}, 1, 1)); //1.58496250072116
            grammar.AddRule(new Rule(0, "CP", new[] {"VP", "C"}, 1, 0));
            grammar.AddRule(new Rule(0, "CP", new[] {"C"}, 0, 0));


            grammar.AddRule(new Rule(0, "VP", new[] {"V1", "DP"}, 0, 1)); //2.80735493280681
            grammar.AddRule(new Rule(0, "VP", new[] {"V2", "P"}, 0, 0));
            grammar.AddRule(new Rule(0, "VP", new[] {"V2"}, 0, 0));
            grammar.AddRule(new Rule(0, "VP", new[] {"V1"}, 0, 0));

            grammar.AddRule(new Rule(0, "VP", new[] {"VP", "NP"}, 0, 1));
            grammar.AddRule(new Rule(0, "VP", new[] {"NP", "V3"}, 1, 0));
            grammar.AddRule(new Rule(0, "VP", new[] {"DP", "V3"}, 1, 1));


            grammar.AddRule(new Rule(0, "NP", new[] {"D", "N"}, 1, 0)); //1
            grammar.AddRule(new Rule(0, "NP", new[] {"CP", "NP"}, 1, 1));
            grammar.AddRule(new Rule(0, "DP", new[] {"D", "N"}, 0, 1)); //0
        }
Example #27
0
        private static void TestMovement2()
        {
            var voc = Vocabulary.GetVocabularyFromFile(@"Vocabulary.json");
            var grammar = new Grammar(voc);
            Program.AddTypesToGrammar(grammar);

            grammar.AddProjectionTypeToTypeDictionary("CP", "V"); // no overt POS here, head of IP is V.
            grammar.AddProjectionTypeToTypeDictionary("IP", "V"); // no overt POS here, head of IP is V.
            grammar.AddRule(new Rule(1, "START", new[] {"VP", "VP"}, 1, 1));
            grammar.AddRule(new Rule(1, "VP", new[] {"VP", "PP"}, 0, 1));
            grammar.AddRule(new Rule(1, "VP", new[] {"NP", "V1"}, 1, 0));
            grammar.AddRule(new Rule(1, "PP", new[] {"V2", "P"}, 1, 0));
            grammar.AddRule(new Rule(1, "NP", new[] {"D", "N"}, 1, 0));


            var parser = new Parser(grammar, true);
            var n = parser.ParseSentence("the man arrived to Mary"); //supposed to fail in parsing!!
            n.Print();
        }
        public bool ChangeRightToLeftBranch(Grammar grammar)
        {
            /* for (var i = 0; i < NUMBER_OF_RETRIES; i++)
             {
                 var outerRule = grammar.GetRandomRule();

                 if (outerRule.Production.Length < 2) continue;

                 var A = outerRule.Production[0];
                 var D = outerRule.Production[1];

                 if (!grammar.Rules.ContainsKey(D.NonTerminal)) continue;

                 var innerRule = grammar.GetRandomRuleForAGivenLHS(D.NonTerminal);
                 if (innerRule.Production.Length < 2) continue;
                 if (outerRule.Equals(innerRule)) continue;

                 var B = innerRule.Production[0];
                 var C = innerRule.Production[1];

                 var oldOuterRuleName = outerRule.Name.NonTerminal;

                 var newOuterRule = new Rule(outerRule);
                 newOuterRule.Production = new[] { D, C };
                 var newInnerRule = new Rule(innerRule);
                 innerRule.Name = D;
                 newInnerRule.Production = new[] { A, B };

                 // the new rules may be inconsistent with head-driven grammar.
                 // for instance, switching branches may create an inconsistent rule  NP -> NP V.
                 if (!grammar.AreHeadRelationsConsistent(newInnerRule) ||
                     !grammar.AreHeadRelationsConsistent(newOuterRule)) continue;
                 //Console.WriteLine("found HeadRelation consistent");

                 var landingSiteInner = grammar.LandingSites.Contains(innerRule);
                 var landingSiteOuter = grammar.LandingSites.Contains(outerRule);

                 grammar.DeleteRule(innerRule);
                 grammar.DeleteRule(outerRule);
                 //the grammar may change the rule name according to the head-driven logic.
                 var newInnerRuleName = grammar.AddRule(newInnerRule, landingSiteInner);
                 newOuterRule.Production = new[] { new NonTerminalObject(newInnerRuleName), C };
                 var newOuterRuleName = grammar.AddRule(newOuterRule, landingSiteOuter);

                 //print("change_left_branch_to_right_branch::new outer rule (inserted): {0} new inner rule (inserted) {1}"\
                 //    .format(new_outer_rule,new_inner_rule))

                 return true;
             }*/
            return false;
        }
Example #29
0
        public Dictionary<int, int> CollectUsages(Grammar currentHypothesis)
        {
            var allParses = ParseAllSentences(currentHypothesis);
            var usagesDic = new Dictionary<int, int>();

            if (allParses != null)
            {
                foreach (var item in allParses)
                    Encoder.CollectRuleUsages(item.Item1, usagesDic, item.Item2);

                return usagesDic;
            }

            return null;
        }
Example #30
0
        internal Grammar GetNeighbor(Grammar currentHypothesis)
        {
            while (true)
            {
                var m = GrammarPermutations.GetWeightedRandomMutation();
                var newGrammar = new Grammar(currentHypothesis);

                var res = m(newGrammar);
                if (res)
                {
                    //Console.WriteLine("mutation accepted: {0}", m);
                    //Console.WriteLine("New grammar {0}",newGrammar.ToString());
                }
                else
                {
                    //if there is no mutation to accept, don't bother re-parsing the current grammar.
                    //Console.WriteLine("mutation rejected");
                    return null;
                }
                var infinite = newGrammar.DoesGrammarAllowInfiniteMovement();
                if (!infinite) return newGrammar;
            }
        }