예제 #1
0
        public void BuildHyperGraph(LAPCFGrammar grammar, HyperEdgePool epool, HyperVertexPool vpool, int[] tagCapacity = null)
        {
            epool.Recycle ();
            vpool.Recycle ();
            this.ROOTID = grammar.ROOTID;

            var maxSubTag = grammar.subTagCounts.Max ();

            lbuf = new double[maxSubTag + 1];

            // CYK
            for (int i = 0; i < wids.Length; ++i) {
                bool isRoot = i == 0 && i == wids.Length - 1;
                chart [i, i] = new HyperCell (i, i + 1, grammar.TotalTagCount);
                MatchLexicon (grammar, chart [i, i], wids [i], epool, vpool, tagCapacity,
                    allowedPoSTags == null ? null : allowedPoSTags[i],
                    rawTagProbs == null ? null : rawTagProbs[i], isRoot);

                MatchUnaryRules (grammar, chart [i, i], epool, vpool, tagCapacity, isRoot);
                chart [i, i].Finish ();
            }

            for (int spanL = 2; spanL <= wids.Length; ++spanL) {
                for (int beg = 0; beg + spanL <= wids.Length; ++beg) {
                    int end = beg + spanL;
                    int l = beg;
                    int r = end - 1;
                    bool isRoot = l == 0 && r == wids.Length - 1;
                    chart [l, r] = new HyperCell (beg, end, grammar.TotalTagCount);
                    for (int mid = l; mid < r; ++mid) {
                        MatchBinaryRules (grammar, chart [l, r], chart [l, mid], chart [mid + 1, r], epool, vpool, tagCapacity, isRoot);
                    }

                    for (int i = 0; i < chart[l, r].l1v.Length; ++i) {
                        var c = chart [l, r].l1v [i];

                        if (c != null) {
                            if (isRoot && c.tag != ROOTID) {
                                continue;
                            }
                            chart [l, r].l2v [i] = vpool.Allocate (false, c.tag, c.beta.Length, c.beta.v.Length);
                            epool.Allocate (chart [l, r].l2v [i], c);
                        }

                    }
                    MatchUnaryRules (grammar, chart [l, r], epool, vpool, tagCapacity, isRoot);
                    chart [l, r].Finish ();
                }
            }
        }
예제 #2
0
        static double EvaluateRawParser()
        {
            string modelfile = //@"/home/nan/Data/PTB/ptb.s1.smoothed.grammar";
                                @"D:\user\nyang\data\treebank\English\pcfg\ptb.s2.smoothed.grammar";

            string testfile = //@"/home/nan/Data/PTB/xbar/wsj.23.flat";
                            @"D:\user\nyang\data\treebank\English\pcfg\xbar\wsj.23.flat";

            string outputfile = @"C:\Users\v-nayang\Downloads\EVALB\EVALB\wsj.23.flat.out";

            string reffile = @"C:\Users\v-nayang\Downloads\EVALB\EVALB\wsj.23.flat.ref";

            string bklex = @"D:\user\nyang\tools\bkparser\wsj.la.grammar.txt.lexicon";
            string bkgrammar = @"D:\user\nyang\tools\bkparser\wsj.la.grammar.txt.grammar";

            int nthread = 16;

            Vocabulary vocab;
            TagSet tagSet;
            LAPCFGrammar grammar;

            using (var s = new TextModelReader(modelfile))
            {
                vocab = Vocabulary.LoadFromStream(s);
                tagSet = TagSet.LoadFromStream(s);
                grammar = LAPCFGrammar.LoadFromStream(s, vocab, tagSet);
                //grammar = new LAPCFGrammar(tagSet, vocab, bklex, bkgrammar);
            }

            //grammar.Smoothing(0.01, 0.1);

            //grammar.Normalize();

            //grammar.PropMaxUnaryPath();

            grammar.MakeCompaction();

            grammar.MakeSubruleCompaction();

            var treebank = new List<PhrasalTree>();

            LoadTrees(treebank, testfile);

            foreach (var tree in treebank)
            {
                foreach (var node in tree.TreeNodes)
                {
                    if (node.Children.Count == 0)
                    {
                        node.Lex = SimpleTokenizor.ETokenize(node.Lex);
                    }
                }

                //PennTreeProcessor.RecoverFromXBarBinarize(tree.Root);

                tree.ComputeStartEnd();
            }

            treebank = treebank.Where(x => x.Root.End <= 20).ToList();

            double ccount = 0;
            double pcount = 0;
            double gcount = 0;
            int failed = 0;
            int sentcount = treebank.Count;

            Console.Error.WriteLine("Start to parse...");
            ConsoleTimer tm = new ConsoleTimer(1);

            PhrasalTree[] ptrees = new PhrasalTree[treebank.Count];

            Parallel.For(0, nthread, thrID =>
            {
                HyperEdgePool epool = new HyperEdgePool();

                HyperVertexPool vpool = new HyperVertexPool(grammar.subTagCounts.Max());
                for (int treeId = thrID; treeId < treebank.Count; treeId += nthread)
                {
                    var tree = treebank[treeId];
                    var words = tree.GetSentence().Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);

                    int[] wids = words.Select(w => vocab.GetId(SimpleTokenizor.ETokenize(w), false)).ToArray();

                    wids[0] = vocab.GetId(SimpleTokenizor.ETokenize(words[0]), true);

                    string[] tags = tree.GetPOS().Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);

                    double[][] tprobs = new double[wids.Length][];

                    //for (int i = 0; i < wids.Length; ++i)
                    //{
                    //    tprobs[i] = rwHanlder.GetLogProbs(SimpleTokenizor.ETokenize(words[i]));
                    //}

                    bool[][] allowedTags = null;
                    //AssignTagConstraints(vocab, tagSet, words, wids);

                    try
                    {
                        //var parser = new ChartParser(wids);
                        var parser = new ChartHyperGraphParser(wids, allowedTags, tprobs);
                        parser.BuildHyperGraph(grammar, epool, vpool, grammar.subTagCounts);
                        parser.SumForward();
                        parser.SumBackward(true);
                        parser.PosteriorViterbi();

                        var ptree = parser.ExtractPosteriorViterbi(words, tagSet);

                        //parser.MaxForward();

                        //var ptree = parser.ExtractViterbi(words, tagSet);

                        ptree.ComputeStartEnd();

                        ptrees[treeId] = ptree;

                    }
                    catch
                    {
                    }
                }
            });

            using (StreamWriter sw = new StreamWriter(outputfile))
            {
                using (StreamWriter swref = new StreamWriter(reffile))
                {
                    for (int treeid = 0; treeid < treebank.Count; ++treeid)
                    {
                        var tree = treebank[treeid];
                        var ptree = ptrees[treeid];

                        swref.WriteLine(tree.GetParseLine());

                        if (ptree == null)
                        {
                            failed += 1;
                            sw.WriteLine("()");
                            continue;
                        }

                        var pbrackets = ptree.GetBracketsIgnorePunc();
                        var gbrackets = tree.GetBracketsIgnorePunc();

                        gcount += gbrackets.Count;
                        pcount += pbrackets.Count;

                        double xxc = 0;

                        foreach (var b in pbrackets)
                        {
                            if (gbrackets.Contains(b))
                            {
                                ccount += 1;
                                xxc += 1;
                            }
                        }

                        if (pbrackets.Count == 0
                            || (pbrackets.Count < gbrackets.Count / 2))
                        {
                            Console.Error.WriteLine("\nStrange prediction: p={0}, g={1}", pbrackets.Count,
                                gbrackets.Count);
                        }

                        string parseline = ptree.GetParseLine();

                        double snt_p = xxc / pbrackets.Count;
                        double snt_r = xxc / gbrackets.Count;

                        double snt_f1 = 2.0 * snt_p * snt_r / (snt_p + snt_r);

                        sw.WriteLine(parseline);

                        //sw.WriteLine(" [Current]\tP: {0:F2} R: {1:F2} F1: {2:F3}", snt_p * 100.0, snt_r * 100.0, snt_f1 * 100.0);

                    }
                }
            }

            tm.Finish();

            Console.Error.WriteLine("SENT: {0}\tFAIL: {1}", sentcount, failed);

            double prec = ccount / pcount;
            double recall = ccount / gcount;

            double f1 = 2.0 * prec * recall / (prec + recall);

            Console.Error.WriteLine("P: {0:F3}\tR: {1:F3}\tF1: {2:F3}", prec, recall, f1);

            return f1;
        }
예제 #3
0
        static void EvaluateParser()
        {
            string modelfile = //@"/home/nan/Data/PTB/ptb.s1.smoothed.grammar";
                                @"D:\user\nyang\data\treebank\English\pcfg\ptb.s1.smoothed.grammar";

            string tagmapfile = @"E:\nyang\TFS\Common\Users\v-nayang\LAPCFGParser\en-tag-map.txt";

            string testfile = //@"/home/nan/Data/PTB/xbar/wsj.23.flat";
                            @"D:\user\nyang\data\treebank\English\pcfg\xbar\wsj.23.flat";

            string trainfile = @"D:\user\nyang\data\treebank\English\pcfg\xbar\wsj.02-21.flat";

            Vocabulary vocab;
            TagSet tagSet;
            LAPCFGrammar grammar;

            var traintrees = new List<PhrasalTree>();

            LoadTrees(traintrees, trainfile);

            var rwHanlder = new RareWordHandler(traintrees, 10);

            using (var s = new TextModelReader(modelfile))
            {
                vocab = Vocabulary.LoadFromStream(s);
                tagSet = TagSet.LoadFromStream(s);
                grammar = LAPCFGrammar.LoadFromStream(s, vocab, tagSet);
            }

            rwHanlder.Build(tagSet, 0.001);

            //grammar.Smoothing(0.1f);

            grammar.MakeCompaction();

            grammar.MakeSubruleCompaction();

            var grammars = new LAPCFGrammar[grammar.subtagTraces.Count + 1];

            var traces = new int[grammars.Length][][];

            grammars [grammars.Length - 1] = grammar;

            for (int i = grammars.Length - 1; i >= 1; --i)
            {
                traces [i] = grammar.subtagTraces [i - 1];
                grammars [i - 1] = grammars [i].ProjectGrammar(traces [i]);
                grammars [i - 1].MakeCompaction();
                grammars [i - 1].MakeSubruleCompaction();
            }

            string[][] tagTiers;

            using (StreamReader sr = new StreamReader(tagmapfile))
            {
                var tt = new List<string[]>();
                while (!sr.EndOfStream)
                {
                    string line = sr.ReadLine();

                    if (string.IsNullOrWhiteSpace(line))
                    {
                        continue;
                    }

                    tt.Add(line.Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries));
                }

                tagTiers = new string[tt [0].Length][];//tt.ToArray();

                for (int i = 0; i < tagTiers.Length; ++i)
                {
                    tagTiers [i] = new string[tt.Count];
                    for (int j = 0; j < tt.Count; ++j)
                    {
                        tagTiers [i] [j] = tt [j] [i];
                    }
                }
            }

            var cbs = new CodeBook32[tagTiers.Length];

            for (int i = 0; i < cbs.Length; ++i)
            {
                cbs [i] = new CodeBook32();

                foreach (var t in tagTiers[i])
                {
                    cbs [i].Add(t);
                }
            }

            int pgcount = cbs.Length - 1;

            int[][] tagMaps = new int[pgcount][];

            for (int i = 0; i < tagMaps.Length; ++i)
            {
                tagMaps [i] = new int[grammars [0].PTCount + 1 + cbs [i + 1].Count];

                for (int j = 0; j < grammars[0].PTCount + 1; ++j)
                {
                    tagMaps [i] [j] = j;
                }
            }

            var lastMap = tagMaps [tagMaps.Length - 1];

            for (int j = grammars[0].PTCount + 1; j < lastMap.Length; ++j)
            {
                string tstr = tagSet.GetTagString(j);
                int id = cbs [cbs.Length - 1] [tstr];
                int pid = cbs [cbs.Length - 2] [tagTiers [tagTiers.Length - 2] [id]];

                lastMap [j] = pid + grammars [0].PTCount + 1;
            }

            for (int i = 0; i < tagMaps.Length - 1; ++i)
            {
                for (int j = grammars[0].PTCount + 1; j < tagMaps[i].Length; ++j)
                {
                    string tstr = cbs [i + 1] [j - grammars [0].PTCount - 1];

                    int xid = Array.IndexOf(tagTiers [i + 1], tstr);

                    string pstr = tagTiers [i] [xid];

                    int pid = cbs [i] [pstr];

                    tagMaps [i] [j] = pid;
                }
            }

            var cgrammars = new LAPCFGrammar[tagMaps.Length];

            cgrammars [cgrammars.Length - 1] = grammars [0].CollapseNonTerminals(tagMaps [cgrammars.Length - 1], 1 + cbs [cgrammars.Length - 1].Count);

            for (int i = cgrammars.Length - 1; i >= 1; --i)
            {
                cgrammars [i - 1] = cgrammars [i].CollapseNonTerminals(tagMaps [i - 1], 1 + cbs [i - 1].Count);
            }

            for (int i = 0; i < cgrammars.Length; ++i)
            {
                cgrammars [i].MakeCompaction();
                cgrammars [i].MakeSubruleCompaction();
            }

            var treebank = new List<PhrasalTree>();

            LoadTrees(treebank, testfile);

            foreach (var tree in treebank)
            {
                foreach (var node in tree.TreeNodes)
                {
                    if (node.Children.Count == 0)
                    {
                        node.Lex = SimpleTokenizor.ETokenize(node.Lex);
                    }
                }

                //PennTreeProcessor.RecoverFromXBarBinarize(tree.Root);

                tree.ComputeStartEnd();
            }

            double ccount = 0;
            double pcount = 0;
            double gcount = 0;
            int failed = 0;
            int sentcount = 0;
            HyperEdgePool epool = new HyperEdgePool(1024 * 1024);

            HyperVertexPool vpool = new HyperVertexPool(grammars [grammars.Length - 1].subTagCounts.Max());

            //EMorph.EnglishMorph.WarmUp();

            Console.Error.WriteLine("Start to parse...");
            ConsoleTimer tm = new ConsoleTimer(1);

            Stopwatch g0bwatch = new Stopwatch();
            Stopwatch g0watch = new Stopwatch();
            Stopwatch bwatch = new Stopwatch();

            Stopwatch[] gwatch = new Stopwatch[grammars.Length];

            for (int i = 0; i < gwatch.Length; ++i)
            {
                gwatch [i] = new Stopwatch();
            }

            Stopwatch vwatch = new Stopwatch();

            foreach (var tree in treebank)
            {
                var words = tree.GetSentence().Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);

                if (words.Length > 20)
                {
                    continue;
                }

                sentcount += 1;

                int[] wids = words.Select(w => vocab.GetId(SimpleTokenizor.ETokenize(w), false)).ToArray();

                wids [0] = vocab.GetId(SimpleTokenizor.ETokenize(words [0]), true);

                string[] tags = tree.GetPOS().Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);

                double[][] tprobs = new double[wids.Length][];

                //for (int i = 0; i < wids.Length; ++i)
                //{
                //    tprobs[i] = rwHanlder.GetLogProbs(SimpleTokenizor.ETokenize(words[i]));
                //}

                bool[][] allowedTags = null;
                    //AssignTagConstraints(vocab, tagSet, words, wids);

                try
                {
                    //var parser = new ChartParser(wids);
                    var parser = new ChartHyperGraphParser(wids, allowedTags, tprobs);
                    g0bwatch.Start();
                    parser.BuildHyperGraph(cgrammars [0], epool, vpool, grammars [grammars.Length - 1].subTagCounts);
                    g0bwatch.Stop();
                    g0watch.Start();
                    parser.SumForward();
                    parser.SumBackward(false);
                    parser.Prune(-15.0);
                    parser.Purge();
                    for (int i = 1; i < cgrammars.Length; ++i)
                    {
                        parser.ExpandHyperGraph(cgrammars [i], tagMaps [i - 1], epool, vpool,
                                                 grammars [grammars.Length - 1].subTagCounts);
                        parser.SumForward();
                        parser.SumBackward(false);
                        parser.Prune(-15.0);
                        parser.Purge();
                    }
                    g0watch.Stop();
            //
                    bwatch.Start();
                    parser.ExpandHyperGraph(grammars [0], tagMaps [2], epool, vpool, grammars [grammars.Length - 1].subTagCounts);
            //					parser.BuildHyperGraph (grammars [0], epool, vpool, grammars [grammars.Length - 1].subTagCounts);
                    bwatch.Stop();

                    for (int i = 0; i < grammars.Length - 1; ++i)
                    {
                        gwatch [i].Start();
                        parser.SumForward();
                        parser.SumBackward(false);

                        parser.Prune(-10.0);

                        parser.Purge();

                        parser.ProjectGrammar(traces [i + 1], grammars [i + 1]);
                        gwatch [i].Stop();
                    }

                    gwatch [grammars.Length - 1].Start();
                    parser.SumForward();

                    parser.SumBackward(true);
                    gwatch [grammars.Length - 1].Stop();

                    vwatch.Start();
                    parser.PosteriorViterbi();

                    var ptree = parser.ExtractPosteriorViterbi(words, tagSet);

                    vwatch.Stop();

                    //PennTreeProcessor.RecoverFromXBarBinarize(ptree.Root);

                    ptree.ComputeStartEnd();

                    var pbrackets = ptree.GetBracketsIgnorePunc();
                    var gbrackets = tree.GetBracketsIgnorePunc();

                    gcount += gbrackets.Count;
                    pcount += pbrackets.Count;

                    foreach (var b in pbrackets)
                    {
                        if (gbrackets.Contains(b))
                        {
                            ccount += 1;
                        }
                    }

                    if (pbrackets.Count == 0
                        || (pbrackets.Count < gbrackets.Count / 2))
                    {
                        Console.Error.WriteLine("\nStrange prediction: p={0}, g={1}", pbrackets.Count,
                            gbrackets.Count);
                    }

                    //Console.Error.WriteLine(tree.TextTree);
                } catch
                {
                    g0bwatch.Stop();
                    g0watch.Stop();
                    bwatch.Stop();
                    foreach (var w in gwatch)
                    {
                        w.Stop();
                    }
                    vwatch.Stop();
                    failed += 1;
                    Console.Error.WriteLine("\nFailure!");
                }

                tm.Up();
            }

            tm.Finish();

            Console.Error.WriteLine("SENT: {0}\tFAIL: {1}", sentcount, failed);

            double prec = ccount / pcount;
            double recall = ccount / gcount;

            double f1 = 2.0 * prec * recall / (prec + recall);

            Console.Error.WriteLine("P: {0:F3}\tR: {1:F3}\tF1: {2:F3}", prec, recall, f1);

            Console.Error.WriteLine("G-1 Build:\t{0:F6} s", g0bwatch.Elapsed.TotalSeconds);

            Console.Error.WriteLine("G-1 Pass:\t{0:F6} s", g0watch.Elapsed.TotalSeconds);

            Console.Error.WriteLine("G0 Build:\t{0:F6} s", bwatch.Elapsed.TotalSeconds);

            for (int i = 0; i < gwatch.Length; ++i)
            {
                Console.Error.WriteLine("G{0} Pass:\t{1:F6} s", i, gwatch [i].Elapsed.TotalSeconds);
            }

            Console.Error.WriteLine("Viterbi:\t{0:F6} s", vwatch.Elapsed.TotalSeconds);
        }
예제 #4
0
        static void TestParse()
        {
            string modelfile = //@"/home/nan/Data/PTB/ptb.s2.smoothed.grammar";
            @"D:\user\nyang\data\treebank\English\pcfg\ptb.s6.smoothed.grammar";
            string tagmapfile = @"E:\nyang\TFS\Common\Users\v-nayang\LAPCFGParser\en-tag-map.txt";

            Vocabulary vocab;
            TagSet tagSet;
            LAPCFGrammar grammar;

            using (var s = new TextModelReader(modelfile))
            {
                vocab = Vocabulary.LoadFromStream(s);
                tagSet = TagSet.LoadFromStream(s);
                grammar = LAPCFGrammar.LoadFromStream(s, vocab, tagSet);
            }

            grammar.Smoothing(0.1f);

            grammar.MakeCompaction();

            grammar.MakeSubruleCompaction();

            var grammars = new LAPCFGrammar[grammar.subtagTraces.Count + 1];

            var traces = new int[grammars.Length][][];

            grammars[grammars.Length - 1] = grammar;

            for (int i = grammars.Length - 1; i >= 1; --i)
            {
                traces[i] = grammar.subtagTraces[i - 1];
                grammars[i - 1] = grammars[i].ProjectGrammar(traces[i]);
                grammars[i - 1].MakeCompaction();
                grammars[i - 1].MakeSubruleCompaction();
            }

            string[][] tagTiers;

            using (StreamReader sr = new StreamReader(tagmapfile))
            {
                var tt = new List<string[]>();
                while (!sr.EndOfStream)
                {
                    string line = sr.ReadLine();

                    if (string.IsNullOrWhiteSpace(line))
                    {
                        continue;
                    }

                    tt.Add(line.Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries));
                }

                tagTiers = new string[tt[0].Length][];//tt.ToArray();

                for (int i = 0; i < tagTiers.Length; ++i)
                {
                    tagTiers[i] = new string[tt.Count];
                    for (int j = 0; j < tt.Count; ++j)
                    {
                        tagTiers[i][j] = tt[j][i];
                    }
                }
            }

            var cbs = new CodeBook32[tagTiers.Length];

            for (int i = 0; i < cbs.Length; ++i)
            {
                cbs[i] = new CodeBook32();

                foreach (var t in tagTiers[i])
                {
                    cbs[i].Add(t);
                }
            }

            int pgcount = cbs.Length - 1;

            int[][] tagMaps = new int[pgcount][];

            for (int i = 0; i < tagMaps.Length; ++i)
            {
                tagMaps[i] = new int[grammars[0].PTCount + 1 + cbs[i + 1].Count];

                for (int j = 0; j < grammars[0].PTCount + 1; ++j)
                {
                    tagMaps[i][j] = j;
                }
            }

            var lastMap = tagMaps[tagMaps.Length - 1];

            for (int j = grammars[0].PTCount + 1; j < lastMap.Length; ++j)
            {
                string tstr = tagSet.GetTagString(j);
                int id = cbs[cbs.Length - 1][tstr];
                int pid = cbs[cbs.Length - 2][tagTiers[tagTiers.Length - 2][id]];

                lastMap[j] = pid + grammars[0].PTCount + 1;
            }

            for (int i = 0; i < tagMaps.Length - 1; ++i)
            {
                for (int j = grammars[0].PTCount + 1; j < tagMaps[i].Length; ++j)
                {
                    string tstr = cbs[i + 1][j - grammars[0].PTCount - 1];

                    int xid = Array.IndexOf(tagTiers[i + 1], tstr);

                    string pstr = tagTiers[i][xid];

                    int pid = cbs[i][pstr];

                    tagMaps[i][j] = pid;
                }
            }

            var cgrammars = new LAPCFGrammar[tagMaps.Length];

            cgrammars[cgrammars.Length - 1] = grammars[0].CollapseNonTerminals(tagMaps[cgrammars.Length - 1], 1 + cbs[cgrammars.Length - 1].Count);

            for (int i = cgrammars.Length - 1; i >= 1; --i)
            {
                cgrammars[i - 1] = cgrammars[i].CollapseNonTerminals(tagMaps[i - 1], 1 + cbs[i - 1].Count);
            }

            for (int i = 0; i < cgrammars.Length; ++i)
            {
                cgrammars[i].MakeCompaction();
                cgrammars[i].MakeSubruleCompaction();
            }

            HyperEdgePool epool = new HyperEdgePool(1024 * 1024);
            HyperVertexPool vpool = new HyperVertexPool(grammars[grammars.Length - 1].subTagCounts.Max());
            EMorph.EnglishMorph.WarmUp();
            Console.Error.WriteLine("READY");

            while(true)
            {
                string line = Console.ReadLine();

                if (string.IsNullOrWhiteSpace(line))
                {
                    continue;
                }

                var words = line.Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);

                int[] wids = words.Select(w => vocab.GetId(SimpleTokenizor.ETokenize(w), false)).ToArray();

                wids[0] = vocab.GetId(SimpleTokenizor.ETokenize(words[0]), true);
                bool[][] allowedTags = new bool[wids.Length][];

                for (int i = 0; i < wids.Length; ++i)
                {
                    if (vocab.IsRareOrUNK(wids[i]))
                    {
                        var lemmas = EMorph.EnglishMorph.GetBaseForm(words[i]);

                        if (lemmas == null || lemmas.Count == 0)
                        {
                            continue;
                        }

                        allowedTags[i] = new bool[tagSet.PTCount];

                        if (char.IsUpper(words[i][0]))
                        {
                            allowedTags[i][tagSet.GetID("NNP")] = true;
                            allowedTags[i][tagSet.GetID("NNPS")] = true;
                        }

                        foreach (var lemma in lemmas)
                        {
                            switch (lemma.PoS)
                            {
                                case EMorph.MorphPoS.NN:
                                    allowedTags[i][tagSet.GetID("NN")] = true;
                                    allowedTags[i][tagSet.GetID("NNS")] = true;
                                    break;
                                case EMorph.MorphPoS.NNS:
                                    allowedTags[i][tagSet.GetID("NNS")] = true;
                                    allowedTags[i][tagSet.GetID("NN")] = true;
                                    break;
                                case EMorph.MorphPoS.JJ:
                                    allowedTags[i][tagSet.GetID("JJ")] = true;
                                    break;
                                case EMorph.MorphPoS.JJR:
                                    allowedTags[i][tagSet.GetID("JJR")] = true;
                                    break;
                                case EMorph.MorphPoS.JJS:
                                    allowedTags[i][tagSet.GetID("JJS")] = true;
                                    break;
                                case EMorph.MorphPoS.RB:
                                    allowedTags[i][tagSet.GetID("RB")] = true;
                                    break;
                                case EMorph.MorphPoS.RBR:
                                    allowedTags[i][tagSet.GetID("RBR")] = true;
                                    break;
                                case EMorph.MorphPoS.RBS:
                                    allowedTags[i][tagSet.GetID("RBS")] = true;
                                    break;
                                case EMorph.MorphPoS.VB:
                                    allowedTags[i][tagSet.GetID("VB")] = true;
                                    allowedTags[i][tagSet.GetID("VBP")] = true;
                                    break;
                                case EMorph.MorphPoS.VBD:
                                    allowedTags[i][tagSet.GetID("VBD")] = true;
                                    allowedTags[i][tagSet.GetID("VBN")] = true;
                                    break;
                                case EMorph.MorphPoS.VBG:
                                    allowedTags[i][tagSet.GetID("VBG")] = true;
                                    break;
                                case EMorph.MorphPoS.VBZ:
                                    allowedTags[i][tagSet.GetID("VBZ")] = true;
                                    break;
                                default:
                                    throw new Exception("not recognized morph lemma!");
                            }
                        }
                    }
                }

                try
                {
                    var parser = new ChartHyperGraphParser(wids, allowedTags);
                    parser.BuildHyperGraph(cgrammars[0], epool, vpool, grammars[grammars.Length - 1].subTagCounts);
                    parser.SumForward();
                    parser.SumBackward(false);
                    parser.Prune(-10.0);
                    parser.Purge();
                    for (int i = 1; i < cgrammars.Length; ++i)
                    {
                        parser.ExpandHyperGraph(cgrammars[i], tagMaps[i - 1], epool, vpool,
                                                 grammars[grammars.Length - 1].subTagCounts);
                        parser.SumForward();
                        parser.SumBackward(false);
                        parser.Prune(-10.0);
                        parser.Purge();
                    }
                    parser.ExpandHyperGraph(grammars[0], tagMaps[2], epool, vpool, grammars[grammars.Length - 1].subTagCounts);

                    for (int i = 0; i < grammars.Length - 1; ++i)
                    {
                        parser.SumForward();
                        parser.SumBackward(false);
                        parser.Prune(-8.0);
                        parser.Purge();
                        parser.ProjectGrammar(traces[i + 1], grammars[i + 1]);
                    }

                    parser.SumForward();
                    parser.SumBackward(true);
                    parser.PosteriorViterbi();

                    var ptree = parser.ExtractPosteriorViterbi(words, tagSet);

                    PennTreeProcessor.RecoverFromXBarBinarize(ptree.Root);

                    ptree.ComputeStartEnd();

                    string treeline = ptree.TextTree;

                    string[] xlines = treeline.Split(new string[] { "\n", "\r", "\r\n" }, StringSplitOptions.RemoveEmptyEntries);

                    foreach (var xline in xlines)
                    {
                        Console.Error.WriteLine(xline);
                    }
                }
                catch
                {
                    Console.Error.WriteLine("Failure to parse!");
                }
            }
        }
예제 #5
0
        public int ExpandHyperGraph(LAPCFGrammar grammar, int[] tagMap, HyperEdgePool epool, HyperVertexPool vpool, int[] tagCapacity = null)
        {
            this.ROOTID = grammar.ROOTID;

            var maxSubTag = grammar.subTagCounts.Max ();

            lbuf = new double[maxSubTag + 1];

            int prunedCell = 0;

            var lv1flags = new TimedArray<bool> (grammar.TotalTagCount);
            var lv2flags = new TimedArray<bool> (grammar.TotalTagCount);

            // CYK
            for (int i = 0; i < wids.Length; ++i) {
                var oldcell = chart [i, i];
                lv1flags.Clear ();
                foreach (var v in oldcell.l1v) {
                    if (v != null) {
                        lv1flags [v.tag] = true;
                    }
                }
                lv2flags.Clear ();
                foreach (var v in oldcell.l2v) {
                    if (v != null) {
                        lv2flags [v.tag] = true;
                    }
                }
                bool isRoot = i == 0 && i == wids.Length - 1;
                chart [i, i] = new HyperCell (i, i + 1, grammar.TotalTagCount);
                ExpandLexicon (
                    grammar, chart [i, i], lv1flags, lv2flags, tagMap,
                    wids [i], epool, vpool, tagCapacity,
                    allowedPoSTags == null ? null: allowedPoSTags[i],
                    rawTagProbs == null ? null : rawTagProbs[i], isRoot);
                ExpandUnaryRules (grammar, chart [i, i], lv1flags, lv2flags, tagMap, epool, vpool, tagCapacity, isRoot);
                chart [i, i].Finish ();
            }

            for (int spanL = 2; spanL <= wids.Length; ++spanL) {
                for (int beg = 0; beg + spanL <= wids.Length; ++beg) {
                    int end = beg + spanL;
                    int l = beg;
                    int r = end - 1;

                    var oldcell = chart [l, r];
                    lv1flags.Clear ();
                    foreach (var v in oldcell.l1v) {
                        if (v != null) {
                            lv1flags [v.tag] = true;
                        }
                    }
                    lv2flags.Clear ();
                    foreach (var v in oldcell.l2v) {
                        if (v != null) {
                            lv2flags [v.tag] = true;
                        }
                    }
                    bool isRoot = l == 0 && r == wids.Length - 1;
                    chart [l, r] = new HyperCell (beg, end, grammar.TotalTagCount);
                    if (!oldcell.IsEmptyCell ()) {
                        for (int mid = l; mid < r; ++mid) {
                            ExpandBinaryRules (grammar, chart [l, r], chart [l, mid], chart [mid + 1, r],
                                               lv1flags, lv2flags, tagMap,
                                               epool, vpool, tagCapacity, isRoot);
                        }

                        for (int i = 0; i < chart[l, r].l1v.Length; ++i) {
                            var c = chart [l, r].l1v [i];

                            if (c != null) {
                                if (isRoot && c.tag != ROOTID) {
                                    continue;
                                }
                                chart [l, r].l2v [i] = vpool.Allocate (false, c.tag, c.beta.Length, c.beta.v.Length);
                                epool.Allocate (chart [l, r].l2v [i], c);
                            }

                        }
                        ExpandUnaryRules (grammar, chart [l, r],
                                          lv1flags, lv2flags, tagMap,
                                          epool, vpool, tagCapacity, isRoot);
                    } else {
                        prunedCell += 1;
                    }
                    chart [l, r].Finish ();
                }
            }
            return prunedCell;
        }
예제 #6
0
        private static void MatchUnaryRules(
            LAPCFGrammar grammar,
            HyperCell cell,
            HyperEdgePool epool,
            HyperVertexPool vpool,
            int[] tagCapacity,
            bool isRoot)
        {
            foreach (var cv in cell.l1v) {
                if (cv == null) {
                    continue;
                }
                var rules = grammar.urules [cv.tag];

                if (rules != null) {
                    foreach (var rule in rules) {
                        if (rule == null) {
                            break;
                        }

                        if (rule.ptag == grammar.ROOTID && !isRoot) {
                            continue;
                        }

                        if (isRoot && rule.ptag != grammar.ROOTID) {
                            continue;
                        }

                        if (cell.l2v [rule.ptag] == null) {
                            var cap = tagCapacity == null ? -1 : tagCapacity [rule.ptag];
                            cell.l2v [rule.ptag] = vpool.Allocate (false, rule.ptag, grammar.GetSubTagCount (rule.ptag), cap);
                        }

                        epool.Allocate (cell.l2v [rule.ptag], cv, rule.scores, null);
                    }
                }
            }
        }
예제 #7
0
        private static void MatchLexicon(
            LAPCFGrammar table,
            HyperCell cell,
            int wid,
            HyperEdgePool epool,
            HyperVertexPool vpool,
            int[] tagCapacity,
            bool[] allowedTags,
            double[] tagProbs,
            bool isRoot)
        {
            var tv = new HyperVertex (true, wid, 1);

            var trules = table.trules [wid];

            foreach (var rule in trules) {
                if (rule == null) {
                    break;
                }
                if (rule.tag == table.ROOTID && !isRoot) {
                    continue;
                }

                if (allowedTags != null && !allowedTags[rule.tag])
                {
                    continue;
                }

                var xrule = rule;

                if (tagProbs != null)
                {
                    var xprob = tagProbs[rule.tag];
                    if (double.IsNegativeInfinity(xprob))
                    {
                        continue;
                    }
                    xrule = rule.Clone();

                    for (int i = 0; i < xrule.scores.Length; ++i)
                    {
                        if (!double.IsNegativeInfinity(xrule.scores[i]))
                        {
                            xrule.scores[i] += xprob;
                        }
                    }
                }
                var cap = tagCapacity == null ? -1 : tagCapacity [rule.tag];
                cell.l1v [rule.tag] = vpool.Allocate (false, rule.tag, table.GetSubTagCount (rule.tag), cap);
                epool.Allocate (cell.l1v [rule.tag], tv, xrule.scores, null);
                if (isRoot && rule.tag != table.ROOTID)
                {
                    continue;
                }
                cell.l2v [rule.tag] = vpool.Allocate (false, rule.tag, table.GetSubTagCount (rule.tag), cap);
                epool.Allocate (cell.l2v [rule.tag], cell.l1v [rule.tag]);
            }
        }
예제 #8
0
        private static void MatchBinaryRules(
            LAPCFGrammar grammar,
            HyperCell pcell,
            HyperCell lcell,
            HyperCell rcell,
            HyperEdgePool epool,
            HyperVertexPool vpool,
            int[] tagCapacity,
            bool isRoot)
        {
            foreach (var lv in lcell.l2v) {
                var rprules = grammar.brules [lv.tag];
                if (rprules == null) {
                    continue;
                }
                foreach (var rv in rcell.l2v) {
                    var prules = rprules [rv.tag];

                    if (prules == null) {
                        continue;
                    }

                    for (int p = 0; p < prules.Length; ++p) {
                        var rule = prules [p];

                        if (rule == null) {
                            break;
                        }

                        if (rule.ptag == grammar.ROOTID && !isRoot) {
                            continue;
                        }

                        if (pcell.l1v [rule.ptag] == null) {
                            var cap = tagCapacity == null ? -1 : tagCapacity [rule.ptag];
                            pcell.l1v [rule.ptag] = vpool.Allocate (false, rule.ptag, grammar.GetSubTagCount (rule.ptag), cap);
                        }

                        epool.Allocate (pcell.l1v [rule.ptag], lv, rv, rule.scores, null);
                    }
                }
            }
        }