Exemplo n.º 1
0
        public MeCabNode Next()
        {
            while (this.agenda.Count != 0)
            {
                QueueElement top   = this.agenda.Pop();
                MeCabNode    rNode = top.Node;

                if (rNode.Stat == MeCabNodeStat.Bos)
                {
                    for (QueueElement n = top; n.Next != null; n = n.Next)
                    {
                        n.Node.Next      = n.Next.Node; // change next & prev
                        n.Next.Node.Prev = n.Node;
                    }
                    return(rNode);
                }

                for (MeCabPath path = rNode.LPath; path != null; path = path.LNext)
                {
                    QueueElement n = new QueueElement()
                    {
                        Node = path.LNode,
                        Gx   = path.Cost + top.Gx,
                        Fx   = path.LNode.Cost + path.Cost + top.Gx,
                        Next = top
                    };
                    this.agenda.Push(n);
                }
            }
            return(null);
        }
Exemplo n.º 2
0
        private MeCabNode BuildAllLattice(ThreadData work)
        {
            if (this.BuildBestLattice(work) == null)
            {
                return(null);
            }

            MeCabNode prev = work.BosNode;

            for (int pos = 0; pos < work.BeginNodeList.Length; pos++)
            {
                for (MeCabNode node = work.BeginNodeList[pos]; node != null; node = node.BNext)
                {
                    prev.Next = node;
                    node.Prev = prev;
                    prev      = node;
                    for (MeCabPath path = node.LPath; path != null; path = path.LNext)
                    {
                        path.Prob = (float)(path.LNode.Alpha
                                            - this.theta * path.Cost
                                            + path.RNode.Beta - work.Z);
                    }
                }
            }

            return(work.BosNode);
        }
Exemplo n.º 3
0
 private void CalcBeta(MeCabNode n, double beta)
 {
     n.Beta = 0f;
     for (MeCabPath path = n.RPath; path != null; path = path.RNext)
     {
         n.Beta = (float)Utils.LogSumExp(n.Beta,
                                         -beta * path.Cost + path.RNode.Beta,
                                         path == n.RPath);
     }
 }
Exemplo n.º 4
0
 private void CalcAlpha(MeCabNode n, double beta)
 {
     n.Alpha = 0f;
     for (MeCabPath path = n.LPath; path != null; path = path.LNext)
     {
         n.Alpha = (float)Utils.LogSumExp(n.Alpha,
                                          -beta * path.Cost + path.LNode.Alpha,
                                          path == n.LPath);
     }
 }
Exemplo n.º 5
0
        public void WriteDump(StringBuilder os, MeCabNode bosNode)
        {
            for (MeCabNode node = bosNode; node != null; node = node.Next)
            {
#if NeedId
                os.Append(node.Id).Append(" ");
#endif
                if (node.Stat == MeCabNodeStat.Bos)
                {
                    os.Append("BOS");
                }
                else if (node.Stat == MeCabNodeStat.Eos)
                {
                    os.Append("EOS");
                }
                else
                {
                    os.Append(node.Surface);
                }

                os.Append(" ").Append(node.Feature);
                os.Append(" ").Append(node.BPos);
                os.Append(" ").Append(node.EPos);
                os.Append(" ").Append(node.RCAttr);
                os.Append(" ").Append(node.LCAttr);
                os.Append(" ").Append(node.PosId);
                os.Append(" ").Append(node.CharType);
                os.Append(" ").Append((int)node.Stat);
                os.Append(" ").Append(node.IsBest ? "1" : "0");
                os.Append(" ").Append(node.Alpha.ToString(FloatFormat));
                os.Append(" ").Append(node.Beta.ToString(FloatFormat));
                os.Append(" ").Append(node.Prob.ToString(FloatFormat));
                os.Append(" ").Append(node.Cost);

                for (MeCabPath path = node.LPath; path != null; path = path.LNext)
                {
#if NeedId
                    os.Append(" ").Append(path.LNode.Id);
#endif
                    os.Append(" ");
                    os.Append(":").Append(path.Cost);
                    os.Append(":").Append(path.Prob.ToString(FloatFormat));
                }

                os.AppendLine();
            }
        }
Exemplo n.º 6
0
        private void ConnectWithAllPath(int pos, MeCabNode rNode, ThreadData work)
        {
            for (; rNode != null; rNode = rNode.BNext)
            {
                long bestCost = int.MaxValue; // 2147483647

                MeCabNode bestNode = null;

                for (MeCabNode lNode = work.EndNodeList[pos]; lNode != null; lNode = lNode.ENext)
                {
                    int  lCost = this.connector.Cost(lNode, rNode); // local cost
                    long cost  = lNode.Cost + lCost;

                    if (cost < bestCost)
                    {
                        bestNode = lNode;
                        bestCost = cost;
                    }

                    MeCabPath path = new MeCabPath()
                    {
                        Cost  = lCost,
                        RNode = rNode,
                        LNode = lNode,
                        LNext = rNode.LPath,
                        RNext = lNode.RPath
                    };
                    rNode.LPath = path;
                    lNode.RPath = path;
                }

                if (bestNode == null)
                {
                    throw new ArgumentException("too long sentence.");
                }

                rNode.Prev = bestNode;
                rNode.Next = null;
                rNode.Cost = bestCost;
                int x = rNode.RLength + pos;
                rNode.ENext         = work.EndNodeList[x];
                work.EndNodeList[x] = rNode;
            }
        }
Exemplo n.º 7
0
        public void WriteEM(StringBuilder os, MeCabNode bosNode)
        {
            const float MinProb = 0.0001f;

            for (MeCabNode node = bosNode; node != null; node = node.Next)
            {
                if (node.Prob >= MinProb)
                {
                    os.Append("U\t");
                    if (node.Stat == MeCabNodeStat.Bos)
                    {
                        os.Append("BOS");
                    }
                    else if (node.Stat == MeCabNodeStat.Eos)
                    {
                        os.Append("EOS");
                    }
                    else
                    {
                        os.Append(node.Surface);
                    }
                    os.Append("\t").Append(node.Feature);
                    os.Append("\t").Append(node.Prob.ToString(FloatFormat));
                    os.AppendLine();
                }
                for (MeCabPath path = node.LPath; path != null; path = path.LNext)
                {
                    if (path.Prob >= MinProb)
                    {
                        os.Append("B\t").Append(path.LNode.Feature);
                        os.Append("\t").Append(node.Feature);
                        os.Append("\t").Append(path.Prob.ToString(FloatFormat));
                        os.AppendLine();
                    }
                }
            }
            os.AppendLine("EOS");
        }
Exemplo n.º 8
0
        public unsafe void WriteNode(StringBuilder os, char *p, string sentence, MeCabNode node)
        {
            for (; *p != 0x0; p++)
            {
                switch (*p)
                {
                default: os.Append(*p); break;

                case '%':
                    switch (*++p)
                    {
                    default: os.Append("unkonwn meta char ").Append(*p); break;

                    case 'S': os.Append(sentence); break;

                    case 'L': os.Append(sentence.Length); break;

                    case 'm': os.Append(node.Surface); break;

                    case 'M': os.Append(sentence, (node.BPos - node.RLength + node.Length), node.RLength); break;

                    case 'h': os.Append(node.PosId); break;

                    case '%': os.Append('%'); break;

                    case 'c': os.Append(node.WCost); break;

                    case 'H': os.Append(node.Feature); break;

                    case 't': os.Append(node.CharType); break;

                    case 's': os.Append(node.Stat); break;

                    case 'P': os.Append(node.Prob); break;

                    case 'p':
                        switch (*++p)
                        {
                        default: throw new ArgumentException("[iseSCwcnblLh] is required after %p");

#if NeedId
                        case 'i': os.Append(node.Id); break;
#else
                        case 'i': throw new ArgumentException("%pi is not supported");
#endif
                        case 'S': os.Append(sentence, node.BPos, (node.RLength - node.Length)); break;

                        case 's': os.Append(node.BPos); break;

                        case 'e': os.Append(node.EPos); break;

                        case 'C': os.Append(node.Cost - node.Prev.Cost - node.WCost); break;

                        case 'w': os.Append(node.WCost); break;

                        case 'c': os.Append(node.Cost); break;

                        case 'n': os.Append(node.Cost - node.Prev.Cost); break;

                        case 'b': os.Append(node.IsBest ? '*' : ' '); break;

                        case 'P': os.Append(node.Prob); break;

                        case 'A': os.Append(node.Alpha); break;

                        case 'B': os.Append(node.Beta); break;

                        case 'l': os.Append(node.Length); break;

                        case 'L': os.Append(node.RLength); break;

                        case 'h':
                            switch (*++p)
                            {
                            default: throw new ArgumentException("lr is required after %ph");

                            case 'l': os.Append(node.LCAttr); break;

                            case 'r': os.Append(node.RCAttr); break;
                            }
                            ; break;

                        case 'p':
                            char mode = *++p;
                            char sep  = *++p;
                            if (sep == '\\')
                            {
                                sep = this.GetEscapedChar(*++p);
                            }
                            if (node.LPath == null)
                            {
                                throw new InvalidOperationException("no path information, use -l option");
                            }
                            for (MeCabPath path = node.LPath; path != null; path = path.LNext)
                            {
                                if (path != node.LPath)
                                {
                                    os.Append(sep);
                                }
                                switch (mode)
                                {
                                case 'i': os.Append(path.LNode.PosId); break;

                                case 'c': os.Append(path.Cost); break;

                                case 'P': os.Append(path.Prob); break;

                                default: throw new ArgumentException("[icP] is required after %pp");
                                }
                            }
                            break;
                        }
                        break;

                    case 'f':
                    case 'F':
                        char separator = '\t';
                        if (*p == 'F')
                        {
                            if (*++p == '\\')
                            {
                                separator = this.GetEscapedChar(*++p);
                            }
                            else
                            {
                                separator = *p;
                            }
                        }
                        if (*++p != '[')
                        {
                            throw new ArgumentException("cannot find '['");
                        }
                        string[] features = node.Feature.Split(',');
                        int      n        = 0;
                        while (true)
                        {
                            if (char.IsDigit(*++p))
                            {
                                n = n * 10 + (*p - '0'); continue;
                            }
                            if (n >= features.Length)
                            {
                                throw new ArgumentException("given index is out of range");
                            }
                            os.Append(features[n]);
                            if (*++p == ',')
                            {
                                os.Append(separator); n = 0; continue;
                            }
                            if (*p == ']')
                            {
                                break;
                            }
                            throw new ArgumentException("cannot find ']'");
                        }
                        break;
                    }
                    break;
                }
            }
        }