Exemplo n.º 1
1
        int seg_termbuf_build(crf_seg_out term_buf)
        {
            term_buf.Clear();

            //build raw result at first
            var iRet = termbuf_build(term_buf);

            if (iRet != Utils.ERROR_SUCCESS)
            {
                return(iRet);
            }

            //Then build token result
            var term_len = 0;
            var weight   = 0.0;
            var num      = 0;

            for (var i = 0; i < x_.Count; i++)
            {
                //Adding the length of current token
                var strTag = term_buf.result_[i];
                term_len += x_[i][0].Length;
                weight   += term_buf.weight_[i];
                num++;

                //Check if current term is the end of a token
                if ((strTag.StartsWith("B_") == false &&
                     strTag.StartsWith("M_") == false) ||
                    i == x_.Count - 1)
                {
                    var tkn = new SegToken();
                    tkn.length = term_len;
                    tkn.offset = term_buf.termTotalLength;

                    var spos = strTag.IndexOf('_');
                    if (spos < 0)
                    {
                        if (strTag == "NOR")
                        {
                            tkn.strTag = "";
                        }
                        else
                        {
                            tkn.strTag = strTag;
                        }
                    }
                    else
                    {
                        tkn.strTag = strTag.Substring(spos + 1);
                    }

                    term_buf.termTotalLength += term_len;
                    //Calculate each token's weight
                    switch (vlevel_)
                    {
                    case 0:
                        tkn.fWeight = 0.0;
                        break;

                    case 2:
                        tkn.fWeight = weight / num;
                        weight      = 0.0;
                        num         = 0;
                        break;
                    }

                    term_buf.tokenList.Add(tkn);
                    term_len = 0;
                }
            }


            return(Utils.ERROR_SUCCESS);
        }
Exemplo n.º 2
0
        int seg_termbuf_build(crf_seg_out term_buf)
        {
            term_buf.Clear();

            //build raw result at first
            var iRet = termbuf_build(term_buf);
            if (iRet != Utils.ERROR_SUCCESS)
            {
                return iRet;
            }

            //Then build token result
            var term_len = 0;
            var weight = 0.0;
            var num = 0;
            for (var i = 0; i < x_.Count; i++)
            {
                //Adding the length of current token
                var strTag = term_buf.result_[i];
                term_len += x_[i][0].Length;
                weight += term_buf.weight_[i];
                num++;

                //Check if current term is the end of a token
                if ((strTag.StartsWith("B_") == false &&
                    strTag.StartsWith("M_") == false) ||
                    i == x_.Count - 1)
                {
                    var tkn = new SegToken();
                    tkn.length = term_len;
                    tkn.offset = term_buf.termTotalLength;

                    var spos = strTag.IndexOf('_');
                    if (spos < 0)
                    {
                        if (strTag == "NOR")
                        {
                            tkn.strTag = "";
                        }
                        else
                        {
                            tkn.strTag = strTag;
                        }
                    }
                    else
                    {
                        tkn.strTag = strTag.Substring(spos + 1);
                    }

                    term_buf.termTotalLength += term_len;
                    //Calculate each token's weight
                    switch (vlevel_)
                    {
                        case 0:
                            tkn.fWeight = 0.0;
                            break;
                        case 2:
                            tkn.fWeight = weight / num;
                            weight = 0.0;
                            num = 0;
                            break;
                    }

                    term_buf.tokenList.Add(tkn);
                    term_len = 0;
                }
            }


            return Utils.ERROR_SUCCESS;
        }