Exemplo n.º 1
0
        public static List <double> getFscore(List <string> goldTagList, List <string> resTagList, List <double> infoList)
        {
            List <double> scoreList = new List <double>();

            if (resTagList.Count != goldTagList.Count)
            {
                throw new Exception("error");
            }

            //convert original tags to 3 tags: B(x), I, O
            getNewTagList(Global.chunkTagMap, ref goldTagList);
            getNewTagList(Global.chunkTagMap, ref resTagList);
            List <string> goldChunkList = getChunks(goldTagList);
            List <string> resChunkList  = getChunks(resTagList);

            int gold_chunk = 0, res_chunk = 0, correct_chunk = 0;

            for (int i = 0; i < goldChunkList.Count; i++)
            {
                string   res          = resChunkList[i];
                string   gold         = goldChunkList[i];
                string[] resChunkAry  = res.Split(Global.commaAry, StringSplitOptions.RemoveEmptyEntries);
                string[] goldChunkAry = gold.Split(Global.commaAry, StringSplitOptions.RemoveEmptyEntries);
                gold_chunk += goldChunkAry.Length;
                res_chunk  += resChunkAry.Length;
                baseHashSet <string> goldChunkSet = new baseHashSet <string>();
                foreach (string im in goldChunkAry)
                {
                    goldChunkSet.Add(im);
                }

                foreach (string im in resChunkAry)
                {
                    if (goldChunkSet.Contains(im))
                    {
                        correct_chunk++;
                    }
                }
            }
            double pre = (double)correct_chunk / (double)res_chunk * 100;
            double rec = (double)correct_chunk / (double)gold_chunk * 100;
            double f1  = 2 * pre * rec / (pre + rec);

            scoreList.Add(f1);
            scoreList.Add(pre);
            scoreList.Add(rec);

            infoList.Add(gold_chunk);
            infoList.Add(res_chunk);
            infoList.Add(correct_chunk);
            return(scoreList);
        }
Exemplo n.º 2
0
        public double getGrad_SGD(List <double> g, model m, dataSeq x, baseHashSet <int> idset)
        {
            if (idset != null)
            {
                idset.Clear();
            }

            if (x == null)
            {
                return(0);
            }

            return(getGradCRF(g, m, x, idset));
        }
Exemplo n.º 3
0
        //the mini-batch version
        public double getGrad_SGD_miniBatch(List <double> g, model m, List <dataSeq> X, baseHashSet <int> idset)
        {
            if (idset != null)
            {
                idset.Clear();
            }
            double error = 0;

            foreach (dataSeq x in X)
            {
                baseHashSet <int> idset2 = new baseHashSet <int>();

                error += getGradCRF(g, m, x, idset2);

                if (idset != null)
                {
                    foreach (int i in idset2)
                    {
                        idset.Add(i);
                    }
                }
            }
            return(error);
        }
Exemplo n.º 4
0
        //ADF training
        public double adf()
        {
            float[]       w                = _model.W;
            int           fsize            = w.Length;
            int           xsize            = _X.Count;
            List <double> grad             = new List <double>(new double[fsize]);
            double        error            = 0;
            List <int>    featureCountList = new List <int>(new int[fsize]);
            List <int>    ri               = randomTool <int> .getShuffledIndexList(xsize);//random shuffle of training samples

            Global.interval = xsize / Global.nUpdate;
            int nSample = 0;//#sample in an update interval

            for (int t = 0; t < xsize; t += Global.miniBatch)
            {
                List <dataSeq> XX  = new List <dataSeq>();
                bool           end = false;
                for (int k = t; k < t + Global.miniBatch; k++)
                {
                    int     i = ri[k];
                    dataSeq x = _X[i];
                    XX.Add(x);
                    if (k == xsize - 1)
                    {
                        end = true;
                        break;
                    }
                }
                int mbSize = XX.Count;
                nSample += mbSize;
                baseHashSet <int> fSet = new baseHashSet <int>();
                double            err  = _grad.getGrad_SGD_miniBatch(grad, _model, XX, fSet);
                error += err;

                foreach (int i in fSet)
                {
                    featureCountList[i]++;
                }

                bool check = false;
                for (int k = t; k < t + Global.miniBatch; k++)
                {
                    if (t != 0 && k % Global.interval == 0)
                    {
                        check = true;
                    }
                }
                //update decay rates
                if (check || end)
                {
                    for (int i = 0; i < fsize; i++)
                    {
                        int    v   = featureCountList[i];
                        double u   = (double)v / (double)nSample;
                        double eta = Global.upper - (Global.upper - Global.lower) * u;
                        Global.decayList[i] *= eta;
                    }
                    //reset
                    for (int i = 0; i < featureCountList.Count; i++)
                    {
                        featureCountList[i] = 0;
                    }
                }
                //update weights
                foreach (int i in fSet)
                {
                    w[i] -= (float)(Global.decayList[i] * grad[i]);
                    //reset
                    grad[i] = 0;
                }
                //reg
                if (check || end)
                {
                    if (Global.reg != 0)
                    {
                        for (int i = 0; i < fsize; i++)
                        {
                            double grad_i = w[i] / (Global.reg * Global.reg) * ((double)nSample / (double)xsize);
                            w[i] -= (float)(Global.decayList[i] * grad_i);
                        }
                    }
                    //reset
                    nSample = 0;
                }
                Global.countWithIter += mbSize;
            }

            if (Global.reg != 0)
            {
                double sum = arrayTool.squareSum(w);
                error += sum / (2.0 * Global.reg * Global.reg);
            }

            Global.diff = convergeTest(error);
            return(error);
        }
Exemplo n.º 5
0
        public double sgd_exactReg()
        {
            double scalar = 1, scalarOld = 1;

            float[] w      = _model.W;
            int     fsize  = w.Length;
            int     xsize  = _X.Count;
            double  newReg = Global.reg * Math.Sqrt(xsize);
            double  oldReg = Global.reg;

            Global.reg = newReg;

            double[]      tmpAry = new double[fsize];
            List <double> grad   = new List <double>(tmpAry);

            List <int> ri = randomTool <int> .getShuffledIndexList(xsize);

            double error = 0;
            double r_k   = 0;

            for (int t = 0; t < xsize; t++)
            {
                int               ii   = ri[t];
                dataSeq           x    = _X[ii];
                baseHashSet <int> fset = new baseHashSet <int>();
                double            err  = _grad.getGrad_SGD(grad, scalar, _model, x, fset);
                error += err;
                //decaying rate: r_k = r_0 * beta^(k/N), with 0<r_0<=1, 0<beta<1
                r_k = Global.rate0 * Math.Pow(Global.decayFactor, (double)Global.countWithIter / (double)xsize);
                if (Global.countWithIter % (xsize / 4) == 0)
                {
                    Global.swLog.WriteLine("iter{0}    decay_rate={1}", Global.glbIter, r_k.ToString("e2"));
                }

                //reg
                if (t % Global.scalarResetStep == 0)
                {
                    //reset
                    for (int i = 0; i < fsize; i++)
                    {
                        w[i] *= (float)scalar;
                    }
                    scalar = scalarOld = 1;
                }
                else
                {
                    scalarOld = scalar;
                    scalar   *= 1 - r_k / (Global.reg * Global.reg);
                }

                foreach (int i in fset)
                {
                    double realWeight = w[i] * scalarOld;
                    double grad_i     = grad[i] + realWeight / (Global.reg * Global.reg);
                    realWeight = realWeight - r_k * grad_i;
                    w[i]       = (float)(realWeight / scalar);

                    //reset
                    grad[i] = 0;
                }
                Global.countWithIter++;
            }

            //recover the real weights
            for (int i = 0; i < fsize; i++)
            {
                w[i] *= (float)scalar;
            }

            if (Global.reg != 0.0)
            {
                double sum = arrayTool.squareSum(w);
                error += sum / (2.0 * Global.reg * Global.reg);
            }

            Global.diff = convergeTest(error);
            Global.reg  = oldReg;
            return(error);
        }
Exemplo n.º 6
0
        //SGD with lazy reg
        public double sgd_lazyReg()
        {
            float[] w     = _model.W;
            int     fsize = w.Length;
            int     xsize = _X.Count;

            double[]      ary  = new double[fsize];
            List <double> grad = new List <double>(ary);

            List <int> ri = randomTool <int> .getShuffledIndexList(xsize);

            double error = 0;
            double r_k   = 0;

            for (int t = 0; t < xsize; t += Global.miniBatch)
            {
                List <dataSeq> XX = new List <dataSeq>();
                for (int k = t; k < t + Global.miniBatch; k++)
                {
                    int     i = ri[k];
                    dataSeq x = _X[i];
                    XX.Add(x);
                    if (k == xsize - 1)
                    {
                        break;
                    }
                }
                int mbSize             = XX.Count;
                baseHashSet <int> fset = new baseHashSet <int>();
                double            err  = _grad.getGrad_SGD_miniBatch(grad, _model, XX, fset);
                error += err;

                //decaying rate: r_k = r_0 * beta^(k/N), with 0<r_0<=1, 0<beta<1
                r_k = Global.rate0 * Math.Pow(Global.decayFactor, (double)Global.countWithIter / (double)xsize);

                if (Global.countWithIter % (xsize / 4) == 0)
                {
                    Global.swLog.WriteLine("iter{0}    decay_rate={1}", Global.glbIter, r_k.ToString("e2"));
                }

                foreach (int i in fset)
                {
                    //because dgrad[i] is the grad of -log(obj), minus the gradient to find the minumum point
                    w[i] -= (float)(r_k * grad[i]);
                    //reset
                    grad[i] = 0;
                }
                Global.countWithIter += mbSize;
            }

            if (Global.reg != 0)
            {
                for (int i = 0; i < fsize; i++)
                {
                    double grad_i = w[i] / (Global.reg * Global.reg);
                    w[i] -= (float)(r_k * grad_i);
                }

                double sum = arrayTool.squareSum(w);
                error += sum / (2.0 * Global.reg * Global.reg);
            }

            Global.diff = convergeTest(error);
            return(error);
        }
Exemplo n.º 7
0
        //the scalar version
        virtual public double getGradCRF(List <double> vecGrad, double scalar, model m, dataSeq x, baseHashSet <int> idSet)
        {
            idSet.Clear();
            int nTag = m.NTag;
            //compute beliefs
            belief bel       = new belief(x.Count, nTag);
            belief belMasked = new belief(x.Count, nTag);

            _inf.getBeliefs(bel, m, x, scalar, false);
            _inf.getBeliefs(belMasked, m, x, scalar, true);
            double ZGold = belMasked.Z;
            double Z     = bel.Z;

            List <featureTemp> fList;

            //Loop over nodes to compute features and update the gradient
            for (int i = 0; i < x.Count; i++)
            {
                fList = _fGene.getFeatureTemp(x, i);
                foreach (featureTemp im in fList)
                {
                    for (int s = 0; s < nTag; s++)
                    {
                        int f = _fGene.getNodeFeatID(im.id, s);
                        idSet.Add(f);

                        vecGrad[f] += bel.belState[i][s] * im.val;
                        vecGrad[f] -= belMasked.belState[i][s] * im.val;
                    }
                }
            }

            //Loop over edges to compute features and update the gradient
            for (int i = 1; i < x.Count; i++)
            {
                for (int s = 0; s < nTag; s++)
                {
                    for (int sPre = 0; sPre < nTag; sPre++)
                    {
                        int f = _fGene.getEdgeFeatID(sPre, s);
                        idSet.Add(f);

                        vecGrad[f] += bel.belEdge[i][sPre, s];
                        vecGrad[f] -= belMasked.belEdge[i][sPre, s];
                    }
                }
            }
            return(Z - ZGold);//-log{P(y*|x,w)}
        }
Exemplo n.º 8
0
 //the scalar version
 public double getGrad_SGD(List <double> g, double scalar, model m, dataSeq x, baseHashSet <int> idset)
 {
     return(getGradCRF(g, scalar, m, x, idset));
 }
Exemplo n.º 9
0
        override public double getGradCRF(List <double> gradList, model m, dataSeq x, baseHashSet <int> idSet)
        {
            if (idSet != null)
            {
                idSet.Clear();
            }
            int nTag = m.NTag;
            //compute beliefs
            belief bel       = new belief(x.Count, nTag);
            belief belMasked = new belief(x.Count, nTag);
            //store the YY and Y
            List <dMatrix>        YYlist = new List <dMatrix>(), maskYYlist = new List <dMatrix>();
            List <List <double> > Ylist = new List <List <double> >(), maskYlist = new List <List <double> >();

            _inf.getYYandY(m, x, YYlist, Ylist, maskYYlist, maskYlist);
            _inf.getBeliefs(bel, m, x, YYlist, Ylist);
            _inf.getBeliefs(belMasked, m, x, maskYYlist, maskYlist);
            double ZGold = belMasked.Z;
            double Z = bel.Z;

            List <featureTemp> fList;

            //Loop over nodes to compute features and update the gradient
            for (int i = 0; i < x.Count; i++)
            {
                fList = _fGene.getFeatureTemp(x, i);
                foreach (featureTemp im in fList)
                {
                    for (int s = 0; s < nTag; s++)
                    {
                        int f = _fGene.getNodeFeatID(im.id, s);
                        if (idSet != null)
                        {
                            idSet.Add(f);
                        }

                        gradList[f] += bel.belState[i][s] * im.val;
                        gradList[f] -= belMasked.belState[i][s] * im.val;
                    }
                }
            }

            //Loop over edges to compute features and update the gradient
            for (int i = 1; i < x.Count; i++)
            {
                //non-rich
                if (Global.useTraditionalEdge)
                {
                    for (int s = 0; s < nTag; s++)
                    {
                        for (int sPre = 0; sPre < nTag; sPre++)
                        {
                            int f = _fGene.getEdgeFeatID(sPre, s);
                            if (idSet != null)
                            {
                                idSet.Add(f);
                            }

                            gradList[f] += bel.belEdge[i][sPre, s];
                            gradList[f] -= belMasked.belEdge[i][sPre, s];
                        }
                    }
                }

                //rich
                fList = _fGene.getFeatureTemp(x, i);
                foreach (featureTemp im in fList)
                {
                    int id = im.id;
                    if (id < _fGene.getNRichFeatTemp())
                    {
                        for (int s = 0; s < nTag; s++)
                        {
                            for (int sPre = 0; sPre < nTag; sPre++)
                            {
                                int f = _fGene.getEdgeFeatID(id, sPre, s);
                                if (idSet != null)
                                {
                                    idSet.Add(f);
                                }

                                gradList[f] += bel.belEdge[i][sPre, s] * im.val;
                                gradList[f] -= belMasked.belEdge[i][sPre, s] * im.val;
                            }
                        }
                    }
                }
            }
            return(Z - ZGold);//-log{P(y*|x,w)}
        }
Exemplo n.º 10
0
        public void getMaps(string file)
        {
            if (!File.Exists(file))
            {
                Console.WriteLine("file {0} no exist!", file);
                return;
            }
            Console.WriteLine("file {0} converting...", file);
            StreamReader sr = new StreamReader(file);

            baseHashMap <string, int> featureFreqMap = new baseHashMap <string, int>();
            baseHashSet <string>      tagSet         = new baseHashSet <string>();

            //get feature-freq info and tagset
            int nFeatTemp = 0;

            while (!sr.EndOfStream)
            {
                string line = sr.ReadLine();
                line = line.Replace("\t", " ");
                line = line.Replace("\r", "");

                if (line == "")
                {
                    continue;
                }

                string[] ary = line.Split(Global.blankAry, StringSplitOptions.RemoveEmptyEntries);
                nFeatTemp = ary.Length - 2;
                for (int i = 1; i < ary.Length - 1; i++)
                {
                    if (ary[i] == "/")//no feature here
                    {
                        continue;
                    }

                    if (Global.weightRegMode == "GL")
                    {
                        if (Global.GL_init == false && Global.groupTrim[i - 1])//this feature is removed in GL 1st step
                        {
                            continue;
                        }
                    }

                    string[] ary2    = ary[i].Split(Global.slashAry, StringSplitOptions.RemoveEmptyEntries);//for real-value features
                    string   feature = i.ToString() + "." + ary2[0];
                    featureFreqMap[feature]++;
                }

                string tag = ary[ary.Length - 1];
                tagSet.Add(tag);
            }

            //sort features
            List <string> sortList = new List <string>();

            foreach (baseHashMap <string, int> .KeyValuePair kv in featureFreqMap)
            {
                sortList.Add(kv.Key + " " + kv.Value);
            }
            if (Global.weightRegMode == "GL")//sort based on feature templates
            {
                sortList.Sort(listSortFunc.compareKV_key);
                //sortList.Reverse();

                StreamWriter sw = new StreamWriter("featureTemp_sorted.txt");
                foreach (string f in sortList)
                {
                    sw.WriteLine(f);
                }
                sw.Close();

                Global.groupStart = new List <int>();
                Global.groupEnd   = new List <int>();
                Global.groupStart.Add(0);
                for (int k = 1; k < sortList.Count; k++)
                {
                    string[] thisAry = sortList[k].Split(Global.dotAry, StringSplitOptions.RemoveEmptyEntries);
                    string[] preAry = sortList[k - 1].Split(Global.dotAry, StringSplitOptions.RemoveEmptyEntries);
                    string   str = thisAry[0], preStr = preAry[0];
                    if (str != preStr)
                    {
                        Global.groupStart.Add(k);
                        Global.groupEnd.Add(k);
                    }
                }
                Global.groupEnd.Add(sortList.Count);
            }
            else//sort based on feature frequency
            {
                sortList.Sort(listSortFunc.compareKV_value);//sort feature based on freq, for 1)compress .txt file 2)better edge features
                sortList.Reverse();
            }

            if (Global.weightRegMode == "GL" && Global.GL_init)
            {
                if (nFeatTemp != Global.groupStart.Count)
                {
                    throw new Exception("inconsistent # of features per line, check the feature file for consistency!");
                }
            }

            //feature index should begin from 0
            StreamWriter swFeat = new StreamWriter(Global.modelDir + "/featureIndex.txt");

            for (int i = 0; i < sortList.Count; i++)
            {
                string[] ary = sortList[i].Split(Global.blankAry);
                featureIndexMap[ary[0]] = i;
                swFeat.WriteLine("{0} {1}", ary[0].Trim(), i);
            }
            swFeat.Close();

            //label index should begin from 0
            StreamWriter  swTag       = new StreamWriter(Global.modelDir + "/tagIndex.txt");
            List <string> tagSortList = new List <string>();

            foreach (string tag in tagSet)
            {
                tagSortList.Add(tag);
            }
            tagSortList.Sort();//sort tags
            for (int i = 0; i < tagSortList.Count; i++)
            {
                tagIndexMap[tagSortList[i]] = i;
                swTag.WriteLine("{0} {1}", tagSortList[i], i);
            }
            swTag.Close();

            sr.Close();
        }