Beispiel #1
0
 public mlpreport(mlptrain.mlpreport obj)
 {
     _innerobj = obj;
 }
Beispiel #2
0
 public mlpreport()
 {
     _innerobj = new mlptrain.mlpreport();
 }
Beispiel #3
0
        /*************************************************************************
        Internal bagging subroutine.

          -- ALGLIB --
             Copyright 19.02.2009 by Bochkanov Sergey
        *************************************************************************/
        private static void mlpebagginginternal(mlpensemble ensemble,
            double[,] xy,
            int npoints,
            double decay,
            int restarts,
            double wstep,
            int maxits,
            bool lmalgorithm,
            ref int info,
            mlptrain.mlpreport rep,
            mlptrain.mlpcvreport ooberrors)
        {
            double[,] xys = new double[0,0];
            bool[] s = new bool[0];
            double[,] oobbuf = new double[0,0];
            int[] oobcntbuf = new int[0];
            double[] x = new double[0];
            double[] y = new double[0];
            double[] dy = new double[0];
            double[] dsbuf = new double[0];
            int ccnt = 0;
            int pcnt = 0;
            int i = 0;
            int j = 0;
            int k = 0;
            double v = 0;
            mlptrain.mlpreport tmprep = new mlptrain.mlpreport();
            int nin = 0;
            int nout = 0;
            int wcount = 0;
            int i_ = 0;
            int i1_ = 0;

            info = 0;

            nin = mlpbase.mlpgetinputscount(ensemble.network);
            nout = mlpbase.mlpgetoutputscount(ensemble.network);
            wcount = mlpbase.mlpgetweightscount(ensemble.network);
            
            //
            // Test for inputs
            //
            if( (!lmalgorithm && (double)(wstep)==(double)(0)) && maxits==0 )
            {
                info = -8;
                return;
            }
            if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 )
            {
                info = -1;
                return;
            }
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                for(i=0; i<=npoints-1; i++)
                {
                    if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
                    {
                        info = -2;
                        return;
                    }
                }
            }
            
            //
            // allocate temporaries
            //
            info = 2;
            rep.ngrad = 0;
            rep.nhess = 0;
            rep.ncholesky = 0;
            ooberrors.relclserror = 0;
            ooberrors.avgce = 0;
            ooberrors.rmserror = 0;
            ooberrors.avgerror = 0;
            ooberrors.avgrelerror = 0;
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                ccnt = nin+1;
                pcnt = nin;
            }
            else
            {
                ccnt = nin+nout;
                pcnt = nin+nout;
            }
            xys = new double[npoints, ccnt];
            s = new bool[npoints];
            oobbuf = new double[npoints, nout];
            oobcntbuf = new int[npoints];
            x = new double[nin];
            y = new double[nout];
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                dy = new double[1];
            }
            else
            {
                dy = new double[nout];
            }
            for(i=0; i<=npoints-1; i++)
            {
                for(j=0; j<=nout-1; j++)
                {
                    oobbuf[i,j] = 0;
                }
            }
            for(i=0; i<=npoints-1; i++)
            {
                oobcntbuf[i] = 0;
            }
            
            //
            // main bagging cycle
            //
            for(k=0; k<=ensemble.ensemblesize-1; k++)
            {
                
                //
                // prepare dataset
                //
                for(i=0; i<=npoints-1; i++)
                {
                    s[i] = false;
                }
                for(i=0; i<=npoints-1; i++)
                {
                    j = math.randominteger(npoints);
                    s[j] = true;
                    for(i_=0; i_<=ccnt-1;i_++)
                    {
                        xys[i,i_] = xy[j,i_];
                    }
                }
                
                //
                // train
                //
                if( lmalgorithm )
                {
                    mlptrain.mlptrainlm(ensemble.network, xys, npoints, decay, restarts, ref info, tmprep);
                }
                else
                {
                    mlptrain.mlptrainlbfgs(ensemble.network, xys, npoints, decay, restarts, wstep, maxits, ref info, tmprep);
                }
                if( info<0 )
                {
                    return;
                }
                
                //
                // save results
                //
                rep.ngrad = rep.ngrad+tmprep.ngrad;
                rep.nhess = rep.nhess+tmprep.nhess;
                rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
                i1_ = (0) - (k*wcount);
                for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
                {
                    ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
                }
                i1_ = (0) - (k*pcnt);
                for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
                {
                    ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
                }
                i1_ = (0) - (k*pcnt);
                for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
                {
                    ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
                }
                
                //
                // OOB estimates
                //
                for(i=0; i<=npoints-1; i++)
                {
                    if( !s[i] )
                    {
                        for(i_=0; i_<=nin-1;i_++)
                        {
                            x[i_] = xy[i,i_];
                        }
                        mlpbase.mlpprocess(ensemble.network, x, ref y);
                        for(i_=0; i_<=nout-1;i_++)
                        {
                            oobbuf[i,i_] = oobbuf[i,i_] + y[i_];
                        }
                        oobcntbuf[i] = oobcntbuf[i]+1;
                    }
                }
            }
            
            //
            // OOB estimates
            //
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                bdss.dserrallocate(nout, ref dsbuf);
            }
            else
            {
                bdss.dserrallocate(-nout, ref dsbuf);
            }
            for(i=0; i<=npoints-1; i++)
            {
                if( oobcntbuf[i]!=0 )
                {
                    v = (double)1/(double)oobcntbuf[i];
                    for(i_=0; i_<=nout-1;i_++)
                    {
                        y[i_] = v*oobbuf[i,i_];
                    }
                    if( mlpbase.mlpissoftmax(ensemble.network) )
                    {
                        dy[0] = xy[i,nin];
                    }
                    else
                    {
                        i1_ = (nin) - (0);
                        for(i_=0; i_<=nout-1;i_++)
                        {
                            dy[i_] = v*xy[i,i_+i1_];
                        }
                    }
                    bdss.dserraccumulate(ref dsbuf, y, dy);
                }
            }
            bdss.dserrfinish(ref dsbuf);
            ooberrors.relclserror = dsbuf[0];
            ooberrors.avgce = dsbuf[1];
            ooberrors.rmserror = dsbuf[2];
            ooberrors.avgerror = dsbuf[3];
            ooberrors.avgrelerror = dsbuf[4];
        }
Beispiel #4
0
        /*************************************************************************
        Training neural networks ensemble using early stopping.

        INPUT PARAMETERS:
            Ensemble    -   model with initialized geometry
            XY          -   training set
            NPoints     -   training set size
            Decay       -   weight decay coefficient, >=0.001
            Restarts    -   restarts, >0.

        OUTPUT PARAMETERS:
            Ensemble    -   trained model
            Info        -   return code:
                            * -2, if there is a point with class number
                                  outside of [0..NClasses-1].
                            * -1, if incorrect parameters was passed
                                  (NPoints<0, Restarts<1).
                            *  6, if task has been solved.
            Rep         -   training report.
            OOBErrors   -   out-of-bag generalization error estimate

          -- ALGLIB --
             Copyright 10.03.2009 by Bochkanov Sergey
        *************************************************************************/
        public static void mlpetraines(mlpensemble ensemble,
            double[,] xy,
            int npoints,
            double decay,
            int restarts,
            ref int info,
            mlptrain.mlpreport rep)
        {
            int i = 0;
            int k = 0;
            int ccount = 0;
            int pcount = 0;
            double[,] trnxy = new double[0,0];
            double[,] valxy = new double[0,0];
            int trnsize = 0;
            int valsize = 0;
            int tmpinfo = 0;
            mlptrain.mlpreport tmprep = new mlptrain.mlpreport();
            int nin = 0;
            int nout = 0;
            int wcount = 0;
            int i_ = 0;
            int i1_ = 0;

            info = 0;

            nin = mlpbase.mlpgetinputscount(ensemble.network);
            nout = mlpbase.mlpgetoutputscount(ensemble.network);
            wcount = mlpbase.mlpgetweightscount(ensemble.network);
            if( (npoints<2 || restarts<1) || (double)(decay)<(double)(0) )
            {
                info = -1;
                return;
            }
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                for(i=0; i<=npoints-1; i++)
                {
                    if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
                    {
                        info = -2;
                        return;
                    }
                }
            }
            info = 6;
            
            //
            // allocate
            //
            if( mlpbase.mlpissoftmax(ensemble.network) )
            {
                ccount = nin+1;
                pcount = nin;
            }
            else
            {
                ccount = nin+nout;
                pcount = nin+nout;
            }
            trnxy = new double[npoints, ccount];
            valxy = new double[npoints, ccount];
            rep.ngrad = 0;
            rep.nhess = 0;
            rep.ncholesky = 0;
            
            //
            // train networks
            //
            for(k=0; k<=ensemble.ensemblesize-1; k++)
            {
                
                //
                // Split set
                //
                do
                {
                    trnsize = 0;
                    valsize = 0;
                    for(i=0; i<=npoints-1; i++)
                    {
                        if( (double)(math.randomreal())<(double)(0.66) )
                        {
                            
                            //
                            // Assign sample to training set
                            //
                            for(i_=0; i_<=ccount-1;i_++)
                            {
                                trnxy[trnsize,i_] = xy[i,i_];
                            }
                            trnsize = trnsize+1;
                        }
                        else
                        {
                            
                            //
                            // Assign sample to validation set
                            //
                            for(i_=0; i_<=ccount-1;i_++)
                            {
                                valxy[valsize,i_] = xy[i,i_];
                            }
                            valsize = valsize+1;
                        }
                    }
                }
                while( !(trnsize!=0 && valsize!=0) );
                
                //
                // Train
                //
                mlptrain.mlptraines(ensemble.network, trnxy, trnsize, valxy, valsize, decay, restarts, ref tmpinfo, tmprep);
                if( tmpinfo<0 )
                {
                    info = tmpinfo;
                    return;
                }
                
                //
                // save results
                //
                i1_ = (0) - (k*wcount);
                for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
                {
                    ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
                }
                i1_ = (0) - (k*pcount);
                for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
                {
                    ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
                }
                i1_ = (0) - (k*pcount);
                for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
                {
                    ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
                }
                rep.ngrad = rep.ngrad+tmprep.ngrad;
                rep.nhess = rep.nhess+tmprep.nhess;
                rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
            }
        }
        public static bool testmlpe(bool silent)
        {
            bool result = new bool();
            bool waserrors = new bool();
            int passcount = 0;
            int maxn = 0;
            int maxhid = 0;
            int nf = 0;
            int nhid = 0;
            int nl = 0;
            int nhid1 = 0;
            int nhid2 = 0;
            int ec = 0;
            int nkind = 0;
            int algtype = 0;
            int tasktype = 0;
            int pass = 0;
            mlpe.mlpensemble ensemble = new mlpe.mlpensemble();
            mlptrain.mlpreport rep = new mlptrain.mlpreport();
            mlptrain.mlpcvreport oobrep = new mlptrain.mlpcvreport();
            double[,] xy = new double[0,0];
            int i = 0;
            int j = 0;
            int nin = 0;
            int nout = 0;
            int npoints = 0;
            double e = 0;
            int info = 0;
            int nless = 0;
            int nall = 0;
            int nclasses = 0;
            bool inferrors = new bool();
            bool procerrors = new bool();
            bool trnerrors = new bool();

            waserrors = false;
            inferrors = false;
            procerrors = false;
            trnerrors = false;
            passcount = 10;
            maxn = 4;
            maxhid = 4;
            
            //
            // General MLP ensembles tests
            //
            for(nf=1; nf<=maxn; nf++)
            {
                for(nl=1; nl<=maxn; nl++)
                {
                    for(nhid1=0; nhid1<=maxhid; nhid1++)
                    {
                        for(nhid2=0; nhid2<=0; nhid2++)
                        {
                            for(nkind=0; nkind<=3; nkind++)
                            {
                                for(ec=1; ec<=3; ec++)
                                {
                                    
                                    //
                                    //  Skip meaningless parameters combinations
                                    //
                                    if( nkind==1 & nl<2 )
                                    {
                                        continue;
                                    }
                                    if( nhid1==0 & nhid2!=0 )
                                    {
                                        continue;
                                    }
                                    
                                    //
                                    // Tests
                                    //
                                    testinformational(nkind, nf, nhid1, nhid2, nl, ec, passcount, ref inferrors);
                                    testprocessing(nkind, nf, nhid1, nhid2, nl, ec, passcount, ref procerrors);
                                }
                            }
                        }
                    }
                }
            }
            
            //
            // network training must reduce error
            // test on random regression task
            //
            nin = 3;
            nout = 2;
            nhid = 5;
            npoints = 100;
            nless = 0;
            nall = 0;
            for(pass=1; pass<=10; pass++)
            {
                for(algtype=0; algtype<=1; algtype++)
                {
                    for(tasktype=0; tasktype<=1; tasktype++)
                    {
                        if( tasktype==0 )
                        {
                            xy = new double[npoints-1+1, nin+nout-1+1];
                            for(i=0; i<=npoints-1; i++)
                            {
                                for(j=0; j<=nin+nout-1; j++)
                                {
                                    xy[i,j] = 2*math.randomreal()-1;
                                }
                            }
                            mlpe.mlpecreate1(nin, nhid, nout, 1+math.randominteger(3), ensemble);
                        }
                        else
                        {
                            xy = new double[npoints-1+1, nin+1];
                            nclasses = 2+math.randominteger(2);
                            for(i=0; i<=npoints-1; i++)
                            {
                                for(j=0; j<=nin-1; j++)
                                {
                                    xy[i,j] = 2*math.randomreal()-1;
                                }
                                xy[i,nin] = math.randominteger(nclasses);
                            }
                            mlpe.mlpecreatec1(nin, nhid, nclasses, 1+math.randominteger(3), ensemble);
                        }
                        e = mlpe.mlpermserror(ensemble, xy, npoints);
                        if( algtype==0 )
                        {
                            mlpe.mlpebagginglm(ensemble, xy, npoints, 0.001, 1, ref info, rep, oobrep);
                        }
                        else
                        {
                            mlpe.mlpebagginglbfgs(ensemble, xy, npoints, 0.001, 1, 0.01, 0, ref info, rep, oobrep);
                        }
                        if( info<0 )
                        {
                            trnerrors = true;
                        }
                        else
                        {
                            if( (double)(mlpe.mlpermserror(ensemble, xy, npoints))<(double)(e) )
                            {
                                nless = nless+1;
                            }
                        }
                        nall = nall+1;
                    }
                }
            }
            trnerrors = trnerrors | (double)(nall-nless)>(double)(0.3*nall);
            
            //
            // Final report
            //
            waserrors = (inferrors | procerrors) | trnerrors;
            if( !silent )
            {
                System.Console.Write("MLP ENSEMBLE TEST");
                System.Console.WriteLine();
                System.Console.Write("INFORMATIONAL FUNCTIONS:                 ");
                if( !inferrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("BASIC PROCESSING:                        ");
                if( !procerrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("TRAINING:                                ");
                if( !trnerrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                if( waserrors )
                {
                    System.Console.Write("TEST SUMMARY: FAILED");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("TEST SUMMARY: PASSED");
                    System.Console.WriteLine();
                }
                System.Console.WriteLine();
                System.Console.WriteLine();
            }
            result = !waserrors;
            return result;
        }
        public static bool testmlptrain(bool silent)
        {
            bool result = new bool();
            bool waserrors = new bool();
            int passcount = 0;
            int maxn = 0;
            int maxhid = 0;
            int info = 0;
            int nf = 0;
            int nl = 0;
            int nhid1 = 0;
            int nhid2 = 0;
            int nkind = 0;
            int i = 0;
            mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron();
            mlpbase.multilayerperceptron network2 = new mlpbase.multilayerperceptron();
            mlptrain.mlpreport rep = new mlptrain.mlpreport();
            mlptrain.mlpcvreport cvrep = new mlptrain.mlpcvreport();
            int ncount = 0;
            double[,] xy = new double[0,0];
            double[,] valxy = new double[0,0];
            bool inferrors = new bool();
            bool procerrors = new bool();
            bool graderrors = new bool();
            bool hesserrors = new bool();
            bool trnerrors = new bool();

            waserrors = false;
            inferrors = false;
            procerrors = false;
            graderrors = false;
            hesserrors = false;
            trnerrors = false;
            passcount = 10;
            maxn = 4;
            maxhid = 4;
            
            //
            // General multilayer network tests
            //
            for(nf=1; nf<=maxn; nf++)
            {
                for(nl=1; nl<=maxn; nl++)
                {
                    for(nhid1=0; nhid1<=maxhid; nhid1++)
                    {
                        for(nhid2=0; nhid2<=0; nhid2++)
                        {
                            for(nkind=0; nkind<=3; nkind++)
                            {
                                
                                //
                                //  Skip meaningless parameters combinations
                                //
                                if( nkind==1 & nl<2 )
                                {
                                    continue;
                                }
                                if( nhid1==0 & nhid2!=0 )
                                {
                                    continue;
                                }
                                
                                //
                                // Tests
                                //
                                testinformational(nkind, nf, nhid1, nhid2, nl, passcount, ref inferrors);
                                testprocessing(nkind, nf, nhid1, nhid2, nl, passcount, ref procerrors);
                                testgradient(nkind, nf, nhid1, nhid2, nl, passcount, ref graderrors);
                                testhessian(nkind, nf, nhid1, nhid2, nl, passcount, ref hesserrors);
                            }
                        }
                    }
                }
            }
            
            //
            // Test network training on simple XOR problem
            //
            xy = new double[3+1, 2+1];
            xy[0,0] = -1;
            xy[0,1] = -1;
            xy[0,2] = -1;
            xy[1,0] = 1;
            xy[1,1] = -1;
            xy[1,2] = 1;
            xy[2,0] = -1;
            xy[2,1] = 1;
            xy[2,2] = 1;
            xy[3,0] = 1;
            xy[3,1] = 1;
            xy[3,2] = -1;
            mlpbase.mlpcreate1(2, 2, 1, network);
            mlptrain.mlptrainlm(network, xy, 4, 0.001, 10, ref info, rep);
            trnerrors = trnerrors | (double)(mlpbase.mlprmserror(network, xy, 4))>(double)(0.1);
            
            //
            // Test CV on random noisy problem
            //
            ncount = 100;
            xy = new double[ncount-1+1, 1+1];
            for(i=0; i<=ncount-1; i++)
            {
                xy[i,0] = 2*math.randomreal()-1;
                xy[i,1] = math.randominteger(4);
            }
            mlpbase.mlpcreatec0(1, 4, network);
            mlptrain.mlpkfoldcvlm(network, xy, ncount, 0.001, 5, 10, ref info, rep, cvrep);
            
            //
            // Final report
            //
            waserrors = (((inferrors | procerrors) | graderrors) | hesserrors) | trnerrors;
            if( !silent )
            {
                System.Console.Write("MLP TEST");
                System.Console.WriteLine();
                System.Console.Write("INFORMATIONAL FUNCTIONS:                 ");
                if( !inferrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("BASIC PROCESSING:                        ");
                if( !procerrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("GRADIENT CALCULATION:                    ");
                if( !graderrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("HESSIAN CALCULATION:                     ");
                if( !hesserrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                System.Console.Write("TRAINING:                                ");
                if( !trnerrors )
                {
                    System.Console.Write("OK");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("FAILED");
                    System.Console.WriteLine();
                }
                if( waserrors )
                {
                    System.Console.Write("TEST SUMMARY: FAILED");
                    System.Console.WriteLine();
                }
                else
                {
                    System.Console.Write("TEST SUMMARY: PASSED");
                    System.Console.WriteLine();
                }
                System.Console.WriteLine();
                System.Console.WriteLine();
            }
            result = !waserrors;
            return result;
        }