示例#1
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="inputWidth"></param>
        /// <param name="inputHeight"></param>
        /// <param name="inputDepth"></param>
        /// <param name="batchSize"></param>
        /// <param name="initializtion"></param>
        /// <param name="random"></param>

        public void Initialize(int inputWidth, int inputHeight, int inputDepth, int batchSize, Initialization initializtion, Random random)
        {
            var fans         = WeightInitialization.GetFans(this, inputWidth, inputHeight, inputDepth);
            var distribution = WeightInitialization.GetWeightDistribution(initializtion, fans, random);

            Weights = Matrix <float> .Build.Random(fans.FanIn, fans.FanOut, distribution);

            Bias = Vector <float> .Build.Dense(fans.FanOut, 0.0f);

            WeightsGradients = Matrix <float> .Build.Dense(fans.FanIn, fans.FanOut);

            BiasGradients = Vector <float> .Build.Dense(fans.FanOut);

            OutputActivations = Matrix <float> .Build.Dense(batchSize, fans.FanOut);

            m_delta = Matrix <float> .Build.Dense(batchSize, fans.FanIn);
        }
示例#2
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="inputWidth"></param>
        /// <param name="inputHeight"></param>
        /// <param name="inputDepth"></param>
        /// <param name="batchSize"></param>
        /// <param name="initializtion"></param>
        /// <param name="random"></param>

        public void Initialize(int inputWidth, int inputHeight, int inputDepth, int batchSize,
                               Initialization initializtion, Random random)
        {
            InputHeight = inputHeight;
            InputWidth  = inputWidth;
            InputDepth  = inputDepth;

            var filterGridWidth = ConvUtils.GetFilterGridLength(InputWidth, FilterWidth,
                                                                m_stride, m_padWidth, BorderMode);
            var filterGridHeight = ConvUtils.GetFilterGridLength(InputHeight, FilterHeight,
                                                                 m_stride, m_padHeight, BorderMode);

            // Calculations of dimensions based on:
            // Nvidia, cuDNN: Efficient Primitives for Deep Learning: https://arxiv.org/pdf/1410.0759.pdf
            var filterCubeSize = InputDepth * FilterWidth * FilterHeight;
            var filterGridSize = filterGridWidth * filterGridHeight;

            Height = filterGridHeight;
            Width  = filterGridWidth;
            Depth  = FilterCount;

            var fans         = WeightInitialization.GetFans(this, InputWidth, InputHeight, inputDepth);
            var distribution = WeightInitialization.GetWeightDistribution(initializtion, fans, random);

            Weights = Matrix <float> .Build.Random(FilterCount, filterCubeSize, distribution);

            WeightsGradients = Matrix <float> .Build.Dense(FilterCount, filterCubeSize);

            Bias = Vector <float> .Build.Dense(FilterCount, 0.0f);

            BiasGradients = Vector <float> .Build.Dense(FilterCount);

            Im2Cols = Matrix <float> .Build.Dense(filterCubeSize, filterGridSize *batchSize);

            Conv = Matrix <float> .Build.Dense(FilterCount, filterGridSize *batchSize);

            OutputActivations = Matrix <float> .Build.Dense(batchSize, FilterCount *filterGridSize);

            m_deltaInReshape = Matrix <float> .Build.Dense(FilterCount, filterGridSize *batchSize);

            var fanIn = inputWidth * inputHeight * inputDepth;

            m_delta = Matrix <float> .Build.Dense(batchSize, fanIn);
        }
示例#3
0
 public OutputLayer(double[,] InputDims, int ThisLayersNeurons, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient) : base(InputDims, ThisLayersNeurons, _MiniBatchSize, _ActivationFunction, _WeightInitialization, _TrainAlgorithm, _LearnRate, _L2_Regularization, _gradient)
 {
 }
示例#4
0
文件: Layer.cs 项目: fontm2/CSharpANN
 public Layer(double[,] InputDims_ZofPreviousLayer, int ThisLayersNeurons, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient)//Constructor For HiddenLayer and OutputLayer
 {
     TrainAlgorithm     = _TrainAlgorithm;
     MiniBatchSize      = _MiniBatchSize;
     gradient           = _gradient;
     L2_Regularization  = _L2_Regularization;
     LearnRate          = _LearnRate;
     ActivationFunction = _ActivationFunction;
     //S is the matrix that holds the inputs to this layer
     S = new double[InputDims_ZofPreviousLayer.GetLength(0), ThisLayersNeurons]; //[Rows=BatchSize,Columns]
     //W is the ingoing weight matrix for this layer
     W = new double[InputDims_ZofPreviousLayer.GetLength(1), ThisLayersNeurons]; //[Rows=PreviousLayersNeurons + Bias,Columns=ThisLayersNeurons]
     if (_WeightInitialization == WeightInitialization.NormalizedGaussianRandom)
     {
         double mean   = 0;
         double stdDev = 1;
         for (int i = 0; i < W.GetLength(0); i++)
         {
             for (int j = 0; j < W.GetLength(1); j++)
             {
                 double u1            = 1.0 - rnd.NextDouble();                                        //uniform(0,1] random doubles
                 double u2            = 1.0 - rnd.NextDouble();
                 double randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); //random normal(0,1)
                 W[i, j] = mean + stdDev * randStdNormal;                                              //random normal(mean,stdDev^2)
             }
         }
     }
     else if (_WeightInitialization == WeightInitialization.Random)
     {
         for (int i = 0; i < W.GetLength(0); i++)
         {
             for (int j = 0; j < W.GetLength(1); j++)
             {
                 W[i, j] = rnd.NextDouble() * 2 - 1;
             }
         }
     }
     //safe previous gradient for adaptive learnrate
     dW_t_minus_1 = new double[W.GetLength(0), W.GetLength(1)];
     //initialize dW_t_minus_1 as zeros-Matrix
     for (int i = 0; i < dW_t_minus_1.GetLength(0); i++)
     {
         for (int j = 0; j < dW_t_minus_1.GetLength(1); j++)
         {
             dW_t_minus_1[i, j] = 0;
         }
     }
     //initialaze the AdaptiveLearnRateGain
     AdaptiveLearnRateGain = new double[W.GetLength(0), W.GetLength(1)];
     //initialize AdaptiveLearnRateGain as ones-Matrix
     for (int i = 0; i < AdaptiveLearnRateGain.GetLength(0); i++)
     {
         for (int j = 0; j < AdaptiveLearnRateGain.GetLength(1); j++)
         {
             AdaptiveLearnRateGain[i, j] = 1;
         }
     }
     if (TrainAlgorithm == TrainAlgorithm.Adam)
     {
         //initialize Velocity_Adam as zeros-Matrix
         Velocity_Adam = new double[W.GetLength(0), W.GetLength(1)];
         //initialize Momentum_Adam as zeros-Matrix
         Momentum_Adam = new double[W.GetLength(0), W.GetLength(1)];
     }
     //Z is the matrix that holds output values
     if (GetType() == typeof(HiddenLayer))
     {
         Z = new double[S.GetLength(0), S.GetLength(1) + Bias]; //[Rows=BatchSize,Columns=ThisLayersNeurons + Bias]
         //Setting Bias-Column (= Last Column) of Z to 1
         for (int i = 0; i < Z.GetLength(0); i++)               //Z.GetLenght(0) = BatchSize
         {
             Z[i, Z.GetLength(1) - Bias] = 1;
         }
     }
     else//if OutputLayer, there's no Bias on Z
     {
         this.Z = new double[this.S.GetLength(0), this.S.GetLength(1)];
     }
     //D is the matrix that holds the deltas for this layer
     this.D = new double[this.S.GetLength(1), this.S.GetLength(0)];  //[Rows=ThisLayersNeurons,Columns=BatchSize]
     //dF is the matrix that holds the derivatives of the activation function. Bias doesent have a derevative
     this.dF = new double[this.S.GetLength(1), this.S.GetLength(0)]; //[Rows=ThisLayersNeurons,Columns=BatchSize]
 }
示例#5
0
文件: Layer.cs 项目: fontm2/CSharpANN
        public Layer(double[,,] InputDims_ZofPreviousLayer, int _FilterSize, int _Stride, int _Depth, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient)//Constructor For ConvolutionalLayer
        {
            TrainAlgorithm    = _TrainAlgorithm;
            MiniBatchSize     = _MiniBatchSize;
            gradient          = _gradient;
            L2_Regularization = _L2_Regularization;
            Stride            = _Stride;
            //InputDims_ZofPreviousLayer.GetLength(0) = Height(i-1)
            //InputDims_ZofPreviousLayer.GetLength(1) =Width(i-1)
            //InputDims_ZofPreviousLayer.GetLength(2) = Depth(i-1)*MiniBacthSize = DepthOfInputLayer*MiniBatchSize
            double dHeight = (InputDims_ZofPreviousLayer.GetLength(0) - 1.0) / Stride + 1.0; //when changing something here, also change in ANNMAth.Im2Mat
            double dWidth  = (InputDims_ZofPreviousLayer.GetLength(1) - 1.0) / Stride + 1.0; //when changing something here, also change in ANNMAth.Im2Mat

            if (dHeight % 1 == 0 && dWidth % 1 == 0)
            {
                Height = (int)dHeight;
                Width  = (int)dWidth;
            }
            else
            {
                throw new Exception("xxxx");
            }
            Depth      = _Depth;
            FilterSize = _FilterSize;
            if (FilterSize % 2 == 0)//Filtersize must be an uneaven number
            {
                throw new Exception("xxxx");
            }
            Stride             = _Stride;
            LearnRate          = _LearnRate;
            ActivationFunction = _ActivationFunction;
            //PreInput is the Matrix that holds the zeropadded, Bias-added and arranged Input for the Computation of S
            PreInput = new double[Height * Width * MiniBatchSize, FilterSize *FilterSize *InputDims_ZofPreviousLayer.GetLength(2) / MiniBatchSize + Bias];
            //S is the matrix that holds the inputs to this layer
            S = new double[Height * Width * MiniBatchSize, Depth];
            //W is the ingoing rotated weight matrix for this layer inclusive a Bias for each Kernel
            W = new double[FilterSize * FilterSize * InputDims_ZofPreviousLayer.GetLength(2) / MiniBatchSize + Bias, Depth];//Have a look at page 11 in Theory, rotated WeightMatrix for ForewardPass
            if (_WeightInitialization == WeightInitialization.NormalizedGaussianRandom)
            {
                double mean   = 0;
                double stdDev = 1;
                for (int i = 0; i < W.GetLength(0); i++)
                {
                    for (int j = 0; j < W.GetLength(1); j++)
                    {
                        //W[i, j] = rnd.NextDouble() * 2 - 1; //Old initialization of weights
                        double u1            = 1.0 - rnd.NextDouble();                                        //uniform(0,1] random doubles
                        double u2            = 1.0 - rnd.NextDouble();
                        double randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); //random normal(0,1)
                        W[i, j] = mean + stdDev * randStdNormal;                                              //random normal(mean,stdDev^2)
                    }
                }
            }
            else if (_WeightInitialization == WeightInitialization.Random)
            {
                for (int i = 0; i < W.GetLength(0); i++)
                {
                    for (int j = 0; j < W.GetLength(1); j++)
                    {
                        W[i, j] = rnd.NextDouble() * 2 - 1;
                    }
                }
            }
            //safe previous gradient for adaptive learnrate
            dW_t_minus_1 = new double[W.GetLength(0), W.GetLength(1)];
            //initialize dW_t_minus_1 as zeros-Matrix
            for (int i = 0; i < dW_t_minus_1.GetLength(0); i++)
            {
                for (int j = 0; j < dW_t_minus_1.GetLength(1); j++)
                {
                    dW_t_minus_1[i, j] = 0;
                }
            }
            //initialaze the AdaptiveLearnRateGain
            AdaptiveLearnRateGain = new double[W.GetLength(0), W.GetLength(1)];
            //initialize AdaptiveLearnRateGain as ones-Matrix
            for (int i = 0; i < AdaptiveLearnRateGain.GetLength(0); i++)
            {
                for (int j = 0; j < AdaptiveLearnRateGain.GetLength(1); j++)
                {
                    AdaptiveLearnRateGain[i, j] = 1;
                }
            }
            if (TrainAlgorithm == TrainAlgorithm.Adam)
            {
                //initialize Velocity_Adam as zeros-Matrix
                Velocity_Adam = new double[W.GetLength(0), W.GetLength(1)];
                //initialize Momentum_Adam as zeros-Matrix
                Momentum_Adam = new double[W.GetLength(0), W.GetLength(1)];
            }
            //Z is the matrix that holds output values
            Z = new double[MiniBatchSize, Height *Width *Depth + Bias]; //[Row = MiniBatchSize, Column=NeuronsperChannel*NumberOfChannels+Bias]
            //Setting Bias of Z to 1
            for (int i = 0; i < Z.GetLength(0); i++)                    //Z.GetLenght(0) = BatchSize
            {
                Z[i, Z.GetLength(1) - Bias] = 1;
            }

            Z_3D = new double[Height, Width, Depth *MiniBatchSize];
            //D is the matrix that holds the deltas for this layer
            D    = new double[Height * Width * MiniBatchSize, Depth];
            D_3D = new double[Height, Width, Depth *MiniBatchSize];
            //dF is the matrix that holds the derivatives of the activation function
            dF = new double[Height * Width * Depth, MiniBatchSize];
        }
示例#6
0
 public ConvolutionalLayer(double[,,] InputDims_ZofPreviousLayer, int FilterSize, int Stride, int Depth, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient) : base(InputDims_ZofPreviousLayer, FilterSize, Stride, Depth, _MiniBatchSize, _ActivationFunction, _WeightInitialization, _TrainAlgorithm, _LearnRate, _L2_Regularization, _gradient)
 {
 }