public OutputLayer(double[,] InputDims, int ThisLayersNeurons, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient) : base(InputDims, ThisLayersNeurons, _MiniBatchSize, _ActivationFunction, _WeightInitialization, _TrainAlgorithm, _LearnRate, _L2_Regularization, _gradient) { }
public Layer(double[,,] InputDims_ZofPreviousLayer, int _FilterSize, int _Stride, int _Depth, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient)//Constructor For ConvolutionalLayer { TrainAlgorithm = _TrainAlgorithm; MiniBatchSize = _MiniBatchSize; gradient = _gradient; L2_Regularization = _L2_Regularization; Stride = _Stride; //InputDims_ZofPreviousLayer.GetLength(0) = Height(i-1) //InputDims_ZofPreviousLayer.GetLength(1) =Width(i-1) //InputDims_ZofPreviousLayer.GetLength(2) = Depth(i-1)*MiniBacthSize = DepthOfInputLayer*MiniBatchSize double dHeight = (InputDims_ZofPreviousLayer.GetLength(0) - 1.0) / Stride + 1.0; //when changing something here, also change in ANNMAth.Im2Mat double dWidth = (InputDims_ZofPreviousLayer.GetLength(1) - 1.0) / Stride + 1.0; //when changing something here, also change in ANNMAth.Im2Mat if (dHeight % 1 == 0 && dWidth % 1 == 0) { Height = (int)dHeight; Width = (int)dWidth; } else { throw new Exception("xxxx"); } Depth = _Depth; FilterSize = _FilterSize; if (FilterSize % 2 == 0)//Filtersize must be an uneaven number { throw new Exception("xxxx"); } Stride = _Stride; LearnRate = _LearnRate; ActivationFunction = _ActivationFunction; //PreInput is the Matrix that holds the zeropadded, Bias-added and arranged Input for the Computation of S PreInput = new double[Height * Width * MiniBatchSize, FilterSize *FilterSize *InputDims_ZofPreviousLayer.GetLength(2) / MiniBatchSize + Bias]; //S is the matrix that holds the inputs to this layer S = new double[Height * Width * MiniBatchSize, Depth]; //W is the ingoing rotated weight matrix for this layer inclusive a Bias for each Kernel W = new double[FilterSize * FilterSize * InputDims_ZofPreviousLayer.GetLength(2) / MiniBatchSize + Bias, Depth];//Have a look at page 11 in Theory, rotated WeightMatrix for ForewardPass if (_WeightInitialization == WeightInitialization.NormalizedGaussianRandom) { double mean = 0; double stdDev = 1; for (int i = 0; i < W.GetLength(0); i++) { for (int j = 0; j < W.GetLength(1); j++) { //W[i, j] = rnd.NextDouble() * 2 - 1; //Old initialization of weights double u1 = 1.0 - rnd.NextDouble(); //uniform(0,1] random doubles double u2 = 1.0 - rnd.NextDouble(); double randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); //random normal(0,1) W[i, j] = mean + stdDev * randStdNormal; //random normal(mean,stdDev^2) } } } else if (_WeightInitialization == WeightInitialization.Random) { for (int i = 0; i < W.GetLength(0); i++) { for (int j = 0; j < W.GetLength(1); j++) { W[i, j] = rnd.NextDouble() * 2 - 1; } } } //safe previous gradient for adaptive learnrate dW_t_minus_1 = new double[W.GetLength(0), W.GetLength(1)]; //initialize dW_t_minus_1 as zeros-Matrix for (int i = 0; i < dW_t_minus_1.GetLength(0); i++) { for (int j = 0; j < dW_t_minus_1.GetLength(1); j++) { dW_t_minus_1[i, j] = 0; } } //initialaze the AdaptiveLearnRateGain AdaptiveLearnRateGain = new double[W.GetLength(0), W.GetLength(1)]; //initialize AdaptiveLearnRateGain as ones-Matrix for (int i = 0; i < AdaptiveLearnRateGain.GetLength(0); i++) { for (int j = 0; j < AdaptiveLearnRateGain.GetLength(1); j++) { AdaptiveLearnRateGain[i, j] = 1; } } if (TrainAlgorithm == TrainAlgorithm.Adam) { //initialize Velocity_Adam as zeros-Matrix Velocity_Adam = new double[W.GetLength(0), W.GetLength(1)]; //initialize Momentum_Adam as zeros-Matrix Momentum_Adam = new double[W.GetLength(0), W.GetLength(1)]; } //Z is the matrix that holds output values Z = new double[MiniBatchSize, Height *Width *Depth + Bias]; //[Row = MiniBatchSize, Column=NeuronsperChannel*NumberOfChannels+Bias] //Setting Bias of Z to 1 for (int i = 0; i < Z.GetLength(0); i++) //Z.GetLenght(0) = BatchSize { Z[i, Z.GetLength(1) - Bias] = 1; } Z_3D = new double[Height, Width, Depth *MiniBatchSize]; //D is the matrix that holds the deltas for this layer D = new double[Height * Width * MiniBatchSize, Depth]; D_3D = new double[Height, Width, Depth *MiniBatchSize]; //dF is the matrix that holds the derivatives of the activation function dF = new double[Height * Width * Depth, MiniBatchSize]; }
public Layer(double[,] InputDims_ZofPreviousLayer, int ThisLayersNeurons, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient)//Constructor For HiddenLayer and OutputLayer { TrainAlgorithm = _TrainAlgorithm; MiniBatchSize = _MiniBatchSize; gradient = _gradient; L2_Regularization = _L2_Regularization; LearnRate = _LearnRate; ActivationFunction = _ActivationFunction; //S is the matrix that holds the inputs to this layer S = new double[InputDims_ZofPreviousLayer.GetLength(0), ThisLayersNeurons]; //[Rows=BatchSize,Columns] //W is the ingoing weight matrix for this layer W = new double[InputDims_ZofPreviousLayer.GetLength(1), ThisLayersNeurons]; //[Rows=PreviousLayersNeurons + Bias,Columns=ThisLayersNeurons] if (_WeightInitialization == WeightInitialization.NormalizedGaussianRandom) { double mean = 0; double stdDev = 1; for (int i = 0; i < W.GetLength(0); i++) { for (int j = 0; j < W.GetLength(1); j++) { double u1 = 1.0 - rnd.NextDouble(); //uniform(0,1] random doubles double u2 = 1.0 - rnd.NextDouble(); double randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); //random normal(0,1) W[i, j] = mean + stdDev * randStdNormal; //random normal(mean,stdDev^2) } } } else if (_WeightInitialization == WeightInitialization.Random) { for (int i = 0; i < W.GetLength(0); i++) { for (int j = 0; j < W.GetLength(1); j++) { W[i, j] = rnd.NextDouble() * 2 - 1; } } } //safe previous gradient for adaptive learnrate dW_t_minus_1 = new double[W.GetLength(0), W.GetLength(1)]; //initialize dW_t_minus_1 as zeros-Matrix for (int i = 0; i < dW_t_minus_1.GetLength(0); i++) { for (int j = 0; j < dW_t_minus_1.GetLength(1); j++) { dW_t_minus_1[i, j] = 0; } } //initialaze the AdaptiveLearnRateGain AdaptiveLearnRateGain = new double[W.GetLength(0), W.GetLength(1)]; //initialize AdaptiveLearnRateGain as ones-Matrix for (int i = 0; i < AdaptiveLearnRateGain.GetLength(0); i++) { for (int j = 0; j < AdaptiveLearnRateGain.GetLength(1); j++) { AdaptiveLearnRateGain[i, j] = 1; } } if (TrainAlgorithm == TrainAlgorithm.Adam) { //initialize Velocity_Adam as zeros-Matrix Velocity_Adam = new double[W.GetLength(0), W.GetLength(1)]; //initialize Momentum_Adam as zeros-Matrix Momentum_Adam = new double[W.GetLength(0), W.GetLength(1)]; } //Z is the matrix that holds output values if (GetType() == typeof(HiddenLayer)) { Z = new double[S.GetLength(0), S.GetLength(1) + Bias]; //[Rows=BatchSize,Columns=ThisLayersNeurons + Bias] //Setting Bias-Column (= Last Column) of Z to 1 for (int i = 0; i < Z.GetLength(0); i++) //Z.GetLenght(0) = BatchSize { Z[i, Z.GetLength(1) - Bias] = 1; } } else//if OutputLayer, there's no Bias on Z { this.Z = new double[this.S.GetLength(0), this.S.GetLength(1)]; } //D is the matrix that holds the deltas for this layer this.D = new double[this.S.GetLength(1), this.S.GetLength(0)]; //[Rows=ThisLayersNeurons,Columns=BatchSize] //dF is the matrix that holds the derivatives of the activation function. Bias doesent have a derevative this.dF = new double[this.S.GetLength(1), this.S.GetLength(0)]; //[Rows=ThisLayersNeurons,Columns=BatchSize] }
public ConvolutionalLayer(double[,,] InputDims_ZofPreviousLayer, int FilterSize, int Stride, int Depth, int _MiniBatchSize, ActivationFunction _ActivationFunction, WeightInitialization _WeightInitialization, TrainAlgorithm _TrainAlgorithm, double _LearnRate, double _L2_Regularization, double _gradient) : base(InputDims_ZofPreviousLayer, FilterSize, Stride, Depth, _MiniBatchSize, _ActivationFunction, _WeightInitialization, _TrainAlgorithm, _LearnRate, _L2_Regularization, _gradient) { }