public override void Execute(MyAbstractWeightLayer layer) { if (layer.Connection == ConnectionType.FULLY_CONNECTED) { ComputeWeightGradientSum(layer); m_adadeltaUpdateKernel.SetupExecution(layer.Weights.Count); m_adadeltaUpdateKernel.Run( layer.Input, layer.Delta, layer.WeightGradient, layer.BiasGradient, layer.Weights, layer.Bias, Owner.L1, Owner.L2, layer.DropoutMask, layer.Neurons, Owner.BatchSize, layer.Weights.Count, layer.MeanSquareWeight, layer.PreviousWeightDelta, layer.MeanSquareBias, layer.PreviousBiasDelta, Owner.Adadelta.Ro, Owner.Adadelta.Epsilon ); } else if (layer.Connection == ConnectionType.GAUSSIAN) { // Gaussian hidden layer just propagates delta, no weight updates } else if (layer.Connection == ConnectionType.CONVOLUTION && layer is MyConvolutionLayer) { MyConvolutionLayer convLayer = (MyConvolutionLayer)layer; m_convAdadeltaUpdateKernel.SetupExecution(convLayer.Weights.Count); m_convAdadeltaUpdateKernel.Run( convLayer.Weights, convLayer.Bias, convLayer.Delta, convLayer.PaddedImage, convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding, (convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding) * (convLayer.InputHeight + convLayer.ZeroPadding + convLayer.ZeroPadding), convLayer.FilterWidth, convLayer.FilterWidth * convLayer.FilterHeight, convLayer.FilterWidth * convLayer.FilterHeight * convLayer.InputDepth, convLayer.OutputWidth, convLayer.OutputHeight, convLayer.OutputWidth * convLayer.OutputHeight, convLayer.HorizontalStride, convLayer.VerticalStride, convLayer.L1Term, convLayer.L2Term, convLayer.MeanSquareWeight, convLayer.PreviousWeightDelta, convLayer.MeanSquareBias, convLayer.PreviousBiasDelta, Owner.Adadelta.Ro, Owner.Adadelta.Epsilon, Owner.BatchSize, convLayer.Weights.Count // should be equal to FilterWidth * FilterHeight * FilterCount * InputDepth ); } else { MyLog.ERROR.WriteLine("No method provided to Adadelta propagate a " + layer.Connection + " connected MyAbstractWeightLayer in " + Owner); } }
private MyAbstractWeightLayer BuildNetworkLayerMirrorConvolution(XmlNode layerItem) { MyConvolutionLayer originalLayer = null; bool layerMissing = true; // Parse foreach (XmlAttribute layerAttribute in layerItem.Attributes) { switch (layerAttribute.Name) { case "original": originalLayer = GetLayerById <MyConvolutionLayer>(layerAttribute.Value); layerMissing = false; break; default: throw new MyXMLBuilderException("Unknown attribute: " + layerAttribute.Name); } } // Validate values if (layerMissing) { throw new MyXMLBuilderException("Missing original layer id"); } // Success! MyAbstractWeightLayer layer = new MyMirrorConvolutionLayer(m_network, originalLayer); m_network.AddLayer(layer); return(layer); }
public override void Execute(MyAbstractWeightLayer layer) { if (layer.Connection == ConnectionType.FULLY_CONNECTED) { ComputeWeightGradientSum(layer); m_SGDupdateKernel.SetupExecution(layer.Weights.Count); m_SGDupdateKernel.Run( layer.Input, layer.Delta, layer.WeightGradient, layer.BiasGradient, layer.Weights, layer.PreviousWeightDelta, layer.Bias, layer.PreviousBiasDelta, Owner.SGD.TrainingRate, Owner.SGD.Momentum, Owner.L1, Owner.L2, layer.DropoutMask, layer.Neurons, Owner.BatchSize, layer.Weights.Count ); } else if (layer.Connection == ConnectionType.GAUSSIAN) { // Gaussian hidden layer just propagates delta, no weight updates } else if (layer.Connection == ConnectionType.PARTIAL_UPDATE && layer is IPartialUpdateLayer) { // Update some but not all of the weights IPartialUpdateLayer partialUpdateLayer = layer as IPartialUpdateLayer; m_partialSGDupdateKernel.SetupExecution(layer.Weights.Count); m_partialSGDupdateKernel.Run( layer.Input, layer.Delta, layer.Weights, layer.PreviousWeightDelta, layer.Bias, layer.PreviousBiasDelta, Owner.SGD.TrainingRate, Owner.SGD.Momentum, Owner.L1, Owner.L2, layer.DropoutMask, layer.Neurons, layer.Weights.Count, partialUpdateLayer.SuppressUpdatesAt(), partialUpdateLayer.SuppressUpdatesCount() ); } else if (layer.Connection == ConnectionType.CONVOLUTION && layer is MyConvolutionLayer) { MyConvolutionLayer convLayer = (MyConvolutionLayer) layer; m_convSGDupdateKernel.SetupExecution(convLayer.Weights.Count); m_convSGDupdateKernel.Run( Owner.SGD.TrainingRate, Owner.SGD.Momentum, convLayer.Weights, convLayer.Bias, convLayer.PreviousBiasDelta, convLayer.Delta, convLayer.PreviousWeightDelta, convLayer.PaddedImage, convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding, (convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding)* (convLayer.InputHeight + convLayer.ZeroPadding + convLayer.ZeroPadding), convLayer.FilterWidth, convLayer.FilterWidth*convLayer.FilterHeight, convLayer.FilterWidth*convLayer.FilterHeight*convLayer.InputDepth, convLayer.OutputWidth, convLayer.OutputHeight, convLayer.OutputWidth*convLayer.OutputHeight, convLayer.HorizontalStride, convLayer.VerticalStride, convLayer.L1Term, convLayer.L2Term, Owner.BatchSize, convLayer.Weights.Count // should be equal to FilterWidth * FilterHeight * FilterCount * InputDepth ); } else { MyLog.ERROR.WriteLine("No method provided to SGD propagate a " + layer.Connection + " connected MyAbstractWeightLayer in " + Owner); } }
private MyAbstractWeightLayer BuildNetworkLayerConvolution(XmlNode layerItem) { uint featuresCount = 0; uint patchWidth = 0; uint patchHeight = 0; uint xStride = 1; uint yStride = 1; bool featuresCountMissing = true; bool patchWidthMissing = true; bool patchHeightMissing = true; // Parse attributes foreach (XmlAttribute layerAttribute in layerItem.Attributes) { switch (layerAttribute.Name) { case "nbfeatures": featuresCount = uint.Parse(layerAttribute.Value); featuresCountMissing = false; break; case "patchwidth": patchWidth = uint.Parse(layerAttribute.Value); patchWidthMissing = false; break; case "patchheight": patchHeight = uint.Parse(layerAttribute.Value); patchHeightMissing = false; break; case "patchsize": patchWidth = uint.Parse(layerAttribute.Value); patchHeight = patchWidth; patchHeightMissing = false; patchWidthMissing = false; break; case "stride": xStride = uint.Parse(layerAttribute.Value); yStride = xStride; break; case "xstride": xStride = uint.Parse(layerAttribute.Value); break; case "ystride": yStride = uint.Parse(layerAttribute.Value); break; default: throw new MyXMLBuilderException("Unknown attribute: " + layerAttribute.Name); } } List <List <uint> > featureMaps = new List <List <uint> >(); // Parse feature map input indexes foreach (XmlNode featureMapItem in layerItem.ChildNodes) { if (featureMapItem.NodeType == XmlNodeType.Element) { switch (featureMapItem.Name) { case "featuremap": List <uint> inputIndexes = new List <uint>(); foreach (XmlAttribute featureMapAttribute in featureMapItem.Attributes) { switch (featureMapAttribute.Name) { case "inputindex": string[] indexStrTab = featureMapAttribute.Value.Split(' '); foreach (string valueStr in indexStrTab) { int value = int.Parse(valueStr); if (value < 0) { throw new MyXMLBuilderException("inputIndex must be null or positive"); } inputIndexes.Add((uint)value); } break; default: throw new MyXMLBuilderException("Unknown attribute: " + featureMapAttribute.Name); } } if (inputIndexes.Count == 0) { throw new MyXMLBuilderException("Missing indexes in featureMap attribute"); } featureMaps.Add(inputIndexes); featuresCountMissing = false; break; default: throw new MyXMLBuilderException("Unknown tag: " + featureMapItem.Name); } } } uint[][] featureMapsArray = null; if (featureMaps.Count > 0) { featuresCount = (uint)featureMaps.Count; featureMapsArray = new uint[featureMaps.Count][]; for (int i = 0; i < featureMaps.Count; i++) { int nbInput = featureMaps[i].Count; featureMapsArray[i] = new uint[nbInput]; for (int j = 0; j < nbInput; j++) { featureMapsArray[i][j] = featureMaps[i][j]; } } } // Validate values if (featuresCountMissing) { throw new MyXMLBuilderException("Missing nbFeatures parameter"); } if (patchWidthMissing) { throw new MyXMLBuilderException("Missing patchWidth parameter"); } if (patchHeightMissing) { throw new MyXMLBuilderException("Missing patchHeight parameter"); } if (featuresCount <= 0) { throw new MyXMLBuilderException("Must have a positive number of features"); } if (patchWidth <= 0) { throw new MyXMLBuilderException("Patch width must be positive"); } if (patchHeight <= 0) { throw new MyXMLBuilderException("Patch height must be positive"); } // Success! float[] initialWeight = null; float[] initialBias = null; if (m_weightLoader != null) { initialWeight = m_weightLoader.ConvWeight[(int)m_convolutionLayerCount]; initialBias = m_weightLoader.ConvBias[(int)m_convolutionLayerCount]; } MyAbstractWeightLayer layer = new MyConvolutionLayer(m_network, featuresCount, patchWidth, patchHeight, xStride, yStride, featureMapsArray, initialWeight, initialBias); m_network.AddLayer(layer); m_convolutionLayerCount++; return(layer); }
/* * Observers not implemented * public override MyOutputView CreateView() { throw new NotImplementedException(); //return new MyWeightView(m_network, this, 0xDDDDBBBB); }*/ public MyMirrorConvolutionLayer(MyAbstractFeedForwardNode network, MyConvolutionLayer originalLayer, float[] initialWeights = null) : base(network) { m_originalLayer = originalLayer; }
/* * Observers not implemented * * public override MyOutputView CreateView() * { * throw new NotImplementedException(); * //return new MyWeightView(m_network, this, 0xDDDDBBBB); * }*/ public MyMirrorConvolutionLayer(MyAbstractFeedForwardNode network, MyConvolutionLayer originalLayer, float[] initialWeights = null) : base(network) { m_originalLayer = originalLayer; }