public override void Execute(MyAbstractWeightLayer layer)
{
if (layer.Connection == ConnectionType.FULLY_CONNECTED)
{
ComputeWeightGradientSum(layer);
m_adadeltaUpdateKernel.SetupExecution(layer.Weights.Count);
m_adadeltaUpdateKernel.Run(
layer.Input,
layer.Delta,
layer.WeightGradient,
layer.BiasGradient,
layer.Weights,
layer.Bias,
Owner.L1,
Owner.L2,
layer.DropoutMask,
layer.Neurons,
Owner.BatchSize,
layer.Weights.Count,
layer.MeanSquareWeight, layer.PreviousWeightDelta, layer.MeanSquareBias, layer.PreviousBiasDelta,
Owner.Adadelta.Ro, Owner.Adadelta.Epsilon
);
}
else if (layer.Connection == ConnectionType.GAUSSIAN)
{
// Gaussian hidden layer just propagates delta, no weight updates
}
else if (layer.Connection == ConnectionType.CONVOLUTION && layer is MyConvolutionLayer)
{
MyConvolutionLayer convLayer = (MyConvolutionLayer)layer;
m_convAdadeltaUpdateKernel.SetupExecution(convLayer.Weights.Count);
m_convAdadeltaUpdateKernel.Run(
convLayer.Weights,
convLayer.Bias,
convLayer.Delta,
convLayer.PaddedImage,
convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding,
(convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding) *
(convLayer.InputHeight + convLayer.ZeroPadding + convLayer.ZeroPadding),
convLayer.FilterWidth,
convLayer.FilterWidth * convLayer.FilterHeight,
convLayer.FilterWidth * convLayer.FilterHeight * convLayer.InputDepth,
convLayer.OutputWidth, convLayer.OutputHeight, convLayer.OutputWidth * convLayer.OutputHeight,
convLayer.HorizontalStride, convLayer.VerticalStride,
convLayer.L1Term, convLayer.L2Term,
convLayer.MeanSquareWeight, convLayer.PreviousWeightDelta, convLayer.MeanSquareBias,
convLayer.PreviousBiasDelta,
Owner.Adadelta.Ro, Owner.Adadelta.Epsilon,
Owner.BatchSize,
convLayer.Weights.Count
// should be equal to FilterWidth * FilterHeight * FilterCount * InputDepth
);
}
else
{
MyLog.ERROR.WriteLine("No method provided to Adadelta propagate a " + layer.Connection +
" connected MyAbstractWeightLayer in " + Owner);
}
}
private MyAbstractWeightLayer BuildNetworkLayerMirrorConvolution(XmlNode layerItem)
{
MyConvolutionLayer originalLayer = null;
bool layerMissing = true;
// Parse
foreach (XmlAttribute layerAttribute in layerItem.Attributes)
{
switch (layerAttribute.Name)
{
case "original":
originalLayer = GetLayerById <MyConvolutionLayer>(layerAttribute.Value);
layerMissing = false;
break;
default:
throw new MyXMLBuilderException("Unknown attribute: " + layerAttribute.Name);
}
}
// Validate values
if (layerMissing)
{
throw new MyXMLBuilderException("Missing original layer id");
}
// Success!
MyAbstractWeightLayer layer = new MyMirrorConvolutionLayer(m_network, originalLayer);
m_network.AddLayer(layer);
return(layer);
}
public override void Execute(MyAbstractWeightLayer layer)
{
if (layer.Connection == ConnectionType.FULLY_CONNECTED)
{
ComputeWeightGradientSum(layer);
m_SGDupdateKernel.SetupExecution(layer.Weights.Count);
m_SGDupdateKernel.Run(
layer.Input,
layer.Delta,
layer.WeightGradient,
layer.BiasGradient,
layer.Weights,
layer.PreviousWeightDelta,
layer.Bias,
layer.PreviousBiasDelta,
Owner.SGD.TrainingRate,
Owner.SGD.Momentum,
Owner.L1,
Owner.L2,
layer.DropoutMask,
layer.Neurons,
Owner.BatchSize,
layer.Weights.Count
);
}
else if (layer.Connection == ConnectionType.GAUSSIAN)
{
// Gaussian hidden layer just propagates delta, no weight updates
}
else if (layer.Connection == ConnectionType.PARTIAL_UPDATE && layer is IPartialUpdateLayer)
{
// Update some but not all of the weights
IPartialUpdateLayer partialUpdateLayer = layer as IPartialUpdateLayer;
m_partialSGDupdateKernel.SetupExecution(layer.Weights.Count);
m_partialSGDupdateKernel.Run(
layer.Input,
layer.Delta,
layer.Weights,
layer.PreviousWeightDelta,
layer.Bias,
layer.PreviousBiasDelta,
Owner.SGD.TrainingRate,
Owner.SGD.Momentum,
Owner.L1,
Owner.L2,
layer.DropoutMask,
layer.Neurons,
layer.Weights.Count,
partialUpdateLayer.SuppressUpdatesAt(),
partialUpdateLayer.SuppressUpdatesCount()
);
}
else if (layer.Connection == ConnectionType.CONVOLUTION && layer is MyConvolutionLayer)
{
MyConvolutionLayer convLayer = (MyConvolutionLayer) layer;
m_convSGDupdateKernel.SetupExecution(convLayer.Weights.Count);
m_convSGDupdateKernel.Run(
Owner.SGD.TrainingRate, Owner.SGD.Momentum,
convLayer.Weights,
convLayer.Bias, convLayer.PreviousBiasDelta,
convLayer.Delta, convLayer.PreviousWeightDelta,
convLayer.PaddedImage,
convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding,
(convLayer.InputWidth + convLayer.ZeroPadding + convLayer.ZeroPadding)*
(convLayer.InputHeight + convLayer.ZeroPadding + convLayer.ZeroPadding),
convLayer.FilterWidth,
convLayer.FilterWidth*convLayer.FilterHeight,
convLayer.FilterWidth*convLayer.FilterHeight*convLayer.InputDepth,
convLayer.OutputWidth, convLayer.OutputHeight, convLayer.OutputWidth*convLayer.OutputHeight,
convLayer.HorizontalStride, convLayer.VerticalStride,
convLayer.L1Term, convLayer.L2Term,
Owner.BatchSize,
convLayer.Weights.Count
// should be equal to FilterWidth * FilterHeight * FilterCount * InputDepth
);
}
else
{
MyLog.ERROR.WriteLine("No method provided to SGD propagate a " + layer.Connection +
" connected MyAbstractWeightLayer in " + Owner);
}
}
private MyAbstractWeightLayer BuildNetworkLayerConvolution(XmlNode layerItem)
{
uint featuresCount = 0;
uint patchWidth = 0;
uint patchHeight = 0;
uint xStride = 1;
uint yStride = 1;
bool featuresCountMissing = true;
bool patchWidthMissing = true;
bool patchHeightMissing = true;
// Parse attributes
foreach (XmlAttribute layerAttribute in layerItem.Attributes)
{
switch (layerAttribute.Name)
{
case "nbfeatures":
featuresCount = uint.Parse(layerAttribute.Value);
featuresCountMissing = false;
break;
case "patchwidth":
patchWidth = uint.Parse(layerAttribute.Value);
patchWidthMissing = false;
break;
case "patchheight":
patchHeight = uint.Parse(layerAttribute.Value);
patchHeightMissing = false;
break;
case "patchsize":
patchWidth = uint.Parse(layerAttribute.Value);
patchHeight = patchWidth;
patchHeightMissing = false;
patchWidthMissing = false;
break;
case "stride":
xStride = uint.Parse(layerAttribute.Value);
yStride = xStride;
break;
case "xstride":
xStride = uint.Parse(layerAttribute.Value);
break;
case "ystride":
yStride = uint.Parse(layerAttribute.Value);
break;
default:
throw new MyXMLBuilderException("Unknown attribute: " + layerAttribute.Name);
}
}
List <List <uint> > featureMaps = new List <List <uint> >();
// Parse feature map input indexes
foreach (XmlNode featureMapItem in layerItem.ChildNodes)
{
if (featureMapItem.NodeType == XmlNodeType.Element)
{
switch (featureMapItem.Name)
{
case "featuremap":
List <uint> inputIndexes = new List <uint>();
foreach (XmlAttribute featureMapAttribute in featureMapItem.Attributes)
{
switch (featureMapAttribute.Name)
{
case "inputindex":
string[] indexStrTab = featureMapAttribute.Value.Split(' ');
foreach (string valueStr in indexStrTab)
{
int value = int.Parse(valueStr);
if (value < 0)
{
throw new MyXMLBuilderException("inputIndex must be null or positive");
}
inputIndexes.Add((uint)value);
}
break;
default:
throw new MyXMLBuilderException("Unknown attribute: " + featureMapAttribute.Name);
}
}
if (inputIndexes.Count == 0)
{
throw new MyXMLBuilderException("Missing indexes in featureMap attribute");
}
featureMaps.Add(inputIndexes);
featuresCountMissing = false;
break;
default:
throw new MyXMLBuilderException("Unknown tag: " + featureMapItem.Name);
}
}
}
uint[][] featureMapsArray = null;
if (featureMaps.Count > 0)
{
featuresCount = (uint)featureMaps.Count;
featureMapsArray = new uint[featureMaps.Count][];
for (int i = 0; i < featureMaps.Count; i++)
{
int nbInput = featureMaps[i].Count;
featureMapsArray[i] = new uint[nbInput];
for (int j = 0; j < nbInput; j++)
{
featureMapsArray[i][j] = featureMaps[i][j];
}
}
}
// Validate values
if (featuresCountMissing)
{
throw new MyXMLBuilderException("Missing nbFeatures parameter");
}
if (patchWidthMissing)
{
throw new MyXMLBuilderException("Missing patchWidth parameter");
}
if (patchHeightMissing)
{
throw new MyXMLBuilderException("Missing patchHeight parameter");
}
if (featuresCount <= 0)
{
throw new MyXMLBuilderException("Must have a positive number of features");
}
if (patchWidth <= 0)
{
throw new MyXMLBuilderException("Patch width must be positive");
}
if (patchHeight <= 0)
{
throw new MyXMLBuilderException("Patch height must be positive");
}
// Success!
float[] initialWeight = null;
float[] initialBias = null;
if (m_weightLoader != null)
{
initialWeight = m_weightLoader.ConvWeight[(int)m_convolutionLayerCount];
initialBias = m_weightLoader.ConvBias[(int)m_convolutionLayerCount];
}
MyAbstractWeightLayer layer = new MyConvolutionLayer(m_network, featuresCount, patchWidth, patchHeight, xStride, yStride, featureMapsArray, initialWeight, initialBias);
m_network.AddLayer(layer);
m_convolutionLayerCount++;
return(layer);
}
/*
* Observers not implemented
*
public override MyOutputView CreateView()
{
throw new NotImplementedException();
//return new MyWeightView(m_network, this, 0xDDDDBBBB);
}*/
public MyMirrorConvolutionLayer(MyAbstractFeedForwardNode network, MyConvolutionLayer originalLayer, float[] initialWeights = null)
: base(network)
{
m_originalLayer = originalLayer;
}
/*
* Observers not implemented
*
* public override MyOutputView CreateView()
* {
* throw new NotImplementedException();
* //return new MyWeightView(m_network, this, 0xDDDDBBBB);
* }*/
public MyMirrorConvolutionLayer(MyAbstractFeedForwardNode network, MyConvolutionLayer originalLayer, float[] initialWeights = null)
: base(network)
{
m_originalLayer = originalLayer;
}
