public override void Execute() //Task execution
{
MyLog.DEBUG.WriteLine("Pooling backward.");
// pointer to previous layer
MyNode node = Owner.Input.Owner;
if (node is MyAbstractLayer)
{
MyAbstractLayer previousLayer = node as MyAbstractLayer;
// reset delta
previousLayer.Delta.Fill(0);
// determine input to previous layer
CUdeviceptr prevInputPtr = MyAbstractLayer.DetermineInput(previousLayer);
m_kernel.SetupExecution(Owner.Neurons);
m_kernel.Run(
(int)previousLayer.ActivationFunction,
Owner.Delta,
previousLayer.Delta,
prevInputPtr,
Owner.ActivatedNeurons,
Owner.Neurons
);
}
}
public override void Execute() //Task execution
{
MyNode node = Owner.Input.Owner;
if (node is MyAbstractLayer)
{
MyAbstractLayer previousLayer = node as MyAbstractLayer;
// reset delta only if next is not Gaussian HACK.
// (Gaussian layer already reseted delta and filled with regularization deltas)
previousLayer.Delta.Fill(0);
// determine input to previous layer
CUdeviceptr prevInputPtr = MyAbstractLayer.DetermineInput(previousLayer);
if (Owner.ParentNetwork.BatchSize == 1)
{
// cuBLAS tends to be slower when BatchSize is 1, use the kernel instead
m_deltaKernel.SetupExecution(previousLayer.Neurons);
m_deltaKernel.Run(
(int)previousLayer.ActivationFunction,
prevInputPtr,
previousLayer.Delta,
Owner.Delta,
Owner.Weights,
Owner.ParentNetwork.Dropout,
previousLayer.Neurons,
Owner.Neurons
);
}
else
{
// previousLayer.Delta = Transpose(Weights) x Delta
MyCublasFactory.Instance.Gemm(Operation.Transpose, Operation.NonTranspose,
previousLayer.Neurons, Owner.ParentNetwork.BatchSize, Owner.Neurons, 1.0f,
Owner.Weights.GetDevice(Owner), Owner.Neurons,
Owner.Delta.GetDevice(Owner), Owner.Neurons,
0.0f, previousLayer.Delta.GetDevice(Owner), previousLayer.Neurons
);
// multiply previousLayer.Delta by activation derivatives of previous layer
m_deltaBatchKernel.SetupExecution(previousLayer.Neurons * Owner.ParentNetwork.BatchSize);
m_deltaBatchKernel.Run(
(int)previousLayer.ActivationFunction,
prevInputPtr,
previousLayer.Delta,
Owner.ParentNetwork.Dropout,
previousLayer.Neurons,
Owner.ParentNetwork.BatchSize
);
}
}
}
public override void Execute()
{
MyLog.DEBUG.WriteLine("Convolution backward.");
// pointer to previous layer
MyNode node = Owner.Input.Owner;
if (node is MyAbstractLayer)
{
MyAbstractLayer previousLayer = node as MyAbstractLayer;
// reset delta
previousLayer.Delta.Fill(0);
// determine input to previous layer
CUdeviceptr prevInputPtr = MyAbstractLayer.DetermineInput(previousLayer);
m_kernel.SetupExecution(previousLayer.Neurons);
m_kernel.Run(
(int)previousLayer.ActivationFunction,
Owner.Weights,
Owner.Delta,
previousLayer.Delta,
prevInputPtr,
Owner.FilterCount,
Owner.InputWidth * Owner.InputHeight, // input slice size without padding
(Owner.InputWidth + Owner.ZeroPadding + Owner.ZeroPadding) * (Owner.InputHeight + Owner.ZeroPadding + Owner.ZeroPadding), // input slice size
Owner.ZeroPadding,
Owner.InputWidth, Owner.InputHeight,
Owner.FilterWidth, Owner.FilterHeight,
Owner.FilterWidth * Owner.FilterHeight,
Owner.FilterWidth * Owner.FilterHeight * Owner.InputDepth,
Owner.OutputWidth, Owner.OutputHeight, Owner.OutputWidth * Owner.OutputHeight,
Owner.HorizontalStride, Owner.VerticalStride,
previousLayer.Neurons
);
}
}
public override void Execute()
{
MyNode node = Owner.Input.Owner;
if (node is MyAbstractLayer)
{
MyAbstractLayer previousLayer = node as MyAbstractLayer;
// Reset delta
previousLayer.Delta.Fill(0);
// Disable backprop when in generative mode
if (!Owner.Generate.IsIncomingRised())
{
// Set locations for mean deltas
CUdeviceptr meanDeltas = previousLayer.Delta.GetDevicePtr(Owner, 0);
// Set locations for sigma deltas
CUdeviceptr sigmaDeltas = previousLayer.Delta.GetDevicePtr(Owner, previousLayer.Delta.Count / 2);
// Determine input to previous layer
CUdeviceptr prevInputPtr = MyAbstractLayer.DetermineInput(previousLayer);
// set locations for sigmas (prev layer or constant
CUdeviceptr sigmas;
if (Owner.UseSigmaConstant)
{
sigmas = Owner.SigmaConstants.GetDevicePtr(Owner);
}
else
{
sigmas = Owner.Input.GetDevicePtr(Owner, Owner.Input.Count / 2);
}
m_samplingDeltaKernel.Run(
Convert.ToInt32(Owner.UseSigmaConstant),
(int)previousLayer.ActivationFunction,
prevInputPtr,
sigmas,
meanDeltas,
sigmaDeltas,
Owner.Delta,
Owner.RandomNormal,
Owner.Neurons
);
// Regularization needs weights to compute gradients
if (Regularize && previousLayer != null && previousLayer is MyAbstractWeightLayer)
{
MyAbstractWeightLayer previousWeightLayer = previousLayer as MyAbstractWeightLayer;
// Try to regularize loss: mean^2 + sigma^2 - log(sigma^2)
// In other words regularize means to 0 and sigmas to 1
int weightCount = previousWeightLayer.Weights.Count;
m_regularizationDeltaKernel.SetConstantVariable <float>("RegularizationCoefficient", RegularizationCoefficient);
m_regularizationDeltaKernel.SetupExecution(weightCount);
m_regularizationDeltaKernel.Run(
Convert.ToInt32(Owner.UseSigmaConstant),
(int)previousLayer.ActivationFunction,
prevInputPtr,
previousLayer.Input,
previousWeightLayer.Weights,
previousLayer.Output.Count,
meanDeltas,
sigmaDeltas
);
}
}
}
}
public override void Execute()
{
if (SimulationStep == 0)
{
return;
}
switch (Owner.LearningTasks)
{
case MyLSTMLayer.LearningTasksType.RTRL:
{
// propagate delta to output gates
m_deltaKernel.Run(
Owner.CellStateErrors,
Owner.OutputGateDeltas,
Owner.CellStates,
Owner.OutputGateActivations,
Owner.OutputGateActivationDerivatives,
Owner.Delta,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
break;
}
case MyLSTMLayer.LearningTasksType.BPTT:
{
// propagate delta to output gates
m_deltaKernel.Run(
Owner.Delta,
Owner.CellStates,
Owner.CellStates.GetTimeShiftedBlock(-1),
Owner.CellStateErrors,
Owner.CellStateErrors.GetTimeShiftedBlock(+1),
Owner.OutputGateDeltas,
Owner.ForgetGateDeltas,
Owner.ForgetGateDeltas.GetTimeShiftedBlock(+1),
Owner.InputGateDeltas,
Owner.InputGateDeltas.GetTimeShiftedBlock(+1),
Owner.CellInputDeltas,
Owner.CellInputActivations,
Owner.CellStateActivations,
Owner.OutputGateActivations,
Owner.ForgetGateActivations.GetTimeShiftedBlock(+1),
Owner.InputGateActivations,
Owner.CellInputActivationDerivatives,
Owner.CellStateActivationDerivatives,
Owner.OutputGateActivationDerivatives,
Owner.ForgetGateActivationDerivatives,
Owner.InputGateActivationDerivatives,
Owner.CellInputWeights,
Owner.OutputGateWeights,
Owner.ForgetGateWeights,
Owner.InputGateWeights,
Owner.Input.Count,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
m_gateGradientKernel.Run(
Owner.Input,
Owner.Output.GetTimeShiftedBlock(-1),
Owner.CellStates,
Owner.InputGateDeltas,
Owner.ForgetGateDeltas,
Owner.OutputGateDeltas,
Owner.OutputGateWeightGradient,
Owner.InputGateWeightGradient,
Owner.ForgetGateWeightGradient,
Owner.Input.Count,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
m_cellInputGradientKernel.Run(
Owner.Input,
Owner.Output.GetTimeShiftedBlock(-1),
Owner.CellInputDeltas,
Owner.CellInputWeightGradient,
Owner.Input.Count,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
break;
}
}
MyNode node = Owner.GetInput(0).Owner;
if (node is MyAbstractLayer)
{
MyAbstractLayer previousLayer = node as MyAbstractLayer;
CUdeviceptr prevInputPtr = nullCUdeviceptr;
// reset delta
if (Owner.ParentNetwork.TimeStep == 0)
{
previousLayer.Delta.Fill(0);
}
// determine input to previous layer
prevInputPtr = MyAbstractLayer.DetermineInput(previousLayer);
switch (Owner.LearningTasks)
{
case MyLSTMLayer.LearningTasksType.RTRL:
{
// propagate delta to previous layer
m_deltaBackKernel.SetupExecution(previousLayer.Neurons);
m_deltaBackKernel.Run(
(int)previousLayer.ActivationFunction,
prevInputPtr,
previousLayer.Delta,
Owner.CellStateErrors,
Owner.PreviousCellStates,
Owner.InputGateActivations,
Owner.CellInputActivationDerivatives,
Owner.InputGateActivationDerivatives,
Owner.ForgetGateActivationDerivatives,
Owner.CellInputWeights,
Owner.InputGateWeights,
Owner.ForgetGateWeights,
Owner.OutputGateWeights,
Owner.OutputGateDeltas,
previousLayer.Neurons,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
break;
}
case MyLSTMLayer.LearningTasksType.BPTT:
{
// propagate delta to previous layer
m_deltaBackKernel.SetupExecution(previousLayer.Neurons);
m_deltaBackKernel.Run(
(int)previousLayer.ActivationFunction,
prevInputPtr,
previousLayer.Delta,
Owner.CellInputDeltas,
Owner.OutputGateDeltas,
Owner.ForgetGateDeltas,
Owner.InputGateDeltas,
Owner.CellInputWeights,
Owner.InputGateWeights,
Owner.ForgetGateWeights,
Owner.OutputGateWeights,
previousLayer.Neurons,
Owner.CellStates.Count,
Owner.CellsPerBlock
);
break;
}
}
}
}