public void ComputeGradientsRTLR2(IDisposable state, Marshaled<RTLRLayerInfo[][]> inputLayerInfosM, Marshaled<IDeviceArray[]> netValueDerivatesM, Marshaled<RTLRComputationData2> dataM, IDeviceArray2 pValuesOfWeightsA, IDeviceArray outputsA, IDeviceArray desiredOutputsA, SequenceMarker seqMark)
{
var data = dataM.Instance();
var inputLayerInfos = inputLayerInfosM.Instance();
var netValueDerivates = netValueDerivatesM.Instance();
var pValuesOfWeights = pValuesOfWeightsA.ToManaged2();
var outputs = outputsA != null ? outputsA.ToManaged() : null;
var desiredOutputs = desiredOutputsA != null ? desiredOutputsA.ToManaged() : null;
var inputs = data.Inputs != null ? data.Inputs().ToManaged() : null;
fixed (float* pOutputs = outputs != null ? outputs.InternalArray : null,
pDesiredOutputs = desiredOutputs != null ? desiredOutputs.InternalArray : null,
pPValuesOfWeights = pValuesOfWeights.InternalArray,
pInputs = inputs != null ? inputs.InternalArray : null)
{
ManagedArrayPtr? outputsPtr = pOutputs != null ? outputs.ToPtr(pOutputs) : default(ManagedArrayPtr?);
ManagedArrayPtr? desiredOutputsPtr = pDesiredOutputs != null ? desiredOutputs.ToPtr(pDesiredOutputs) : default(ManagedArrayPtr?);
ManagedArrayPtr? inputsPtr = pInputs != null ? inputs.ToPtr(pInputs) : default(ManagedArrayPtr?);
int inputsSize = inputs == null ? 1 : inputs.Size;
for (int ijValueIndex = 0; ijValueIndex < pValuesOfWeights.Size1; ijValueIndex++) // group Id
{
float gradient = 0.0f;
int iValueIndex = ijValueIndex / inputsSize;
int jValueIndex = ijValueIndex % inputsSize;
float inputValue = inputsPtr.HasValue ? inputsPtr.Value[jValueIndex] : 1.0f;
for (int kLayerIndex = 0; kLayerIndex < data.ULayersCount; kLayerIndex++)
{
int kLayerSize = netValueDerivates[kLayerIndex].Size;
for (int kValueIndex = 0; kValueIndex < kLayerSize; kValueIndex++)
{
var layerNetValueDerivates = netValueDerivates[kLayerIndex].ToManaged();
int outputLayerIndex = layerNetValueDerivates.Size - 1;
bool computeGradient = kLayerIndex == outputLayerIndex && outputs != null && desiredOutputs != null;
var p_i_j_k_Ptr = GetPValuesPtr(pValuesOfWeights, pPValuesOfWeights, ijValueIndex, data, kLayerIndex);
float sum = 0.0f;
var upperInfos_k = inputLayerInfos[kLayerIndex];
foreach (var lLayerInfo in upperInfos_k)
{
if (lLayerInfo.IsElementOfU)
{
Debug.Assert(lLayerInfo.Weights != null);
int lLayerIndex = lLayerInfo.Index;
var p_i_j_l_Ptr = GetPValuesPtr(pValuesOfWeights, pPValuesOfWeights, ijValueIndex, data, lLayerIndex);
var weights = lLayerInfo.Weights.ToManaged2();
fixed (float* pWeights = weights.InternalArray)
{
var weightsPtr = weights.ToPtr2(pWeights);
for (int lValueIndex = 0; lValueIndex < lLayerInfo.Size; lValueIndex++)
{
sum += weightsPtr[lValueIndex, kValueIndex] * p_i_j_l_Ptr[lValueIndex];
}
}
}
}
if (data.ILayerIndex == kLayerIndex && iValueIndex == kValueIndex) sum += inputValue;
fixed (float* pLayerNetValueDerivates = layerNetValueDerivates.InternalArray)
{
p_i_j_k_Ptr[kValueIndex] = layerNetValueDerivates.ToPtr(pLayerNetValueDerivates)[kValueIndex] * sum;
}
if (computeGradient)
{
Debug.Assert(outputsPtr.HasValue && desiredOutputsPtr.HasValue);
gradient += (desiredOutputsPtr.Value[kValueIndex] - outputsPtr.Value[kValueIndex]) * p_i_j_k_Ptr[kValueIndex];
}
}
}
SetGradientsRTLR(data, ijValueIndex, gradient);
}
}
}
unsafe public void ComputeGradientsRTLR(IDisposable state, Marshaled<RTLRLayerInfo[][]> inputLayerInfosM, Marshaled<IDeviceArray[]> netValueDerivatesM, Marshaled<RTLRComputationData> dataM, Marshaled<IDeviceArray[]> valueRelatedPBuffsM, IDeviceArray outputsA, IDeviceArray desiredOutputsA, SequenceMarker seqMark)
{
var data = dataM.Instance();
var inputLayerInfos = inputLayerInfosM.Instance();
var netValueDerivates = netValueDerivatesM.Instance();
var outputs = outputsA != null ? outputsA.ToManaged() : null;
var desiredOutputs = desiredOutputsA != null ? desiredOutputsA.ToManaged() : null;
var inputs = data.Inputs != null ? data.Inputs().ToManaged() : null;
var valueRelatedPBuffs = valueRelatedPBuffsM.Instance();
float gradient = 0.0f;
fixed (float* pOutputs = outputs != null ? outputs.InternalArray : null,
pDesiredOutputs = desiredOutputs != null ? desiredOutputs.InternalArray : null)
{
float inputValue = inputs != null ? inputs.InternalArray[data.JValueIndex] : 1.0f;
int outputLayerIndex = valueRelatedPBuffs.Length - 1;
for (int kLayerIndex = 0; kLayerIndex < valueRelatedPBuffs.Length; kLayerIndex++)
{
var layerNetValueDerivates = netValueDerivates[kLayerIndex].ToManaged();
var p_i_j_k_Values = valueRelatedPBuffs[kLayerIndex].ToManaged();
bool computeGradient = kLayerIndex == outputLayerIndex && pOutputs != null && pDesiredOutputs != null;
fixed (float* pLayerNetValueDerivates = layerNetValueDerivates.InternalArray, pp_i_j_k_Values = p_i_j_k_Values.InternalArray)
{
var layerNetValueDerivatesPtr = layerNetValueDerivates.ToPtr(pLayerNetValueDerivates);
var p_i_j_k_ValuesPtr = p_i_j_k_Values.ToPtr(pp_i_j_k_Values);
for (int kValueIndex = 0; kValueIndex < p_i_j_k_Values.Size; kValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
float sum = 0.0f;
var upperInfos_k = inputLayerInfos[kLayerIndex];
foreach (var upperInputLayerInfo in upperInfos_k)
{
if (upperInputLayerInfo.IsElementOfU)
{
Debug.Assert(upperInputLayerInfo.Weights != null);
int lLayerIndex = upperInputLayerInfo.Index;
var p_i_j_l_Values = valueRelatedPBuffs[lLayerIndex].ToManaged();
var weights = upperInputLayerInfo.Weights.ToManaged2();
Debug.Assert(p_i_j_l_Values.Size == weights.Size1);
Debug.Assert(weights.Size2 == p_i_j_k_Values.Size);
fixed (float* pp_i_j_l = p_i_j_l_Values.InternalArray, pWeights = weights.InternalArray)
{
var p_i_j_l_ValuesPtr = p_i_j_l_Values.ToPtr(pp_i_j_l);
var weightsPtr = weights.ToPtr2(pWeights);
for (int lValueIndex = 0; lValueIndex < p_i_j_l_Values.Size; lValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
// l: lLayerIndex, lValueIndex
sum += weightsPtr[lValueIndex, kValueIndex] * p_i_j_l_ValuesPtr[lValueIndex];
}
}
}
}
if (data.ILayerIndex == kLayerIndex && data.IValueIndex == kValueIndex) sum += inputValue;
p_i_j_k_ValuesPtr[kValueIndex] = layerNetValueDerivatesPtr[kValueIndex] * sum;
if (computeGradient)
{
var outputsPtr = outputs.ToPtr(pOutputs);
var desiredOutputsPtr = desiredOutputs.ToPtr(pDesiredOutputs);
gradient += (desiredOutputsPtr[kValueIndex] - outputsPtr[kValueIndex]) * p_i_j_k_ValuesPtr[kValueIndex];
}
}
}
}
}
if (gradient != 0.0f) SetGradientsRTLR(data, gradient);
}
private List <SequenceMarker> SolveSequence(Random random, List <Point> markers, InterferencePattern.Direction[,] connections)
{
var directions = new List <Coord>(CardinalOffsets);
Shuffle(directions, random);
var results = new List <SequenceMarker>();
var startPosition = markers.First();
var currentPosition = startPosition;
var prevPosition = startPosition;
var prevDirection = InterferencePattern.Direction.None;
var stepsToNextMarker = 0;
SequenceMarker prevMarker = new SequenceMarker
{
SortOrder = 0,
Cell = currentPosition,
};
do
{
stepsToNextMarker++;
// Find a direction we can move in that isn't back to where we came from.
var currentConnections = connections[currentPosition.X, currentPosition.Y];
var direction = directions.First
(
direction => currentConnections.HasFlag(direction.Direction) &&
direction.OppositeDirection != prevDirection
);
prevPosition = currentPosition;
currentPosition.Offset(direction.X, direction.Y);
prevDirection = direction.Direction;
if (prevMarker.SubsequentCellDirection == InterferencePattern.Direction.None)
{
prevMarker.SubsequentCell = currentPosition;
prevMarker.SubsequentCellDirection = prevDirection;
}
var markerId = markers.IndexOf(currentPosition);
if (markerId != -1)
{
var test = markers[markerId];
// We've reached a new marker, so add it to the results.
prevMarker = new SequenceMarker
{
SortOrder = markerId,
Cell = currentPosition,
PreviousCell = prevPosition,
PreviousCellDirection = prevDirection,
};
results.Add(prevMarker);
// Update the previous marker with its # steps.
prevMarker.StepsToNextMarker = stepsToNextMarker;
stepsToNextMarker = 0;
}
} while (currentPosition != startPosition);
var firstStep = results.First();
firstStep.PreviousCell = prevPosition;
firstStep.PreviousCellDirection = prevDirection;
return(results);
}
private void ComputeGradients(int computationIndex, Marshaled<IDeviceArray[]> valueRelatedPBuffs, int iLayerIndex, int iValueIndex, int jLayerIndex, int jValueIndex, int ijValueIndex, IDeviceArray outputs, IDeviceArray desiredOutputs, SequenceMarker seqMark)
{
#if DEBUG
int outputLayerIndex = valueRelatedPBuffs.Instance().Length - 1;
Debug.Assert(outputLayerIndex == mlp.Layers.Count - 2);
Debug.Assert(outputLayerIndex == mlp.Layers[mlp.Layers.Count - 1].Index - 1);
#endif
if (codes.Count > computationIndex)
{
var code = codes[computationIndex];
if (code != null) code(outputs, desiredOutputs);
}
else
{
codes.EnsureSize(computationIndex + 1);
Action<IDeviceArray, IDeviceArray> code = null;
bool forBias = jValueIndex == -1;
var dataM = mlp.AsMarshaled(new RTLRComputationData());
var data = dataM.ManagedObject;
int iLayerIndexN = iLayerIndex + 1;
var iLayer = mlp.Layers[iLayerIndexN];
data.ILayerIndex = iLayerIndex;
data.IValueIndex = iValueIndex;
data.JLayerIndex = jLayerIndex;
data.JValueIndex = jValueIndex;
data.IJValueIndex = ijValueIndex;
if (forBias)
{
Debug.Assert(jLayerIndex == 0);
data.BiasGradients = mlp.GetBiasGradients(iLayerIndexN);
data.BiasGradientSums = mlp.GetBiasGradientSums(iLayerIndexN);
}
else
{
Debug.Assert(jLayerIndex > 0);
var inputLayerOfILayer = iLayer.Layer.GetInputLayer(jLayerIndex - 1);
var inputLayerOfILayerIndex = mlp.GetLayerIndex(inputLayerOfILayer);
var weightKey = Tuple.Create(inputLayerOfILayerIndex, iLayerIndexN);
data.Inputs = () => mlp.GetNetValues(inputLayerOfILayerIndex);
data.Gradients = mlp.GetGradients(weightKey);
data.GradientSums = mlp.GetGradientSums(weightKey);
}
Debug.Assert(!(data.BiasGradients == null && data.BiasGradientSums == null && data.Gradients == null && data.GradientSums == null));
var state = mlp.CreateComputationState();
code = (os, dos) => mlp.Adapter.ComputeActivation.ComputeGradientsRTLR(state, inputLayerInfos, netValueDerivates, dataM, valueRelatedPBuffs, os, dos, seqMark);
codes[computationIndex] = code;
code(outputs, desiredOutputs);
}
}
private void ComputeGradients(int iLayerIndex, int jLayerIndex, IDeviceArray2 pValuesOfWeights, IDeviceArray outputs, IDeviceArray desiredOutputs, int computationIndex, SequenceMarker seqMark)
{
// jLayerIndex: 0: Bias, 1..: Weights
if (codes.Count > computationIndex)
{
var code = codes[computationIndex];
if (code != null) code(outputs, desiredOutputs);
}
else
{
codes.EnsureSize(computationIndex + 1);
Action<IDeviceArray, IDeviceArray> code = null;
bool forBias = jLayerIndex == 0;
var dataM = mlp.AsMarshaled(new RTLRComputationData2());
var data = dataM.ManagedObject;
data.MaxULayerSize = maxULayerSize;
data.ULayersCount = uLayersCount;
int iLayerIndexN = iLayerIndex + 1;
var iLayer = mlp.Layers[iLayerIndexN];
data.ILayerIndex = iLayerIndex;
data.JLayerIndex = jLayerIndex;
if (forBias)
{
Debug.Assert(jLayerIndex == 0);
data.BiasGradients = mlp.GetBiasGradients(iLayerIndexN);
data.BiasGradientSums = mlp.GetBiasGradientSums(iLayerIndexN);
}
else
{
Debug.Assert(jLayerIndex > 0);
var inputLayerOfILayer = iLayer.Layer.GetInputLayer(jLayerIndex - 1);
var inputLayerOfILayerIndex = mlp.GetLayerIndex(inputLayerOfILayer);
var weightKey = Tuple.Create(inputLayerOfILayerIndex, iLayerIndexN);
data.Inputs = () => mlp.GetNetValues(inputLayerOfILayerIndex);
data.Gradients = mlp.GetGradients(weightKey);
data.GradientSums = mlp.GetGradientSums(weightKey);
}
Debug.Assert(!(data.BiasGradients == null && data.BiasGradientSums == null && data.Gradients == null && data.GradientSums == null));
var state = mlp.CreateComputationState();
code = (os, dos) => mlp.Adapter.ComputeActivation.ComputeGradientsRTLR2(state, inputLayerInfos, netValueDerivates, dataM, pValuesOfWeights, os, dos, seqMark);
codes[computationIndex] = code;
code(outputs, desiredOutputs);
}
}
