public void ComputeForwardRTLR(IDisposable state, Marshaled<DeviceArrayFactory[]> inputsM, Marshaled<IDeviceArray2[]> weightsM, IDeviceArray biases, IDeviceArray outputs, IDeviceArray netValueDerivates, ActivationFunction function, float alpha)
{
var inputs = inputsM.Instance();
var weights = weightsM.Instance();
Debug.Assert(inputs.Length != 0 && inputs.Length == weights.Length);
var mOutputs = outputs.ToManaged();
var mNVDerivs = netValueDerivates.ToManaged();
var mBiases = (ManagedArray)biases;
fixed (float* pOutputs = mOutputs.InternalArray, pBiases = mBiases.InternalArray, pNVDerivs = mNVDerivs.InternalArray)
{
var outputsPtr = mOutputs.ToPtr(pOutputs);
var biasesPtr = mBiases.ToPtr(pBiases);
var nvDerivsPtr = mNVDerivs.ToPtr(pNVDerivs);
if (function == ActivationFunction.Sigmoid)
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
float sum = biasesPtr[oIdx];
for (int lIdx = 0; lIdx < inputs.Length; lIdx++)
{
var inputsMA = (inputs[lIdx]()).ToManaged();
var weightsMA = (ManagedArray2)weights[lIdx];
Debug.Assert(inputsMA.Size != 0 && inputsMA.Size == weightsMA.Size1);
Debug.Assert(outputs.Size == weightsMA.Size2);
fixed (float* pInputs = inputsMA.InternalArray, pWeights = weightsMA.InternalArray)
{
sum += ComputeForward_Sum(inputsMA.ToPtr(pInputs), weightsMA.ToPtr2(pWeights), oIdx);
}
}
outputsPtr[oIdx] = Sigmoid(sum, alpha);
nvDerivsPtr[oIdx] = SigmoidD(sum, alpha);
}
}
else
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
float sum = biasesPtr[oIdx];
for (int lIdx = 0; lIdx < inputs.Length; lIdx++)
{
var inputsMA = (inputs[lIdx]()).ToManaged();
var weightsMA = (ManagedArray2)weights[lIdx];
Debug.Assert(inputsMA.Size != 0 && inputsMA.Size == weightsMA.Size1);
Debug.Assert(outputs.Size == weightsMA.Size2);
fixed (float* pInputs = inputsMA.InternalArray, pWeights = weightsMA.InternalArray)
{
sum += ComputeForward_Sum(inputsMA.ToPtr(pInputs), weightsMA.ToPtr2(pWeights), oIdx);
}
}
outputsPtr[oIdx] = Math.Min(Math.Max(sum * alpha, -alpha), alpha);
nvDerivsPtr[oIdx] = alpha;
}
}
}
}
public void CalculateGlobalError(IDisposable state, IDeviceArray desiredOutputsA, IDeviceArray actualOutputsA, IDeviceArray errorValueA, IDeviceArray errorSumValueA)
{
var actualOutputs = actualOutputsA.ToManaged();
var desiredOutputs = desiredOutputsA.ToManaged();
var errorValue = errorValueA.ToManaged();
var errorSumValue = errorSumValueA != null ? errorSumValueA.ToManaged() : null;
fixed (float* pActualOutputs = actualOutputs.InternalArray,
pDesiredOutputs = desiredOutputs.InternalArray,
pErrorValue = errorValue.InternalArray,
pErrorSumValue = errorSumValue != null ? errorSumValue.InternalArray : null)
{
var actualOutputsPtr = actualOutputs.ToPtr(pActualOutputs);
var desiredOutputsPtr = desiredOutputs.ToPtr(pDesiredOutputs);
var errorValuePtr = errorValue.ToPtr(pErrorValue);
float cMse = 0.0f;
for (int x = 0; x < actualOutputs.Size; x++)
{
float error = (desiredOutputsPtr[x] - actualOutputsPtr[x]) * 0.5f;
cMse += error * error;
}
errorValuePtr[0] = cMse / (float)actualOutputs.Size;
if (pErrorSumValue != null)
{
var errorSumValuePtr = errorSumValue.ToPtr(pErrorSumValue);
errorSumValuePtr[0] += errorValuePtr[0];
}
}
}
public void ComputeGradientsRTLR2(IDisposable state, Marshaled<RTLRLayerInfo[][]> inputLayerInfosM, Marshaled<IDeviceArray[]> netValueDerivatesM, Marshaled<RTLRComputationData2> dataM, IDeviceArray2 pValuesOfWeightsA, IDeviceArray outputsA, IDeviceArray desiredOutputsA, SequenceMarker seqMark)
{
var data = dataM.Instance();
var inputLayerInfos = inputLayerInfosM.Instance();
var netValueDerivates = netValueDerivatesM.Instance();
var pValuesOfWeights = pValuesOfWeightsA.ToManaged2();
var outputs = outputsA != null ? outputsA.ToManaged() : null;
var desiredOutputs = desiredOutputsA != null ? desiredOutputsA.ToManaged() : null;
var inputs = data.Inputs != null ? data.Inputs().ToManaged() : null;
fixed (float* pOutputs = outputs != null ? outputs.InternalArray : null,
pDesiredOutputs = desiredOutputs != null ? desiredOutputs.InternalArray : null,
pPValuesOfWeights = pValuesOfWeights.InternalArray,
pInputs = inputs != null ? inputs.InternalArray : null)
{
ManagedArrayPtr? outputsPtr = pOutputs != null ? outputs.ToPtr(pOutputs) : default(ManagedArrayPtr?);
ManagedArrayPtr? desiredOutputsPtr = pDesiredOutputs != null ? desiredOutputs.ToPtr(pDesiredOutputs) : default(ManagedArrayPtr?);
ManagedArrayPtr? inputsPtr = pInputs != null ? inputs.ToPtr(pInputs) : default(ManagedArrayPtr?);
int inputsSize = inputs == null ? 1 : inputs.Size;
for (int ijValueIndex = 0; ijValueIndex < pValuesOfWeights.Size1; ijValueIndex++) // group Id
{
float gradient = 0.0f;
int iValueIndex = ijValueIndex / inputsSize;
int jValueIndex = ijValueIndex % inputsSize;
float inputValue = inputsPtr.HasValue ? inputsPtr.Value[jValueIndex] : 1.0f;
for (int kLayerIndex = 0; kLayerIndex < data.ULayersCount; kLayerIndex++)
{
int kLayerSize = netValueDerivates[kLayerIndex].Size;
for (int kValueIndex = 0; kValueIndex < kLayerSize; kValueIndex++)
{
var layerNetValueDerivates = netValueDerivates[kLayerIndex].ToManaged();
int outputLayerIndex = layerNetValueDerivates.Size - 1;
bool computeGradient = kLayerIndex == outputLayerIndex && outputs != null && desiredOutputs != null;
var p_i_j_k_Ptr = GetPValuesPtr(pValuesOfWeights, pPValuesOfWeights, ijValueIndex, data, kLayerIndex);
float sum = 0.0f;
var upperInfos_k = inputLayerInfos[kLayerIndex];
foreach (var lLayerInfo in upperInfos_k)
{
if (lLayerInfo.IsElementOfU)
{
Debug.Assert(lLayerInfo.Weights != null);
int lLayerIndex = lLayerInfo.Index;
var p_i_j_l_Ptr = GetPValuesPtr(pValuesOfWeights, pPValuesOfWeights, ijValueIndex, data, lLayerIndex);
var weights = lLayerInfo.Weights.ToManaged2();
fixed (float* pWeights = weights.InternalArray)
{
var weightsPtr = weights.ToPtr2(pWeights);
for (int lValueIndex = 0; lValueIndex < lLayerInfo.Size; lValueIndex++)
{
sum += weightsPtr[lValueIndex, kValueIndex] * p_i_j_l_Ptr[lValueIndex];
}
}
}
}
if (data.ILayerIndex == kLayerIndex && iValueIndex == kValueIndex) sum += inputValue;
fixed (float* pLayerNetValueDerivates = layerNetValueDerivates.InternalArray)
{
p_i_j_k_Ptr[kValueIndex] = layerNetValueDerivates.ToPtr(pLayerNetValueDerivates)[kValueIndex] * sum;
}
if (computeGradient)
{
Debug.Assert(outputsPtr.HasValue && desiredOutputsPtr.HasValue);
gradient += (desiredOutputsPtr.Value[kValueIndex] - outputsPtr.Value[kValueIndex]) * p_i_j_k_Ptr[kValueIndex];
}
}
}
SetGradientsRTLR(data, ijValueIndex, gradient);
}
}
}
public void ComputeErrors(IDisposable state, IDeviceArray outputs, IDeviceArray errors, Marshaled<IDeviceArray2[]> lowerWeightsM, Marshaled<IDeviceArray[]> lowerErrorsM, ActivationFunction function, float alpha)
{
var lowerWeights = lowerWeightsM.Instance();
var lowerErrors = lowerErrorsM.Instance();
var mOutputs = outputs.ToManaged();
var mErrors = (ManagedArray)errors;
Debug.Assert(lowerWeights.Length != 0 && lowerWeights.Length == lowerErrors.Length);
fixed (float* pOutputs = mOutputs.InternalArray, pErrors = mErrors.InternalArray)
{
var outputsPtr = mOutputs.ToPtr(pOutputs);
var errorsPtr = mErrors.ToPtr(pErrors);
if (function == ActivationFunction.Sigmoid)
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
float sum = 0.0f;
for (int lIdx = 0; lIdx < lowerErrors.Length; lIdx++)
{
var lowerWeightsMA = (ManagedArray2)lowerWeights[lIdx];
var lowerErrorsMA = (ManagedArray)lowerErrors[lIdx];
Debug.Assert(lowerWeightsMA.Size2 == lowerErrorsMA.Size);
Debug.Assert(lowerWeightsMA.Size1 == outputs.Size);
fixed (float* pLowerWeights = lowerWeightsMA.InternalArray, pLowerErrors = lowerErrorsMA.InternalArray)
{
sum += ComputeErrors_LowerErrorSum(lowerErrorsMA.ToPtr(pLowerErrors), lowerWeightsMA.ToPtr2(pLowerWeights), oIdx);
}
}
errorsPtr[oIdx] = sum * SigmoidD(outputsPtr[oIdx], alpha);
}
}
else
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
float sum = 0.0f;
for (int lIdx = 0; lIdx < lowerErrors.Length; lIdx++)
{
var lowerWeightsMA = (ManagedArray2)lowerWeights[lIdx];
var lowerErrorsMA = (ManagedArray)lowerErrors[lIdx];
Debug.Assert(lowerWeightsMA.Size2 == lowerErrorsMA.Size);
Debug.Assert(lowerWeightsMA.Size1 == outputs.Size);
fixed (float* plw = lowerWeightsMA.InternalArray, ple = lowerWeightsMA.InternalArray)
{
sum += ComputeErrors_LowerErrorSum(lowerErrorsMA.ToPtr(ple), lowerWeightsMA.ToPtr2(plw), oIdx);
}
}
errorsPtr[oIdx] = sum * alpha;
}
}
}
}
unsafe public void ComputeGradientsRTLR(IDisposable state, Marshaled<RTLRLayerInfo[][]> inputLayerInfosM, Marshaled<IDeviceArray[]> netValueDerivatesM, Marshaled<RTLRComputationData> dataM, Marshaled<IDeviceArray[]> valueRelatedPBuffsM, IDeviceArray outputsA, IDeviceArray desiredOutputsA, SequenceMarker seqMark)
{
var data = dataM.Instance();
var inputLayerInfos = inputLayerInfosM.Instance();
var netValueDerivates = netValueDerivatesM.Instance();
var outputs = outputsA != null ? outputsA.ToManaged() : null;
var desiredOutputs = desiredOutputsA != null ? desiredOutputsA.ToManaged() : null;
var inputs = data.Inputs != null ? data.Inputs().ToManaged() : null;
var valueRelatedPBuffs = valueRelatedPBuffsM.Instance();
float gradient = 0.0f;
fixed (float* pOutputs = outputs != null ? outputs.InternalArray : null,
pDesiredOutputs = desiredOutputs != null ? desiredOutputs.InternalArray : null)
{
float inputValue = inputs != null ? inputs.InternalArray[data.JValueIndex] : 1.0f;
int outputLayerIndex = valueRelatedPBuffs.Length - 1;
for (int kLayerIndex = 0; kLayerIndex < valueRelatedPBuffs.Length; kLayerIndex++)
{
var layerNetValueDerivates = netValueDerivates[kLayerIndex].ToManaged();
var p_i_j_k_Values = valueRelatedPBuffs[kLayerIndex].ToManaged();
bool computeGradient = kLayerIndex == outputLayerIndex && pOutputs != null && pDesiredOutputs != null;
fixed (float* pLayerNetValueDerivates = layerNetValueDerivates.InternalArray, pp_i_j_k_Values = p_i_j_k_Values.InternalArray)
{
var layerNetValueDerivatesPtr = layerNetValueDerivates.ToPtr(pLayerNetValueDerivates);
var p_i_j_k_ValuesPtr = p_i_j_k_Values.ToPtr(pp_i_j_k_Values);
for (int kValueIndex = 0; kValueIndex < p_i_j_k_Values.Size; kValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
float sum = 0.0f;
var upperInfos_k = inputLayerInfos[kLayerIndex];
foreach (var upperInputLayerInfo in upperInfos_k)
{
if (upperInputLayerInfo.IsElementOfU)
{
Debug.Assert(upperInputLayerInfo.Weights != null);
int lLayerIndex = upperInputLayerInfo.Index;
var p_i_j_l_Values = valueRelatedPBuffs[lLayerIndex].ToManaged();
var weights = upperInputLayerInfo.Weights.ToManaged2();
Debug.Assert(p_i_j_l_Values.Size == weights.Size1);
Debug.Assert(weights.Size2 == p_i_j_k_Values.Size);
fixed (float* pp_i_j_l = p_i_j_l_Values.InternalArray, pWeights = weights.InternalArray)
{
var p_i_j_l_ValuesPtr = p_i_j_l_Values.ToPtr(pp_i_j_l);
var weightsPtr = weights.ToPtr2(pWeights);
for (int lValueIndex = 0; lValueIndex < p_i_j_l_Values.Size; lValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
// l: lLayerIndex, lValueIndex
sum += weightsPtr[lValueIndex, kValueIndex] * p_i_j_l_ValuesPtr[lValueIndex];
}
}
}
}
if (data.ILayerIndex == kLayerIndex && data.IValueIndex == kValueIndex) sum += inputValue;
p_i_j_k_ValuesPtr[kValueIndex] = layerNetValueDerivatesPtr[kValueIndex] * sum;
if (computeGradient)
{
var outputsPtr = outputs.ToPtr(pOutputs);
var desiredOutputsPtr = desiredOutputs.ToPtr(pDesiredOutputs);
gradient += (desiredOutputsPtr[kValueIndex] - outputsPtr[kValueIndex]) * p_i_j_k_ValuesPtr[kValueIndex];
}
}
}
}
}
if (gradient != 0.0f) SetGradientsRTLR(data, gradient);
}
public unsafe void ComputeErrors(IDisposable state, IDeviceArray outputs, IDeviceArray errors, IDeviceArray desiredOutputs, ActivationFunction function, float alpha)
{
var mOutputs = outputs.ToManaged();
var mErrors = (ManagedArray)errors;
var mDesiredOutputs = desiredOutputs.ToManaged();
fixed (float* pOutputs = mOutputs.InternalArray, pErrors = mErrors.InternalArray, pDesiredOutputs = mDesiredOutputs.InternalArray)
{
var outputsPtr = mOutputs.ToPtr(pOutputs);
var errorsPtr = mErrors.ToPtr(pErrors);
var desiredOutputsPtr = mDesiredOutputs.ToPtr(pDesiredOutputs);
if (function == ActivationFunction.Sigmoid)
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
errorsPtr[oIdx] = (desiredOutputsPtr[oIdx] - outputsPtr[oIdx]) * SigmoidD(outputsPtr[oIdx], alpha);
}
}
else
{
for (int oIdx = 0; oIdx < outputs.Size; oIdx++)
{
errorsPtr[oIdx] = (desiredOutputsPtr[oIdx] - outputsPtr[oIdx]) * alpha;
}
}
}
}
public unsafe void Copy(IDeviceArray from, int fromIndex, IDeviceArray to, int toIndex, int size)
{
Array.Copy(from.ToManaged().InternalArray, fromIndex, to.ToManaged().InternalArray, toIndex, size);
}
override public void Zero(IDeviceArray deviceArray)
{
var a = deviceArray.ToManaged().InternalArray;
ZeroMemory(a);
}
public unsafe void Copy(IDeviceArray from, int fromIndex, IDeviceArray to, int toIndex, int size)
{
var fm = from.ToManaged();
var tm = to.ToManaged();
Array.Copy(fm.InternalArray, fm.BeginIndex + fromIndex, tm.InternalArray, tm.BeginIndex + toIndex, size);
}
unsafe public void ComputeGradientsRTLR(Marshaled<RTLRComputationData> dataM, Marshaled<IDeviceArray[]> valueRelatedPBuffsM, IDeviceArray outputsA, IDeviceArray desiredOutputsA)
{
var data = dataM.Instance();
var outputs = outputsA != null ? outputsA.ToManaged() : null;
var desiredOutputs = desiredOutputsA != null ? desiredOutputsA.ToManaged() : null;
var inputs = data.Inputs != null ? data.Inputs().ToManaged() : null;
var valueRelatedPBuffs = valueRelatedPBuffsM.Instance();
float gradient = 0.0f;
fixed (float* pOutputs = outputs != null ? outputs.InternalArray : null,
pDesiredOutputs = desiredOutputs != null ? desiredOutputs.InternalArray : null,
pInputs = inputs != null ? inputs.InternalArray : null)
{
int outputLayerIndex = valueRelatedPBuffs.Length - 1;
for (int kLayerIndex = 0; kLayerIndex < valueRelatedPBuffs.Length; kLayerIndex++)
{
var layerNetValueDerivates = data.NetValueDerivates[kLayerIndex].ToManaged();
var p_i_j_k_Values = valueRelatedPBuffs[kLayerIndex].ToManaged();
bool computeGradient = kLayerIndex == outputLayerIndex && pOutputs != null && pDesiredOutputs != null;
fixed (float* pLayerNetValueDerivates = layerNetValueDerivates.InternalArray, pp_i_j_k_Values = p_i_j_k_Values.InternalArray)
{
var layerNetValueDerivatesPtr = layerNetValueDerivates.ToPtr(pLayerNetValueDerivates);
var p_i_j_k_ValuesPtr = p_i_j_k_Values.ToPtr(pp_i_j_k_Values);
for (int kValueIndex = 0; kValueIndex < p_i_j_k_Values.Size; kValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
float netDeriv_k = layerNetValueDerivatesPtr[kValueIndex];
float sum = 0.0f;
var upperInfos_k = data.InputLayerInfos[kLayerIndex];
foreach (var upperNonInputLayerInfo in upperInfos_k)
{
Debug.Assert(upperNonInputLayerInfo.Weights != null);
int lLayerIndex = upperNonInputLayerInfo.Index;
var p_i_j_l_Values = valueRelatedPBuffs[lLayerIndex].ToManaged();
var weights = upperNonInputLayerInfo.Weights.ToManaged2();
Debug.Assert(p_i_j_l_Values.Size == weights.Size1);
Debug.Assert(weights.Size2 == p_i_j_k_Values.Size);
fixed (float* pp_i_j_l = p_i_j_l_Values.InternalArray, pWeights = weights.InternalArray)
{
var p_i_j_l_ValuesPtr = p_i_j_l_Values.ToPtr(pp_i_j_l);
var weightsPtr = weights.ToPtr(pWeights);
for (int lValueIndex = 0; lValueIndex < p_i_j_l_Values.Size; lValueIndex++)
{
// i: iLayerIndex, iValueIndex
// j: jLayerIndex, jValueIndex
// k: kLayerIndex, kValueIndex
// l: lLayerIndex, lValueIndex
sum += weightsPtr[lValueIndex, kValueIndex] * p_i_j_l_ValuesPtr[lValueIndex];
}
}
}
if (data.ILayerIndex == kLayerIndex && data.IValueIndex == kValueIndex)
{
if (inputs != null)
{
// Weighted connection
var inputsPtr = inputs.ToPtr(pInputs);
sum += inputsPtr[data.JValueIndex];
}
else //if (kValueIndex == 0)
{
Debug.Assert(data.JValueIndex == -1);
// Biased connection
sum += 1.0f;
}
}
p_i_j_k_ValuesPtr[kValueIndex] = netDeriv_k * sum;
if (computeGradient)
{
var outputsPtr = outputs.ToPtr(pOutputs);
var desiredOutputsPtr = desiredOutputs.ToPtr(pDesiredOutputs);
gradient += (desiredOutputsPtr[kValueIndex] - outputsPtr[kValueIndex]) * p_i_j_k_ValuesPtr[kValueIndex];
}
//p_i_j_k_ValuesPtr[kValueIndex] = netDeriv_k * sum;
}
}
}
}
if (gradient != 0.0f) SetGradientsRTLR(data, gradient);
}