public static Tensor SRN(
this Session session,
Tensor x,
Tensor w,
Tensor u,
Tensor b,
int numberOfNeurons,
MatrixLayout matrixLayout)
{
const string ActionName = "srn";
// calculate gates = W * x + b
Tensor y = session.FullyConnected(x, w, b, matrixLayout);
int tt = y.Shape.GetAxis(Axis.B); // number of vectors in time sequence
float[] uw = u.Weights;
float[] yw = y.Weights;
// add hidden layer to the output tensor
// y += U * y(t-1) (product of hidden weight matrix and hidden vector)
Nonlinearity.ReLU(numberOfNeurons, yw, 0, yw, 0);
for (int t = 1, yi = numberOfNeurons; t < tt; t++, yi += numberOfNeurons)
{
Matrix.MxV(matrixLayout, numberOfNeurons, numberOfNeurons, uw, 0, false, yw, yi - numberOfNeurons, yw, yi, false);
// TODO: customize activation function
Nonlinearity.ReLU(numberOfNeurons, yw, yi, yw, yi);
}
if (session.CalculateGradients)
{
session.Push(
ActionName,
() =>
{
float[] duw = u.Gradient;
float[] dyw = y.Gradient;
for (int t = tt - 1, yi = t * numberOfNeurons; t > 0; t--, yi -= numberOfNeurons)
{
Nonlinearity.ReLUGradient(numberOfNeurons, dyw, yi, true, yw, yi, dyw, yi);
// dA += dy * x'
lock (u)
{
Matrix.VxV(matrixLayout, numberOfNeurons, numberOfNeurons, dyw, yi, yw, yi - numberOfNeurons, duw, 0, false);
}
// dx += A' * dy
Matrix.MxV(matrixLayout, numberOfNeurons, numberOfNeurons, uw, 0, true, dyw, yi, dyw, yi - numberOfNeurons, false);
}
Nonlinearity.ReLUGradient(numberOfNeurons, dyw, 0, true, yw, 0, dyw, 0);
});
}
return(y);
}
public static Dictionary <UnLinearityType, Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> > > GetStatisticsOfDiagramOfTransformation(GlobalUnit unit, int numbers)
{
Dictionary <UnLinearityType, Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> > > result =
new Dictionary <UnLinearityType, Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> > >(2);
Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> > dictIntegralUnLinearity =
new Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> >(4);
Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> > dictDiferentialUnLinearity =
new Dictionary <ModellingDataTypeWithStrategies, Dictionary <StatisticParametrType, double> >(4);
double[][] integral = new double[4][];
for (int i = 0; i < 4; i++)
{
integral[i] = new double[numbers];
}
double[][] differential = new double[4][];
for (int i = 0; i < 4; i++)
{
differential[i] = new double[numbers];
}
CharacteristoOfTransformationParametrs param =
new CharacteristoOfTransformationParametrs(int.MaxValue, 1,
new AnalogSignal(TypesOfSignalEnum.KxB, new LineParametrs(1.0, 0.0)));
for (int index = 0; index < numbers; index++)
{
PointPairList[] graphArrays = GetDiagramOfTransformationOfAllStrategies(unit);
// graphArrays.Length==4 без калібрування + 3 стратегії
for (int i = 0; i < graphArrays.Length; i++)
{
PointPairList integralUnLinearityCurve = Nonlinearity.CalculateIntegralUnLinearity(graphArrays[i]);
PointPairList diferentialUnLinearityCurve = Nonlinearity.CalculateDifferentialUnLinearity(graphArrays[i]);
CalculateStatisticsForOneParametr(integralUnLinearityCurve, ref integral[i][index]);
CalculateStatisticsForOneParametr(diferentialUnLinearityCurve, ref differential[i][index]);
}
}
dictIntegralUnLinearity.Add(ModellingDataTypeWithStrategies.Simple, MakeLowestDictionary(integral[0]));
dictIntegralUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithFirstStrategy, MakeLowestDictionary(integral[1]));
dictIntegralUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithSecondStrategy, MakeLowestDictionary(integral[2]));
dictIntegralUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithThirdStrategy, MakeLowestDictionary(integral[3]));
dictDiferentialUnLinearity.Add(ModellingDataTypeWithStrategies.Simple, MakeLowestDictionary(differential[0]));
dictDiferentialUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithFirstStrategy, MakeLowestDictionary(differential[1]));
dictDiferentialUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithSecondStrategy, MakeLowestDictionary(differential[2]));
dictDiferentialUnLinearity.Add(ModellingDataTypeWithStrategies.CalibratedWithThirdStrategy, MakeLowestDictionary(differential[3]));
result.Add(UnLinearityType.Integral, dictIntegralUnLinearity);
result.Add(UnLinearityType.Differential, dictDiferentialUnLinearity);
return(result);
}
public static Dictionary <UnLinearityType, Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> > > GetStatisticsOfDiagramOfTransformation(GlobalUnit unit, int numbers)
{
var result =
new Dictionary <UnLinearityType, Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> > >(2);
Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> > dictIntegralUnLinearity =
new Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> >(3);
Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> > dictDiferentialUnLinearity =
new Dictionary <ModellingDataType, Dictionary <StatisticParametrType, double> >(3);
Dictionary <int, int[]> regTrackWithoutCalibration;
Dictionary <int, int[]> regTrackWithCalibration;
var simpleIntegral = new double[numbers];
//0- tracking
//1- cletching
var calibratedIntegral = new double[2][];
calibratedIntegral[0] = new double[numbers];
calibratedIntegral[1] = new double[numbers];
var simpleDifferential = new double[numbers];
//0- tracking
//1- cletching
var calibratedDifferential = new double[2][];
calibratedDifferential[0] = new double[numbers];
calibratedDifferential[1] = new double[numbers];
CharacteristoOfTransformationParametrs param =
new CharacteristoOfTransformationParametrs(int.MaxValue, 1,
new AnalogSignal(TypesOfSignalEnum.KxB, new LineParametrs(1.0, 0.0)));
for (int index = 0; index < numbers; index++)
{
var graphArraysTracking = GetDiagramOfTransformation(unit, param, CalibrationType.Tracking, out regTrackWithoutCalibration, out regTrackWithCalibration);
var graphArraysCletching = GetDiagramOfTransformation(unit, param, CalibrationType.Cletching, out regTrackWithoutCalibration, out regTrackWithCalibration);
var res = new PointPairList [] { graphArraysTracking[1], graphArraysTracking[2], graphArraysCletching[2] };
GetValueForCurve(Nonlinearity.CalculateDifferentialUnLinearity(res[0]), ref simpleIntegral[index]);
GetValueForCurve(Nonlinearity.CalculateDifferentialUnLinearity(res[1]), ref calibratedDifferential[0][index]);
GetValueForCurve(Nonlinearity.CalculateDifferentialUnLinearity(res[2]), ref calibratedDifferential[1][index]);
GetValueForCurve(Nonlinearity.CalculateIntegralUnLinearity(res[0]), ref simpleDifferential[index]);
GetValueForCurve(Nonlinearity.CalculateIntegralUnLinearity(res[1]), ref calibratedIntegral[0][index]);
GetValueForCurve(Nonlinearity.CalculateIntegralUnLinearity(res[2]), ref calibratedIntegral[1][index]);
}
dictIntegralUnLinearity.Add(ModellingDataType.Simple, MakeLowestDictionary(simpleIntegral));
dictIntegralUnLinearity.Add(ModellingDataType.CalibratedTracking, MakeLowestDictionary(calibratedIntegral[0]));
dictIntegralUnLinearity.Add(ModellingDataType.CalibratedCletching, MakeLowestDictionary(calibratedIntegral[1]));
dictDiferentialUnLinearity.Add(ModellingDataType.Simple, MakeLowestDictionary(simpleDifferential));
dictDiferentialUnLinearity.Add(ModellingDataType.CalibratedTracking, MakeLowestDictionary(calibratedDifferential[0]));
dictDiferentialUnLinearity.Add(ModellingDataType.CalibratedCletching, MakeLowestDictionary(calibratedDifferential[1]));
result.Add(UnLinearityType.Integral, dictIntegralUnLinearity);
result.Add(UnLinearityType.Differential, dictDiferentialUnLinearity);
return(result);
}
public void ForwardBackwardTest1()
{
const int T = 2;
const int N = 3;
Session session = new Session();
SRNCell layer = new SRNCell(new Shape(new[] { -1, N }), RNNDirection.ForwardOnly, 2, MatrixLayout.RowMajor, null);
layer.W.Randomize(this.random);
layer.U.Randomize(this.random);
layer.B.Randomize(this.random);
////layer.W.Set(new float[] { 0.1f, 0.2f, -0.3f, 0.4f, 0.5f, 0.6f }); // 3x2 matrix
////layer.U.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f }); // 2x2 matrix
////layer.B.Set(new float[] { 0.1f, 0.2f }); // 2x1 vector
Tensor x = new Tensor(null, new[] { T, N });
x.Randomize(this.random);
////x.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f });
IList <Tensor> xs = new[] { x };
IList <Tensor> ys = layer.Forward(session, xs);
float[] bw = layer.B.Weights;
Tensor expected = new Tensor(null, new[] { 2, 2 });
expected.Weights[0] = Matrix.DotProduct(3, layer.W.Weights, 0, x.Weights, 0) + bw[0];
expected.Weights[1] = Matrix.DotProduct(3, layer.W.Weights, 3, x.Weights, 0) + bw[1];
Nonlinearity.ReLU(2, expected.Weights, 0, expected.Weights, 0);
expected.Weights[2] = Matrix.DotProduct(3, layer.W.Weights, 0, x.Weights, 3) + bw[0] + Matrix.DotProduct(2, layer.U.Weights, 0, expected.Weights, 0);
expected.Weights[3] = Matrix.DotProduct(3, layer.W.Weights, 3, x.Weights, 3) + bw[1] + Matrix.DotProduct(2, layer.U.Weights, 2, expected.Weights, 0);
Nonlinearity.ReLU(2, expected.Weights, 2, expected.Weights, 2);
Helpers.AreTensorsEqual(expected, ys[0]);
// unroll the graph
////session.GetGradient(ys[0]).Randomize(this.random);
ys[0].SetGradient(new float[] { 0.1f, 0.2f, 0.3f, 0.4f });
session.Unroll();
////float[] dy = session.GetGradient(ys[0]).Weights.ToArray();
float[] dy = new float[] { 0.1f, 0.2f, 0.3f, 0.4f };
float[] expectedWG = new float[layer.W.Length];
float[] expectedUG = new float[layer.U.Length];
float[] expectedBG = new float[layer.B.Length];
float[] expectedDx = new float[x.Length];
for (int oi = 2, ii = 3; oi >= 0; oi -= 2, ii -= 3)
{
Nonlinearity.ReLUGradient(2, dy, oi, true, expected.Weights, oi, dy, oi);
// should be x' * dy
Matrix.VxV(MatrixLayout.ColumnMajor, 3, 2, x.Weights, ii, dy, oi, expectedWG, 0, false);
// should be W' * dy
Matrix.MxV(MatrixLayout.ColumnMajor, 3, 2, layer.W.Weights, 0, false, dy, oi, expectedDx, ii, true);
if (oi > 0)
{
// should be x(t-1)' * dy
Matrix.VxV(MatrixLayout.ColumnMajor, 2, 2, expected.Weights, oi - 2, dy, oi, expectedUG, 0, false);
// should be U' * dy
Matrix.MxV(MatrixLayout.ColumnMajor, 2, 2, layer.U.Weights, 0, false, dy, oi, dy, oi - 2, false);
////MKL.MxV(MatrixLayout.RowMajor, 2, 2, layer.U.Weights, 0, false, dy, oi, dy, oi - 2, false);
}
// should be dy
Mathematics.Add(2, dy, oi, expectedBG, 0);
}
Helpers.AreArraysEqual(expectedWG, layer.W.Gradient);
////Helpers.AreArraysEqual(expectedUG, layer.U.Gradient);
Helpers.AreArraysEqual(expectedBG, layer.B.Gradient);
Helpers.AreArraysEqual(expectedDx, x.Gradient);
}
