private LMatrix _forward(LMatrix x, bool train_flg)
{
if (running_mean == null)
{
running_mean = new double[x.Column];
running_var = new double[x.Column];
}
if (train_flg)
{
//获取相同列的平均值
LMatrix mu = x.Mean(0);
//每行的元素值-平均值
xc = x - mu;
//每个元素平方后去均值,得到方差
LMatrix var = xc.Pow.Mean(0);
//获取标准差
std = var.Sqrt;
//均值除以标准差
xn = xc / std;
batch_size = x.Row;
running_mean = running_mean * momentum + mu * (1 - momentum);
running_var = running_var * momentum + var * (1 - momentum);
}
else
{
xc = x - running_mean;
xn = xc / running_var.Sqrt;
}
return(xn * gamma + beta);
}
/// <summary>
/// 计算微分数值，返回偏导组成的梯度
/// </summary>
/// <param name="fun"></param>
/// <param name="number"></param>
/// <param name="lable"></param>
/// <returns></returns>
public double[][] numerical_gradient(Func <double[][], LMatrix, double> fun,
double[][] number, LMatrix lable)
{
double h = 1e-4; // 0.0001
double[][] grad = new double[number.Length][];
double[][] copy = number;
for (int r = 0; r < number.Length; r++)
{
int len = number[r].Length;
grad[r] = new double[len];
for (int i = 0; i < len; i++)
{
double temp = number[r][i];
copy[r][i] = temp + h;
double res1 = fun(copy, lable);
copy[r][i] = temp - h;
double res2 = fun(copy, lable);
grad[r][i] = (res1 - res2) / (2 * h);
copy[r][i] = temp;
}
//Debug.Write(string.Format("当前步数：{0}\n", r));
}
return(grad);
}
public double[,,,] forward(double[,,,] input)
{
//获取样本的总数
int sampleCount = input.GetLength(0);
//获取单个样本的深度
int sampleSingleDepth = input.GetLength(1);
//获取特征图的行数（高）
int inputRow = input.GetLength(2);
//获取特征图的列数（宽）
int inputColumn = input.GetLength(3);
//创建最大值的
SingleMaxIndex = new int[sampleCount][][];
int row = (inputRow - PadRow) / Stride + 1;
int column = (inputColumn - PadColumn) / Stride + 1;
var result = new double[sampleCount, sampleSingleDepth, row, column];
for (int sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++)
{
//初始化特征图最大索引对象
SingleMaxIndex[sampleIndex] = new int[sampleSingleDepth][];
//记录特征图的行和列信息
InputShape = new MatrixShape(inputRow, inputColumn);
for (int depth = 0; depth < sampleSingleDepth; depth++)
{
LMatrix pad = im2col(input.GetNextDimVal(sampleIndex, depth, inputRow, inputColumn), row, column, PadRow, PadColumn, Stride);
SingleMaxIndex[sampleIndex][depth] = pad.MaxIndex();
//LMatrix data = pad.Matrix.Select(m => m.Max()).ToArray();
LMatrix data = pad.Max(1);
result.SetDimVal(data.ReShape(row, column), sampleIndex, depth, row, column);
}
}
return(result);
}
/// <summary>
/// 梯度下降法更新参数
/// </summary>
/// <param name="param"></param>
/// <param name="grads"></param>
public override void Update(List <LMatrix> param, List <LMatrix> grads)
{
if (M == null)
{
M = new LMatrix[grads.Count];
}
if (V == null)
{
V = new LMatrix[grads.Count];
}
iter += 1;
double lr_t = lr * Math.Sqrt((1 - Math.Pow(beta2, iter))) / (1 - Math.Pow(beta1, iter));
for (int k = 0; k < grads.Count; k++)
{
M[k] = grads[k].Zero;
V[k] = grads[k].Zero;
}
for (int k = 0; k < grads.Count; k++)
{
M[k] = (grads[k] - M[k]) * (1 - beta1) + M[k];
V[k] = (grads[k].Pow - V[k]) * (1 - beta2) + V[k];
param[k].Matrix = (param[k] - M[k] * lr_t / (V[k].Sqrt + 1e-7)).Matrix;
}
}
/// <summary>Linear algebraic matrix multiplication, A * B</summary>
/// <param name="B"> another matrix
/// </param>
/// <returns> Matrix product, A * B
/// </returns>
/// <exception cref="System.ArgumentException"> Matrix inner dimensions must agree.
/// </exception>
public LMatrix Multiply(LMatrix B)
{
if (B.m != n)
{
throw new System.ArgumentException("GeneralMatrix inner dimensions must agree.");
}
LMatrix X = new LMatrix(m, B.n);
LFloat[][] C = X.Array;
LFloat[] Bcolj = new LFloat[n];
for (int j = 0; j < B.n; j++)
{
for (int k = 0; k < n; k++)
{
Bcolj[k] = B.A[k][j];
}
for (int i = 0; i < m; i++)
{
LFloat[] Arowi = A[i];
LFloat s = 0;
for (int k = 0; k < n; k++)
{
s += Arowi[k] * Bcolj[k];
}
C[i][j] = s;
}
}
return(X);
}
/// <summary>
/// Check if size(A) == size(B)
/// </summary>
/// <param name="B"></param>
private void CheckMatrixDimensions(LMatrix B)
{
if (B.m != m || B.n != n)
{
throw new System.ArgumentException("GeneralMatrix dimensions must agree.");
}
}
public override LMatrix forward(LMatrix x, bool train_flg)
{
Row = x.Row;
Column = x.Column;
LMatrix res = _forward(x, train_flg);
return(res.ReShape(x.Row, x.Column));
}
public override void Update(List <LMatrix> param, List <LMatrix> grads)
{
for (int k = 0; k < grads.Count; k++)
{
var grad = grads[k];
LMatrix temp = grad * lr;
param[k].Matrix = (param[k] - temp).Matrix;
}
}
public double forward(LMatrix lable)
{
var y = Predict();
LLossMethod method = new LLossMethod();
double res = method.loss_entropyError(y, lable);//评分
Index++;
return(res);
}
public override LMatrix backward(LMatrix dout)
{
dB = dout.Sum(0);
dW = X.T * dout;
return(dout * W.T);
//LMatrix dx = dout * W.T;
//dW = X.T * dout;
//dB = dout.Sum(0);
//return dx.ReShape(X.Row, X.Column);
}
public LMatrix softmax(LMatrix mat)
{
LMatrix result = new double[mat.Row, mat.Column];
for (int row = 0; row < mat.Row; row++)
{
result[row] = softmax(mat[row]);
}
return(result);
}
public override LMatrix forward(LMatrix x, bool train_flg = true)
{
if (train_flg)
{
LMatrix temp = this.Distribution(x.Row, x.Column);
mask = temp > ratio;
return(x & mask);
}
return(x & (1 - ratio));
}
//protected LMatrix _forward(LMatrix x, WehtInfo info, int kerDepIndex)
//{
// int kernelRow = info.W.Length;
// int kernelColumn = info.W[0].Length;
// int row = (x.Row + 2 * this.Padding - kernelRow) / this.Stride + 1;
// int column = (x.Column + 2 * this.Padding - kernelColumn) / this.Stride + 1;
// if (this._col[kerDepIndex] == null)
// this._col[kerDepIndex] = im2col(x.Matrix, row, column, kernelRow, kernelColumn, Stride);
// LMatrix tW = info.W.Flatten();
// LMatrix _out = _col[kerDepIndex] * tW.T;
// LMatrix res = _out.ReShape(row, column);
// return res;
//}
protected double[,] _forward(double[,] mat, WehtInfo info, int kerDepIndex, int resRow, int resCol)
{
if (this._col[kerDepIndex] == null)
{
this._col[kerDepIndex] = im2col(mat, resRow, resCol, kernelRow, kernelColumn, Stride);
}
LMatrix tW = info.W.Flatten();
LMatrix _out = _col[kerDepIndex] * tW.T;
return(_out.ReShape(resRow, resCol));
}
public LBN(double gamma, double beta, double momentum = 0.9,
LMatrix running_mean = null, LMatrix running_var = null)
{
this.gamma = gamma;
this.beta = beta;
this.momentum = momentum;
//测试时使用的平均值和方差
this.running_mean = running_mean;
this.running_var = running_var;
}
public override void ApplyConstraintImpluse(LFloat deltaTime)
{
LVector2 ca = objA.GetWorldCenterOfMass();
LVector2 ra = anchorA - ca;
LVector2 cb = objB.GetWorldCenterOfMass();
LVector2 rb = anchorB - cb;
// 1 X 6
LMatrix jacobian = new LMatrix(new LFloat[] {
-normal.x, -normal.y, -LVector3.Cross(ra, normal).z, normal.x, normal.y, LVector3.Cross(rb, normal).z
}, 1);
// 6 X 1
LMatrix jacobian_t = jacobian.Transpose();
// 6 X 6
LMatrix massMatrixInverse = new LMatrix(6, 6);
massMatrixInverse[0][0] = objA.massInverse;
massMatrixInverse[1][1] = objA.massInverse;
massMatrixInverse[2][2] = objA.inertiaInverse;
massMatrixInverse[3][3] = objB.massInverse;
massMatrixInverse[4][4] = objB.massInverse;
massMatrixInverse[5][5] = objB.inertiaInverse;
// 6 X 1
LMatrix v = new LMatrix(new LFloat[] {
objA.linearVelocity.x,
objA.linearVelocity.y,
objA.angularVelocity,
objB.linearVelocity.x,
objB.linearVelocity.y,
objB.angularVelocity,
}, 6);
//GeneralMatrix m1 = jacobian * massMatrixInverse;
//GeneralMatrix m2 = jacobian_t;
//double[][] m1s = m1.Array;
//Debug.Log(m1s[0][0] + " " + m1s[0][1] + " " + m1s[0][2] + " " + m1s[0][3] + " " + m1s[0][4] + " " + m1s[0][5]);
//double[][] m2s = m1.Array;
//Debug.Log(m2s[0][0] + " " + m2s[1][0] + " " + m2s[2][0] + " " + m2s[3][0] + " " + m2s[4][0] + " " + m2s[5][0]);
LMatrix lamdaMatrix = (jacobian * v * -1) * (jacobian * massMatrixInverse * jacobian_t).Inverse();
LFloat lamda = lamdaMatrix[0][0];
// 6 X 1
LMatrix dv = massMatrixInverse * jacobian_t * lamda;
objA.linearVelocity += new LVector2(dv[0][0], dv[1][0]);
objA.angularVelocity += dv[2][0];
objB.linearVelocity += new LVector2(dv[3][0], dv[4][0]);
objB.angularVelocity += dv[5][0];
}
/// <summary>
/// 反向传播
/// </summary>
/// <param name="dout"></param>
/// <returns></returns>
public override LMatrix backward(LMatrix dout)
{
var res = new double[dout.Row, dout.Column];
for (int row = 0; row < dout.Row; row++)
{
for (int col = 0; col < dout.Column; col++)
{
res[row, col] = back_value(dout.Matrix[row, col]);
}
}
return(res);
}
/// <summary>
/// 正向传播
/// </summary>
/// <param name="mat"></param>
/// <param name="train_flg"></param>
/// <returns></returns>
public override LMatrix forward(LMatrix mat, bool train_flg = true)
{
var res = new double[mat.Row, mat.Column];
for (int row = 0; row < mat.Row; row++)
{
for (int col = 0; col < mat.Column; col++)
{
res[row, col] = forward_value(mat.Matrix[row, col]);
}
}
return(res);
}
/// <summary>Least squares solution of A*X = B</summary>
/// <param name="B"> A Matrix with as many rows as A and any number of columns.
/// </param>
/// <returns> X that minimizes the two norm of Q*R*X-B.
/// </returns>
/// <exception cref="System.ArgumentException"> Matrix row dimensions must agree.
/// </exception>
/// <exception cref="System.SystemException"> Matrix is rank deficient.
/// </exception>
public virtual LMatrix Solve(LMatrix B)
{
if (B.Rows != m)
{
throw new System.ArgumentException("GeneralMatrix row dimensions must agree.");
}
if (!this.FullRank)
{
throw new System.SystemException("Matrix is rank deficient.");
}
// Copy right hand side
int nx = B.Columns;
LFloat[][] X = B.ArrayCopy;
// Compute Y = transpose(Q)*B
for (int k = 0; k < n; k++)
{
for (int j = 0; j < nx; j++)
{
LFloat s = 0;
for (int i = k; i < m; i++)
{
s += QR[i][k] * X[i][j];
}
s = (-s) / QR[k][k];
for (int i = k; i < m; i++)
{
X[i][j] += s * QR[i][k];
}
}
}
// Solve R*X = Y;
for (int k = n - 1; k >= 0; k--)
{
for (int j = 0; j < nx; j++)
{
X[k][j] /= Rdiag[k];
}
for (int i = 0; i < k; i++)
{
for (int j = 0; j < nx; j++)
{
X[i][j] -= X[k][j] * QR[i][k];
}
}
}
return(new LMatrix(X, n, nx).GetMatrix(0, n - 1, 0, nx - 1));
}
private void TestMatrix()
{
LMatrix matrix = new LMatrix(new LFloat[] {
1, 2,
3, 4,
5, 6
}, 3);
Debug.Log(matrix);
Debug.Log(matrix.Inverse() * matrix);
Debug.Log(matrix.Transpose());
Debug.Log(matrix * 3);
Debug.Log("----------------------");
}
public LMatrix Transpose()
{
LMatrix X = new LMatrix(n, m);
LFloat[][] C = X.Array;
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
C[j][i] = A[i][j];
}
}
return(X);
}
public static LMatrix Identity(int m, int n)
{
LMatrix A = new LMatrix(m, n);
LFloat[][] X = A.Array;
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
X[i][j] = (i == j ? 1 : 0);
}
}
return(A);
}
/// <summary>Multiply a matrix by a scalar, C = s*A</summary>
/// <param name="s"> scalar
/// </param>
/// <returns> s*A
/// </returns>
public LMatrix Multiply(LFloat s)
{
LMatrix X = new LMatrix(m, n);
LFloat[][] C = X.Array;
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
C[i][j] = s * A[i][j];
}
}
return(X);
}
/// <summary>QR Decomposition, computed by Householder reflections.</summary>
/// <param name="A"> Rectangular matrix
/// </param>
/// <returns> Structure to access R and the Householder vectors and compute Q.
/// </returns>
public QRDecomposition(LMatrix A)
{
// Initialize.
QR = A.ArrayCopy;
m = A.Rows;
n = A.Columns;
Rdiag = new LFloat[n];
// Main loop.
for (int k = 0; k < n; k++)
{
// Compute 2-norm of k-th column without under/overflow.
LFloat nrm = 0;
for (int i = k; i < m; i++)
{
nrm = LMatrixUtil.Hypot(nrm, QR[i][k]);
}
if (nrm != 0)
{
// Form k-th Householder vector.
if (QR[k][k] < 0)
{
nrm = -nrm;
}
for (int i = k; i < m; i++)
{
QR[i][k] /= nrm;
}
QR[k][k] += 1;
// Apply transformation to remaining columns.
for (int j = k + 1; j < n; j++)
{
LFloat s = 0;
for (int i = k; i < m; i++)
{
s += QR[i][k] * QR[i][j];
}
s = (-s) / QR[k][k];
for (int i = k; i < m; i++)
{
QR[i][j] += s * QR[i][k];
}
}
}
Rdiag[k] = -nrm;
}
}
/// <summary>C = A - B</summary>
/// <param name="B"> another matrix
/// </param>
/// <returns> A - B
/// </returns>
public virtual LMatrix Subtract(LMatrix B)
{
CheckMatrixDimensions(B);
LMatrix X = new LMatrix(m, n);
LFloat[][] C = X.Array;
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
C[i][j] = A[i][j] - B.A[i][j];
}
}
return(X);
}
/// <summary>C = A + B</summary>
/// <param name="B"> another matrix
/// </param>
/// <returns> A + B
/// </returns>
public LMatrix Add(LMatrix B)
{
CheckMatrixDimensions(B);
LMatrix X = new LMatrix(m, n);
LFloat[][] C = X.Array;
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
C[i][j] = A[i][j] + B.A[i][j];
}
}
return(X);
}
public double Accuracy(LMatrix lable)
{
var y = Predict();
int len = y.GetLength(0);
LMatrix maxtrx = y;
var res = maxtrx.MaxIndex();
var lab = lable.MaxIndex();
int sum = 0;
for (int i = 0; i < len; i++)
{
sum += (res[i] == lab[i]) ? 1 : 0;
}
return(sum * 1.0d / len);
}
/// <summary>
/// 反向传播函数
/// </summary>
/// <param name="lable">正确标签值</param>
/// <returns></returns>
public LMatrix backward(LMatrix lable)
{
if (Result == null)
{
Predict();
}
LMatrix dy = (Result - lable) / Input.Row;
//定义层结构
foreach (var layer in layers.Reverse())
{
//按照层结构一步步往前传递
dy = layer.backward(dy);
}
return(dy);
}
public override LMatrix backward(LMatrix dout)
{
LMatrix _out = dout.Copy();
for (int row = 0; row < _out.Row; row++)
{
for (int col = 0; col < _out.Column; col++)
{
if (mask.Matrix[row, col] <= 0)
{
_out.Matrix[row, col] = 0;
}
}
}
return(_out);
}
public double loss_entropyError(double[,] res, LMatrix lable)
{
int _row = res.GetLength(0);
if (res.GetLength(0) != lable.Row && res.GetLength(1) != lable.Column)
{
throw new Exception("正确标签值矩阵行列不一致，禁止计算误差");
}
double sum = 0;
for (int i = 0; i < _row; i++)
{
sum += loss_entropyError(res.GetNextDimVal(i), lable[i]);
}
return(sum / _row);
}
private LMatrix _backward(LMatrix dout)
{
dbeta = dout.Sum(0);
dgamma = (xn & dout).Sum(0);
LMatrix dxn = dout & gamma;
LMatrix dxc = dxn / this.std;
LMatrix dstd = ((dxn & xc) / (std & std)).Sum(0) * -1;
LMatrix dvar = (dstd & 0.5) / this.std;
dxc = dxc + (this.xc & dvar & (2.0 / this.batch_size));
LMatrix dmu = dxc.Sum(0);
LMatrix dx = dxc - dmu / this.batch_size;
this.dgamma = dgamma;
this.dbeta = dbeta;
return(dx);
}
