public void Add_Always_ReturnsSumOfAandB(int a, int b, int expected)
{
// Arrange
// Act
var sum = math.Add(a, b);
// Assert
//Assert.That(sum, Is.EqualTo(expected));
Assert.AreEqual(expected, sum);
}
private static void Stack(IList <Tensor> xs, int axis, Tensor y, bool useGradients)
{
int xdim = xs.Count;
int ylen = y.Length;
int ystride = y.Strides[axis];
float[] yw = useGradients ? y.Gradient : y.Weights;
for (int offx = 0, offy = 0; offy < ylen; offx += ystride)
{
for (int i = 0; i < xdim; i++, offy += ystride)
{
Tensor x = xs[i];
float[] xw = useGradients ? x.Gradient : x.Weights;
if (useGradients)
{
Mathematics.Add(ystride, xw, offx, yw, offy);
}
else
{
Vectors.Copy(ystride, xw, offx, yw, offy);
}
}
}
}
public void AddTest_float()
{
const int offx = 5;
const int offy0 = 8;
const int offy = 10;
foreach (int length in new[] { 24, 128 })
{
float[] x = this.random.Generate(length, null);
float[] y0 = this.random.Generate(length, null);
float[] y = y0.ToArray();
Mathematics.Add(length, x, 0, y, 0);
GenixAssert.AreArraysEqual(x.Zip(y0, (a, b) => a + b).ToArray(), y);
y = y0.ToArray();
int count = length - Math.Max(offx, offy);
Mathematics.Add(count, x, offx, y, offy);
GenixAssert.AreArraysEqual(offy, y0, 0, y, 0);
GenixAssert.AreArraysEqual(count, x.Skip(offx).Zip(y0.Skip(offy), (a, b) => a + b).ToArray(), 0, y, offy);
y = y0.ToArray();
Mathematics.Add(length, x, 0, y0, 0, y, 0);
GenixAssert.AreArraysEqual(x.Zip(y0, (a, b) => a + b).ToArray(), y);
y = y0.ToArray();
count = length - Math.Max(offx, Math.Max(offy, offy0));
Mathematics.Add(count, x, offx, y0, offy0, y, offy);
GenixAssert.AreArraysEqual(offy, y0, 0, y, 0);
GenixAssert.AreArraysEqual(count, x.Skip(offx).Zip(y0.Skip(offy0), (a, b) => a + b).ToArray(), 0, y, offy);
}
}
private static void Split(Tensor x, int axis, IList <Tensor> ys, bool useGradients)
{
int xstride = axis == 0 ? x.Length : x.Strides[axis - 1];
int ydim = axis == 0 ? 1 : x.Length / xstride;
float[] xw = useGradients ? x.Gradient : x.Weights;
for (int i = 0, offx = 0, ii = ys.Count; i < ii; i++)
{
Tensor y = ys[i];
float[] yw = useGradients ? y.Gradient : y.Weights;
int ystride = axis == 0 ? y.Length : y.Strides[axis - 1];
for (int n = 0, offxx = offx, offy = 0; n < ydim; n++, offxx += xstride, offy += ystride)
{
if (useGradients)
{
Mathematics.Add(ystride, xw, offxx, yw, offy);
}
else
{
Vectors.Copy(ystride, xw, offxx, yw, offy);
}
}
offx += ystride;
}
}
public void Update(float dt)
{
SteeringForce = new Point(0, 0);
foreach (var behavior in behaviorList)
{
behavior.Update();
SteeringForce = Mathematics.Add(SteeringForce, behavior.SteeringForce);
}
SteeringForce = Mathematics.Truncate(SteeringForce, MaxForce);
if (Mathematics.Length(SteeringForce) > _steeringForceThreshold)
{
Point acceleration = Mathematics.Multiply(SteeringForce, 1f / Mass);
Velocity = Mathematics.Add(Velocity, acceleration);
Position = Mathematics.Add(Position, Velocity);
float speed = Mathematics.Length(Velocity);
if (speed > _speedThreshold)
{
Point velocity = Mathematics.Normalize(Velocity);
velocity = Mathematics.Lerp(Front, velocity, VelocityAlignmentFactor);
Front = Mathematics.Normalize(velocity);
}
}
else
{
Velocity = new Point(0, 0);
}
}
public void Update(GameTime gameTime)
{
SteeringForce = new Vector2(0, 0);
vectorDifference = Mathematics.Subtract(TargetAgent.Position, Agent.Position);
distance = Mathematics.Length(vectorDifference);
Vector2 predictedPosition = Mathematics.Multiply(TargetAgent.Velocity, distance * predictionFactor);
Vector2 targetPosition = Mathematics.Add(TargetAgent.Position, predictedPosition);
vectorDifference = Mathematics.Subtract(targetPosition, Agent.Position);
distance = Mathematics.Length(vectorDifference);
if (distance < 2)
{
return;
}
rampSpeed = Agent.MaxSpeed * (distance / rampDistance);
rampSpeed = Math.Min(rampSpeed, Agent.MaxSpeed);
vectorDifference = Mathematics.Multiply(vectorDifference, rampSpeed / distance);
force = Mathematics.Subtract(vectorDifference, Agent.Velocity);
SteeringForce = force;
}
private static void Unstack(Tensor x, int axis, IList <Tensor> ys, bool useGradients)
{
int xlen = x.Length;
int xdim = x.Axes[axis];
int xstride0 = x.Strides[axis]; // axis inner stride
int xstride1 = axis == 0 ? xlen : x.Strides[axis - 1]; // axis outer stride
float[] xw = useGradients ? x.Gradient : x.Weights;
for (int i = 0, offx = 0; i < xdim; i++, offx += xstride0)
{
Tensor y = ys[i];
float[] yw = useGradients ? y.Gradient : y.Weights;
for (int offxx = offx, offy = 0; offxx < xlen; offxx += xstride1, offy += xstride0)
{
if (useGradients)
{
Mathematics.Add(xstride0, xw, offxx, yw, offy);
}
else
{
Vectors.Copy(xstride0, xw, offxx, yw, offy);
}
}
}
}
public static Tensor AddBias(this Session session, Tensor x, Tensor b)
{
int mb = x.Axes[0]; // number of items in a mini-batch
if (mb == 1)
{
return(session.Add(x, b));
}
Tensor y = session.Allocate("addBias", x.Axes);
int stride = x.Strides[0]; // item length
Debug.Assert(stride == b.Length, "Biases tensor has invalid size.");
// repeat for each item in mini-batch
for (int i = 0, off = 0; i < mb; i++, off += stride)
{
Mathematics.Add(stride, x.Weights, off, b.Weights, 0, y.Weights, off);
}
if (session.CalculateGradients && (x.CalculateGradient || b.CalculateGradient))
{
session.Push(() =>
{
Tensor dy = session.GetGradient(y);
// dx += dy
if (x.CalculateGradient)
{
Tensor dx = session.GetGradient(x);
lock (dx)
{
dx.Add(dy);
}
}
// db += sum(dy)
if (b.CalculateGradient)
{
float[] ones = new float[mb];
MKL.Set(mb, 1.0f, ones, 0);
Tensor db = session.GetGradient(b);
lock (db)
{
MKL.MxV(MatrixLayout.ColumnMajor, stride, mb, y.Gradient, 0, false, ones, 0, db.Weights, 0, false);
/*for (int i = 0, off = 0; i < mb; i++, off += stride)
* {
* Mathematics.Add(stride, y.Gradient, off, db.Weights, 0);
* }*/
}
}
});
}
return(y);
}
public void AddTest()
{
expresionValue.value = new string[2];
expresionValue.value[0] = "6";
expresionValue.value[1] = "5";
string plus = math.Add(expresionValue, 1);
//Assert.Fail();
}
private static int Calculate(MathematicModel model)
{
switch (model.Operation)
{
default:
return(Mathematics.Add(model.First, model.Second));
}
}
public static void Main(string[] args)
{
var objMath = new Mathematics();
Console.WriteLine(objMath.Add(10, 20));
Console.WriteLine(objMath.Subract(10, 20));
Console.WriteLine(objMath.Multiply(10, 20));
Console.WriteLine(objMath.Divide(10, 5));
}
public void ShouldAddNumbers(int a, int b, int sum)
{
// Arrange
var maths = new Mathematics();
// Act
int result = maths.Add(a, b);
// Assert
Assert.Equal(sum, result);
}
public void ShouldAddTwoNumbers()
{
// Arrange
int a = 5;
int b = 10;
var maths = new Mathematics();
// Act
int result = maths.Add(a, b);
// Assert
Assert.Equal(15, result);
}
public void Update()
{
SteeringForce = new Point(0, 0);
foreach (var agent in agentList)
{
vectorDifference = Mathematics.Subtract(Agent.Position, agent.Position);
distance = Mathematics.Length(vectorDifference);
if (distance < seperationDistance && distance > 0)
{
vectorDifference = Mathematics.Normalize(vectorDifference);
vectorDifference = Mathematics.Multiply(vectorDifference, 50 / (float)(distance));
SteeringForce = Mathematics.Add(SteeringForce, vectorDifference);
}
}
}
private static Tensor CalculateDW(Tensor w, Tensor x, Tensor y, Kernel kernel, int numberOfFilters, MatrixLayout matrixLayout)
{
Tensor dw = new Tensor(null, w.Axes);
for (int ib = 0, iib = y.Shape.GetAxis(Axis.B); ib < iib; ib++)
{
for (int ix = 0, xpos = -kernel.PaddingX, iix = y.Shape.GetAxis(Axis.X); ix < iix; ix++, xpos += kernel.StrideX)
{
for (int iy = 0, ypos = -kernel.PaddingY, iiy = y.Shape.GetAxis(Axis.Y); iy < iiy; iy++, ypos += kernel.StrideY)
{
Tensor k = x.CropKernel(ib, xpos, ypos, kernel, false, out int kernelArea);
Tensor kdy = y.CropKernel(ib, ix, iy, new Kernel(1, 1, 1, 1), true, out kernelArea);
float[] dww = FullyConnectedLayerTest.CalculateDW(k, kdy, matrixLayout);
Mathematics.Add(dw.Length, dww, 0, dw.Gradient, 0);
}
}
}
return(dw);
}
static void Main(string[] args)
{
Car Yugo = new Car("Zastava", "55");
Yugo.Color = "white";
Console.WriteLine("Yugo is {0}.", Yugo.Color);
Console.WriteLine("Yugo is manufactured by {0}", Yugo.Manufacturer);
Console.WriteLine("Yugo model is {0}", Yugo.Model);
CarEngine engine = Yugo.Engine;
Console.WriteLine("Yugo has a {0} HP engine, model {1}", engine.Power, engine.Name);
Yugo.Drive();
Mathematics Math = new Mathematics();
List <int> nums = new List <int>();
nums.Add(1);
nums.Add(9);
Console.WriteLine("Sum = {0}", Mathematics.Sum(nums));
Console.WriteLine("Add = {0}", Math.Add(1, 9));
double n1 = 5, n2 = 0;
string oper;
oper = "+";
Console.WriteLine("[SimpleCalc] {0} {1} {2} = {3}", n1, oper, n2, SimpleCalc.Calc(oper, n1, n2));
oper = "-";
Console.WriteLine("[SimpleCalc] {0} {1} {2} = {3}", n1, oper, n2, SimpleCalc.Calc(oper, n1, n2));
oper = "*";
Console.WriteLine("[SimpleCalc] {0} {1} {2} = {3}", n1, oper, n2, SimpleCalc.Calc(oper, n1, n2));
oper = "/";
Console.WriteLine("[SimpleCalc] {0} {1} {2} = {3}", n1, oper, n2, SimpleCalc.Calc(oper, n1, n2));
Console.ReadKey();
}
private static Tensor CalculateDB(Tensor y)
{
int y0 = y.Shape.GetAxis(Axis.B);
int y1 = y.Shape.GetAxis(Axis.X);
int y2 = y.Shape.GetAxis(Axis.Y);
int y3 = y.Shape.GetAxis(Axis.C);
int ystride3 = y.Shape.GetStride(Axis.C);
Tensor db = new Tensor(null, new[] { y3 });
for (int iy0 = 0; iy0 < y0; iy0++)
{
for (int iy1 = 0; iy1 < y1; iy1++)
{
for (int iy2 = 0; iy2 < y2; iy2++)
{
Mathematics.Add(y3, y.Gradient, y.Shape.Position(iy0, iy1, iy2, 0), ystride3, db.Gradient, 0, 1);
}
}
}
return(db);
}
private static void Untile(Tensor x, int axis, int count, Tensor y, bool useGradients)
{
int ylen = y.Length;
int ystride = axis == 0 ? ylen : y.Strides[axis - 1];
float[] xw = useGradients ? x.Gradient : x.Weights;
float[] yw = useGradients ? y.Gradient : y.Weights;
for (int offx = 0, offy = 0; offy < ylen; offy += ystride)
{
for (int i = 0; i < count; i++, offx += ystride)
{
if (useGradients || i > 0)
{
Mathematics.Add(ystride, xw, offx, yw, offy);
}
else
{
Vectors.Copy(ystride, xw, offx, yw, offy);
}
}
}
}
public void FiveTwo_Add_ReturnsSeven()
{
int current = Mathematics.Add(5, 2);
Assert.AreEqual(7, current, "the mathematic operation add should add first number to second.");
}
public static Tensor Slice(this Session session, Tensor x, int[] begin, int[] size)
{
const string ActionName = "slice";
int dims = x.Axes.Length;
if (begin.Length != dims)
{
throw new ArgumentException("The parameter dimension must match the number of axes in the tensor.", nameof(begin));
}
if (size.Length != dims)
{
throw new ArgumentException("The parameter dimension must match the number of axes in the tensor.", nameof(size));
}
Shape yshape = x.Shape.Slice(begin, size);
if (x.Length == yshape.Length)
{
// copy entire tensor
return(session.Copy(x));
}
return(session.RunOperation(
ActionName,
() =>
{
bool calculateGradient = session.CalculateGradients && x.CalculateGradient;
// allocate destination
Tensor y = session.AllocateTensor(ActionName, yshape, calculateGradient);
int[] xaxes = x.Axes;
int[] yaxes = yshape.Axes;
// 1. find last axis of the slice that occupies the entire tensor
int blocksize = 1;
int lastaxis = dims;
while (lastaxis > 0 && xaxes[lastaxis - 1] == yaxes[lastaxis - 1])
{
blocksize *= yaxes[lastaxis - 1];
lastaxis--;
}
if (lastaxis == dims)
{
blocksize = yaxes[dims - 1];
}
// now point to the axis where the copying should start
lastaxis--;
// 2. do slicing
Slice();
void Slice()
{
float[] xw = x.Weights;
float[] yw = y.Weights;
int[] xstrides = x.Strides;
int offy = 0;
Do(0, 0);
Debug.Assert(offy == y.Length, "Entire tensor must be filled.");
void Do(int axis, int offx)
{
offx += begin[axis] * xstrides[axis];
if (axis == lastaxis)
{
Vectors.Copy(blocksize, xw, offx, yw, offy);
offy += blocksize;
}
else
{
for (int i = 0; i < yaxes[axis]; i++, offx += xstrides[axis])
{
Do(axis + 1, offx);
}
}
}
}
#if !NOLEARNING
if (calculateGradient)
{
session.Push(
ActionName,
() =>
{
float[] dxw = x.Gradient;
float[] dyw = y.Gradient;
int[] xstrides = x.Strides;
int offy = 0;
Unslice(0, 0);
Debug.Assert(offy == y.Length, "Entire tensor must be filled.");
void Unslice(int axis, int offx)
{
offx += begin[axis] * xstrides[axis];
if (axis == lastaxis)
{
Mathematics.Add(blocksize, dyw, offy, dxw, offx);
offy += blocksize;
}
else
{
for (int i = 0; i < yaxes[axis]; i++, offx += xstrides[axis])
{
Unslice(axis + 1, offx);
}
}
}
});
}
#endif
return y;
}));
}
[Test] public void Add()
{
Assert.AreEqual(6, _obj.Add(2, 4), 6, "Addition of simple numbers");
}
public double Add(double x, double y)
{
return(_math.Add(x, y));
}
public bool Optimize(
int numberOfVariables,
float[] c,
float[] p,
int[] y,
Func <int, int[], int, float[], float[]> q,
out float[] solution,
out float rho)
{
// make copies, these array will be modified
p = p.ToArray();
y = y.ToArray();
float[] temp = new float[numberOfVariables];
float[] g = new float[numberOfVariables]; // gradient
float[] gbar = new float[numberOfVariables]; // gradient, if we treat free variables as 0
float[] qd = new float[numberOfVariables];
for (int k = 0; k < qd.Length; k++)
{
qd[k] = q(k, new[] { k }, 1, temp)[0];
}
float[] qi = new float[numberOfVariables];
float[] qj = new float[numberOfVariables];
bool unshrink = false;
// Lagrange multipliers
float[] alpha = new float[numberOfVariables];
// initialize alpha_status
Status[] alphaStatus = new Status[numberOfVariables];
for (int i = 0; i < numberOfVariables; i++)
{
UpdateAlphaStatus(i);
}
// initialize active set (for shrinking)
int activeSize = numberOfVariables;
int[] activeSet = Arrays.Indexes(numberOfVariables);
// initialize index lookup vector
int[] indices = Arrays.Indexes(numberOfVariables);
// initialize gradient
Vectors.Copy(numberOfVariables, p, 0, g, 0);
Vectors.Set(numberOfVariables, 0.0f, gbar, 0);
/*for (int i = 0; i < numberOfVariables; i++)
* {
* g[i] = p[i];
* gbar[i] = 0;
* }*/
for (int i = 0; i < numberOfVariables; i++)
{
if (!IsLowerBound(i))
{
q(i, indices, numberOfVariables, qi);
Mathematics.AddProductC(numberOfVariables, qi, 0, alpha[i], g, 0);
/*float alpha_i = alpha[i];
* for (int j = 0; j < numberOfVariables; j++)
* {
* g[j] += alpha_i * qi[j];
* }*/
if (IsUpperBound(i))
{
Mathematics.AddProductC(numberOfVariables, qi, 0, c[i], gbar, 0);
/*for (int j = 0; j < numberOfVariables; j++)
* {
* gbar[j] += c[i] * qi[j];
* }*/
}
}
}
// optimization step
int iter = 0;
int max_iter = MinMax.Max(10000000, numberOfVariables > int.MaxValue / 100 ? int.MaxValue : 100 * numberOfVariables);
int counter = MinMax.Min(numberOfVariables, 1000) + 1;
while (iter < max_iter)
{
// show progress and do shrinking
if (--counter == 0)
{
counter = MinMax.Min(numberOfVariables, 1000);
if (this.Shrinking)
{
Shrink();
}
Trace.WriteLine(".");
}
if (SelectWorkingSet(out int i, out int j) != 0)
{
// reconstruct the whole gradient
ReconstructGradient();
// reset active set size and check
activeSize = numberOfVariables;
Trace.WriteLine("*");
if (SelectWorkingSet(out i, out j) != 0)
{
break;
}
else
{
counter = 1; // do shrinking next iteration
}
}
iter++;
// update alpha[i] and alpha[j], handle bounds carefully
q(i, indices, activeSize, qi);
q(j, indices, activeSize, qj);
float ci = c[i];
float cj = c[j];
float old_alpha_i = alpha[i];
float old_alpha_j = alpha[j];
if (y[i] != y[j])
{
float quad_coef = qd[i] + qd[j] + (2.0f * qi[j]);
if (quad_coef <= 0)
{
quad_coef = TAU;
}
float delta = (-g[i] - g[j]) / quad_coef;
float diff = alpha[i] - alpha[j];
alpha[i] += delta;
alpha[j] += delta;
if (diff > 0)
{
if (alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = diff;
}
}
else
{
if (alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = -diff;
}
}
if (diff > ci - cj)
{
if (alpha[i] > ci)
{
alpha[i] = ci;
alpha[j] = ci - diff;
}
}
else
{
if (alpha[j] > cj)
{
alpha[j] = cj;
alpha[i] = cj + diff;
}
}
}
else
{
float quad_coef = qd[i] + qd[j] - (2.0f * qi[j]);
if (quad_coef <= 0)
{
quad_coef = TAU;
}
float delta = (g[i] - g[j]) / quad_coef;
float sum = alpha[i] + alpha[j];
alpha[i] -= delta;
alpha[j] += delta;
if (sum > ci)
{
if (alpha[i] > ci)
{
alpha[i] = ci;
alpha[j] = sum - ci;
}
}
else
{
if (alpha[j] < 0)
{
alpha[j] = 0;
alpha[i] = sum;
}
}
if (sum > cj)
{
if (alpha[j] > cj)
{
alpha[j] = cj;
alpha[i] = sum - cj;
}
}
else
{
if (alpha[i] < 0)
{
alpha[i] = 0;
alpha[j] = sum;
}
}
}
// update G
float delta_alpha_i = alpha[i] - old_alpha_i;
float delta_alpha_j = alpha[j] - old_alpha_j;
for (int k = 0; k < activeSize; k++)
{
g[k] += (qi[k] * delta_alpha_i) + (qj[k] * delta_alpha_j);
}
// update alpha_status and G_bar
{
bool ui = IsUpperBound(i);
bool uj = IsUpperBound(j);
UpdateAlphaStatus(i);
UpdateAlphaStatus(j);
if (ui != IsUpperBound(i))
{
q(i, indices, numberOfVariables, qi);
Mathematics.AddProductC(numberOfVariables, qi, 0, ui ? -ci : ci, gbar, 0);
/*if (ui)
* {
* for (int k = 0; k < numberOfVariables; k++)
* {
* gbar[k] -= ci * qi[k];
* }
* }
* else
* {
* for (int k = 0; k < numberOfVariables; k++)
* {
* gbar[k] += ci * qi[k];
* }
* }*/
}
if (uj != IsUpperBound(j))
{
q(j, indices, numberOfVariables, qj);
Mathematics.AddProductC(numberOfVariables, qj, 0, uj ? -cj : cj, gbar, 0);
/*if (uj)
* {
* for (int k = 0; k < numberOfVariables; k++)
* {
* gbar[k] -= cj * qj[k];
* }
* }
* else
* {
* for (int k = 0; k < numberOfVariables; k++)
* {
* gbar[k] += cj * qj[k];
* }
* }*/
}
}
}
if (iter >= max_iter)
{
if (activeSize < numberOfVariables)
{
// reconstruct the whole gradient to calculate objective value
ReconstructGradient();
activeSize = numberOfVariables;
Trace.WriteLine("*");
}
Trace.WriteLine("WARNING: reaching max number of iterations.");
}
// calculate rho
rho = CalculateRho();
// calculate objective value
/*float v = 0;
* for (int i = 0; i < numberOfVariables; i++)
* {
* v += alpha[i] * (g[i] + p[i]);
* }
*
* si->obj = v / 2;*/
// put back the solution
solution = new float[numberOfVariables];
for (int i = 0; i < numberOfVariables; i++)
{
solution[activeSet[i]] = alpha[i];
}
////si->upper_bound_p = Cp;
////si->upper_bound_n = Cn;
Trace.WriteLine(string.Format(
CultureInfo.InvariantCulture,
"optimization finished, #iter = {0}",
iter));
return(iter < max_iter);
// return 1 if already optimal, return 0 otherwise
int SelectWorkingSet(out int out_i, out int out_j)
{
// return i,j such that
// i: maximizes -y_i * grad(f)_i, i in I_up(\alpha)
// j: minimizes the decrease of obj value
// (if quadratic coefficient <= 0, replace it with tau)
// -y_j*grad(f)_j < -y_i*grad(f)_i, j in I_low(\alpha)
float gmax = float.NegativeInfinity;
float gmax2 = float.NegativeInfinity;
int gmax_idx = -1;
int gmin_idx = -1;
float obj_diff_min = float.PositiveInfinity;
for (int t = 0; t < activeSize; t++)
{
if (y[t] == +1)
{
if (!IsUpperBound(t))
{
if (-g[t] >= gmax)
{
gmax = -g[t];
gmax_idx = t;
}
}
}
else
{
if (!IsLowerBound(t))
{
if (g[t] >= gmax)
{
gmax = g[t];
gmax_idx = t;
}
}
}
}
int i = gmax_idx;
if (i != -1)
{
q(i, indices, activeSize, temp); // NULL Q_i not accessed: Gmax=-INF if i=-1
}
for (int j = 0; j < activeSize; j++)
{
if (y[j] == +1)
{
if (!IsLowerBound(j))
{
float grad_diff = gmax + g[j];
if (g[j] >= gmax2)
{
gmax2 = g[j];
}
if (grad_diff > 0)
{
float obj_diff;
float quad_coef = qd[i] + qd[j] - (2.0f * y[i] * temp[j]);
if (quad_coef > 0)
{
obj_diff = -(grad_diff * grad_diff) / quad_coef;
}
else
{
obj_diff = -(grad_diff * grad_diff) / TAU;
}
if (obj_diff <= obj_diff_min)
{
gmin_idx = j;
obj_diff_min = obj_diff;
}
}
}
}
else
{
if (!IsUpperBound(j))
{
float grad_diff = gmax - g[j];
if (-g[j] >= gmax2)
{
gmax2 = -g[j];
}
if (grad_diff > 0)
{
float obj_diff;
float quad_coef = qd[i] + qd[j] + (2.0f * y[i] * temp[j]);
if (quad_coef > 0)
{
obj_diff = -(grad_diff * grad_diff) / quad_coef;
}
else
{
obj_diff = -(grad_diff * grad_diff) / TAU;
}
if (obj_diff <= obj_diff_min)
{
gmin_idx = j;
obj_diff_min = obj_diff;
}
}
}
}
}
if (gmax + gmax2 < this.Tolerance)
{
out_i = 0;
out_j = 0;
return(1);
}
out_i = gmax_idx;
out_j = gmin_idx;
return(0);
}
void Shrink()
{
float gmax1 = float.NegativeInfinity; // max { -y_i * grad(f)_i | i in I_up(\alpha) }
float gmax2 = float.NegativeInfinity; // max { y_i * grad(f)_i | i in I_low(\alpha) }
// find maximal violating pair first
for (int i = 0; i < activeSize; i++)
{
if (y[i] == 1)
{
if (!IsUpperBound(i))
{
if (-g[i] >= gmax1)
{
gmax1 = -g[i];
}
}
if (!IsLowerBound(i))
{
if (g[i] >= gmax2)
{
gmax2 = g[i];
}
}
}
else
{
if (!IsUpperBound(i))
{
if (-g[i] >= gmax2)
{
gmax2 = -g[i];
}
}
if (!IsLowerBound(i))
{
if (g[i] >= gmax1)
{
gmax1 = g[i];
}
}
}
}
if (!unshrink && gmax1 + gmax2 <= this.Tolerance * 10)
{
unshrink = true;
ReconstructGradient();
activeSize = numberOfVariables;
Trace.WriteLine("*");
}
for (int i = 0; i < activeSize; i++)
{
if (IsShrunk(i, gmax1, gmax2))
{
activeSize--;
while (activeSize > i)
{
if (!IsShrunk(activeSize, gmax1, gmax2))
{
SwapIndex(i, activeSize);
break;
}
activeSize--;
}
}
}
}
void ReconstructGradient()
{
// reconstruct inactive elements of G from G_bar and free variables
if (activeSize == numberOfVariables)
{
return;
}
Mathematics.Add(
numberOfVariables - activeSize,
gbar,
activeSize,
p,
activeSize,
g,
activeSize);
/*for (int j = activeSize; j < numberOfVariables; j++)
* {
* g[j] = gbar[j] + p[j];
* }*/
int freeCount = 0;
for (int j = 0; j < activeSize; j++)
{
if (IsFree(j))
{
freeCount++;
}
}
if (2 * freeCount < activeSize)
{
Trace.WriteLine("WARNING: using -h 0 may be faster");
}
if (freeCount * numberOfVariables > 2 * activeSize * (numberOfVariables - activeSize))
{
for (int i = activeSize; i < numberOfVariables; i++)
{
q(indices[i], indices, activeSize, temp);
for (int j = 0; j < activeSize; j++)
{
if (IsFree(j))
{
g[i] += alpha[j] * temp[j];
}
}
}
}
else
{
for (int i = 0; i < activeSize; i++)
{
if (IsFree(i))
{
q(indices[i], indices, numberOfVariables, temp);
Mathematics.AddProductC(
numberOfVariables - activeSize,
temp,
activeSize,
alpha[i],
g,
activeSize);
/*float alpha_i = alpha[i];
* for (int j = activeSize; j < numberOfVariables; j++)
* {
* g[j] += alpha_i * temp[j];
* }*/
}
}
}
}
void SwapIndex(int i, int j)
{
Swap(indices);
Swap(y);
Swap(g);
Swap(alphaStatus);
Swap(alpha);
Swap(p);
Swap(activeSet);
Swap(gbar);
void Swap <T>(T[] array)
{
T t = array[i];
array[i] = array[j];
array[j] = t;
}
}
/// <inheritdoc />
public float Loss(Tensor y, int[] expected, bool calculateGradient)
{
if (y == null)
{
throw new ArgumentNullException(nameof(y));
}
if (expected == null)
{
throw new ArgumentNullException(nameof(expected));
}
#pragma warning disable SA1312 // Variable names must begin with lower-case letter
int L = expected.Length; // Number of labels
int T = y.Axes[0]; // Number of mini-batches (time)
int A = y.Strides[0]; // Number of classes (alphabet size)
#pragma warning restore SA1312 // Variable names must begin with lower-case letter
int[] labels = CTCLoss.InsertBlanks(expected, A, this.BlankLabelIndex, out int repeats);
if (L + repeats > T)
{
if (calculateGradient)
{
Vectors.Copy(y.Length, y.Weights, 0, y.Gradient, 0);
}
// not enough elements to compute
return(float.PositiveInfinity);
}
#pragma warning disable SA1312 // Variable names must begin with lower-case letter
int S = labels.Length; // Number of labels with blanks
#pragma warning restore SA1312 // Variable names must begin with lower-case letter
// convert predicted probabilities into log space
float[] ylog = new float[y.Length];
Vectors.Log(y.Length, y.Weights, 0, ylog, 0);
// compute alphas
float[] alphas = new float[T * S];
////CTCLoss.CTCComputeAlphas(T, A, S, ylog, labels, alphas);
NativeMethods.CTCComputeAlphas(T, A, S, ylog, labels, alphas);
float logLossA = Mathematics.LogSumExp(alphas[alphas.Length - 1], alphas[alphas.Length - 2]);
if (float.IsNegativeInfinity(logLossA))
{
if (calculateGradient)
{
Vectors.Copy(y.Length, y.Weights, 0, y.Gradient, 0);
}
return(float.PositiveInfinity);
}
if (calculateGradient)
{
// compute betas
float[] betas = new float[T * S];
NativeMethods.CTCComputeBetas(T, A, S, ylog, labels, betas);
////float logLossB = MKL.LogSumExp(betas.Weights[0], betas.Weights[1]);
// compute unnormalized gradient
Mathematics.Add(alphas.Length, betas, 0, alphas, 0);
NativeMethods.CTCReduceAlphasBetas(T, A, S, alphas, labels, y.Gradient);
Mathematics.Sub(y.Length, ylog, 0, y.Gradient, 0);
Mathematics.SubC(y.Length, logLossA, y.Gradient, 0);
Vectors.Exp(y.Length, y.Gradient, 0);
// remove NaN
// NaN may come from various sources (for instance log(y) where y = 0)
Arrays.Replace(y.Length, y.Gradient, 0, float.NaN, 0.0f, y.Gradient, 0);
Debug.Assert(!float.IsNaN(y.Gradient[0]), "Tensor contains invalid weight.");
}
Debug.Assert(!float.IsNaN(logLossA), "Calculated loss is invalid.");
return(-logLossA);
}
/// <summary>
/// De-skews the <see cref="Image"/> and aligns it horizontally.
/// </summary>
/// <param name="dst">The destination <see cref="Image"/>. Can be <b>null</b>.</param>
/// <returns>
/// The destination <see cref="Image"/>.
/// </returns>
/// <exception cref="NotImplementedException">
/// <see cref="Image{T}.BitsPerPixel"/> is not one.
/// </exception>
/// <remarks>
/// <para>This method works with binary (1bpp) images only.</para>
/// </remarks>
public Image Deskew(Image dst)
{
if (this.BitsPerPixel != 1)
{
throw new NotImplementedException(Properties.Resources.E_UnsupportedDepth_1bpp);
}
int width = this.Width;
int height = this.Height;
int stride = this.Stride;
ulong[] bits = this.Bits;
// build histogram
ulong endMask = this.EndMask;
float[][] histogram = new float[stride][];
for (int ix = 0; ix < stride; ix++)
{
float[] h = histogram[ix] = new float[height];
ulong mask = ix == stride - 1 ? endMask : ulong.MaxValue;
for (int iy = 0, off = ix; iy < height; iy++, off += stride)
{
h[iy] = BitUtils.CountOneBits(bits[off] & mask);
}
}
// calculate image variance
float angleBest = 0.0f;
float varianceBest = EstimateSkewAngle(angleBest);
// move up or down with 1 degree interval
// move counterclockwise
for (float angle = -1.0f; angle >= -10.0f; angle -= 1.0f)
{
float variance = EstimateSkewAngle(angle);
if (variance <= varianceBest)
{
break;
}
varianceBest = variance;
angleBest = angle;
}
if (angleBest == 0.0f)
{
// move clockwise
for (float angle = 1.0f; angle <= 10.0f; angle += 1.0f)
{
float variance = EstimateSkewAngle(angle);
if (variance <= varianceBest)
{
break;
}
varianceBest = variance;
angleBest = angle;
}
}
// move up or down with 0.1 degree interval
// move counterclockwise
float originalAngle = angleBest;
for (float angle = angleBest - 0.1f, max = angleBest - 0.9f; angle >= max; angle -= 0.1f)
{
float variance = EstimateSkewAngle(angle);
if (variance <= varianceBest)
{
break;
}
varianceBest = variance;
angleBest = angle;
}
if (originalAngle == angleBest)
{
// move clockwise
for (float angle = angleBest + 0.1f, max = angleBest + 0.9f; angle <= max; angle += 0.1f)
{
float variance = EstimateSkewAngle(angle);
if (variance <= varianceBest)
{
break;
}
varianceBest = variance;
angleBest = angle;
}
}
return(this.Rotate(dst, -angleBest, BorderType.BorderRepl, 0));
float EstimateSkewAngle(float angle)
{
const float PiConv = 3.1415926535f / 180.0f;
int centerX = width / 2;
float dblTanA = (float)Math.Tan(angle * PiConv);
float[] ds = new float[height];
for (int ix = 0; ix < stride; ix++)
{
// negative shift is down
int shiftY = (dblTanA * (centerX - (ix * 64) - 32)).Round();
Mathematics.Add(
height - Math.Abs(shiftY),
histogram[ix],
shiftY < 0 ? 0 : shiftY,
ds,
shiftY < 0 ? -shiftY : 0);
}
return(ds.Variance());
}
}
public void FivePlusSix_Returns_Eleven()
{
int result = mathematics.Add(5, 6);
Assert.AreEqual(11, result, "Add method has to add second argument to the first");
}
private static void Concat(IList <Tensor> xs, int axis, Tensor y, bool useGradients)
{
float[] yw = useGradients ? y.Gradient : y.Weights;
if (axis == 0)
{
for (int i = 0, ii = xs.Count, offy = 0; i < ii; i++)
{
Tensor x = xs[i];
float[] xw = useGradients ? x.Gradient : x.Weights;
int xstride = x.Length;
if (useGradients)
{
Mathematics.Add(xstride, xw, 0, yw, offy);
}
else
{
Vectors.Copy(xstride, xw, 0, yw, offy);
}
offy += xstride;
}
}
else
{
/*for (int n = 0, nn = y.Length / y.Strides[axis - 1], offy = 0; n < nn; n++)
* {
* for (int i = 0, ii = xs.Count; i < ii; i++)
* {
* Tensor x = xs[i];
* int xstride = x.Strides[axis - 1];
*
* if (useGradients)
* {
* Mathematics.Add(xstride, x.Gradient, n * xstride, yw, offy);
* }
* else
* {
* SetCopy.Copy(xstride, x.Weights, n * xstride, yw, offy);
* }
*
* offy += xstride;
* }
* }*/
int ylen = y.Length;
int ystride = y.Strides[axis - 1];
for (int i = 0, offy = 0, ii = xs.Count; i < ii; i++)
{
Tensor x = xs[i];
float[] xw = useGradients ? x.Gradient : x.Weights;
int xstride = x.Strides[axis - 1];
if (useGradients)
{
for (int offx = 0, offyy = offy; offyy < ylen; offx += xstride, offyy += ystride)
{
Mathematics.Add(xstride, xw, offx, yw, offyy);
}
}
else
{
for (int offx = 0, offyy = offy; offyy < ylen; offx += xstride, offyy += ystride)
{
Vectors.Copy(xstride, xw, offx, yw, offyy);
}
}
offy += xstride;
}
}
}
/// <summary>
/// Learns a <see cref="KMeans"/> model that can map the given inputs to the desired outputs.
/// </summary>
/// <param name="k">The number of clusters.</param>
/// <param name="seeding">The cluster initialization algorithm.</param>
/// <param name="maxiter">The maximum number of iterations.</param>
/// <param name="distance">The distance function.</param>
/// <param name="x">The data points <paramref name="x"/> to clusterize.</param>
/// <param name="weights">The <c>weight</c> of importance for each data point.</param>
/// <param name="cancellationToken">The cancellationToken token used to notify the classifier that the operation should be canceled.</param>
/// <returns>
/// The <see cref="KMeans"/> clusterizer learned by this method.
/// </returns>
/// <exception cref="ArgumentNullException">
/// <para><paramref name="x"/> is <b>null</b>.</para>
/// <para>-or-</para>
/// <para><paramref name="distance"/> is <b>null</b>.</para>
/// </exception>
/// <exception cref="ArgumentException">
/// <para><paramref name="weights"/> is not <b>null</b> and the number of elements in <paramref name="weights"/> does not match the number of elements in <paramref name="x"/>.</para>
/// </exception>
public static KMeans Learn(
int k,
KMeansSeeding seeding,
int maxiter,
IVectorDistance <float, IVector <float>, float> distance,
IList <IVector <float> > x,
IList <float> weights,
CancellationToken cancellationToken)
{
if (x == null)
{
throw new ArgumentNullException(nameof(x));
}
if (weights != null && weights.Count != x.Count)
{
throw new ArgumentException("The number of weights must match the number of input vectors.", nameof(weights));
}
int sampleCount = x.Count;
int dimension = x[0].Length;
KMeansClusterCollection clusters = new KMeansClusterCollection(k, dimension, distance);
switch (seeding)
{
case KMeansSeeding.KMeansPlusPlus:
clusters.KMeansPlusPlusSeeding(x, weights, cancellationToken);
break;
default:
clusters.RandomSeeding(x, weights, cancellationToken);
break;
}
float[] counts = new float[k];
float[] means = new float[k * dimension];
object sync = new object();
for (int iter = 0; iter < maxiter; iter++)
{
cancellationToken.ThrowIfCancellationRequested();
// reset means and counts
if (iter > 0)
{
Vectors.Set(counts.Length, 0.0f, counts, 0);
Vectors.Set(means.Length, 0.0f, means, 0);
}
// assign vectors to new clusters
CommonParallel.For(
0,
sampleCount,
(a, b) =>
{
float[] lcounts = new float[counts.Length];
float[] lmeans = new float[means.Length];
for (int i = a; i < b; i++)
{
int index = clusters.Assign(x[i]);
float weight = weights?[i] ?? 1.0f;
lcounts[index] += weight;
x[i].AddProductC(weight, lmeans, index * dimension);
}
lock (sync)
{
Mathematics.Add(lcounts.Length, lcounts, 0, counts, 0);
Mathematics.Add(lmeans.Length, lmeans, 0, means, 0);
}
},
new ParallelOptions());
// calculate new centroids
for (int i = 0, off = 0; i < k; i++, off += dimension)
{
if (counts[i] != 0)
{
Mathematics.DivC(dimension, means, off, counts[i], clusters[i].Centroid, 0);
}
}
}
return(new KMeans(clusters)
{
Seeding = seeding,
});
}
public void ForwardBackwardTest1()
{
const int T = 2;
const int N = 3;
Session session = new Session();
SRNCell layer = new SRNCell(new Shape(new[] { -1, N }), RNNDirection.ForwardOnly, 2, MatrixLayout.RowMajor, null);
layer.W.Randomize(this.random);
layer.U.Randomize(this.random);
layer.B.Randomize(this.random);
////layer.W.Set(new float[] { 0.1f, 0.2f, -0.3f, 0.4f, 0.5f, 0.6f }); // 3x2 matrix
////layer.U.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f }); // 2x2 matrix
////layer.B.Set(new float[] { 0.1f, 0.2f }); // 2x1 vector
Tensor x = new Tensor(null, new[] { T, N });
x.Randomize(this.random);
////x.Set(new float[] { 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f });
IList <Tensor> xs = new[] { x };
IList <Tensor> ys = layer.Forward(session, xs);
float[] bw = layer.B.Weights;
Tensor expected = new Tensor(null, new[] { 2, 2 });
expected.Weights[0] = Matrix.DotProduct(3, layer.W.Weights, 0, x.Weights, 0) + bw[0];
expected.Weights[1] = Matrix.DotProduct(3, layer.W.Weights, 3, x.Weights, 0) + bw[1];
Nonlinearity.ReLU(2, expected.Weights, 0, expected.Weights, 0);
expected.Weights[2] = Matrix.DotProduct(3, layer.W.Weights, 0, x.Weights, 3) + bw[0] + Matrix.DotProduct(2, layer.U.Weights, 0, expected.Weights, 0);
expected.Weights[3] = Matrix.DotProduct(3, layer.W.Weights, 3, x.Weights, 3) + bw[1] + Matrix.DotProduct(2, layer.U.Weights, 2, expected.Weights, 0);
Nonlinearity.ReLU(2, expected.Weights, 2, expected.Weights, 2);
Helpers.AreTensorsEqual(expected, ys[0]);
// unroll the graph
////session.GetGradient(ys[0]).Randomize(this.random);
ys[0].SetGradient(new float[] { 0.1f, 0.2f, 0.3f, 0.4f });
session.Unroll();
////float[] dy = session.GetGradient(ys[0]).Weights.ToArray();
float[] dy = new float[] { 0.1f, 0.2f, 0.3f, 0.4f };
float[] expectedWG = new float[layer.W.Length];
float[] expectedUG = new float[layer.U.Length];
float[] expectedBG = new float[layer.B.Length];
float[] expectedDx = new float[x.Length];
for (int oi = 2, ii = 3; oi >= 0; oi -= 2, ii -= 3)
{
Nonlinearity.ReLUGradient(2, dy, oi, true, expected.Weights, oi, dy, oi);
// should be x' * dy
Matrix.VxV(MatrixLayout.ColumnMajor, 3, 2, x.Weights, ii, dy, oi, expectedWG, 0, false);
// should be W' * dy
Matrix.MxV(MatrixLayout.ColumnMajor, 3, 2, layer.W.Weights, 0, false, dy, oi, expectedDx, ii, true);
if (oi > 0)
{
// should be x(t-1)' * dy
Matrix.VxV(MatrixLayout.ColumnMajor, 2, 2, expected.Weights, oi - 2, dy, oi, expectedUG, 0, false);
// should be U' * dy
Matrix.MxV(MatrixLayout.ColumnMajor, 2, 2, layer.U.Weights, 0, false, dy, oi, dy, oi - 2, false);
////MKL.MxV(MatrixLayout.RowMajor, 2, 2, layer.U.Weights, 0, false, dy, oi, dy, oi - 2, false);
}
// should be dy
Mathematics.Add(2, dy, oi, expectedBG, 0);
}
Helpers.AreArraysEqual(expectedWG, layer.W.Gradient);
////Helpers.AreArraysEqual(expectedUG, layer.U.Gradient);
Helpers.AreArraysEqual(expectedBG, layer.B.Gradient);
Helpers.AreArraysEqual(expectedDx, x.Gradient);
}