public void AsumTest()
{
var xf = new[] { 1.0f, 1.0f, 1.0f };
var xd = new[] { 1.0, 1.0, 1.0 };
Assert.AreEqual(3.0f, Blas1.asum(3, xf, 1));
Assert.AreEqual(3.0, Blas1.asum(3, xd, 1));
}
public void IamaxTest()
{
var xf = new[] { 1.0f, -2.0f, 5.0f, 12.0f };
var xd = new[] { 1.0, -2.0, 5.0, 12.0, -3.0 };
Assert.AreEqual(3, Blas1.iamax(xf.Length, xf, 1));
Assert.AreEqual(3, Blas1.iamax(xd.Length, xd, 1));
}
/// <summary>
/// Gets the normalized component vector.
/// </summary>
public RowVector NormalizedVector()
{
//get {
double[] pc = new double[analysis.cols];
Blas1.dCopy(analysis.vStore, analysis.cols * index, 1, pc, 0, 1, analysis.cols);
return(new RowVector(pc, pc.Length));
//}
}
internal void Update(Layer preLayer, Connection connection)
{
Blas2.gemv(BlasLayout.RowMajor, BlasTranspose.NoTrans,
connection.PostLayerSize, connection.PreLayerSize, 1.0f, connection.Weight, connection.PreLayerSize,
preLayer.Unit, 1, 0.0f, Input, 1);
Blas1.copy(PureSize, Function(Input), 1, Unit, 1);
}
internal override void Update(float[] weight, float[] delta)
{
var velocity = _velocity ?? new float[delta.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(weight.Length, 1.0f, velocity, 1, weight, 1);
}
public void DotTest()
{
var xf = new[] { 1.0f, 1.0f, 1.0f };
var yf = new[] { 1.0f, 1.0f, 1.0f };
var xd = new[] { 1.0, 1.0, 1.0 };
var yd = new[] { 1.0, 1.0, 1.0 };
Assert.AreEqual(3.0f, Blas1.dot(3, xf, 1, yf, 1));
Assert.AreEqual(3.0, Blas1.dot(3, xd, 1, yd, 1));
}
internal override void Update(float[] weight, float[] delta)
{
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
Blas1.scal(v.Length, RememberRate, v, 1);
Blas1.axpy(sq.Length, 1.0f - RememberRate, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => { weight[i] -= LearningRate / (MathF.Sqrt(v[i]) + 1e-8f) * delta[i]; });
_v = v;
}
public void CopyTest()
{
var xf = new[] { 1.0f, -1.0f, 0.0f };
var yf = new float[xf.Length];
var xd = new[] { 1.0, -1.0, 0.0 };
var yd = new double[xd.Length];
Blas1.copy(3, xf, 1, yf, 1);
Blas1.copy(3, xd, 1, yd, 1);
for (var i = 0; i < yf.Length; i++)
{
Assert.AreEqual(xf[i], yf[i]);
Assert.AreEqual(xd[i], yd[i]);
}
}
internal override void Update(float[] weight, float[] delta)
{
if (_weightMemory == null)
{
Blas1.copy(weight.Length, weight, 1, out _weightMemory, 1);
}
var velocity = _velocity ?? new float[weight.Length];
Blas1.scal(velocity.Length, MomentumValue, velocity, 1);
Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1);
_velocity = velocity;
Blas1.axpy(_weightMemory.Length, 1.0f, velocity, 1, _weightMemory, 1);
Blas1.copy(_velocity.Length, _velocity, 1, out var ahead, 1);
Blas1.scal(ahead.Length, MomentumValue, ahead, 1);
Blas1.copy(_weightMemory.Length, _weightMemory, 1, weight, 1);
Blas1.axpy(weight.Length, 1.0f, ahead, 1, weight, 1);
}
public void AxpyTest()
{
var xf = new[] { 1.0f, 1.0f, 1.0f };
var yf = new[] { 1.0f, 1.0f, 1.0f };
var xd = new[] { 1.0, 1.0, 1.0 };
var yd = new[] { 1.0, 1.0, 1.0 };
Blas1.axpy(3, 2.0f, xf, 1, yf, 1);
foreach (var t in yf)
{
Assert.AreEqual(3.0f, t);
}
Blas1.axpy(3, 2.0, xd, 1, yd, 1);
foreach (var t in yd)
{
Assert.AreEqual(3.0, t);
}
}
static void Main(string[] args)
{
var x = new float[] { 1.0f, 1.0f, 1.0f };
WriteLine("Level1 BLAS sasum call test.");
WriteLine(Blas1.sasum(x.Length, x, 1));
WriteLine("Level1 BLAS scopy call test.");
Blas1.scopy(x.Length, x, 1, out var y, 1);
for (var i = 0; i < y.Length; i++)
{
Write(y[i] + " ");
}
WriteLine("\n");
WriteLine("Level2 BLAS dgemv call test.");
var ad = new double[] { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 };
var xd = new double[] { 1.0, 1.0, 1.0 };
var yd = new double[] { 0.0, 0.0, 0.0 };
Blas2.dgemv(CBlasLayout.RowMajor, CBlasTranspose.NoTrans, 3, 3, 1.0, ad, 3, xd, 1, 1.0, yd, 1);
for (var i = 0; i < yd.Length; i++)
{
Write(yd[i] + " ");
}
WriteLine("\n");
WriteLine("LAPACK General Matrix call test.");
var ag = new double[] { 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0 };
var bg = new double[] { 6.0, 7.0, 12.0, 15.0 };
Lapack.dgetrf(LapackLayout.RowMajor, 4, 4, ag, 4, out var ipiv);
Lapack.dgetrs(LapackLayout.RowMajor, LapackTranspose.N, 4, 1, ag, 4, ipiv, bg, 1);
for (var i = 0; i < bg.Length; i++)
{
Write(bg[i] + " ");
}
WriteLine();
WriteLine("Please press Enter key...");
ReadLine();
}
public static void ClusteringTest(Network network, DataSet.DataSet data)
{
var correct = 0.0f;
var count = 0.0f;
WriteLine("Testing.");
Write($"Success Rate : {0.0f:##0.00%}");
foreach (var datum in data.TestData())
{
count += 1.0f;
network.SetInputs(datum.Input);
network.ForwardPropagation();
var maxIdx = Blas1.iamax(network.Output.Length, network.Output, 1);
if (maxIdx == Blas1.iamax(datum.Output.Length, datum.Output, 1))
{
correct += 1.0f;
}
Write($"\rSuccess Rate : {correct / count:##0.00%}");
}
WriteLine();
}
public void ScalTest()
{
const float af = 2.0f;
const double ad = -1.0;
var xf = new[] { 1.0f, 1.0f, 1.0f };
Blas1.copy(xf.Length, xf, 1, out var mxf, 1);
var xd = new[] { 1.0, 1.0, 1.0 };
Blas1.copy(xd.Length, xd, 1, out var mxd, 1);
Blas1.scal(mxf.Length, af, mxf, 1);
Blas1.scal(mxd.Length, ad, mxd, 1);
for (var i = 0; i < mxf.Length; i++)
{
Assert.AreEqual(af * xf[i], mxf[i]);
}
for (var i = 0; i < mxd.Length; i++)
{
Assert.AreEqual(ad * xd[i], mxd[i]);
}
}
internal override void Update(float[] weight, float[] delta)
{
var m = _m ?? new float[delta.Length];
var v = _v ?? new float[delta.Length];
var sq = new float[delta.Length];
Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; });
_beta1 *= Beta1;
_beta2 *= Beta2;
Blas1.scal(m.Length, Beta1, m, 1);
Blas1.scal(v.Length, Beta2, v, 1);
Blas1.axpy(m.Length, 1.0f - Beta1, delta, 1, m, 1);
Blas1.axpy(v.Length, 1.0f - Beta2, sq, 1, v, 1);
Parallel.For(0, weight.Length, i => {
var mHat = m[i] / (1.0f - _beta1);
var vHat = v[i] / (1.0f - _beta2);
weight[i] -= LearningRate / (MathF.Sqrt(vHat) + Eps) * mHat;
});
_m = m;
_v = v;
}
private static void Main()
{
CompareTimeDot(10);
CompareTimeDot(100);
CompareTimeDot(100000);
CompareTimeLU();
void CompareTimeDot(int size)
{
var sw = new Stopwatch();
(var x, var y) = GenerateVector();
WriteLine($"Calc dot product by raw C# : size = {size}");
sw.Reset();
var res = 0.0;
for (var i = 0; i < LoopDot; i++)
{
sw.Start();
res = Dot(x, y);
sw.Stop();
}
WriteLine($"Result : {res}\tTime : {sw.Elapsed / (double) LoopDot}");
WriteLine($"Calc dot product by BLAS : size = {size}");
sw.Reset();
for (var i = 0; i < LoopDot; i++)
{
sw.Start();
res = Blas1.dot(size, x, 1, y, 1);
sw.Stop();
}
WriteLine($"Result : {res}\tTime : {sw.Elapsed / (double) LoopDot}\n");
(double[] x, double[] y) GenerateVector()
{
x = new double[size];
y = new double[size];
for (var i = 0; i < size; i++)
{
x[i] = 1.0;
y[i] = 1.0;
}
return(x, y);
}
}
void CompareTimeLU()
{
const int M = 49;
const int N = M * M;
const double h = 1.0 / (M + 1);
const double Heat = 4.0;
var aBase = new double[N * N];
var bBase = new double[N];
var ipiv = new int[N];
for (var i = 1; i <= M; i++)
{
for (var j = 1; j <= M; j++)
{
var k = (j - 1) * M + i - 1;
aBase[k * N + k] = 4.0 / (h * h);
if (i > 1)
{
var kl = k - 1;
aBase[kl * N + k] = -1.0 / (h * h);
}
if (i < M)
{
var kr = k + 1;
aBase[kr * N + k] = -1.0 / (h * h);
}
if (j > 1)
{
var kd = k - M;
aBase[kd * N + k] = -1.0 / (h * h);
}
if (j < M)
{
var ku = k + M;
aBase[ku * N + k] = -1.0 / (h * h);
}
bBase[k] = Heat;
}
}
var sw = new Stopwatch();
WriteLine("Calc Poisson eq by raw C#");
sw.Reset();
var res = new double[bBase.Length];
for (var i = 0; i < LoopLU; i++)
{
Blas1.copy(aBase.Length, aBase, 1, out var a, 1);
Blas1.copy(bBase.Length, bBase, 1, out var b, 1);
sw.Start();
Decomp(N, N, a, ipiv);
Solve(N, N, a, b, ipiv);
sw.Stop();
if (i == LoopLU - 1)
{
Blas1.copy(b.Length, b, 1, res, 1);
}
}
WriteLine($"Result : {res[((M + 1) / 2 - 1) * M + M + 1]}\tTime : {sw.Elapsed / (double) LoopLU}");
WriteLine("Calc Poisson eq by LAPACK");
sw.Reset();
for (var i = 0; i < LoopLU; i++)
{
Blas1.copy(aBase.Length, aBase, 1, out var a, 1);
Blas1.copy(bBase.Length, bBase, 1, out var b, 1);
sw.Start();
Lapack.getrf(LapackLayout.RowMajor, N, N, a, N, ipiv);
Lapack.getrs(LapackLayout.RowMajor, LapackTranspose.NoTrans, N, 1, a, N, ipiv, b, 1);
sw.Stop();
if (i == LoopLU - 1)
{
Blas1.copy(b.Length, b, 1, res, 1);
}
}
WriteLine($"Result : {res[((M + 1) / 2 - 1) * M + M + 1]}\tTime : {sw.Elapsed / (double) LoopLU}");
}
}
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what
// is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell
// developed that method in the 1960s, it was included in Numerical Recipes and is very popular.
// This is a model trust algorithm developed by Powell in the 2000s. It typically uses many
// fewer function evaluations, but does more intensive calculations between each evaluation.
// This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model
// that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted
// within a given radius. At each step, it moves to the minimum of the model (or the boundary of
// the trust region in that direction) and evaluates the function. The new value is incorporated
// into the model and the trust region expanded or contracted depending on how accurate its
// prediction of the function value was.
// Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software.
// The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756.
// The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf.
// The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas.
// The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful.
// It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into
// the step limits or by mapping hyper-space into a hyper-cube.
private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IReadOnlyList <double> x, double s, MultiExtremumSettings settings)
{
// Construct an initial model.
QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s);
double trustRadius = s;
while (f.EvaluationCount < settings.EvaluationBudget)
{
// Find the minimum point of the model within the trust radius
double[] z = model.FindMinimum(trustRadius);
double expectedValue = model.Evaluate(z);
double deltaExpected = model.MinimumValue - expectedValue;
// Evaluate the function at the suggested minimum
double[] point = model.ConvertPoint(z);
double value = f.Evaluate(point);
double delta = model.MinimumValue - value;
double tol = settings.ComputePrecision(Math.Min(model.MinimumValue, value));
// Note value can be way off, so use better of old best and new value to compute tol.
// When we didn't do this before, we got value = infinity, so tol = infinity, and thus terminated!
if (delta > 0.0 && settings.Listener != null)
{
MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian());
settings.Listener(report);
}
// To terminate, we demand: a reduction, that the reduction be small, that the reduction be in line with
// its expected value, that we have not run up against the trust boundary, and that the gradient is small.
// I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly
// negative, typically orders of magnitude less than tol, causing the trust radius to shrink in an
// endless cycle that causes our approximation to ultimately go sour, even though terminating on the original
// very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now.
if ((delta <= tol) && (-0.25 * tol <= delta))
{
// We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta.
if (Math.Abs(delta - deltaExpected) <= tol)
{
// We demand that the step not just be small because it ran up against the trust radius.
// If it ran up against the trust radius, there is probably more to be hand by continuing.
double zm = Blas1.dNrm2(z, 0, 1, z.Length);
if (zm < trustRadius)
{
// Finally, we demand that the gradient be small. You might think this was obvious since
// z was small, but if the Hessian is not positive definite
// the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum.
double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length);
if (gm * zm <= tol)
{
if (f.IsNegated)
{
value = -value;
}
return(new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()));
}
}
}
}
// There are now three decisions to be made:
// 1. How to change the trust radius
// 2. Whether to accept the new point
// 3. Which existing point to replace
// If the actual change was very far from the expected change, reduce the trust radius.
// If the expected change did a good job of predicting the actual change, increase the trust radius.
if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/)
{
trustRadius = 0.5 * trustRadius;
}
else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/)
{
trustRadius = 2.0 * trustRadius;
}
// It appears that the limits on delta being too large don't help, and even hurt if made too stringent.
// Replace an old point with the new point.
int iMax = 0; double fMax = model.values[0];
int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value);
for (int i = 1; i < model.values.Length; i++)
{
if (model.values[i] > fMax)
{
iMax = i; fMax = model.values[i];
}
double bad = model.ComputeBadness(i, z, point, value);
if (bad > fBad)
{
iBad = i; fBad = bad;
}
}
// Use the new point as long as it is better than our worst existing point.
if (value < fMax)
{
Debug.Assert(!Double.IsPositiveInfinity(value) && !Double.IsNaN(value));
model.ReplacePoint(iBad, point, z, value);
}
// There is some question about how best to choose which point to replace.
// The largest value? The furthest away? The one closest to new min?
}
throw new NonconvergenceException();
}
