public void AsumTest() { var xf = new[] { 1.0f, 1.0f, 1.0f }; var xd = new[] { 1.0, 1.0, 1.0 }; Assert.AreEqual(3.0f, Blas1.asum(3, xf, 1)); Assert.AreEqual(3.0, Blas1.asum(3, xd, 1)); }
public void IamaxTest() { var xf = new[] { 1.0f, -2.0f, 5.0f, 12.0f }; var xd = new[] { 1.0, -2.0, 5.0, 12.0, -3.0 }; Assert.AreEqual(3, Blas1.iamax(xf.Length, xf, 1)); Assert.AreEqual(3, Blas1.iamax(xd.Length, xd, 1)); }
/// <summary> /// Gets the normalized component vector. /// </summary> public RowVector NormalizedVector() { //get { double[] pc = new double[analysis.cols]; Blas1.dCopy(analysis.vStore, analysis.cols * index, 1, pc, 0, 1, analysis.cols); return(new RowVector(pc, pc.Length)); //} }
internal void Update(Layer preLayer, Connection connection) { Blas2.gemv(BlasLayout.RowMajor, BlasTranspose.NoTrans, connection.PostLayerSize, connection.PreLayerSize, 1.0f, connection.Weight, connection.PreLayerSize, preLayer.Unit, 1, 0.0f, Input, 1); Blas1.copy(PureSize, Function(Input), 1, Unit, 1); }
internal override void Update(float[] weight, float[] delta) { var velocity = _velocity ?? new float[delta.Length]; Blas1.scal(velocity.Length, MomentumValue, velocity, 1); Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1); _velocity = velocity; Blas1.axpy(weight.Length, 1.0f, velocity, 1, weight, 1); }
public void DotTest() { var xf = new[] { 1.0f, 1.0f, 1.0f }; var yf = new[] { 1.0f, 1.0f, 1.0f }; var xd = new[] { 1.0, 1.0, 1.0 }; var yd = new[] { 1.0, 1.0, 1.0 }; Assert.AreEqual(3.0f, Blas1.dot(3, xf, 1, yf, 1)); Assert.AreEqual(3.0, Blas1.dot(3, xd, 1, yd, 1)); }
internal override void Update(float[] weight, float[] delta) { var v = _v ?? new float[delta.Length]; var sq = new float[delta.Length]; Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; }); Blas1.scal(v.Length, RememberRate, v, 1); Blas1.axpy(sq.Length, 1.0f - RememberRate, sq, 1, v, 1); Parallel.For(0, weight.Length, i => { weight[i] -= LearningRate / (MathF.Sqrt(v[i]) + 1e-8f) * delta[i]; }); _v = v; }
public void CopyTest() { var xf = new[] { 1.0f, -1.0f, 0.0f }; var yf = new float[xf.Length]; var xd = new[] { 1.0, -1.0, 0.0 }; var yd = new double[xd.Length]; Blas1.copy(3, xf, 1, yf, 1); Blas1.copy(3, xd, 1, yd, 1); for (var i = 0; i < yf.Length; i++) { Assert.AreEqual(xf[i], yf[i]); Assert.AreEqual(xd[i], yd[i]); } }
internal override void Update(float[] weight, float[] delta) { if (_weightMemory == null) { Blas1.copy(weight.Length, weight, 1, out _weightMemory, 1); } var velocity = _velocity ?? new float[weight.Length]; Blas1.scal(velocity.Length, MomentumValue, velocity, 1); Blas1.axpy(velocity.Length, -LearningRate, delta, 1, velocity, 1); _velocity = velocity; Blas1.axpy(_weightMemory.Length, 1.0f, velocity, 1, _weightMemory, 1); Blas1.copy(_velocity.Length, _velocity, 1, out var ahead, 1); Blas1.scal(ahead.Length, MomentumValue, ahead, 1); Blas1.copy(_weightMemory.Length, _weightMemory, 1, weight, 1); Blas1.axpy(weight.Length, 1.0f, ahead, 1, weight, 1); }
public void AxpyTest() { var xf = new[] { 1.0f, 1.0f, 1.0f }; var yf = new[] { 1.0f, 1.0f, 1.0f }; var xd = new[] { 1.0, 1.0, 1.0 }; var yd = new[] { 1.0, 1.0, 1.0 }; Blas1.axpy(3, 2.0f, xf, 1, yf, 1); foreach (var t in yf) { Assert.AreEqual(3.0f, t); } Blas1.axpy(3, 2.0, xd, 1, yd, 1); foreach (var t in yd) { Assert.AreEqual(3.0, t); } }
static void Main(string[] args) { var x = new float[] { 1.0f, 1.0f, 1.0f }; WriteLine("Level1 BLAS sasum call test."); WriteLine(Blas1.sasum(x.Length, x, 1)); WriteLine("Level1 BLAS scopy call test."); Blas1.scopy(x.Length, x, 1, out var y, 1); for (var i = 0; i < y.Length; i++) { Write(y[i] + " "); } WriteLine("\n"); WriteLine("Level2 BLAS dgemv call test."); var ad = new double[] { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 }; var xd = new double[] { 1.0, 1.0, 1.0 }; var yd = new double[] { 0.0, 0.0, 0.0 }; Blas2.dgemv(CBlasLayout.RowMajor, CBlasTranspose.NoTrans, 3, 3, 1.0, ad, 3, xd, 1, 1.0, yd, 1); for (var i = 0; i < yd.Length; i++) { Write(yd[i] + " "); } WriteLine("\n"); WriteLine("LAPACK General Matrix call test."); var ag = new double[] { 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 2.0 }; var bg = new double[] { 6.0, 7.0, 12.0, 15.0 }; Lapack.dgetrf(LapackLayout.RowMajor, 4, 4, ag, 4, out var ipiv); Lapack.dgetrs(LapackLayout.RowMajor, LapackTranspose.N, 4, 1, ag, 4, ipiv, bg, 1); for (var i = 0; i < bg.Length; i++) { Write(bg[i] + " "); } WriteLine(); WriteLine("Please press Enter key..."); ReadLine(); }
public static void ClusteringTest(Network network, DataSet.DataSet data) { var correct = 0.0f; var count = 0.0f; WriteLine("Testing."); Write($"Success Rate : {0.0f:##0.00%}"); foreach (var datum in data.TestData()) { count += 1.0f; network.SetInputs(datum.Input); network.ForwardPropagation(); var maxIdx = Blas1.iamax(network.Output.Length, network.Output, 1); if (maxIdx == Blas1.iamax(datum.Output.Length, datum.Output, 1)) { correct += 1.0f; } Write($"\rSuccess Rate : {correct / count:##0.00%}"); } WriteLine(); }
public void ScalTest() { const float af = 2.0f; const double ad = -1.0; var xf = new[] { 1.0f, 1.0f, 1.0f }; Blas1.copy(xf.Length, xf, 1, out var mxf, 1); var xd = new[] { 1.0, 1.0, 1.0 }; Blas1.copy(xd.Length, xd, 1, out var mxd, 1); Blas1.scal(mxf.Length, af, mxf, 1); Blas1.scal(mxd.Length, ad, mxd, 1); for (var i = 0; i < mxf.Length; i++) { Assert.AreEqual(af * xf[i], mxf[i]); } for (var i = 0; i < mxd.Length; i++) { Assert.AreEqual(ad * xd[i], mxd[i]); } }
internal override void Update(float[] weight, float[] delta) { var m = _m ?? new float[delta.Length]; var v = _v ?? new float[delta.Length]; var sq = new float[delta.Length]; Parallel.For(0, sq.Length, i => { sq[i] = delta[i] * delta[i]; }); _beta1 *= Beta1; _beta2 *= Beta2; Blas1.scal(m.Length, Beta1, m, 1); Blas1.scal(v.Length, Beta2, v, 1); Blas1.axpy(m.Length, 1.0f - Beta1, delta, 1, m, 1); Blas1.axpy(v.Length, 1.0f - Beta2, sq, 1, v, 1); Parallel.For(0, weight.Length, i => { var mHat = m[i] / (1.0f - _beta1); var vHat = v[i] / (1.0f - _beta2); weight[i] -= LearningRate / (MathF.Sqrt(vHat) + Eps) * mHat; }); _m = m; _v = v; }
private static void Main() { CompareTimeDot(10); CompareTimeDot(100); CompareTimeDot(100000); CompareTimeLU(); void CompareTimeDot(int size) { var sw = new Stopwatch(); (var x, var y) = GenerateVector(); WriteLine($"Calc dot product by raw C# : size = {size}"); sw.Reset(); var res = 0.0; for (var i = 0; i < LoopDot; i++) { sw.Start(); res = Dot(x, y); sw.Stop(); } WriteLine($"Result : {res}\tTime : {sw.Elapsed / (double) LoopDot}"); WriteLine($"Calc dot product by BLAS : size = {size}"); sw.Reset(); for (var i = 0; i < LoopDot; i++) { sw.Start(); res = Blas1.dot(size, x, 1, y, 1); sw.Stop(); } WriteLine($"Result : {res}\tTime : {sw.Elapsed / (double) LoopDot}\n"); (double[] x, double[] y) GenerateVector() { x = new double[size]; y = new double[size]; for (var i = 0; i < size; i++) { x[i] = 1.0; y[i] = 1.0; } return(x, y); } } void CompareTimeLU() { const int M = 49; const int N = M * M; const double h = 1.0 / (M + 1); const double Heat = 4.0; var aBase = new double[N * N]; var bBase = new double[N]; var ipiv = new int[N]; for (var i = 1; i <= M; i++) { for (var j = 1; j <= M; j++) { var k = (j - 1) * M + i - 1; aBase[k * N + k] = 4.0 / (h * h); if (i > 1) { var kl = k - 1; aBase[kl * N + k] = -1.0 / (h * h); } if (i < M) { var kr = k + 1; aBase[kr * N + k] = -1.0 / (h * h); } if (j > 1) { var kd = k - M; aBase[kd * N + k] = -1.0 / (h * h); } if (j < M) { var ku = k + M; aBase[ku * N + k] = -1.0 / (h * h); } bBase[k] = Heat; } } var sw = new Stopwatch(); WriteLine("Calc Poisson eq by raw C#"); sw.Reset(); var res = new double[bBase.Length]; for (var i = 0; i < LoopLU; i++) { Blas1.copy(aBase.Length, aBase, 1, out var a, 1); Blas1.copy(bBase.Length, bBase, 1, out var b, 1); sw.Start(); Decomp(N, N, a, ipiv); Solve(N, N, a, b, ipiv); sw.Stop(); if (i == LoopLU - 1) { Blas1.copy(b.Length, b, 1, res, 1); } } WriteLine($"Result : {res[((M + 1) / 2 - 1) * M + M + 1]}\tTime : {sw.Elapsed / (double) LoopLU}"); WriteLine("Calc Poisson eq by LAPACK"); sw.Reset(); for (var i = 0; i < LoopLU; i++) { Blas1.copy(aBase.Length, aBase, 1, out var a, 1); Blas1.copy(bBase.Length, bBase, 1, out var b, 1); sw.Start(); Lapack.getrf(LapackLayout.RowMajor, N, N, a, N, ipiv); Lapack.getrs(LapackLayout.RowMajor, LapackTranspose.NoTrans, N, 1, a, N, ipiv, b, 1); sw.Stop(); if (i == LoopLU - 1) { Blas1.copy(b.Length, b, 1, res, 1); } } WriteLine($"Result : {res[((M + 1) / 2 - 1) * M + M + 1]}\tTime : {sw.Elapsed / (double) LoopLU}"); } }
// This method is due to Powell (http://en.wikipedia.org/wiki/Michael_J._D._Powell), but it is not what // is usually called Powell's Method (http://en.wikipedia.org/wiki/Powell%27s_method); Powell // developed that method in the 1960s, it was included in Numerical Recipes and is very popular. // This is a model trust algorithm developed by Powell in the 2000s. It typically uses many // fewer function evaluations, but does more intensive calculations between each evaluation. // This is basically the UOBYQA variant of Powell's new methods. It maintains a quadratic model // that interpolates between (d + 1) (d + 2) / 2 points. The model is trusted // within a given radius. At each step, it moves to the minimum of the model (or the boundary of // the trust region in that direction) and evaluates the function. The new value is incorporated // into the model and the trust region expanded or contracted depending on how accurate its // prediction of the function value was. // Papers on these methods are collected at http://mat.uc.pt/~zhang/software.html#powell_software. // The UOBYQA paper is here: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.28.1756. // The NEWUOA paper is here: http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2004_08.pdf. // The CONDOR system (http://www.applied-mathematics.net/optimization/CONDORdownload.html) is based on these same ideas. // The thesis of CONDOR's author (http://www.applied-mathematics.net/mythesis/index.html) was also helpful. // It should be very easy to extend this method to constrained optimization, either by incorporating the bounds into // the step limits or by mapping hyper-space into a hyper-cube. private static MultiExtremum FindMinimum_ModelTrust(MultiFunctor f, IReadOnlyList <double> x, double s, MultiExtremumSettings settings) { // Construct an initial model. QuadraticInterpolationModel model = QuadraticInterpolationModel.Construct(f, x, s); double trustRadius = s; while (f.EvaluationCount < settings.EvaluationBudget) { // Find the minimum point of the model within the trust radius double[] z = model.FindMinimum(trustRadius); double expectedValue = model.Evaluate(z); double deltaExpected = model.MinimumValue - expectedValue; // Evaluate the function at the suggested minimum double[] point = model.ConvertPoint(z); double value = f.Evaluate(point); double delta = model.MinimumValue - value; double tol = settings.ComputePrecision(Math.Min(model.MinimumValue, value)); // Note value can be way off, so use better of old best and new value to compute tol. // When we didn't do this before, we got value = infinity, so tol = infinity, and thus terminated! if (delta > 0.0 && settings.Listener != null) { MultiExtremum report = new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian()); settings.Listener(report); } // To terminate, we demand: a reduction, that the reduction be small, that the reduction be in line with // its expected value, that we have not run up against the trust boundary, and that the gradient is small. // I had wanted to demand delta > 0, but we run into some cases where delta keeps being very slightly // negative, typically orders of magnitude less than tol, causing the trust radius to shrink in an // endless cycle that causes our approximation to ultimately go sour, even though terminating on the original // very slightly negative delta would have produced an accurate estimate. So we tolerate this case for now. if ((delta <= tol) && (-0.25 * tol <= delta)) { // We demand that the model be decent, i.e. that the expected delta was within tol of the measured delta. if (Math.Abs(delta - deltaExpected) <= tol) { // We demand that the step not just be small because it ran up against the trust radius. // If it ran up against the trust radius, there is probably more to be hand by continuing. double zm = Blas1.dNrm2(z, 0, 1, z.Length); if (zm < trustRadius) { // Finally, we demand that the gradient be small. You might think this was obvious since // z was small, but if the Hessian is not positive definite // the interplay of the Hessian and the gradient can produce a small z even if the model looks nothing like a quadratic minimum. double gm = Blas1.dNrm2(model.GetGradient(), 0, 1, z.Length); if (gm * zm <= tol) { if (f.IsNegated) { value = -value; } return(new MultiExtremum(f.EvaluationCount, settings, point, value, Math.Max(Math.Abs(delta), 0.75 * tol), model.GetHessian())); } } } } // There are now three decisions to be made: // 1. How to change the trust radius // 2. Whether to accept the new point // 3. Which existing point to replace // If the actual change was very far from the expected change, reduce the trust radius. // If the expected change did a good job of predicting the actual change, increase the trust radius. if ((delta < 0.25 * deltaExpected) /*|| (8.0 * deltaExpected < delta)*/) { trustRadius = 0.5 * trustRadius; } else if ((0.75 * deltaExpected <= delta) /*&& (delta <= 2.0 * deltaExpected)*/) { trustRadius = 2.0 * trustRadius; } // It appears that the limits on delta being too large don't help, and even hurt if made too stringent. // Replace an old point with the new point. int iMax = 0; double fMax = model.values[0]; int iBad = 0; double fBad = model.ComputeBadness(0, z, point, value); for (int i = 1; i < model.values.Length; i++) { if (model.values[i] > fMax) { iMax = i; fMax = model.values[i]; } double bad = model.ComputeBadness(i, z, point, value); if (bad > fBad) { iBad = i; fBad = bad; } } // Use the new point as long as it is better than our worst existing point. if (value < fMax) { Debug.Assert(!Double.IsPositiveInfinity(value) && !Double.IsNaN(value)); model.ReplacePoint(iBad, point, z, value); } // There is some question about how best to choose which point to replace. // The largest value? The furthest away? The one closest to new min? } throw new NonconvergenceException(); }