public StochasticDiffFunctionTester(IFunction function) { // check for derivatives if (!(function is AbstractStochasticCachingDiffFunction)) { log.Info("Attempt to test non stochastic function using StochasticDiffFunctionTester"); throw new NotSupportedException(); } thisFunc = (AbstractStochasticCachingDiffFunction)function; // Make sure the function is Stochastic generator = new Random(Runtime.CurrentTimeMillis()); // used to generate random test vectors // Look for a good batchSize to test with by getting factors testBatchSize = (int)GetTestBatchSize(thisFunc.DataDimension()); // Again make sure that our calculated batchSize is actually valid if (testBatchSize < 0 || testBatchSize > thisFunc.DataDimension() || (thisFunc.DataDimension() % testBatchSize != 0)) { log.Info("Invalid testBatchSize found, testing aborted. Data size: " + thisFunc.DataDimension() + " batchSize: " + testBatchSize); System.Environment.Exit(1); } numBatches = thisFunc.DataDimension() / testBatchSize; Sayln("StochasticDiffFunctionTester created with:"); Sayln(" data dimension = " + thisFunc.DataDimension()); Sayln(" batch size = " + testBatchSize); Sayln(" number of batches = " + numBatches); }
public virtual double[] GetVariance(double[] x, int batchSize) { double[] ret = new double[4]; double[] fullHx = new double[thisFunc.DomainDimension()]; double[] thisHx = new double[x.Length]; double[] thisGrad = new double[x.Length]; IList <double[]> HxList = new List <double[]>(); /* * PrintWriter file = null; * NumberFormat nf = new DecimalFormat("0.000E0"); * * try{ * file = new PrintWriter(new FileOutputStream("var.out"),true); * } * catch (IOException e){ * log.info("Caught IOException outputing List to file: " + e.getMessage()); * System.exit(1); * } */ //get the full hessian thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.Ordered; System.Array.Copy(thisFunc.DerivativeAt(x, x, thisFunc.DataDimension()), 0, thisGrad, 0, thisGrad.Length); System.Array.Copy(thisFunc.HdotVAt(x, x, thisGrad, thisFunc.DataDimension()), 0, fullHx, 0, fullHx.Length); double fullNorm = ArrayMath.Norm(fullHx); double hessScale = ((double)thisFunc.DataDimension()) / ((double)batchSize); thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.RandomWithReplacement; int n = 100; double simDelta; double ratDelta; double simMean = 0; double ratMean = 0; double simS = 0; double ratS = 0; int k = 0; log.Info(fullHx[4] + " " + x[4]); for (int i = 0; i < n; i++) { System.Array.Copy(thisFunc.DerivativeAt(x, x, batchSize), 0, thisGrad, 0, thisGrad.Length); System.Array.Copy(thisFunc.HdotVAt(x, x, thisGrad, batchSize), 0, thisHx, 0, thisHx.Length); ArrayMath.MultiplyInPlace(thisHx, hessScale); double thisNorm = ArrayMath.Norm(thisHx); double sim = ArrayMath.InnerProduct(thisHx, fullHx) / (thisNorm * fullNorm); double rat = thisNorm / fullNorm; k += 1; simDelta = sim - simMean; simMean += simDelta / k; simS += simDelta * (sim - simMean); ratDelta = rat - ratMean; ratMean += ratDelta / k; ratS += ratDelta * (rat - ratMean); } //file.println( nf.format(sim) + " , " + nf.format(rat)); double simVar = simS / (k - 1); double ratVar = ratS / (k - 1); //file.close(); ret[0] = simMean; ret[1] = simVar; ret[2] = ratMean; ret[3] = ratVar; return(ret); }
public virtual double[] Minimize(Func function, double functionTolerance, double[] initial, int maxIterations) { // check for stochastic derivatives if (!(function is AbstractStochasticCachingDiffFunction)) { throw new NotSupportedException(); } AbstractStochasticCachingDiffFunction dfunction = (AbstractStochasticCachingDiffFunction)function; dfunction.method = StochasticCalculateMethods.GradientOnly; /* --- * StochasticDiffFunctionTester sdft = new StochasticDiffFunctionTester(dfunction); * ArrayMath.add(initial, gen.nextDouble() ); // to make sure that priors are working. * sdft.testSumOfBatches(initial, 1e-4); * System.exit(1); * --- */ x = initial; grad = new double[x.Length]; newX = new double[x.Length]; gradList = new List <double[]>(); numBatches = dfunction.DataDimension() / bSize; outputFrequency = (int)System.Math.Ceil(((double)numBatches) / ((double)outputFrequency)); Init(dfunction); InitFiles(); bool have_max = (maxIterations > 0 || numPasses > 0); if (!have_max) { throw new NotSupportedException("No maximum number of iterations has been specified."); } else { maxIterations = System.Math.Max(maxIterations, numPasses) * numBatches; } Sayln(" Batchsize of: " + bSize); Sayln(" Data dimension of: " + dfunction.DataDimension()); Sayln(" Batches per pass through data: " + numBatches); Sayln(" Max iterations is = " + maxIterations); if (outputIterationsToFile) { infoFile.Println(function.DomainDimension() + "; DomainDimension "); infoFile.Println(bSize + "; batchSize "); infoFile.Println(maxIterations + "; maxIterations"); infoFile.Println(numBatches + "; numBatches "); infoFile.Println(outputFrequency + "; outputFrequency"); } //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // Loop //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Timing total = new Timing(); Timing current = new Timing(); total.Start(); current.Start(); for (k = 0; k < maxIterations; k++) { try { bool doEval = (k > 0 && evaluateIters > 0 && k % evaluateIters == 0); if (doEval) { DoEvaluation(x); } int pass = k / numBatches; int batch = k % numBatches; Say("Iter: " + k + " pass " + pass + " batch " + batch); // restrict number of saved gradients // (recycle memory of first gradient in list for new gradient) if (k > 0 && gradList.Count >= memory) { newGrad = gradList.Remove(0); } else { newGrad = new double[grad.Length]; } dfunction.hasNewVals = true; System.Array.Copy(dfunction.DerivativeAt(x, v, bSize), 0, newGrad, 0, newGrad.Length); ArrayMath.AssertFinite(newGrad, "newGrad"); gradList.Add(newGrad); grad = Smooth(gradList); //Get the next X TakeStep(dfunction); ArrayMath.AssertFinite(newX, "newX"); //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // THIS IS FOR DEBUG ONLY //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if (outputIterationsToFile && (k % outputFrequency == 0) && k != 0) { double curVal = dfunction.ValueAt(x); Say(" TrueValue{ " + curVal + " } "); file.Println(k + " , " + curVal + " , " + total.Report()); } //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // END OF DEBUG STUFF //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if (k >= maxIterations) { Sayln("Stochastic Optimization complete. Stopped after max iterations"); x = newX; break; } if (total.Report() >= maxTime) { Sayln("Stochastic Optimization complete. Stopped after max time"); x = newX; break; } System.Array.Copy(newX, 0, x, 0, x.Length); Say("[" + (total.Report()) / 1000.0 + " s "); Say("{" + (current.Restart() / 1000.0) + " s}] "); Say(" " + dfunction.LastValue()); if (quiet) { log.Info("."); } else { Sayln(string.Empty); } } catch (ArrayMath.InvalidElementException e) { log.Info(e.ToString()); for (int i = 0; i < x.Length; i++) { x[i] = double.NaN; } break; } } if (evaluateIters > 0) { // do final evaluation DoEvaluation(x); } if (outputIterationsToFile) { infoFile.Println(k + "; Iterations"); infoFile.Println((total.Report()) / 1000.0 + "; Completion Time"); infoFile.Println(dfunction.ValueAt(x) + "; Finalvalue"); infoFile.Close(); file.Close(); log.Info("Output Files Closed"); } //System.exit(1); Say("Completed in: " + (total.Report()) / 1000.0 + " s"); return(x); }