public StochasticDiffFunctionTester(IFunction function)
{
// check for derivatives
if (!(function is AbstractStochasticCachingDiffFunction))
{
log.Info("Attempt to test non stochastic function using StochasticDiffFunctionTester");
throw new NotSupportedException();
}
thisFunc = (AbstractStochasticCachingDiffFunction)function;
// Make sure the function is Stochastic
generator = new Random(Runtime.CurrentTimeMillis());
// used to generate random test vectors
// Look for a good batchSize to test with by getting factors
testBatchSize = (int)GetTestBatchSize(thisFunc.DataDimension());
// Again make sure that our calculated batchSize is actually valid
if (testBatchSize < 0 || testBatchSize > thisFunc.DataDimension() || (thisFunc.DataDimension() % testBatchSize != 0))
{
log.Info("Invalid testBatchSize found, testing aborted. Data size: " + thisFunc.DataDimension() + " batchSize: " + testBatchSize);
System.Environment.Exit(1);
}
numBatches = thisFunc.DataDimension() / testBatchSize;
Sayln("StochasticDiffFunctionTester created with:");
Sayln(" data dimension = " + thisFunc.DataDimension());
Sayln(" batch size = " + testBatchSize);
Sayln(" number of batches = " + numBatches);
}
public virtual double[] GetVariance(double[] x, int batchSize)
{
double[] ret = new double[4];
double[] fullHx = new double[thisFunc.DomainDimension()];
double[] thisHx = new double[x.Length];
double[] thisGrad = new double[x.Length];
IList <double[]> HxList = new List <double[]>();
/*
* PrintWriter file = null;
* NumberFormat nf = new DecimalFormat("0.000E0");
*
* try{
* file = new PrintWriter(new FileOutputStream("var.out"),true);
* }
* catch (IOException e){
* log.info("Caught IOException outputing List to file: " + e.getMessage());
* System.exit(1);
* }
*/
//get the full hessian
thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.Ordered;
System.Array.Copy(thisFunc.DerivativeAt(x, x, thisFunc.DataDimension()), 0, thisGrad, 0, thisGrad.Length);
System.Array.Copy(thisFunc.HdotVAt(x, x, thisGrad, thisFunc.DataDimension()), 0, fullHx, 0, fullHx.Length);
double fullNorm = ArrayMath.Norm(fullHx);
double hessScale = ((double)thisFunc.DataDimension()) / ((double)batchSize);
thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.RandomWithReplacement;
int n = 100;
double simDelta;
double ratDelta;
double simMean = 0;
double ratMean = 0;
double simS = 0;
double ratS = 0;
int k = 0;
log.Info(fullHx[4] + " " + x[4]);
for (int i = 0; i < n; i++)
{
System.Array.Copy(thisFunc.DerivativeAt(x, x, batchSize), 0, thisGrad, 0, thisGrad.Length);
System.Array.Copy(thisFunc.HdotVAt(x, x, thisGrad, batchSize), 0, thisHx, 0, thisHx.Length);
ArrayMath.MultiplyInPlace(thisHx, hessScale);
double thisNorm = ArrayMath.Norm(thisHx);
double sim = ArrayMath.InnerProduct(thisHx, fullHx) / (thisNorm * fullNorm);
double rat = thisNorm / fullNorm;
k += 1;
simDelta = sim - simMean;
simMean += simDelta / k;
simS += simDelta * (sim - simMean);
ratDelta = rat - ratMean;
ratMean += ratDelta / k;
ratS += ratDelta * (rat - ratMean);
}
//file.println( nf.format(sim) + " , " + nf.format(rat));
double simVar = simS / (k - 1);
double ratVar = ratS / (k - 1);
//file.close();
ret[0] = simMean;
ret[1] = simVar;
ret[2] = ratMean;
ret[3] = ratVar;
return(ret);
}
public virtual double[] Minimize(Func function, double functionTolerance, double[] initial, int maxIterations)
{
// check for stochastic derivatives
if (!(function is AbstractStochasticCachingDiffFunction))
{
throw new NotSupportedException();
}
AbstractStochasticCachingDiffFunction dfunction = (AbstractStochasticCachingDiffFunction)function;
dfunction.method = StochasticCalculateMethods.GradientOnly;
/* ---
* StochasticDiffFunctionTester sdft = new StochasticDiffFunctionTester(dfunction);
* ArrayMath.add(initial, gen.nextDouble() ); // to make sure that priors are working.
* sdft.testSumOfBatches(initial, 1e-4);
* System.exit(1);
* --- */
x = initial;
grad = new double[x.Length];
newX = new double[x.Length];
gradList = new List <double[]>();
numBatches = dfunction.DataDimension() / bSize;
outputFrequency = (int)System.Math.Ceil(((double)numBatches) / ((double)outputFrequency));
Init(dfunction);
InitFiles();
bool have_max = (maxIterations > 0 || numPasses > 0);
if (!have_max)
{
throw new NotSupportedException("No maximum number of iterations has been specified.");
}
else
{
maxIterations = System.Math.Max(maxIterations, numPasses) * numBatches;
}
Sayln(" Batchsize of: " + bSize);
Sayln(" Data dimension of: " + dfunction.DataDimension());
Sayln(" Batches per pass through data: " + numBatches);
Sayln(" Max iterations is = " + maxIterations);
if (outputIterationsToFile)
{
infoFile.Println(function.DomainDimension() + "; DomainDimension ");
infoFile.Println(bSize + "; batchSize ");
infoFile.Println(maxIterations + "; maxIterations");
infoFile.Println(numBatches + "; numBatches ");
infoFile.Println(outputFrequency + "; outputFrequency");
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// Loop
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Timing total = new Timing();
Timing current = new Timing();
total.Start();
current.Start();
for (k = 0; k < maxIterations; k++)
{
try
{
bool doEval = (k > 0 && evaluateIters > 0 && k % evaluateIters == 0);
if (doEval)
{
DoEvaluation(x);
}
int pass = k / numBatches;
int batch = k % numBatches;
Say("Iter: " + k + " pass " + pass + " batch " + batch);
// restrict number of saved gradients
// (recycle memory of first gradient in list for new gradient)
if (k > 0 && gradList.Count >= memory)
{
newGrad = gradList.Remove(0);
}
else
{
newGrad = new double[grad.Length];
}
dfunction.hasNewVals = true;
System.Array.Copy(dfunction.DerivativeAt(x, v, bSize), 0, newGrad, 0, newGrad.Length);
ArrayMath.AssertFinite(newGrad, "newGrad");
gradList.Add(newGrad);
grad = Smooth(gradList);
//Get the next X
TakeStep(dfunction);
ArrayMath.AssertFinite(newX, "newX");
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// THIS IS FOR DEBUG ONLY
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if (outputIterationsToFile && (k % outputFrequency == 0) && k != 0)
{
double curVal = dfunction.ValueAt(x);
Say(" TrueValue{ " + curVal + " } ");
file.Println(k + " , " + curVal + " , " + total.Report());
}
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// END OF DEBUG STUFF
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if (k >= maxIterations)
{
Sayln("Stochastic Optimization complete. Stopped after max iterations");
x = newX;
break;
}
if (total.Report() >= maxTime)
{
Sayln("Stochastic Optimization complete. Stopped after max time");
x = newX;
break;
}
System.Array.Copy(newX, 0, x, 0, x.Length);
Say("[" + (total.Report()) / 1000.0 + " s ");
Say("{" + (current.Restart() / 1000.0) + " s}] ");
Say(" " + dfunction.LastValue());
if (quiet)
{
log.Info(".");
}
else
{
Sayln(string.Empty);
}
}
catch (ArrayMath.InvalidElementException e)
{
log.Info(e.ToString());
for (int i = 0; i < x.Length; i++)
{
x[i] = double.NaN;
}
break;
}
}
if (evaluateIters > 0)
{
// do final evaluation
DoEvaluation(x);
}
if (outputIterationsToFile)
{
infoFile.Println(k + "; Iterations");
infoFile.Println((total.Report()) / 1000.0 + "; Completion Time");
infoFile.Println(dfunction.ValueAt(x) + "; Finalvalue");
infoFile.Close();
file.Close();
log.Info("Output Files Closed");
}
//System.exit(1);
Say("Completed in: " + (total.Report()) / 1000.0 + " s");
return(x);
}
