/// <summary>
/// This function tests to make sure that the sum of the stochastic calculated gradients is equal to the
/// full gradient.
/// </summary>
/// <remarks>
/// This function tests to make sure that the sum of the stochastic calculated gradients is equal to the
/// full gradient. This requires using ordered sampling, so if the ObjectiveFunction itself randomizes
/// the inputs this function will likely fail.
/// </remarks>
/// <param name="x">is the point to evaluate the function at</param>
/// <param name="functionTolerance">is the tolerance to place on the infinity norm of the gradient and value</param>
/// <returns>boolean indicating success or failure.</returns>
public virtual bool TestDerivatives(double[] x, double functionTolerance)
{
bool ret = false;
bool compareHess = true;
log.Info("Making sure that the stochastic derivatives are ok.");
AbstractStochasticCachingDiffFunction.SamplingMethod tmpSampleMethod = thisFunc.sampleMethod;
StochasticCalculateMethods tmpMethod = thisFunc.method;
//Make sure that our function is using ordered sampling. Otherwise we have no gaurentees.
thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.Ordered;
if (thisFunc.method == StochasticCalculateMethods.NoneSpecified)
{
log.Info("No calculate method has been specified");
}
else
{
if (!thisFunc.method.CalculatesHessianVectorProduct())
{
compareHess = false;
}
}
approxValue = 0;
approxGrad = new double[x.Length];
curGrad = new double[x.Length];
Hv = new double[x.Length];
double percent = 0.0;
//This loop runs through all the batches and sums of the calculations to compare against the full gradient
for (int i = 0; i < numBatches; i++)
{
percent = 100 * ((double)i) / (numBatches);
//Can't figure out how to get a carriage return??? ohh well
System.Console.Error.Printf("%5.1f percent complete\n", percent);
// update the "hopefully" correct Hessian
thisFunc.method = tmpMethod;
System.Array.Copy(thisFunc.HdotVAt(x, v, testBatchSize), 0, Hv, 0, Hv.Length);
// Now get the hessian through finite difference
thisFunc.method = StochasticCalculateMethods.ExternalFiniteDifference;
System.Array.Copy(thisFunc.DerivativeAt(x, v, testBatchSize), 0, gradFD, 0, gradFD.Length);
thisFunc.recalculatePrevBatch = true;
System.Array.Copy(thisFunc.HdotVAt(x, v, gradFD, testBatchSize), 0, HvFD, 0, HvFD.Length);
//Compare the difference
double DiffHv = ArrayMath.Norm_inf(ArrayMath.PairwiseSubtract(Hv, HvFD));
//Keep track of the biggest H.v error
if (DiffHv > maxHvDiff)
{
maxHvDiff = DiffHv;
}
}
if (maxHvDiff < functionTolerance)
{
Sayln(string.Empty);
Sayln("Success: Hessian approximations lined up");
ret = true;
}
else
{
Sayln(string.Empty);
Sayln("Failure: Hessian approximation at somepoint was off by " + maxHvDiff);
ret = false;
}
thisFunc.sampleMethod = tmpSampleMethod;
thisFunc.method = tmpMethod;
return(ret);
}
/// <summary>
/// This function tests to make sure that the sum of the stochastic calculated gradients is equal to the
/// full gradient.
/// </summary>
/// <remarks>
/// This function tests to make sure that the sum of the stochastic calculated gradients is equal to the
/// full gradient. This requires using ordered sampling, so if the ObjectiveFunction itself randomizes
/// the inputs this function will likely fail.
/// </remarks>
/// <param name="x">is the point to evaluate the function at</param>
/// <param name="functionTolerance">is the tolerance to place on the infinity norm of the gradient and value</param>
/// <returns>boolean indicating success or failure.</returns>
public virtual bool TestSumOfBatches(double[] x, double functionTolerance)
{
bool ret = false;
log.Info("Making sure that the sum of stochastic gradients equals the full gradient");
AbstractStochasticCachingDiffFunction.SamplingMethod tmpSampleMethod = thisFunc.sampleMethod;
StochasticCalculateMethods tmpMethod = thisFunc.method;
//Make sure that our function is using ordered sampling. Otherwise we have no gaurentees.
thisFunc.sampleMethod = AbstractStochasticCachingDiffFunction.SamplingMethod.Ordered;
if (thisFunc.method == StochasticCalculateMethods.NoneSpecified)
{
log.Info("No calculate method has been specified");
}
approxValue = 0;
approxGrad = new double[x.Length];
curGrad = new double[x.Length];
fullGrad = new double[x.Length];
double percent = 0.0;
//This loop runs through all the batches and sums of the calculations to compare against the full gradient
for (int i = 0; i < numBatches; i++)
{
percent = 100 * ((double)i) / (numBatches);
// update the value
approxValue += thisFunc.ValueAt(x, v, testBatchSize);
// update the gradient
thisFunc.returnPreviousValues = true;
System.Array.Copy(thisFunc.DerivativeAt(x, v, testBatchSize), 0, curGrad, 0, curGrad.Length);
//Update Approximate
approxGrad = ArrayMath.PairwiseAdd(approxGrad, curGrad);
double norm = ArrayMath.Norm(approxGrad);
System.Console.Error.Printf("%5.1f percent complete %6.2f \n", percent, norm);
}
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// Get the full gradient and value, these should equal the approximates
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
log.Info("About to calculate the full derivative and value");
System.Array.Copy(thisFunc.DerivativeAt(x), 0, fullGrad, 0, fullGrad.Length);
thisFunc.returnPreviousValues = true;
fullValue = thisFunc.ValueAt(x);
diff = new double[x.Length];
if ((ArrayMath.Norm_inf(diff = ArrayMath.PairwiseSubtract(fullGrad, approxGrad))) < functionTolerance)
{
Sayln(string.Empty);
Sayln("Success: sum of batch gradients equals full gradient");
ret = true;
}
else
{
diffNorm = ArrayMath.Norm(diff);
Sayln(string.Empty);
Sayln("Failure: sum of batch gradients minus full gradient has norm " + diffNorm);
ret = false;
}
if (System.Math.Abs(approxValue - fullValue) < functionTolerance)
{
Sayln(string.Empty);
Sayln("Success: sum of batch values equals full value");
ret = true;
}
else
{
Sayln(string.Empty);
Sayln("Failure: sum of batch values minus full value has norm " + System.Math.Abs(approxValue - fullValue));
ret = false;
}
thisFunc.sampleMethod = tmpSampleMethod;
thisFunc.method = tmpMethod;
return(ret);
}