public MinimizationResult Train()
{
Vector <double> theta = Vector <double> .Build.Dense(X.ColumnCount);
LinearRegression lr = new LinearRegression(this.X, this.y, this.Lambda);
var obj = ObjectiveFunction.Gradient(lr.Cost, lr.Gradient);
var solver = new BfgsMinimizer(1e-5, 1e-5, 1e-5, 200);
MinimizationResult result = solver.FindMinimum(obj, theta);
return(result);
}
private void TuneLanguageModel(string lmPrefix, IList <IReadOnlyList <string> > tuneTargetCorpus, int ngramSize)
{
if (tuneTargetCorpus.Count == 0)
{
return;
}
var simplex = new NelderMeadSimplex(0.1, 200, 1.0);
MinimizationResult result = simplex.FindMinimum(w =>
CalculatePerplexity(tuneTargetCorpus, lmPrefix, ngramSize, w), Enumerable.Repeat(0.5, ngramSize * 3));
WriteLanguageModelWeightsFile(lmPrefix, ngramSize, result.MinimizingPoint);
Stats.LanguageModelPerplexity = result.ErrorValue;
}
public override int Extend(List <Point> points, List <double> weights, bool[] extended)
{
data.AddRange(points);
weightdata.AddRange(weights);
BfgsMinimizer minimizer = new BfgsMinimizer(1e-6, 1e-6, 1e-5);
MinimizationResult result = minimizer.FindMinimum(new Funcelu(data, weightdata), Vector <double> .Build.DenseOfArray(new double[3] {
this.middle.x, this.middle.y, this.middle.y
}));
//Console.WriteLine(result.MinimizingPoint);
this.middle = new Point(result.MinimizingPoint.ToArray());
this.radius = RadiusFromPoints(data, weightdata, this.middle);
//Console.WriteLine("Radius {0}", this.radius);
return(result.Iterations);
}
/// <summary>
/// Refines an intersection pair for two curves given an initial guess.<br/>
/// This is an unconstrained minimization, so the caller is responsible for providing a very good initial guess.
/// </summary>
/// <param name="crv0">The first curve.</param>
/// <param name="crv1">The second curve.</param>
/// <param name="firstGuess">The first guess parameter.</param>
/// <param name="secondGuess">The second guess parameter.</param>
/// <param name="tolerance">The value tolerance for the intersection.</param>
/// <returns>The results collected into the object <see cref="CurvesIntersectionResult"/>.</returns>
internal static CurvesIntersectionResult CurvesWithEstimation(NurbsBase crv0, NurbsBase crv1,
double firstGuess, double secondGuess, double tolerance)
{
IObjectiveFunction objectiveFunctions = new CurvesIntersectionObjectives(crv0, crv1);
Minimizer min = new Minimizer(objectiveFunctions);
MinimizationResult solution = min.UnconstrainedMinimizer(new Vector {
firstGuess, secondGuess
}, tolerance * tolerance);
// These are not the same points, also are used to filter where the intersection is not happening.
Point3 pt1 = crv0.PointAt(solution.SolutionPoint[0]);
Point3 pt2 = crv1.PointAt(solution.SolutionPoint[1]);
return(new CurvesIntersectionResult(pt1, pt2, solution.SolutionPoint[0], solution.SolutionPoint[1]));
}
public override bool ApplyOn()
{
//string outputString = GetOutputString(d => d.Name);
using (var csvFile = new CSVFiler(CsvPath))
{
Vector <double> x0 = SetInitialConditionsAndparameters();
WriteCsvRow(csvFile);
//MinimizationResult result = bfgsMinimizer.FindMinimum(objectiveFunctionWithGradient, x0);
MinimizationResult result = minimizer.FindMinimum(ObjectiveFunction, x0);
SetOptimum(result.MinimizingPoint);
WriteCsvRow(csvFile);
return(result.ReasonForExit == ExitCondition.Converged);
}
}
/// <summary>
/// Refines an intersection between a curve and a plane given an initial guess.<br/>
/// This is an unconstrained minimization, so the caller is responsible for providing a very good initial guess.
/// </summary>
/// <param name="crv">The curve to intersect.</param>
/// <param name="plane">The plane to intersect with the curve.</param>
/// <param name="firstGuess">The first guess parameter.</param>
/// <param name="secondGuess">The second guess parameter.</param>
/// <param name="tolerance">The value tolerance for the intersection.</param>
/// <returns>The results collected into the object <see cref="CurvePlaneIntersectionResult"/>.</returns>
internal static CurvePlaneIntersectionResult CurvePlaneWithEstimation(NurbsBase crv, Plane plane,
double firstGuess, double secondGuess, double tolerance)
{
IObjectiveFunction objectiveFunctions = new CurvePlaneIntersectionObjectives(crv, plane);
Minimizer min = new Minimizer(objectiveFunctions);
MinimizationResult solution = min.UnconstrainedMinimizer(new Vector {
firstGuess, secondGuess
}, tolerance * tolerance);
Point3 pt = crv.PointAt(solution.SolutionPoint[0]);
(double u, double v)parameters = plane.ClosestParameters(pt);
Vector uv = new Vector {
parameters.u, parameters.v, 0.0
};
return(new CurvePlaneIntersectionResult(pt, solution.SolutionPoint[0], uv));
}
public ThotSmtParameters Tune(ThotSmtParameters parameters,
IReadOnlyList <IReadOnlyList <string> > tuneSourceCorpus,
IReadOnlyList <IReadOnlyList <string> > tuneTargetCorpus, ThotTrainProgressReporter reporter,
SmtBatchTrainStats stats)
{
float sentLenWeight = parameters.ModelWeights[7];
int numFuncEvals = 0;
double Evaluate(Vector weights)
{
ThotSmtParameters newParameters = parameters.Clone();
newParameters.ModelWeights = weights.Select(w => (float)w).Concat(sentLenWeight).ToArray();
newParameters.Freeze();
double quality = CalculateBleu(newParameters, tuneSourceCorpus, tuneTargetCorpus);
numFuncEvals++;
if (numFuncEvals < MaxFunctionEvaluations && ProgressIncrementInterval > 0 &&
numFuncEvals % ProgressIncrementInterval == 0)
{
reporter.Step();
}
else
{
reporter.CheckCanceled();
}
return(quality);
};
var simplex = new NelderMeadSimplex(ConvergenceTolerance, MaxFunctionEvaluations, 1.0);
MinimizationResult result = simplex.FindMinimum(Evaluate,
parameters.ModelWeights.Select(w => (double)w).Take(7));
stats.TranslationModelBleu = 1.0 - result.ErrorValue;
ThotSmtParameters bestParameters = parameters.Clone();
bestParameters.ModelWeights = result.MinimizingPoint.Select(w => (float)w).Concat(sentLenWeight).ToArray();
bestParameters.Freeze();
return(bestParameters);
}
public ThotSmtParameters Tune(ThotSmtParameters parameters,
IReadOnlyList <IReadOnlyList <string> > tuneSourceCorpus,
IReadOnlyList <IReadOnlyList <string> > tuneTargetCorpus, SmtBatchTrainStats stats,
IProgress <ProgressStatus> progress)
{
float sentLenWeight = parameters.ModelWeights[7];
int numFuncEvals = 0;
double Evaluate(Vector weights)
{
ThotSmtParameters newParameters = parameters.Clone();
newParameters.ModelWeights = weights.Select(w => (float)w).Concat(sentLenWeight).ToArray();
newParameters.Freeze();
double quality = CalculateBleu(newParameters, tuneSourceCorpus, tuneTargetCorpus);
numFuncEvals++;
int currentStep = Math.Min(numFuncEvals, MaxProgressFunctionEvaluations);
progress.Report(new ProgressStatus(currentStep, MaxProgressFunctionEvaluations));
return(quality);
};
progress.Report(new ProgressStatus(0, MaxFunctionEvaluations));
var simplex = new NelderMeadSimplex(ConvergenceTolerance, MaxFunctionEvaluations, 1.0);
MinimizationResult result = simplex.FindMinimum(Evaluate,
parameters.ModelWeights.Select(w => (double)w).Take(7));
stats.TranslationModelBleu = 1.0 - result.ErrorValue;
ThotSmtParameters bestParameters = parameters.Clone();
bestParameters.ModelWeights = result.MinimizingPoint.Select(w => (float)w).Concat(sentLenWeight).ToArray();
bestParameters.Freeze();
if (result.FunctionEvaluationCount < MaxProgressFunctionEvaluations)
{
progress.Report(new ProgressStatus(1.0));
}
return(bestParameters);
}
public double ParameterAtChordLength(double t, double chordLength)
{
if (chordLength <= 0)
{
throw new ArgumentException("Chord length must be greater than 0.");
}
if (chordLength >= Length)
{
return(1.0);
}
IObjectiveFunction objectiveFunction = new ChordLengthObjective(this, t, chordLength);
Minimizer min = new Minimizer(objectiveFunction);
var lengthAtPrevious = LengthAt(t);
var initialGuess = ParameterAtLength(lengthAtPrevious + chordLength);
MinimizationResult solution = min.UnconstrainedMinimizer(new Vector {
initialGuess, initialGuess
});
return(solution.SolutionPoint[0]);
}
// BFGF Minimizer
public static Value Argmin(Value function, Value initial, Value tolerance, Netlist netlist, Style style, int s)
{
if (!(initial is ListValue <Value>))
{
throw new Error("argmin: expecting a list for second argument");
}
Vector <double> initialGuess = CreateVector.Dense((initial as ListValue <Value>).ToDoubleArray("argmin: expecting a list of numbers for second argument"));
if (!(tolerance is NumberValue))
{
throw new Error("argmin: expecting a number for third argument");
}
double toler = (tolerance as NumberValue).value;
if (!(function is FunctionValue))
{
throw new Error("argmin: expecting a function as first argument");
}
FunctionValue closure = function as FunctionValue;
if (closure.parameters.parameters.Count != 1)
{
throw new Error("argmin: initial values and function parameters have different lengths");
}
IObjectiveFunction objectiveFunction = ObjectiveFunction.Gradient(
(Vector <double> objParameters) => {
const string badResult = "argmin: objective function should return a list with a number (cost) and a list of numbers (partial derivatives of cost)";
List <Value> parameters = new List <Value>(); foreach (double parameter in objParameters)
{
parameters.Add(new NumberValue(parameter));
}
ListValue <Value> arg1 = new ListValue <Value>(parameters);
List <Value> arguments = new List <Value>(); arguments.Add(arg1);
bool autoContinue = netlist.autoContinue; netlist.autoContinue = true;
Value result = closure.ApplyReject(arguments, netlist, style, s);
if (result == null)
{
throw new Error(badResult);
}
netlist.autoContinue = autoContinue;
if (!(result is ListValue <Value>))
{
throw new Error(badResult);
}
List <Value> results = (result as ListValue <Value>).elements;
if (results.Count != 2 || !(results[0] is NumberValue) || !(results[1] is ListValue <Value>))
{
throw new Error(badResult);
}
double cost = (results[0] as NumberValue).value;
ListValue <Value> gradients = results[1] as ListValue <Value>;
KGui.gui.GuiOutputAppendText("argmin: parameters=" + arg1.Format(style) + " => cost=" + style.FormatDouble(cost) + ", gradients=" + results[1].Format(style) + Environment.NewLine);
return(new Tuple <double, Vector <double> >(cost, CreateVector.Dense(gradients.ToDoubleArray(badResult))));
});
try {
BfgsMinimizer minimizer = new BfgsMinimizer(toler, toler, toler);
MinimizationResult result = minimizer.FindMinimum(objectiveFunction, initialGuess);
if (result.ReasonForExit == ExitCondition.Converged || result.ReasonForExit == ExitCondition.AbsoluteGradient || result.ReasonForExit == ExitCondition.RelativeGradient)
{
List <Value> elements = new List <Value>();
for (int i = 0; i < result.MinimizingPoint.Count; i++)
{
elements.Add(new NumberValue(result.MinimizingPoint[i]));
}
ListValue <Value> list = new ListValue <Value>(elements);
KGui.gui.GuiOutputAppendText("argmin: converged with parameters " + list.Format(style) + " and reason '" + result.ReasonForExit + "'" + Environment.NewLine);
return(list);
}
else
{
throw new Error("reason '" + result.ReasonForExit.ToString() + "'");
}
} catch (Exception e) { throw new Error("argmin ended: " + ((e.InnerException == null) ? e.Message : e.InnerException.Message)); } // somehow we need to recatch the inner exception coming from CostAndGradient
}
/// <summary>
/// Finds the minimum of the objective function with an initial perturbation
/// </summary>
/// <param name="objectiveFunction">The objective function, no gradient or hessian needed</param>
/// <param name="initialGuess">The initial guess</param>
/// <param name="initalPertubation">The initial perturbation</param>
/// <returns>The minimum point</returns>
public static MinimizationResult Minimum(IObjectiveFunction objectiveFunction, Vector <double> initialGuess, Vector <double> initalPertubation, double convergenceTolerance = 1e-8, int maximumIterations = 1000)
{
// confirm that we are in a position to commence
if (objectiveFunction == null)
{
throw new ArgumentNullException(nameof(objectiveFunction), "ObjectiveFunction must be set to a valid ObjectiveFunctionDelegate");
}
if (initialGuess == null)
{
throw new ArgumentNullException(nameof(initialGuess), "initialGuess must be initialized");
}
if (initalPertubation == null)
{
throw new ArgumentNullException(nameof(initalPertubation), "initalPertubation must be initialized, if unknown use overloaded version of FindMinimum()");
}
SimplexConstant[] simplexConstants = SimplexConstant.CreateSimplexConstantsFromVectors(initialGuess, initalPertubation);
// create the initial simplex
int numDimensions = simplexConstants.Length;
int numVertices = numDimensions + 1;
Vector <double>[] vertices = InitializeVertices(simplexConstants);
double[] errorValues = new double[numVertices];
int evaluationCount = 0;
ExitCondition exitCondition = ExitCondition.None;
ErrorProfile errorProfile;
errorValues = InitializeErrorValues(vertices, objectiveFunction);
int numTimesHasConverged = 0;
// iterate until we converge, or complete our permitted number of iterations
while (true)
{
errorProfile = EvaluateSimplex(errorValues);
// see if the range in point heights is small enough to exit
// to handle the case when the function is symmetrical and extra iteration is performed
if (HasConverged(convergenceTolerance, errorProfile, errorValues))
{
numTimesHasConverged++;
}
else
{
numTimesHasConverged = 0;
}
if (numTimesHasConverged == 2)
{
exitCondition = ExitCondition.Converged;
break;
}
// attempt a reflection of the simplex
double reflectionPointValue = TryToScaleSimplex(-1.0, ref errorProfile, vertices, errorValues, objectiveFunction);
++evaluationCount;
if (reflectionPointValue <= errorValues[errorProfile.LowestIndex])
{
// it's better than the best point, so attempt an expansion of the simplex
double expansionPointValue = TryToScaleSimplex(2.0, ref errorProfile, vertices, errorValues, objectiveFunction);
++evaluationCount;
}
else if (reflectionPointValue >= errorValues[errorProfile.NextHighestIndex])
{
// it would be worse than the second best point, so attempt a contraction to look
// for an intermediate point
double currentWorst = errorValues[errorProfile.HighestIndex];
double contractionPointValue = TryToScaleSimplex(0.5, ref errorProfile, vertices, errorValues, objectiveFunction);
++evaluationCount;
if (contractionPointValue >= currentWorst)
{
// that would be even worse, so let's try to contract uniformly towards the low point;
// don't bother to update the error profile, we'll do it at the start of the
// next iteration
ShrinkSimplex(errorProfile, vertices, errorValues, objectiveFunction);
evaluationCount += numVertices; // that required one function evaluation for each vertex; keep track
}
}
// check to see if we have exceeded our alloted number of evaluations
if (evaluationCount >= maximumIterations)
{
throw new MaximumIterationsException(String.Format("Maximum iterations ({0}) reached.", maximumIterations));
}
}
objectiveFunction.EvaluateAt(vertices[errorProfile.LowestIndex]);
var regressionResult = new MinimizationResult(objectiveFunction, evaluationCount, exitCondition);
return(regressionResult);
}
