public SegmentationSolution SegmentImage(Image2D <Color> image, ObjectBackgroundColorModels colorModels)
{
if (image == null)
{
throw new ArgumentNullException("image");
}
if (colorModels == null)
{
throw new ArgumentNullException("colorModels");
}
this.ImageSegmentator = new ImageSegmentator(
image,
colorModels,
this.ColorDifferencePairwiseTermCutoff,
this.ColorDifferencePairwiseTermWeight,
this.ConstantPairwiseTermWeight,
this.ObjectColorUnaryTermWeight,
this.BackgroundColorUnaryTermWeight,
this.ObjectShapeUnaryTermWeight,
this.BackgroundShapeUnaryTermWeight);
DebugConfiguration.WriteImportantDebugText(
"Segmented image size is {0}x{1}.",
this.ImageSegmentator.ImageSize.Width,
this.ImageSegmentator.ImageSize.Height);
SegmentationSolution solution = null;
try
{
if (this.ShapeModel == null)
{
throw new InvalidOperationException("Shape model must be specified before segmenting image.");
}
DebugConfiguration.WriteImportantDebugText("Running segmentation algorithm...");
this.IsRunning = true;
solution = this.SegmentCurrentImage();
if (solution == null)
{
throw new InvalidOperationException("Segmentation solution can not be null.");
}
}
finally
{
if (this.IsStopping)
{
this.WasStopped = true;
}
this.IsRunning = false;
this.IsStopping = false;
}
DebugConfiguration.WriteImportantDebugText("Finished");
return(solution);
}
protected override SegmentationSolution SegmentCurrentImage()
{
DebugConfiguration.WriteImportantDebugText("Performing initial segmentation...");
this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => ObjectBackgroundTerm.Zero);
Image2D <bool> prevMask = this.ImageSegmentator.GetLastSegmentationMask();
Shape prevShape = this.ShapeModel.FitMeanShape(
this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
double prevEnergy = 0;
for (int iteration = 1; iteration <= this.MaxIterationCount && !this.IsStopping; ++iteration)
{
this.WaitIfPaused();
DebugConfiguration.WriteImportantDebugText("Iteration {0}", iteration);
Image2D <bool> prevMaskCopy = prevMask;
Shape currentShape = this.ShapeFitter.Run(
prevShape,
(s, t) => this.ShapeMutator.MutateShape(s, this.ShapeModel, this.ImageSegmentator.ImageSize, t / this.ShapeFitter.StartTemperature),
s => this.CalcObjective(s, prevMaskCopy));
double currentEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.ShapeModel.CalculatePenalties(currentShape, new Vector(x, y)));
Image2D <bool> currentMask = this.ImageSegmentator.GetLastSegmentationMask();
int differentValues = Image2D <bool> .DifferentValueCount(prevMask, currentMask);
double changedPixelRate = (double)differentValues / (this.ImageSegmentator.ImageSize.Width * this.ImageSegmentator.ImageSize.Height);
DebugConfiguration.WriteImportantDebugText("On iteration {0}:", iteration);
DebugConfiguration.WriteImportantDebugText("Energy is {0:0.000}", currentEnergy);
DebugConfiguration.WriteImportantDebugText("Changed pixel rate is {0:0.000000}", changedPixelRate);
DebugConfiguration.WriteImportantDebugText();
if (iteration > this.MinIterationCount && changedPixelRate < this.MinChangeRate)
{
DebugConfiguration.WriteImportantDebugText("Changed pixel rate is too low, breaking...");
break;
}
prevMask = currentMask;
prevShape = currentShape;
prevEnergy = currentEnergy;
if (IterationFinished != null)
{
IterationFinished(this, new SegmentationIterationFinishedEventArgs(
iteration,
currentShape,
this.ImageSegmentator.GetLastSegmentationMask(),
this.ImageSegmentator.GetLastUnaryTerms(),
this.ImageSegmentator.GetLastShapeTerms()));
}
}
return(new SegmentationSolution(prevShape, prevMask, prevEnergy));
}
protected override SegmentationSolution SegmentCurrentImage()
{
if (this.Shape != null && this.Shape.Structure != this.ShapeModel.Structure)
{
throw new InvalidOperationException("Specified shape differs in structure from shape model.");
}
double segmentationEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms(
(x, y) => this.Shape == null ? ObjectBackgroundTerm.Zero : CalculateShapeTerms(new Vector(x, y)));
double shapeEnergy = this.Shape == null ? 0 : this.ShapeModel.CalculateEnergy(this.Shape);
double totalEnergy = segmentationEnergy + shapeEnergy * this.ShapeEnergyWeight;
DebugConfiguration.WriteImportantDebugText(
"Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000})",
totalEnergy,
segmentationEnergy,
this.ShapeEnergyWeight,
shapeEnergy);
return(new SegmentationSolution(this.Shape, this.ImageSegmentator.GetLastSegmentationMask(), totalEnergy));
}
private double CalcObjective(Shape shape, bool report)
{
this.UpdateShapeTerms(shape);
double shapeEnergy = this.ShapeModel.CalculateEnergy(shape);
double labelingEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.shapeTerms[x, y]);
double energy = shapeEnergy * this.ShapeEnergyWeight + labelingEnergy;
double additionalPenalty = this.AdditionalShapePenalty == null ? 0 : this.AdditionalShapePenalty(shape);
double totalEnergy = energy + additionalPenalty;
if (report)
{
DebugConfiguration.WriteImportantDebugText(
"Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000} + {4:0.0000})",
totalEnergy,
labelingEnergy,
this.ShapeEnergyWeight,
shapeEnergy,
additionalPenalty);
}
return(totalEnergy);
}
public static Mixture <VectorGaussian> Fit(MicrosoftResearch.Infer.Maths.Vector[] data, int componentCount, int retryCount, double tolerance = 1e-4)
{
Debug.Assert(data != null);
Debug.Assert(data.Length > componentCount * 3);
Debug.Assert(componentCount > 1);
Debug.Assert(retryCount >= 0);
int dimensions = data[0].Count;
// Find point boundary
MicrosoftResearch.Infer.Maths.Vector min = data[0].Clone();
MicrosoftResearch.Infer.Maths.Vector max = min.Clone();
for (int i = 1; i < data.Length; ++i)
{
Debug.Assert(dimensions == data[i].Count);
for (int j = 0; j < dimensions; ++j)
{
min[j] = Math.Min(min[j], data[i][j]);
max[j] = Math.Max(max[j], data[i][j]);
}
}
// Initialize solution
MicrosoftResearch.Infer.Maths.Vector[] means = new MicrosoftResearch.Infer.Maths.Vector[componentCount];
PositiveDefiniteMatrix[] covariances = new PositiveDefiniteMatrix[componentCount];
for (int i = 0; i < componentCount; ++i)
{
GenerateRandomMixtureComponent(min, max, out means[i], out covariances[i]);
}
double[] weights = Enumerable.Repeat(1.0 / componentCount, componentCount).ToArray();
// EM algorithm for GMM
double[,] expectations = new double[data.Length, componentCount];
double lastEstimate;
const double negativeInfinity = -1e+20;
bool convergenceDetected;
double currentEstimate = negativeInfinity;
do
{
lastEstimate = currentEstimate;
convergenceDetected = false;
// E-step: estimate expectations on hidden variables
for (int i = 0; i < data.Length; ++i)
{
double sum = 0;
for (int j = 0; j < componentCount; ++j)
{
expectations[i, j] =
Math.Exp(VectorGaussian.GetLogProb(data[i], means[j], covariances[j])) * weights[j];
sum += expectations[i, j];
}
for (int j = 0; j < componentCount; ++j)
{
expectations[i, j] /= sum;
}
}
// M-step:
// Re-estimate means
for (int j = 0; j < componentCount; ++j)
{
means[j] = MicrosoftResearch.Infer.Maths.Vector.Zero(dimensions);
double sum = 0;
for (int i = 0; i < data.Length; ++i)
{
means[j] += data[i] * expectations[i, j];
sum += expectations[i, j];
}
means[j] *= 1.0 / sum;
}
// Re-estimate covariances
for (int j = 0; j < componentCount; ++j)
{
Matrix covariance = new Matrix(dimensions, dimensions);
double sum = 0;
for (int i = 0; i < data.Length; ++i)
{
MicrosoftResearch.Infer.Maths.Vector dataDiff = data[i] - means[j];
covariance += dataDiff.Outer(dataDiff) * expectations[i, j];
sum += expectations[i, j];
}
covariance *= 1.0 / sum;
covariances[j] = new PositiveDefiniteMatrix(covariance);
if (covariances[j].LogDeterminant() < -30)
{
DebugConfiguration.WriteDebugText("Convergence detected for component {0}", j);
if (retryCount == 0)
{
throw new InvalidOperationException("Can't fit GMM. Retry number exceeded.");
}
retryCount -= 1;
GenerateRandomMixtureComponent(min, max, out means[j], out covariances[j]);
DebugConfiguration.WriteDebugText("Component {0} regenerated", j);
convergenceDetected = true;
}
}
if (convergenceDetected)
{
currentEstimate = negativeInfinity;
continue;
}
// Re-estimate weights
double expectationSum = 0;
for (int j = 0; j < componentCount; ++j)
{
weights[j] = 0;
for (int i = 0; i < data.Length; ++i)
{
weights[j] += expectations[i, j];
expectationSum += expectations[i, j];
}
}
for (int j = 0; j < componentCount; ++j)
{
weights[j] /= expectationSum;
}
// Compute likelihood estimate
currentEstimate = 0;
for (int i = 0; i < data.Length; ++i)
{
for (int j = 0; j < componentCount; ++j)
{
currentEstimate +=
expectations[i, j] * (VectorGaussian.GetLogProb(data[i], means[j], covariances[j]) + Math.Log(weights[j]));
}
}
DebugConfiguration.WriteDebugText("L={0:0.000000}", currentEstimate);
} while (convergenceDetected || (currentEstimate - lastEstimate > tolerance));
Mixture <VectorGaussian> result = new Mixture <VectorGaussian>();
for (int j = 0; j < componentCount; ++j)
{
result.Add(VectorGaussian.FromMeanAndVariance(means[j], covariances[j]), weights[j]);
}
DebugConfiguration.WriteDebugText("GMM successfully fitted.");
return(result);
}
public T Run(T startSolution, Func <T, double, T> mutationFunction, Func <T, double> objectiveFunction)
{
if (startSolution == null)
{
throw new ArgumentNullException("startSolution");
}
if (mutationFunction == null)
{
throw new ArgumentNullException("mutationFunction");
}
if (objectiveFunction == null)
{
throw new ArgumentNullException("objectiveFunction");
}
T bestSolution = startSolution;
double minObjective = objectiveFunction(bestSolution);
int lastUpdateIteration = 0;
int currentIteration = 0;
int iterationsFromLastReannealing = 0;
double prevObjective = minObjective;
T prevSolution = bestSolution;
while (currentIteration < this.MaxIterations && currentIteration - lastUpdateIteration < this.MaxStallingIterations)
{
double temperature = CalcTemperature(iterationsFromLastReannealing);
T currentSolution = mutationFunction(prevSolution, temperature);
double currentObjective = objectiveFunction(currentSolution);
double acceptanceProb = CalcAcceptanceProbability(prevObjective, currentObjective, temperature);
if (Random.Double() < acceptanceProb)
{
prevObjective = currentObjective;
prevSolution = currentSolution;
}
if (currentObjective < minObjective)
{
lastUpdateIteration = currentIteration;
minObjective = currentObjective;
bestSolution = currentSolution;
DebugConfiguration.WriteDebugText("Best solution update: {0:0.000}", minObjective);
}
++currentIteration;
if (currentIteration % this.reportRate == 0)
{
DebugConfiguration.WriteDebugText("Iteration {0}", currentIteration);
if (this.AnnealingProgress != null)
{
this.AnnealingProgress(this, new SimulatedAnnealingProgressEventArgs <T>(currentSolution, bestSolution));
}
}
if (iterationsFromLastReannealing >= this.ReannealingInterval)
{
iterationsFromLastReannealing = 0;
prevSolution = bestSolution;
DebugConfiguration.WriteDebugText("Reannealing");
}
else
{
++iterationsFromLastReannealing;
}
}
return(bestSolution);
}
private SortedSet <EnergyBound> BreadthFirstBranchAndBoundTraverse(ShapeConstraints constraints)
{
SortedSet <EnergyBound> front = new SortedSet <EnergyBound> {
this.CalculateEnergyBound(constraints)
};
int currentIteration = 1;
DateTime lastOutputTime = startTime;
int processedConstraintSets = 0;
while (!front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom) && !this.IsStopping)
{
this.WaitIfPaused();
EnergyBound parentLowerBound = front.Min;
front.Remove(parentLowerBound);
List <ShapeConstraints> expandedConstraints = parentLowerBound.Constraints.SplitMostFree(this.maxCoordFreedom, this.maxWidthFreedom);
foreach (ShapeConstraints constraintsSet in expandedConstraints)
{
EnergyBound lowerBound = this.CalculateEnergyBound(constraintsSet);
front.Add(lowerBound);
// Uncomment for strong invariants check
//ObjectBackgroundTerm[,] lowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height];
//for (int i = 0; i < this.segmentedImage.Width; ++i)
// for (int j = 0; j < this.segmentedImage.Height; ++j)
// lowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(lowerBound.Constraints, new Point(i, j));
//ObjectBackgroundTerm[,] parentLowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height];
//for (int i = 0; i < this.segmentedImage.Width; ++i)
// for (int j = 0; j < this.segmentedImage.Height; ++j)
// parentLowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(parentLowerBound.Constraints, new Point(i, j));
//for (int i = 0; i < this.segmentedImage.Width; ++i)
// for (int j = 0; j < this.segmentedImage.Height; ++j)
// {
// Debug.Assert(lowerBoundShapeTerm[i, j].ObjectTerm >= parentLowerBoundShapeTerm[i, j].ObjectTerm - 1e-7);
// Debug.Assert(lowerBoundShapeTerm[i, j].BackgroundTerm >= parentLowerBoundShapeTerm[i, j].BackgroundTerm - 1e-7);
// //CalculateShapeTerm(lowerBound.Constraints, new Point(0, 67));
// //CalculateShapeTerm(parentLowerBound.Constraints, new Point(0, 67));
// }
// Lower bound should not decrease (check always, it's important!)
Trace.Assert(lowerBound.SegmentationEnergy >= parentLowerBound.SegmentationEnergy - 1e-6);
Trace.Assert(lowerBound.ShapeEnergy >= parentLowerBound.ShapeEnergy - 1e-6);
//this.CalculateEnergyBound(lowerBound.Constraints);
//this.CalculateEnergyBound(parentLowerBound.Constraints);
++processedConstraintSets;
}
// Some debug output
if (currentIteration % this.ProgressReportRate == 0)
{
DateTime currentTime = DateTime.Now;
EnergyBound currentMin = front.Min;
DebugConfiguration.WriteDebugText(
"On iteration {0} front contains {1} constraint sets.", currentIteration, front.Count);
DebugConfiguration.WriteDebugText(
"Current lower bound is {0:0.0000} ({1:0.0000} + {2:0.0000}).",
currentMin.Bound,
currentMin.SegmentationEnergy,
currentMin.ShapeEnergy * this.ShapeEnergyWeight);
double processingSpeed = processedConstraintSets / (currentTime - lastOutputTime).TotalSeconds;
DebugConfiguration.WriteDebugText("Processing speed is {0:0.000} items per sec", processingSpeed);
double maxVertexConstraintsFreedom = currentMin.Constraints.VertexConstraints.Max(c => c.Freedom);
double maxEdgeConstraintsFreedom = currentMin.Constraints.EdgeConstraints.Max(c => c.Freedom);
DebugConfiguration.WriteDebugText(
"Max vertex freedom: {0:0.00}, max edge freedom: {1:0.00}",
maxVertexConstraintsFreedom,
maxEdgeConstraintsFreedom);
DebugConfiguration.WriteDebugText("Elapsed time: {0}", DateTime.Now - this.startTime);
DebugConfiguration.WriteDebugText();
this.ReportBranchAndBoundProgress(front);
lastOutputTime = currentTime;
processedConstraintSets = 0;
}
currentIteration += 1;
}
return(front);
}
protected override SegmentationSolution SegmentCurrentImage()
{
if (this.minEdgeWidth >= this.maxEdgeWidth)
{
throw new InvalidOperationException("Min edge width should be less than max edge width.");
}
if (this.startConstraints != null)
{
if (this.startConstraints.ShapeStructure != this.ShapeModel.Structure)
{
throw new InvalidOperationException("Given start constraints have shape structure different from the one specified in shape model.");
}
// TODO: make this check work
//foreach (VertexConstraints vertexConstraints in this.startConstraints.VertexConstraints)
//{
// RectangleF imageRectangle =
// new RectangleF(0, 0, this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
// if (!imageRectangle.Contains(vertexConstraints.CoordRectangle))
// throw new InvalidOperationException("Given start constraints are not fully inside the segmented image.");
//}
}
this.shapeUnaryTerms = new Image2D <ObjectBackgroundTerm>(
this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
ShapeConstraints constraints = this.startConstraints;
if (constraints == null)
{
constraints = ShapeConstraints.CreateFromBounds(
this.ShapeModel.Structure,
Vector.Zero,
new Vector(this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height),
this.minEdgeWidth,
this.maxEdgeWidth);
}
if (this.BranchAndBoundStarted != null)
{
this.BranchAndBoundStarted(this, EventArgs.Empty);
}
this.startTime = DateTime.Now;
DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound started.");
SortedSet <EnergyBound> front = this.BreadthFirstBranchAndBoundTraverse(constraints);
ReportBranchAndBoundCompletion(front.Min);
if (front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom))
{
DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound finished in {0}.",
DateTime.Now - this.startTime);
DebugConfiguration.WriteImportantDebugText("Best lower bound is {0:0.0000}", front.Min.Bound);
}
else
{
DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound forced to stop after {0}.", DateTime.Now - this.startTime);
DebugConfiguration.WriteImportantDebugText("Min energy value achieved is {0:0.0000}", front.Min.Bound);
}
EnergyBound collapsedBfsSolution = this.CalculateEnergyBound(front.Min.Constraints.Collapse());
Shape resultShape = front.Min.Constraints.CollapseToShape();
DebugConfiguration.WriteImportantDebugText(
"Collapsed solution energy value is {0:0.0000} ({1:0.0000} + {2:0.0000})",
collapsedBfsSolution.Bound,
collapsedBfsSolution.SegmentationEnergy,
collapsedBfsSolution.ShapeEnergy * this.ShapeEnergyWeight);
return(new SegmentationSolution(resultShape, this.ImageSegmentator.GetLastSegmentationMask(), collapsedBfsSolution.Bound));
}