public SegmentationSolution SegmentImage(Image2D <Color> image, ObjectBackgroundColorModels colorModels) { if (image == null) { throw new ArgumentNullException("image"); } if (colorModels == null) { throw new ArgumentNullException("colorModels"); } this.ImageSegmentator = new ImageSegmentator( image, colorModels, this.ColorDifferencePairwiseTermCutoff, this.ColorDifferencePairwiseTermWeight, this.ConstantPairwiseTermWeight, this.ObjectColorUnaryTermWeight, this.BackgroundColorUnaryTermWeight, this.ObjectShapeUnaryTermWeight, this.BackgroundShapeUnaryTermWeight); DebugConfiguration.WriteImportantDebugText( "Segmented image size is {0}x{1}.", this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height); SegmentationSolution solution = null; try { if (this.ShapeModel == null) { throw new InvalidOperationException("Shape model must be specified before segmenting image."); } DebugConfiguration.WriteImportantDebugText("Running segmentation algorithm..."); this.IsRunning = true; solution = this.SegmentCurrentImage(); if (solution == null) { throw new InvalidOperationException("Segmentation solution can not be null."); } } finally { if (this.IsStopping) { this.WasStopped = true; } this.IsRunning = false; this.IsStopping = false; } DebugConfiguration.WriteImportantDebugText("Finished"); return(solution); }
protected override SegmentationSolution SegmentCurrentImage() { DebugConfiguration.WriteImportantDebugText("Performing initial segmentation..."); this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => ObjectBackgroundTerm.Zero); Image2D <bool> prevMask = this.ImageSegmentator.GetLastSegmentationMask(); Shape prevShape = this.ShapeModel.FitMeanShape( this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height); double prevEnergy = 0; for (int iteration = 1; iteration <= this.MaxIterationCount && !this.IsStopping; ++iteration) { this.WaitIfPaused(); DebugConfiguration.WriteImportantDebugText("Iteration {0}", iteration); Image2D <bool> prevMaskCopy = prevMask; Shape currentShape = this.ShapeFitter.Run( prevShape, (s, t) => this.ShapeMutator.MutateShape(s, this.ShapeModel, this.ImageSegmentator.ImageSize, t / this.ShapeFitter.StartTemperature), s => this.CalcObjective(s, prevMaskCopy)); double currentEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.ShapeModel.CalculatePenalties(currentShape, new Vector(x, y))); Image2D <bool> currentMask = this.ImageSegmentator.GetLastSegmentationMask(); int differentValues = Image2D <bool> .DifferentValueCount(prevMask, currentMask); double changedPixelRate = (double)differentValues / (this.ImageSegmentator.ImageSize.Width * this.ImageSegmentator.ImageSize.Height); DebugConfiguration.WriteImportantDebugText("On iteration {0}:", iteration); DebugConfiguration.WriteImportantDebugText("Energy is {0:0.000}", currentEnergy); DebugConfiguration.WriteImportantDebugText("Changed pixel rate is {0:0.000000}", changedPixelRate); DebugConfiguration.WriteImportantDebugText(); if (iteration > this.MinIterationCount && changedPixelRate < this.MinChangeRate) { DebugConfiguration.WriteImportantDebugText("Changed pixel rate is too low, breaking..."); break; } prevMask = currentMask; prevShape = currentShape; prevEnergy = currentEnergy; if (IterationFinished != null) { IterationFinished(this, new SegmentationIterationFinishedEventArgs( iteration, currentShape, this.ImageSegmentator.GetLastSegmentationMask(), this.ImageSegmentator.GetLastUnaryTerms(), this.ImageSegmentator.GetLastShapeTerms())); } } return(new SegmentationSolution(prevShape, prevMask, prevEnergy)); }
protected override SegmentationSolution SegmentCurrentImage() { if (this.Shape != null && this.Shape.Structure != this.ShapeModel.Structure) { throw new InvalidOperationException("Specified shape differs in structure from shape model."); } double segmentationEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms( (x, y) => this.Shape == null ? ObjectBackgroundTerm.Zero : CalculateShapeTerms(new Vector(x, y))); double shapeEnergy = this.Shape == null ? 0 : this.ShapeModel.CalculateEnergy(this.Shape); double totalEnergy = segmentationEnergy + shapeEnergy * this.ShapeEnergyWeight; DebugConfiguration.WriteImportantDebugText( "Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000})", totalEnergy, segmentationEnergy, this.ShapeEnergyWeight, shapeEnergy); return(new SegmentationSolution(this.Shape, this.ImageSegmentator.GetLastSegmentationMask(), totalEnergy)); }
private double CalcObjective(Shape shape, bool report) { this.UpdateShapeTerms(shape); double shapeEnergy = this.ShapeModel.CalculateEnergy(shape); double labelingEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.shapeTerms[x, y]); double energy = shapeEnergy * this.ShapeEnergyWeight + labelingEnergy; double additionalPenalty = this.AdditionalShapePenalty == null ? 0 : this.AdditionalShapePenalty(shape); double totalEnergy = energy + additionalPenalty; if (report) { DebugConfiguration.WriteImportantDebugText( "Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000} + {4:0.0000})", totalEnergy, labelingEnergy, this.ShapeEnergyWeight, shapeEnergy, additionalPenalty); } return(totalEnergy); }
public static Mixture <VectorGaussian> Fit(MicrosoftResearch.Infer.Maths.Vector[] data, int componentCount, int retryCount, double tolerance = 1e-4) { Debug.Assert(data != null); Debug.Assert(data.Length > componentCount * 3); Debug.Assert(componentCount > 1); Debug.Assert(retryCount >= 0); int dimensions = data[0].Count; // Find point boundary MicrosoftResearch.Infer.Maths.Vector min = data[0].Clone(); MicrosoftResearch.Infer.Maths.Vector max = min.Clone(); for (int i = 1; i < data.Length; ++i) { Debug.Assert(dimensions == data[i].Count); for (int j = 0; j < dimensions; ++j) { min[j] = Math.Min(min[j], data[i][j]); max[j] = Math.Max(max[j], data[i][j]); } } // Initialize solution MicrosoftResearch.Infer.Maths.Vector[] means = new MicrosoftResearch.Infer.Maths.Vector[componentCount]; PositiveDefiniteMatrix[] covariances = new PositiveDefiniteMatrix[componentCount]; for (int i = 0; i < componentCount; ++i) { GenerateRandomMixtureComponent(min, max, out means[i], out covariances[i]); } double[] weights = Enumerable.Repeat(1.0 / componentCount, componentCount).ToArray(); // EM algorithm for GMM double[,] expectations = new double[data.Length, componentCount]; double lastEstimate; const double negativeInfinity = -1e+20; bool convergenceDetected; double currentEstimate = negativeInfinity; do { lastEstimate = currentEstimate; convergenceDetected = false; // E-step: estimate expectations on hidden variables for (int i = 0; i < data.Length; ++i) { double sum = 0; for (int j = 0; j < componentCount; ++j) { expectations[i, j] = Math.Exp(VectorGaussian.GetLogProb(data[i], means[j], covariances[j])) * weights[j]; sum += expectations[i, j]; } for (int j = 0; j < componentCount; ++j) { expectations[i, j] /= sum; } } // M-step: // Re-estimate means for (int j = 0; j < componentCount; ++j) { means[j] = MicrosoftResearch.Infer.Maths.Vector.Zero(dimensions); double sum = 0; for (int i = 0; i < data.Length; ++i) { means[j] += data[i] * expectations[i, j]; sum += expectations[i, j]; } means[j] *= 1.0 / sum; } // Re-estimate covariances for (int j = 0; j < componentCount; ++j) { Matrix covariance = new Matrix(dimensions, dimensions); double sum = 0; for (int i = 0; i < data.Length; ++i) { MicrosoftResearch.Infer.Maths.Vector dataDiff = data[i] - means[j]; covariance += dataDiff.Outer(dataDiff) * expectations[i, j]; sum += expectations[i, j]; } covariance *= 1.0 / sum; covariances[j] = new PositiveDefiniteMatrix(covariance); if (covariances[j].LogDeterminant() < -30) { DebugConfiguration.WriteDebugText("Convergence detected for component {0}", j); if (retryCount == 0) { throw new InvalidOperationException("Can't fit GMM. Retry number exceeded."); } retryCount -= 1; GenerateRandomMixtureComponent(min, max, out means[j], out covariances[j]); DebugConfiguration.WriteDebugText("Component {0} regenerated", j); convergenceDetected = true; } } if (convergenceDetected) { currentEstimate = negativeInfinity; continue; } // Re-estimate weights double expectationSum = 0; for (int j = 0; j < componentCount; ++j) { weights[j] = 0; for (int i = 0; i < data.Length; ++i) { weights[j] += expectations[i, j]; expectationSum += expectations[i, j]; } } for (int j = 0; j < componentCount; ++j) { weights[j] /= expectationSum; } // Compute likelihood estimate currentEstimate = 0; for (int i = 0; i < data.Length; ++i) { for (int j = 0; j < componentCount; ++j) { currentEstimate += expectations[i, j] * (VectorGaussian.GetLogProb(data[i], means[j], covariances[j]) + Math.Log(weights[j])); } } DebugConfiguration.WriteDebugText("L={0:0.000000}", currentEstimate); } while (convergenceDetected || (currentEstimate - lastEstimate > tolerance)); Mixture <VectorGaussian> result = new Mixture <VectorGaussian>(); for (int j = 0; j < componentCount; ++j) { result.Add(VectorGaussian.FromMeanAndVariance(means[j], covariances[j]), weights[j]); } DebugConfiguration.WriteDebugText("GMM successfully fitted."); return(result); }
public T Run(T startSolution, Func <T, double, T> mutationFunction, Func <T, double> objectiveFunction) { if (startSolution == null) { throw new ArgumentNullException("startSolution"); } if (mutationFunction == null) { throw new ArgumentNullException("mutationFunction"); } if (objectiveFunction == null) { throw new ArgumentNullException("objectiveFunction"); } T bestSolution = startSolution; double minObjective = objectiveFunction(bestSolution); int lastUpdateIteration = 0; int currentIteration = 0; int iterationsFromLastReannealing = 0; double prevObjective = minObjective; T prevSolution = bestSolution; while (currentIteration < this.MaxIterations && currentIteration - lastUpdateIteration < this.MaxStallingIterations) { double temperature = CalcTemperature(iterationsFromLastReannealing); T currentSolution = mutationFunction(prevSolution, temperature); double currentObjective = objectiveFunction(currentSolution); double acceptanceProb = CalcAcceptanceProbability(prevObjective, currentObjective, temperature); if (Random.Double() < acceptanceProb) { prevObjective = currentObjective; prevSolution = currentSolution; } if (currentObjective < minObjective) { lastUpdateIteration = currentIteration; minObjective = currentObjective; bestSolution = currentSolution; DebugConfiguration.WriteDebugText("Best solution update: {0:0.000}", minObjective); } ++currentIteration; if (currentIteration % this.reportRate == 0) { DebugConfiguration.WriteDebugText("Iteration {0}", currentIteration); if (this.AnnealingProgress != null) { this.AnnealingProgress(this, new SimulatedAnnealingProgressEventArgs <T>(currentSolution, bestSolution)); } } if (iterationsFromLastReannealing >= this.ReannealingInterval) { iterationsFromLastReannealing = 0; prevSolution = bestSolution; DebugConfiguration.WriteDebugText("Reannealing"); } else { ++iterationsFromLastReannealing; } } return(bestSolution); }
private SortedSet <EnergyBound> BreadthFirstBranchAndBoundTraverse(ShapeConstraints constraints) { SortedSet <EnergyBound> front = new SortedSet <EnergyBound> { this.CalculateEnergyBound(constraints) }; int currentIteration = 1; DateTime lastOutputTime = startTime; int processedConstraintSets = 0; while (!front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom) && !this.IsStopping) { this.WaitIfPaused(); EnergyBound parentLowerBound = front.Min; front.Remove(parentLowerBound); List <ShapeConstraints> expandedConstraints = parentLowerBound.Constraints.SplitMostFree(this.maxCoordFreedom, this.maxWidthFreedom); foreach (ShapeConstraints constraintsSet in expandedConstraints) { EnergyBound lowerBound = this.CalculateEnergyBound(constraintsSet); front.Add(lowerBound); // Uncomment for strong invariants check //ObjectBackgroundTerm[,] lowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height]; //for (int i = 0; i < this.segmentedImage.Width; ++i) // for (int j = 0; j < this.segmentedImage.Height; ++j) // lowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(lowerBound.Constraints, new Point(i, j)); //ObjectBackgroundTerm[,] parentLowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height]; //for (int i = 0; i < this.segmentedImage.Width; ++i) // for (int j = 0; j < this.segmentedImage.Height; ++j) // parentLowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(parentLowerBound.Constraints, new Point(i, j)); //for (int i = 0; i < this.segmentedImage.Width; ++i) // for (int j = 0; j < this.segmentedImage.Height; ++j) // { // Debug.Assert(lowerBoundShapeTerm[i, j].ObjectTerm >= parentLowerBoundShapeTerm[i, j].ObjectTerm - 1e-7); // Debug.Assert(lowerBoundShapeTerm[i, j].BackgroundTerm >= parentLowerBoundShapeTerm[i, j].BackgroundTerm - 1e-7); // //CalculateShapeTerm(lowerBound.Constraints, new Point(0, 67)); // //CalculateShapeTerm(parentLowerBound.Constraints, new Point(0, 67)); // } // Lower bound should not decrease (check always, it's important!) Trace.Assert(lowerBound.SegmentationEnergy >= parentLowerBound.SegmentationEnergy - 1e-6); Trace.Assert(lowerBound.ShapeEnergy >= parentLowerBound.ShapeEnergy - 1e-6); //this.CalculateEnergyBound(lowerBound.Constraints); //this.CalculateEnergyBound(parentLowerBound.Constraints); ++processedConstraintSets; } // Some debug output if (currentIteration % this.ProgressReportRate == 0) { DateTime currentTime = DateTime.Now; EnergyBound currentMin = front.Min; DebugConfiguration.WriteDebugText( "On iteration {0} front contains {1} constraint sets.", currentIteration, front.Count); DebugConfiguration.WriteDebugText( "Current lower bound is {0:0.0000} ({1:0.0000} + {2:0.0000}).", currentMin.Bound, currentMin.SegmentationEnergy, currentMin.ShapeEnergy * this.ShapeEnergyWeight); double processingSpeed = processedConstraintSets / (currentTime - lastOutputTime).TotalSeconds; DebugConfiguration.WriteDebugText("Processing speed is {0:0.000} items per sec", processingSpeed); double maxVertexConstraintsFreedom = currentMin.Constraints.VertexConstraints.Max(c => c.Freedom); double maxEdgeConstraintsFreedom = currentMin.Constraints.EdgeConstraints.Max(c => c.Freedom); DebugConfiguration.WriteDebugText( "Max vertex freedom: {0:0.00}, max edge freedom: {1:0.00}", maxVertexConstraintsFreedom, maxEdgeConstraintsFreedom); DebugConfiguration.WriteDebugText("Elapsed time: {0}", DateTime.Now - this.startTime); DebugConfiguration.WriteDebugText(); this.ReportBranchAndBoundProgress(front); lastOutputTime = currentTime; processedConstraintSets = 0; } currentIteration += 1; } return(front); }
protected override SegmentationSolution SegmentCurrentImage() { if (this.minEdgeWidth >= this.maxEdgeWidth) { throw new InvalidOperationException("Min edge width should be less than max edge width."); } if (this.startConstraints != null) { if (this.startConstraints.ShapeStructure != this.ShapeModel.Structure) { throw new InvalidOperationException("Given start constraints have shape structure different from the one specified in shape model."); } // TODO: make this check work //foreach (VertexConstraints vertexConstraints in this.startConstraints.VertexConstraints) //{ // RectangleF imageRectangle = // new RectangleF(0, 0, this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height); // if (!imageRectangle.Contains(vertexConstraints.CoordRectangle)) // throw new InvalidOperationException("Given start constraints are not fully inside the segmented image."); //} } this.shapeUnaryTerms = new Image2D <ObjectBackgroundTerm>( this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height); ShapeConstraints constraints = this.startConstraints; if (constraints == null) { constraints = ShapeConstraints.CreateFromBounds( this.ShapeModel.Structure, Vector.Zero, new Vector(this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height), this.minEdgeWidth, this.maxEdgeWidth); } if (this.BranchAndBoundStarted != null) { this.BranchAndBoundStarted(this, EventArgs.Empty); } this.startTime = DateTime.Now; DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound started."); SortedSet <EnergyBound> front = this.BreadthFirstBranchAndBoundTraverse(constraints); ReportBranchAndBoundCompletion(front.Min); if (front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom)) { DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound finished in {0}.", DateTime.Now - this.startTime); DebugConfiguration.WriteImportantDebugText("Best lower bound is {0:0.0000}", front.Min.Bound); } else { DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound forced to stop after {0}.", DateTime.Now - this.startTime); DebugConfiguration.WriteImportantDebugText("Min energy value achieved is {0:0.0000}", front.Min.Bound); } EnergyBound collapsedBfsSolution = this.CalculateEnergyBound(front.Min.Constraints.Collapse()); Shape resultShape = front.Min.Constraints.CollapseToShape(); DebugConfiguration.WriteImportantDebugText( "Collapsed solution energy value is {0:0.0000} ({1:0.0000} + {2:0.0000})", collapsedBfsSolution.Bound, collapsedBfsSolution.SegmentationEnergy, collapsedBfsSolution.ShapeEnergy * this.ShapeEnergyWeight); return(new SegmentationSolution(resultShape, this.ImageSegmentator.GetLastSegmentationMask(), collapsedBfsSolution.Bound)); }