Ejemplo n.º 1
0
        public SegmentationSolution SegmentImage(Image2D <Color> image, ObjectBackgroundColorModels colorModels)
        {
            if (image == null)
            {
                throw new ArgumentNullException("image");
            }
            if (colorModels == null)
            {
                throw new ArgumentNullException("colorModels");
            }

            this.ImageSegmentator = new ImageSegmentator(
                image,
                colorModels,
                this.ColorDifferencePairwiseTermCutoff,
                this.ColorDifferencePairwiseTermWeight,
                this.ConstantPairwiseTermWeight,
                this.ObjectColorUnaryTermWeight,
                this.BackgroundColorUnaryTermWeight,
                this.ObjectShapeUnaryTermWeight,
                this.BackgroundShapeUnaryTermWeight);

            DebugConfiguration.WriteImportantDebugText(
                "Segmented image size is {0}x{1}.",
                this.ImageSegmentator.ImageSize.Width,
                this.ImageSegmentator.ImageSize.Height);

            SegmentationSolution solution = null;

            try
            {
                if (this.ShapeModel == null)
                {
                    throw new InvalidOperationException("Shape model must be specified before segmenting image.");
                }

                DebugConfiguration.WriteImportantDebugText("Running segmentation algorithm...");
                this.IsRunning = true;
                solution       = this.SegmentCurrentImage();

                if (solution == null)
                {
                    throw new InvalidOperationException("Segmentation solution can not be null.");
                }
            }
            finally
            {
                if (this.IsStopping)
                {
                    this.WasStopped = true;
                }

                this.IsRunning  = false;
                this.IsStopping = false;
            }

            DebugConfiguration.WriteImportantDebugText("Finished");

            return(solution);
        }
Ejemplo n.º 2
0
        protected override SegmentationSolution SegmentCurrentImage()
        {
            DebugConfiguration.WriteImportantDebugText("Performing initial segmentation...");
            this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => ObjectBackgroundTerm.Zero);
            Image2D <bool> prevMask  = this.ImageSegmentator.GetLastSegmentationMask();
            Shape          prevShape = this.ShapeModel.FitMeanShape(
                this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
            double prevEnergy = 0;

            for (int iteration = 1; iteration <= this.MaxIterationCount && !this.IsStopping; ++iteration)
            {
                this.WaitIfPaused();

                DebugConfiguration.WriteImportantDebugText("Iteration {0}", iteration);

                Image2D <bool> prevMaskCopy = prevMask;
                Shape          currentShape = this.ShapeFitter.Run(
                    prevShape,
                    (s, t) => this.ShapeMutator.MutateShape(s, this.ShapeModel, this.ImageSegmentator.ImageSize, t / this.ShapeFitter.StartTemperature),
                    s => this.CalcObjective(s, prevMaskCopy));

                double         currentEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.ShapeModel.CalculatePenalties(currentShape, new Vector(x, y)));
                Image2D <bool> currentMask   = this.ImageSegmentator.GetLastSegmentationMask();

                int differentValues = Image2D <bool> .DifferentValueCount(prevMask, currentMask);

                double changedPixelRate = (double)differentValues / (this.ImageSegmentator.ImageSize.Width * this.ImageSegmentator.ImageSize.Height);

                DebugConfiguration.WriteImportantDebugText("On iteration {0}:", iteration);
                DebugConfiguration.WriteImportantDebugText("Energy is {0:0.000}", currentEnergy);
                DebugConfiguration.WriteImportantDebugText("Changed pixel rate is {0:0.000000}", changedPixelRate);
                DebugConfiguration.WriteImportantDebugText();

                if (iteration > this.MinIterationCount && changedPixelRate < this.MinChangeRate)
                {
                    DebugConfiguration.WriteImportantDebugText("Changed pixel rate is too low, breaking...");
                    break;
                }

                prevMask   = currentMask;
                prevShape  = currentShape;
                prevEnergy = currentEnergy;

                if (IterationFinished != null)
                {
                    IterationFinished(this, new SegmentationIterationFinishedEventArgs(
                                          iteration,
                                          currentShape,
                                          this.ImageSegmentator.GetLastSegmentationMask(),
                                          this.ImageSegmentator.GetLastUnaryTerms(),
                                          this.ImageSegmentator.GetLastShapeTerms()));
                }
            }

            return(new SegmentationSolution(prevShape, prevMask, prevEnergy));
        }
Ejemplo n.º 3
0
        protected override SegmentationSolution SegmentCurrentImage()
        {
            if (this.Shape != null && this.Shape.Structure != this.ShapeModel.Structure)
            {
                throw new InvalidOperationException("Specified shape differs in structure from shape model.");
            }

            double segmentationEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms(
                (x, y) => this.Shape == null ? ObjectBackgroundTerm.Zero : CalculateShapeTerms(new Vector(x, y)));
            double shapeEnergy = this.Shape == null ? 0 : this.ShapeModel.CalculateEnergy(this.Shape);
            double totalEnergy = segmentationEnergy + shapeEnergy * this.ShapeEnergyWeight;

            DebugConfiguration.WriteImportantDebugText(
                "Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000})",
                totalEnergy,
                segmentationEnergy,
                this.ShapeEnergyWeight,
                shapeEnergy);

            return(new SegmentationSolution(this.Shape, this.ImageSegmentator.GetLastSegmentationMask(), totalEnergy));
        }
Ejemplo n.º 4
0
        private double CalcObjective(Shape shape, bool report)
        {
            this.UpdateShapeTerms(shape);

            double shapeEnergy       = this.ShapeModel.CalculateEnergy(shape);
            double labelingEnergy    = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.shapeTerms[x, y]);
            double energy            = shapeEnergy * this.ShapeEnergyWeight + labelingEnergy;
            double additionalPenalty = this.AdditionalShapePenalty == null ? 0 : this.AdditionalShapePenalty(shape);
            double totalEnergy       = energy + additionalPenalty;

            if (report)
            {
                DebugConfiguration.WriteImportantDebugText(
                    "Solution energy: {0:0.0000} ({1:0.0000} + {2:0.0000} * {3:0.0000} + {4:0.0000})",
                    totalEnergy,
                    labelingEnergy,
                    this.ShapeEnergyWeight,
                    shapeEnergy,
                    additionalPenalty);
            }

            return(totalEnergy);
        }
        public static Mixture <VectorGaussian> Fit(MicrosoftResearch.Infer.Maths.Vector[] data, int componentCount, int retryCount, double tolerance = 1e-4)
        {
            Debug.Assert(data != null);
            Debug.Assert(data.Length > componentCount * 3);
            Debug.Assert(componentCount > 1);
            Debug.Assert(retryCount >= 0);

            int dimensions = data[0].Count;

            // Find point boundary
            MicrosoftResearch.Infer.Maths.Vector min = data[0].Clone();
            MicrosoftResearch.Infer.Maths.Vector max = min.Clone();
            for (int i = 1; i < data.Length; ++i)
            {
                Debug.Assert(dimensions == data[i].Count);
                for (int j = 0; j < dimensions; ++j)
                {
                    min[j] = Math.Min(min[j], data[i][j]);
                    max[j] = Math.Max(max[j], data[i][j]);
                }
            }

            // Initialize solution
            MicrosoftResearch.Infer.Maths.Vector[] means = new MicrosoftResearch.Infer.Maths.Vector[componentCount];
            PositiveDefiniteMatrix[] covariances         = new PositiveDefiniteMatrix[componentCount];
            for (int i = 0; i < componentCount; ++i)
            {
                GenerateRandomMixtureComponent(min, max, out means[i], out covariances[i]);
            }
            double[] weights = Enumerable.Repeat(1.0 / componentCount, componentCount).ToArray();

            // EM algorithm for GMM
            double[,] expectations = new double[data.Length, componentCount];
            double       lastEstimate;
            const double negativeInfinity = -1e+20;
            bool         convergenceDetected;
            double       currentEstimate = negativeInfinity;

            do
            {
                lastEstimate        = currentEstimate;
                convergenceDetected = false;

                // E-step: estimate expectations on hidden variables
                for (int i = 0; i < data.Length; ++i)
                {
                    double sum = 0;
                    for (int j = 0; j < componentCount; ++j)
                    {
                        expectations[i, j] =
                            Math.Exp(VectorGaussian.GetLogProb(data[i], means[j], covariances[j])) * weights[j];
                        sum += expectations[i, j];
                    }
                    for (int j = 0; j < componentCount; ++j)
                    {
                        expectations[i, j] /= sum;
                    }
                }

                // M-step:

                // Re-estimate means
                for (int j = 0; j < componentCount; ++j)
                {
                    means[j] = MicrosoftResearch.Infer.Maths.Vector.Zero(dimensions);
                    double sum = 0;
                    for (int i = 0; i < data.Length; ++i)
                    {
                        means[j] += data[i] * expectations[i, j];
                        sum      += expectations[i, j];
                    }
                    means[j] *= 1.0 / sum;
                }

                // Re-estimate covariances
                for (int j = 0; j < componentCount; ++j)
                {
                    Matrix covariance = new Matrix(dimensions, dimensions);
                    double sum        = 0;
                    for (int i = 0; i < data.Length; ++i)
                    {
                        MicrosoftResearch.Infer.Maths.Vector dataDiff = data[i] - means[j];
                        covariance += dataDiff.Outer(dataDiff) * expectations[i, j];
                        sum        += expectations[i, j];
                    }
                    covariance    *= 1.0 / sum;
                    covariances[j] = new PositiveDefiniteMatrix(covariance);

                    if (covariances[j].LogDeterminant() < -30)
                    {
                        DebugConfiguration.WriteDebugText("Convergence detected for component {0}", j);
                        if (retryCount == 0)
                        {
                            throw new InvalidOperationException("Can't fit GMM. Retry number exceeded.");
                        }

                        retryCount -= 1;
                        GenerateRandomMixtureComponent(min, max, out means[j], out covariances[j]);
                        DebugConfiguration.WriteDebugText("Component {0} regenerated", j);

                        convergenceDetected = true;
                    }
                }

                if (convergenceDetected)
                {
                    currentEstimate = negativeInfinity;
                    continue;
                }

                // Re-estimate weights
                double expectationSum = 0;
                for (int j = 0; j < componentCount; ++j)
                {
                    weights[j] = 0;
                    for (int i = 0; i < data.Length; ++i)
                    {
                        weights[j]     += expectations[i, j];
                        expectationSum += expectations[i, j];
                    }
                }
                for (int j = 0; j < componentCount; ++j)
                {
                    weights[j] /= expectationSum;
                }

                // Compute likelihood estimate
                currentEstimate = 0;
                for (int i = 0; i < data.Length; ++i)
                {
                    for (int j = 0; j < componentCount; ++j)
                    {
                        currentEstimate +=
                            expectations[i, j] * (VectorGaussian.GetLogProb(data[i], means[j], covariances[j]) + Math.Log(weights[j]));
                    }
                }

                DebugConfiguration.WriteDebugText("L={0:0.000000}", currentEstimate);
            } while (convergenceDetected || (currentEstimate - lastEstimate > tolerance));

            Mixture <VectorGaussian> result = new Mixture <VectorGaussian>();

            for (int j = 0; j < componentCount; ++j)
            {
                result.Add(VectorGaussian.FromMeanAndVariance(means[j], covariances[j]), weights[j]);
            }

            DebugConfiguration.WriteDebugText("GMM successfully fitted.");

            return(result);
        }
        public T Run(T startSolution, Func <T, double, T> mutationFunction, Func <T, double> objectiveFunction)
        {
            if (startSolution == null)
            {
                throw new ArgumentNullException("startSolution");
            }
            if (mutationFunction == null)
            {
                throw new ArgumentNullException("mutationFunction");
            }
            if (objectiveFunction == null)
            {
                throw new ArgumentNullException("objectiveFunction");
            }

            T      bestSolution = startSolution;
            double minObjective = objectiveFunction(bestSolution);

            int    lastUpdateIteration           = 0;
            int    currentIteration              = 0;
            int    iterationsFromLastReannealing = 0;
            double prevObjective = minObjective;
            T      prevSolution  = bestSolution;

            while (currentIteration < this.MaxIterations && currentIteration - lastUpdateIteration < this.MaxStallingIterations)
            {
                double temperature      = CalcTemperature(iterationsFromLastReannealing);
                T      currentSolution  = mutationFunction(prevSolution, temperature);
                double currentObjective = objectiveFunction(currentSolution);
                double acceptanceProb   = CalcAcceptanceProbability(prevObjective, currentObjective, temperature);

                if (Random.Double() < acceptanceProb)
                {
                    prevObjective = currentObjective;
                    prevSolution  = currentSolution;
                }

                if (currentObjective < minObjective)
                {
                    lastUpdateIteration = currentIteration;
                    minObjective        = currentObjective;
                    bestSolution        = currentSolution;

                    DebugConfiguration.WriteDebugText("Best solution update: {0:0.000}", minObjective);
                }

                ++currentIteration;
                if (currentIteration % this.reportRate == 0)
                {
                    DebugConfiguration.WriteDebugText("Iteration {0}", currentIteration);

                    if (this.AnnealingProgress != null)
                    {
                        this.AnnealingProgress(this, new SimulatedAnnealingProgressEventArgs <T>(currentSolution, bestSolution));
                    }
                }

                if (iterationsFromLastReannealing >= this.ReannealingInterval)
                {
                    iterationsFromLastReannealing = 0;
                    prevSolution = bestSolution;
                    DebugConfiguration.WriteDebugText("Reannealing");
                }
                else
                {
                    ++iterationsFromLastReannealing;
                }
            }

            return(bestSolution);
        }
Ejemplo n.º 7
0
        private SortedSet <EnergyBound> BreadthFirstBranchAndBoundTraverse(ShapeConstraints constraints)
        {
            SortedSet <EnergyBound> front = new SortedSet <EnergyBound> {
                this.CalculateEnergyBound(constraints)
            };

            int      currentIteration        = 1;
            DateTime lastOutputTime          = startTime;
            int      processedConstraintSets = 0;

            while (!front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom) && !this.IsStopping)
            {
                this.WaitIfPaused();

                EnergyBound parentLowerBound = front.Min;
                front.Remove(parentLowerBound);

                List <ShapeConstraints> expandedConstraints = parentLowerBound.Constraints.SplitMostFree(this.maxCoordFreedom, this.maxWidthFreedom);
                foreach (ShapeConstraints constraintsSet in expandedConstraints)
                {
                    EnergyBound lowerBound = this.CalculateEnergyBound(constraintsSet);
                    front.Add(lowerBound);

                    // Uncomment for strong invariants check
                    //ObjectBackgroundTerm[,] lowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height];
                    //for (int i = 0; i < this.segmentedImage.Width; ++i)
                    //    for (int j = 0; j < this.segmentedImage.Height; ++j)
                    //        lowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(lowerBound.Constraints, new Point(i, j));
                    //ObjectBackgroundTerm[,] parentLowerBoundShapeTerm = new ObjectBackgroundTerm[this.segmentedImage.Width, this.segmentedImage.Height];
                    //for (int i = 0; i < this.segmentedImage.Width; ++i)
                    //    for (int j = 0; j < this.segmentedImage.Height; ++j)
                    //        parentLowerBoundShapeTerm[i, j] = CpuBranchAndBoundShapeTermsCalculator.CalculateShapeTerm(parentLowerBound.Constraints, new Point(i, j));
                    //for (int i = 0; i < this.segmentedImage.Width; ++i)
                    //    for (int j = 0; j < this.segmentedImage.Height; ++j)
                    //    {
                    //        Debug.Assert(lowerBoundShapeTerm[i, j].ObjectTerm >= parentLowerBoundShapeTerm[i, j].ObjectTerm - 1e-7);
                    //        Debug.Assert(lowerBoundShapeTerm[i, j].BackgroundTerm >= parentLowerBoundShapeTerm[i, j].BackgroundTerm - 1e-7);
                    //        //CalculateShapeTerm(lowerBound.Constraints, new Point(0, 67));
                    //        //CalculateShapeTerm(parentLowerBound.Constraints, new Point(0, 67));
                    //    }

                    // Lower bound should not decrease (check always, it's important!)
                    Trace.Assert(lowerBound.SegmentationEnergy >= parentLowerBound.SegmentationEnergy - 1e-6);
                    Trace.Assert(lowerBound.ShapeEnergy >= parentLowerBound.ShapeEnergy - 1e-6);

                    //this.CalculateEnergyBound(lowerBound.Constraints);
                    //this.CalculateEnergyBound(parentLowerBound.Constraints);

                    ++processedConstraintSets;
                }

                // Some debug output
                if (currentIteration % this.ProgressReportRate == 0)
                {
                    DateTime    currentTime = DateTime.Now;
                    EnergyBound currentMin  = front.Min;

                    DebugConfiguration.WriteDebugText(
                        "On iteration {0} front contains {1} constraint sets.", currentIteration, front.Count);
                    DebugConfiguration.WriteDebugText(
                        "Current lower bound is {0:0.0000} ({1:0.0000} + {2:0.0000}).",
                        currentMin.Bound,
                        currentMin.SegmentationEnergy,
                        currentMin.ShapeEnergy * this.ShapeEnergyWeight);
                    double processingSpeed = processedConstraintSets / (currentTime - lastOutputTime).TotalSeconds;
                    DebugConfiguration.WriteDebugText("Processing speed is {0:0.000} items per sec", processingSpeed);

                    double maxVertexConstraintsFreedom = currentMin.Constraints.VertexConstraints.Max(c => c.Freedom);
                    double maxEdgeConstraintsFreedom   = currentMin.Constraints.EdgeConstraints.Max(c => c.Freedom);
                    DebugConfiguration.WriteDebugText(
                        "Max vertex freedom: {0:0.00}, max edge freedom: {1:0.00}",
                        maxVertexConstraintsFreedom,
                        maxEdgeConstraintsFreedom);

                    DebugConfiguration.WriteDebugText("Elapsed time: {0}", DateTime.Now - this.startTime);

                    DebugConfiguration.WriteDebugText();

                    this.ReportBranchAndBoundProgress(front);

                    lastOutputTime          = currentTime;
                    processedConstraintSets = 0;
                }

                currentIteration += 1;
            }

            return(front);
        }
Ejemplo n.º 8
0
        protected override SegmentationSolution SegmentCurrentImage()
        {
            if (this.minEdgeWidth >= this.maxEdgeWidth)
            {
                throw new InvalidOperationException("Min edge width should be less than max edge width.");
            }
            if (this.startConstraints != null)
            {
                if (this.startConstraints.ShapeStructure != this.ShapeModel.Structure)
                {
                    throw new InvalidOperationException("Given start constraints have shape structure different from the one specified in shape model.");
                }
                // TODO: make this check work
                //foreach (VertexConstraints vertexConstraints in this.startConstraints.VertexConstraints)
                //{
                //    RectangleF imageRectangle =
                //        new RectangleF(0, 0, this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
                //    if (!imageRectangle.Contains(vertexConstraints.CoordRectangle))
                //        throw new InvalidOperationException("Given start constraints are not fully inside the segmented image.");
                //}
            }

            this.shapeUnaryTerms = new Image2D <ObjectBackgroundTerm>(
                this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);

            ShapeConstraints constraints = this.startConstraints;

            if (constraints == null)
            {
                constraints = ShapeConstraints.CreateFromBounds(
                    this.ShapeModel.Structure,
                    Vector.Zero,
                    new Vector(this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height),
                    this.minEdgeWidth,
                    this.maxEdgeWidth);
            }

            if (this.BranchAndBoundStarted != null)
            {
                this.BranchAndBoundStarted(this, EventArgs.Empty);
            }

            this.startTime = DateTime.Now;
            DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound started.");

            SortedSet <EnergyBound> front = this.BreadthFirstBranchAndBoundTraverse(constraints);

            ReportBranchAndBoundCompletion(front.Min);

            if (front.Min.Constraints.CheckIfSatisfied(this.maxCoordFreedom, this.maxWidthFreedom))
            {
                DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound finished in {0}.",
                                                           DateTime.Now - this.startTime);
                DebugConfiguration.WriteImportantDebugText("Best lower bound is {0:0.0000}", front.Min.Bound);
            }
            else
            {
                DebugConfiguration.WriteImportantDebugText("Breadth-first branch-and-bound forced to stop after {0}.", DateTime.Now - this.startTime);
                DebugConfiguration.WriteImportantDebugText("Min energy value achieved is {0:0.0000}", front.Min.Bound);
            }

            EnergyBound collapsedBfsSolution = this.CalculateEnergyBound(front.Min.Constraints.Collapse());
            Shape       resultShape          = front.Min.Constraints.CollapseToShape();

            DebugConfiguration.WriteImportantDebugText(
                "Collapsed solution energy value is {0:0.0000} ({1:0.0000} + {2:0.0000})",
                collapsedBfsSolution.Bound,
                collapsedBfsSolution.SegmentationEnergy,
                collapsedBfsSolution.ShapeEnergy * this.ShapeEnergyWeight);
            return(new SegmentationSolution(resultShape, this.ImageSegmentator.GetLastSegmentationMask(), collapsedBfsSolution.Bound));
        }