Ejemplo n.º 1
0
        public List <int> AcceptedWindows(Image <Gray, byte> frame, List <int> scanningWindows)
        {
            _acceptedWindows.Clear();
            IBoundingBox[] allScanningWindows = _scanningWindowGenerator.ScanningWindows;

            _relativeSimilarities.Clear();
            _patches.Clear();
            CurrentState.RelativeSimilarities.Clear();
            for (int i = 0; i < scanningWindows.Count; i++)
            {
                // get relative similarity of the patch to the object model
                int                windowIndex = scanningWindows[i];
                IBoundingBox       bb          = allScanningWindows[windowIndex];
                Image <Gray, byte> patch       = Service.GeneratePatch(frame, bb, null, CurrentState.ObjectModelGaussianSigma, _objectModel.PatchSize);

                float pnn, nnn;
                float relativeSimilarity = _objectModel.RelativeSimilarity(patch, out pnn, out nnn);
                if (pnn > _pnnSimilarityThreshold && relativeSimilarity > _relativeSimilarityThreshold)
                {
                    _acceptedWindows.Add(windowIndex);
                }

                CurrentState.RelativeSimilarities.Add(bb, relativeSimilarity);
                _patches.Add(bb, patch);
            }
            frame.ROI = Rectangle.Empty;

            return(_acceptedWindows);
        }
Ejemplo n.º 2
0
        public void TrainDetector(Image <Gray, byte> currentFrame, IBoundingBox currentBb, out bool valid)
        {
            // if patch is outside the image
            if (!currentBb.InsideFrame(currentFrame.Size))
            {
                valid = false;
                return;
            }

            // get normalized image patch
            currentFrame.ROI = Rectangle.Round(currentBb.GetRectangle());
            Image <Gray, byte> currentPatch = currentFrame.Resize(_objectModel.PatchSize.Width, _objectModel.PatchSize.Height, INTER.CV_INTER_LINEAR);

            currentFrame.ROI = Rectangle.Empty;

            // if appearance changed too fast
            float pnnSimilarity, nnnSimilarity;

            if (_objectModel.RelativeSimilarity(currentPatch, out pnnSimilarity, out nnnSimilarity) < 0.5f)
            {
                valid = false;
                return;
            }
            if (nnnSimilarity > _sameSimilarityThreshold)
            {
                valid = false;
                return;
            }

            #region ensemble classifier

            // generate positive patches
            List <Image <Gray, byte> > positivePatchesForEnsemble = GeneratePatches(
                _ensembleClassifier.SmoothFrame,
                currentBb,
                _runtimePosPatchSynthesisInfo.EnsembleCount,
                _runtimePosPatchSynthesisInfo.WarpInfo,
                0,
                Size.Empty
                );

            // pick negative patches
            List <IBoundingBox> allEnsembleNegativeBbs = new List <IBoundingBox>();
            foreach (KeyValuePair <IBoundingBox, double> pair in _ensembleClassifier.PositivePosteriors)
            {
                IBoundingBox bb = pair.Key;
                if (bb.GetOverlap(currentBb) < _runtimeNegativePatchPickingInfo.Overlap)
                {
                    allEnsembleNegativeBbs.Add(bb);
                }
            }

            List <Image <Gray, byte> > negativePatchesForEnsemble = PickBoundingBoxesAndGeneratePatches(
                _ensembleClassifier.SmoothFrame,
                allEnsembleNegativeBbs,
                _runtimeNegativePatchPickingInfo.EnsembleCount,
                Size.Empty
                );

            // train ensemble classifier
            int bootstrap = 1;
            for (int i = 0; i < bootstrap; i++)
            {
                _ensembleClassifier.TrainWithUnseenPatches(positivePatchesForEnsemble, negativePatchesForEnsemble);
            }

            #endregion

            #region nn classifier

            // generate positive patches
            List <Image <Gray, byte> > positivePatchesForNn = GeneratePatches(
                currentFrame,
                currentBb,
                _runtimePosPatchSynthesisInfo.NnCount,
                _runtimePosPatchSynthesisInfo.WarpInfo,
                _runtimePosPatchSynthesisInfo.GaussianSigma,
                _objectModel.PatchSize
                );

            int width  = _objectModel.PatchSize.Width;
            int height = _objectModel.PatchSize.Height;
            for (int i = 0; i < positivePatchesForNn.Count; i++)
            {
                positivePatchesForNn[i] = positivePatchesForNn[i].Resize(width, height, INTER.CV_INTER_LINEAR);
            }

            /*
             * // pick negative patches
             * List<IBoundingBox> allNegativeNnBbs = new List<IBoundingBox>();
             * IBoundingBox[] allBoundingBoxes = _nnClassifier.ScanningWindowGenerator.ScanningWindows;
             * List<int> nnAcceptedPatches = _nnClassifier.AcceptedPatches;
             * int nnAcceptedPatchesCount = nnAcceptedPatches.Count;
             *
             * for (int i = 0; i < nnAcceptedPatchesCount; i++)
             * {
             *  int windowIndex = nnAcceptedPatches[i];
             *  IBoundingBox bb = allBoundingBoxes[windowIndex];
             *  if (bb.GetOverlap(currentBb) < _runtimeNegativePatchPickingInfo.Overlap)
             *  {
             *      allNegativeNnBbs.Add(bb);
             *  }
             * }
             *
             * List<Image<Gray, byte>> negativePatchesForNn = PickBoundingBoxesAndGeneratePatches(
             *  currentFrame,
             *  allNegativeNnBbs,
             *  _runtimeNegativePatchPickingInfo.NnCount,
             *  _objectModel.PatchSize
             * );
             */

            List <Image <Gray, byte> > negativePatchesForNn = PickFromList <Image <Gray, byte> >(negativePatchesForEnsemble, _runtimeNegativePatchPickingInfo.NnCount);
            for (int i = 0; i < negativePatchesForNn.Count; i++)
            {
                negativePatchesForNn[i] = negativePatchesForNn[i].Resize(_objectModel.PatchSize.Width, ObjectModel.PatchSize.Height, INTER.CV_INTER_LINEAR);
            }

            // train nn classifier (update object model)
            TrainNnClassifier(positivePatchesForNn, negativePatchesForNn);

            #endregion

            valid = true;

            currentFrame.ROI = Rectangle.Empty;
        }
Ejemplo n.º 3
0
        public IBoundingBox DetermineBoundingBox(IBoundingBox trackerBoundingBox, List <IBoundingBox> detectorBoundingBoxes, Image <Gray, byte> frame, out bool reinitializeTracker, bool prevValid, out bool currValid)
        {
            reinitializeTracker = false;
            currValid           = false;

            /* If neither the tracker nor the detector output a bounding box,
             * the object is declared as not visible
             */
            if (trackerBoundingBox == null && detectorBoundingBoxes == null)
            {
                return(null);
            }

            // calculate relative similarities for all bounding boxes
            Dictionary <IBoundingBox, float> relativeSimilarities = new Dictionary <IBoundingBox, float>();
            List <IBoundingBox> boundingBoxes = new List <IBoundingBox>();

            if (trackerBoundingBox != null)
            {
                boundingBoxes.Add(trackerBoundingBox);
            }
            if (detectorBoundingBoxes != null)
            {
                boundingBoxes.AddRange(detectorBoundingBoxes);
            }
            float        maxConservativeSimilarity = -1;
            IBoundingBox maxConfidentBoundingBox   = null;

            foreach (IBoundingBox boundingBox in boundingBoxes)
            {
                frame.ROI = Rectangle.Round(boundingBox.GetRectangle());
                Image <Gray, byte> patch     = frame.Resize(_objectModel.PatchSize.Width, _objectModel.PatchSize.Height, INTER.CV_INTER_LINEAR);
                float conservativeSimilarity = _objectModel.RelativeSimilarity(patch);
                relativeSimilarities.Add(boundingBox, conservativeSimilarity);

                if (conservativeSimilarity > maxConservativeSimilarity)
                {
                    maxConservativeSimilarity = conservativeSimilarity;
                    maxConfidentBoundingBox   = boundingBox;
                }
            }
            frame.ROI = Rectangle.Empty;

            IBoundingBox output = null;

            // if tracker is defined
            if (trackerBoundingBox != null)
            {
                output = trackerBoundingBox;

                if (detectorBoundingBoxes != null)
                {
                    // get detector bounding boxes
                    // that are more confident then the tracker
                    // and have sufficient rel. similarity
                    List <IBoundingBox> bbs = new List <IBoundingBox>();
                    foreach (IBoundingBox bb in detectorBoundingBoxes)
                    {
                        if (relativeSimilarities[bb] > relativeSimilarities[trackerBoundingBox] &&
                            relativeSimilarities[bb] > 0.55f)
                        {
                            bbs.Add(bb);
                        }
                    }

                    // if there is only only one such bounding box,
                    // reinitialize the tracker
                    if (bbs.Count == 1)
                    {
                        output = bbs[0];
                        reinitializeTracker = true;
                    }
                }
            }

            // if tracker is not defined
            else
            {
                if (detectorBoundingBoxes != null)
                {
                    // get detector bounding boxes
                    // that have sufficient rel. similarity
                    List <IBoundingBox> bbs = new List <IBoundingBox>();
                    foreach (IBoundingBox bb in detectorBoundingBoxes)
                    {
                        if (relativeSimilarities[bb] > 0.55f)
                        {
                            bbs.Add(bb);
                        }
                    }

                    // if there is only only one such bounding box,
                    // reinitialize the tracker
                    if (bbs.Count == 1)
                    {
                        output = bbs[0];
                        reinitializeTracker = true;
                    }
                }
            }

            return(output);
        }
Ejemplo n.º 4
0
        public IBoundingBox DetermineBoundingBox(IBoundingBox trackerBoundingBox, List <IBoundingBox> detectorBoundingBoxes, Image <Gray, byte> frame, out bool reinitializeTracker, bool prevValid, out bool currValid)
        {
            IBoundingBox output = null;

            reinitializeTracker = false;
            currValid           = false;

            /* [Zdenek] If neither the tracker nor the detector output a bounding box,
             * the object is declared as not visible
             */
            if (trackerBoundingBox == null && detectorBoundingBoxes == null)
            {
                return(null);
            }

            // calculate relative similarities for all bounding boxes
            Dictionary <IBoundingBox, float> relativeSimilarities = new Dictionary <IBoundingBox, float>();

            // -> tracker
            if (trackerBoundingBox != null)
            {
                frame.ROI = Rectangle.Round(trackerBoundingBox.GetRectangle());
                Size patchSize = _objectModel.PatchSize;
                Image <Gray, byte> trackerPatch = frame.Resize(patchSize.Width, patchSize.Height, INTER.CV_INTER_LINEAR);
                relativeSimilarities.Add(trackerBoundingBox, _objectModel.RelativeSimilarity(trackerPatch));
                frame.ROI = Rectangle.Empty;
            }

            // -> detector
            if (detectorBoundingBoxes != null)
            {
                for (int i = 0; i < detectorBoundingBoxes.Count; i++)
                {
                    IBoundingBox bb = detectorBoundingBoxes[i];
                    relativeSimilarities.Add(bb, CurrentState.RelativeSimilarities[bb]);
                }
            }

            // if tracker is defined
            if (trackerBoundingBox != null)
            {
                output = trackerBoundingBox;

                if (detectorBoundingBoxes != null)
                {
                    float bigOverlap = 0.8f;

                    // get detector bounding boxes
                    // that are far from tracker
                    // and are more confident then the tracker
                    List <IBoundingBox> bbs = new List <IBoundingBox>();
                    foreach (IBoundingBox bb in detectorBoundingBoxes)
                    {
                        if (bb.GetOverlap(trackerBoundingBox) < bigOverlap &&
                            relativeSimilarities[bb] > relativeSimilarities[trackerBoundingBox])
                        {
                            bbs.Add(bb);
                        }
                    }

                    // if there is only only one such bounding box,
                    // reinitialize the tracker
                    if (bbs.Count == 1)
                    {
                        output = bbs[0];
                        reinitializeTracker = true;
                        currValid           = false;
                    }

                    // otherwise calculate weighted average with close detections
                    else
                    {
                        int    trackerRepeat = 1;
                        PointF center        = trackerBoundingBox.Center.Multiply(trackerRepeat, trackerRepeat);
                        SizeF  size          = trackerBoundingBox.Size.Multiply(trackerRepeat, trackerRepeat);

                        // consider detector bounding boxes that are close to the tracker
                        int bbCount = trackerRepeat;
                        foreach (IBoundingBox bb in detectorBoundingBoxes)
                        {
                            if (bb.GetOverlap(trackerBoundingBox) >= bigOverlap)
                            {
                                center = center.Add(bb.Center);
                                size   = size.Add(bb.Size);
                                bbCount++;
                            }
                        }

                        output = trackerBoundingBox.CreateInstance(
                            center.Divide(bbCount, bbCount),
                            size.Divide(bbCount, bbCount)
                            );
                    }
                }
            }

            // if tracker is not defined
            else
            {
                if (detectorBoundingBoxes != null)
                {
                    List <IBoundingBox> suppressedBbs = Service.NonMaximalBoundingBoxSuppress(detectorBoundingBoxes);

                    // if there is a single detection, reinitialize the tracker
                    if (suppressedBbs.Count == 1)
                    {
                        output = suppressedBbs[0];
                        reinitializeTracker = true;
                        currValid           = false;
                    }
                }
            }

            if (output != null)
            {
                if (prevValid == true)
                {
                    currValid = true;
                }
                else
                {
                    Image <Gray, byte> outputPatch = frame.GetPatch(output.Center, Size.Round(output.Size))
                                                     .Resize(_objectModel.PatchSize.Width, _objectModel.PatchSize.Height, INTER.CV_INTER_LINEAR);

                    float pnn, nnn;
                    float conservativeSimilarity = _objectModel.ConservativeSimilarity(outputPatch, out pnn, out nnn);
                    if (pnn > 0.8f && conservativeSimilarity >= ConsSimValidThreshold)
                    {
                        currValid = true;
                    }
                }
            }

            return(output);
        }