/// <summary>
/// Processes one frame during the gradient
/// </summary>
private void RunGradient(BlobWithMoments fish)
{
//compute laser power based on distance from dish center
if (fish != null)
{
var radius = Math.Sqrt((fish.xc - DishCenter.x) * (fish.xc - DishCenter.x) + (fish.yc - DishCenter.y) * (fish.yc - DishCenter.y));
var radFraction = radius / Radius;
if (radFraction > 1)
{
radFraction = 1;
}
_laser.LaserPower = radFraction * (EdgeLaserPower - CenterLaserPower) + CenterLaserPower;
}
if (_currentPhaseFrame >= _gradFrames)
{
if (_currentTrial < _nTrials)
{
SwitchToPrePhase();
}
else
{
_laser.LaserPower = 0;
_experimentPhase = ExperimentPhase.Done;
}
}
else
{
_currentPhaseFrame++;
}
}
/// <summary>
/// Finds the location of the laser beam in the camera image
/// </summary>
/// <param name="image">The current image in which to find the beam</param>
/// <param name="background">The calculated background image</param>
/// <param name="calc">Image buffer for intermediate calculations</param>
/// <param name="foreground">Image buffer for foreground extraction</param>
/// <returns></returns>
private IppiPoint FindBeamLocation(Image8 image, Image8 background, Image8 calc, Image8 foreground)
{
IppiSize imageSize = new IppiSize(image.Width, image.Height);
//Clear foreground and calc
ip.ippiSet_8u_C1R(0, foreground.Image, foreground.Stride, imageSize);
ip.ippiSet_8u_C1R(0, calc.Image, calc.Stride, imageSize);
//Perform background subtraction
cv.ippiAbsDiff_8u_C1R(image.Image, image.Stride, background.Image, background.Stride, calc.Image, calc.Stride, imageSize);
//remove noise via 3x3 median filtering
ip.ippiFilterMedianWeightedCenter3x3_8u_C1R(calc[2, 2], calc.Stride, foreground[2, 2], foreground.Stride, new IppiSize(image.Width - 4, image.Height - 4), 1);
//threshold difference
ip.ippiThreshold_LTVal_8u_C1IR(foreground.Image, foreground.Stride, imageSize, 111, 0);
ip.ippiThreshold_GTVal_8u_C1IR(foreground.Image, foreground.Stride, imageSize, 110, 255);
//perform closing operation - 2 step to put result back into foreground
_strel3x3.Dilate(foreground, calc, imageSize);
_strel3x3.Erode(calc, foreground, imageSize);
//feature extraction
int nMarkers = 0;
cv.ippiLabelMarkers_8u_C1IR(foreground.Image, foreground.Stride, imageSize, 1, 254, IppiNorm.ippiNormInf, &nMarkers, _markerBuffer);
BlobWithMoments[] blobsDetected = BlobWithMoments.ExtractBlobs(foreground, nMarkers, new IppiROI(0, 0, image.Width, image.Height));
//re-threshold foreground for display
ip.ippiThreshold_LTVal_8u_C1IR(foreground.Image, foreground.Stride, imageSize, 1, 0); //6
ip.ippiThreshold_GTVal_8u_C1IR(foreground.Image, foreground.Stride, imageSize, 0, 255); //5
long maxArea = 0;
int maxIndex = -1;
if (blobsDetected != null)
{
for (int i = 0; i < blobsDetected.Length; i++)
{
if (blobsDetected[i] == null)
{
break;
}
//Simply note down the largest blob
if (blobsDetected[i].Area > maxArea)
{
maxArea = blobsDetected[i].Area;
maxIndex = i;
}
}
// System.Diagnostics.Debug.WriteLine("Area: {0}", blobsDetected[maxIndex].Area);
// System.Diagnostics.Debug.WriteLine("Eccentricity: {0}", blobsDetected[maxIndex].Eccentricity);
if (maxArea > 30)
{
return(blobsDetected[maxIndex].Centroid);
}
else
{
return(new IppiPoint(-1, -1));
}
}
else
{
return(new IppiPoint(-1, -1));
}
}
/// <summary>
/// Process the next frame
/// </summary>
/// <param name="frameNumber">The frame index</param>
/// <param name="camImage">The camera image</param>
/// <param name="poi">The fish location</param>
/// <returns>Whether experiment should continue or not</returns>
public override bool ProcessNext(int frameNumber, Image8 camImage, out IppiPoint?poi)
{
base.ProcessNext(frameNumber, camImage, out poi);
if (_scanner == null)
{
System.Diagnostics.Debug.WriteLine("Scanner was not initialized. Terminating experiment");
return(false);
}
int writeEvery = Properties.Settings.Default.FrameRate * 4;
if (frameNumber % writeEvery == writeEvery - 1)
{
_laser.LaserPower = Properties.Settings.Default.LaserCalibPowermW;
}
else
{
_laser.LaserPower = 0;
}
_lastFrame = frameNumber;
if (frameNumber >= _totalFrames)
{
return(false);
}
BlobWithMoments fish = null;
// Every 4s we turn on the laser - those frames won't be tracked but the image will be saved
if (frameNumber % writeEvery != 0)
{
fish = Tracker.Track(camImage);
}
if (fish != null)
{
poi = new IppiPoint(fish.Centroid.x, fish.Centroid.y);
if (_scanner != null)
{
_scanner.Hit(poi.GetValueOrDefault());
}
}
if (_trackWriter != null)
{
if (fish != null)
{
_trackWriter.WriteLine("{0}\t{1}\t{2}\t{3}", frameNumber, fish.Centroid.x, fish.Centroid.y, fish.Angle);
}
else
{
_trackWriter.WriteLine("NaN\tNaN\tNaN\tNaN");
}
// Write image of Laser On frames and the two surrounding frames on each side to file
if (frameNumber % writeEvery <= 2 || frameNumber % writeEvery >= writeEvery - 2)
{
_imageWriter.WriteFrame(camImage);
}
}
return(true);
}
/// <summary>
/// Write information to track file
/// </summary>
/// <param name="fish"></param>
private void WriteTrackInfo(int frameNumber, BlobWithMoments fish, int originalPhase, int originalTrial)
{
if (fish != null)
{
_trackWriter.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}", frameNumber, fish.Centroid.x, fish.Centroid.y, fish.Angle, originalPhase, originalTrial, _laser.LaserPower);
}
else
{
_trackWriter.WriteLine("NaN\tNaN\tNaN\tNaN\t{0}\t{1}\tNaN", originalPhase, originalTrial);
}
}
/// <summary>
/// Write information to track file
/// </summary>
/// <param name="fish"></param>
private void WriteTrackInfo(int frameNumber, BlobWithMoments fish)
{
if (fish != null)
{
_trackWriter.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}", frameNumber, fish.Centroid.x, fish.Centroid.y, fish.Angle, (int)_experimentPhase, _currentTrial, _laser.LaserPower);
}
else
{
_trackWriter.WriteLine("NaN\tNaN\tNaN\tNaN\t{0}\t{1}\tNaN", (int)_experimentPhase, _currentTrial);
}
}
/// <summary>
/// Extracts a fish (candidate) from an image by performing background subtraction, noise filtering, thresholding and closing to obtain a foreground
/// followed by marker extraction.
/// </summary>
/// <param name="im">The image to extract the fish from</param>
/// <param name="region">The ROI to search</param>
/// <returns>The most likely fish blob or null if no suitable candidate was found</returns>
protected BlobWithMoments ExtractFish(/*Image8 im,*/ IppiROI region)
{
int nMarkers = 0;
BlobWithMoments[] blobsDetected;
//Copy foreground to marker and label connected components
//IppHelper.IppCheckCall(ip.ippiCopy_8u_C1R(_foreground[region.TopLeft], _foreground.Stride, _labelMarkers[region.TopLeft], _labelMarkers.Stride, region.Size));
IppHelper.IppCheckCall(cv.ippiLabelMarkers_8u_C1IR(_foreground[region.TopLeft], _foreground.Stride, region.Size, 1, 254, IppiNorm.ippiNormInf, &nMarkers, _markerBuffer));
//loop over returned markers and use ipp to extract blobs
if (nMarkers > 0)
{
if (nMarkers > 254)
{
nMarkers = 254;
}
blobsDetected = new BlobWithMoments[nMarkers];
for (int i = 1; i <= nMarkers; i++)
{
//label all pixels with the current marker as 255 and others as 0
IppHelper.IppCheckCall(ip.ippiCompareC_8u_C1R(_foreground[region.TopLeft], _foreground.Stride, (byte)i, _calc[region.TopLeft], _calc.Stride, region.Size, IppCmpOp.ippCmpEq));
//calculate image moments
IppHelper.IppCheckCall(ip.ippiMoments64s_8u_C1R(_calc[region.TopLeft], _calc.Stride, region.Size, _momentState));
//retrieve moments
long m00 = 0;
long m10 = 0;
long m01 = 0;
long m20 = 0;
long m02 = 0;
long m11 = 0;
long m30 = 0;
long m03 = 0;
long m21 = 0;
long m12 = 0;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 0, 0, new IppiPoint(region.X, region.Y), &m00, 0);
//since our input image is not 0s and 1s but 0s and 255s we have to divide by 255 in order to re-normalize our moments
System.Diagnostics.Debug.Assert(m00 % 255 == 0, "M00 was not a multiple of 255");
m00 /= 255;
//only retrieve other moments if this is a "fish candidate"
if (m00 > MinArea && m00 <= MaxArea)
{
ip.ippiGetSpatialMoment_64s(_momentState, 1, 0, 0, new IppiPoint(region.X, region.Y), &m10, 0);
m10 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 1, 0, new IppiPoint(region.X, region.Y), &m01, 0);
m01 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 2, 0, 0, new IppiPoint(region.X, region.Y), &m20, 0);
m20 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 2, 0, new IppiPoint(region.X, region.Y), &m02, 0);
m02 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 1, 1, 0, new IppiPoint(region.X, region.Y), &m11, 0);
m11 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 3, 0, 0, new IppiPoint(region.X, region.Y), &m30, 0);
m30 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 3, 0, new IppiPoint(region.X, region.Y), &m03, 0);
m03 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 2, 1, 0, new IppiPoint(region.X, region.Y), &m21, 0);
m21 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 1, 2, 0, new IppiPoint(region.X, region.Y), &m12, 0);
m12 /= 255;
blobsDetected[i - 1] = new BlobWithMoments(m00, m10, m01, m20, m11, m02, m30, m03, m21, m12);
//Determine bounding box of the blob. The following seems kinda retarded as Ipp must already
//have obtained that information before so maybe there is some way to actually retrieve it??
//Do linescans using ipp's sum function starting from the blobs centroid until we hit a line
//the sum of which is 0
int xStart, xEnd, yStart, yEnd;
double sum = 1;
IppiPoint centroid = blobsDetected[i - 1].Centroid;
xStart = centroid.x - 5;
xEnd = centroid.x + 5;
yStart = centroid.y - 5;
yEnd = centroid.y + 5;
//in the following loops we PRE-increment, whence we stop the loop if we are at one coordinate short of the ends
//find xStart
while (sum > 0 && xStart > region.X + 4)
{
xStart -= 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, region.Y], _calc.Stride, new IppiSize(1, region.Height), &sum));
}
xStart += 1;//we have a sum of 0, so go back one line towards the centroid
//find xEnd
sum = 1;
while (sum > 0 && xEnd < region.X + region.Width - 6)
{
xEnd += 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xEnd, region.Y], _calc.Stride, new IppiSize(1, region.Height), &sum));
}
xEnd -= 1;//we have sum of 0, so go back one line towards the centroid
//find yStart - we can limit our x-search-space as we already have those boundaries
sum = 1;
while (sum > 0 && yStart > region.Y + 4)
{
yStart -= 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, yStart], _calc.Stride, new IppiSize(xEnd - xStart + 1, 1), &sum));
}
yStart += 1;
//find yEnd - again limit summation to x-search-space
sum = 1;
while (sum > 0 && yEnd < region.Y + region.Height - 6)
{
yEnd += 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, yEnd], _calc.Stride, new IppiSize(xEnd - xStart + 1, 1), &sum));
}
yEnd -= 1;
blobsDetected[i - 1].BoundingBox = new IppiRect(xStart, yStart, xEnd - xStart + 1, yEnd - yStart + 1);
}
else
{
blobsDetected[i - 1] = new BlobWithMoments();
}
}
}
else
{
return(null);
}
long maxArea = 0;
int maxIndex = -1;
for (int i = 0; i < blobsDetected.Length; i++)
{
if (blobsDetected[i] == null)
{
break;
}
//Simply note down the largest blob
if (blobsDetected[i].Area > maxArea)
{
maxArea = blobsDetected[i].Area;
maxIndex = i;
}
}
if (maxArea < MinArea)
{
return(null);
}
else
{
return(blobsDetected[maxIndex]);
}
}
/// <summary>
/// Process the next frame
/// </summary>
/// <param name="frameNumber">The frame index</param>
/// <param name="camImage">The camera image</param>
/// <param name="poi">The fish location</param>
/// <returns>Whether experiment should continue or not</returns>
public override bool ProcessNext(int frameNumber, Image8 camImage, out IppiPoint?poi)
{
base.ProcessNext(frameNumber, camImage, out poi);
_lastFrame = frameNumber;
if (_originalType == OriginalType.Unknown)
{
//This should never happen...
System.Diagnostics.Debug.WriteLine("Unknown original experiment type. Terminating experiment");
return(false);
}
if (_scanner == null)
{
System.Diagnostics.Debug.WriteLine("Scanner was not initialized. Terminating experiment");
return(false);
}
if (_experimentPhase == ExperimentPhase.Done)
{
return(false);
}
BlobWithMoments fish = null;
fish = Tracker.Track(camImage);
if (fish != null)
{
poi = new IppiPoint(fish.Centroid.x, fish.Centroid.y);
try
{
if (_scanner != null)
{
_scanner.Hit(poi.GetValueOrDefault());
}
}
catch (ArgumentOutOfRangeException)
{
System.Diagnostics.Debug.WriteLine("Tried to hit coordinates outside scan table area.");
System.Diagnostics.Debug.WriteLine("Coordinates were: x={0}, y={1}", fish.xc, fish.yc);
System.Diagnostics.Debug.WriteLine("Terminating Experiment");
_laser.LaserPower = 0;
return(false);
}
}
//Get the next data item
string currentData = _originalTrackData.Dequeue();
string[] currentItems = currentData.Split('\t');
int phase, trial;
double power = ProcessTrackItems(currentItems, out phase, out trial);
if (double.IsNaN(power))
{
power = 0;
}
//Write track information
WriteTrackInfo(frameNumber, fish, phase, trial);
//Write images
ip.ippiSet_8u_C1R(0, _camRegion.Image, _camRegion.Stride, _camRegion.Size);
if (fish != null)
{
MainViewModel.CopyRegionImage(fish.Centroid, _camRegion, camImage);
}
_imageWriter.WriteFrame(_camRegion);
if (fish != null)
{
MainViewModel.CopyRegionImage(fish.Centroid, _camRegion, Tracker.Background);
}
_backgroundWriter.WriteFrame(_camRegion);
RunReplay(power);
//If we have lost the fish, disengage the laser
if (fish == null)
{
_laser.LaserPower = 0;
}
return(true);
}
/// <summary>
/// Tracks fish on a multiwell plate
/// </summary>
/// <param name="image">The current image to track and update the background with</param>
/// <param name="updateBackground">If true the current image will be used to update the background</param>
/// <param name="forceFullFrame">If set to true background updates won't exclude fish locations</param>
/// <returns>A list of fish-blobs, one for each well or null if no fish was detected</returns>
public BlobWithMoments[] TrackMultiWell(Image8 image, bool updateBackground = true, bool forceFullFrame = false)
{
if (_frame == 0)
{
ip.ippiSet_8u_C1R(0, Foreground.Image, Foreground.Stride, Foreground.Size);
ip.ippiSet_8u_C1R(0, _calc.Image, _calc.Stride, _calc.Size);
}
BlobWithMoments[] currentFish = new BlobWithMoments[_wellnumber];
if (_frame > FramesInitialBackground && !forceFullFrame)
{
//do global stuff (everything except labelMarkers if we don't do parallel tracking otherwise only the inherently parallel background subtraction)
//cache 8-bit representation of the background
Image8 bg = _bgModel.Background;
//Perform background subtraction
if (DoMedianFiltering)
{
IppHelper.IppCheckCall(cv.ippiAbsDiff_8u_C1R(image.Image, image.Stride, bg.Image, bg.Stride, _calc.Image, _calc.Stride, image.Size));
}
else
{
IppHelper.IppCheckCall(cv.ippiAbsDiff_8u_C1R(image.Image, image.Stride, bg.Image, bg.Stride, _foreground.Image, _foreground.Stride, image.Size));
}
if (_trackMethod == TrackMethods.Parallel && _parallelChunks > 1)
{
if (!Parallel.For(0, _parallelChunks, TrackChunk).IsCompleted)
{
System.Diagnostics.Debug.WriteLine("Error in parallel tracking. Not all regions completed");
}
//copy our result into currentFish
for (int i = 0; i < _wellnumber; i++)
{
currentFish[i] = _parallelTrackResults[i];
}
}
else
{
if (DoMedianFiltering)
{
//remove noise via median filtering
IppHelper.IppCheckCall(ip.ippiFilterMedianWeightedCenter3x3_8u_C1R(_calc[1, 1], _calc.Stride,
_foreground[1, 1], _foreground.Stride, new IppiSize(image.Width - 2, image.Height - 2), 1));
}
//Threshold and close
Im2Bw(_foreground, new IppiROI(new IppiPoint(0, 0), image.Size));
Close3x3(_foreground, new IppiROI(new IppiPoint(0, 0), image.Size));
//label markers and extract - depending on the method selector
//we label components on the whole image (new) or individual wells(old)
if (_trackMethod == TrackMethods.OnWholeImage)
{
currentFish = ExtractAll();
}
else
{
for (int i = 0; i < _wellnumber; i++)
{
currentFish[i] = ExtractFish(_wells[i]);
}
}
}
}//If We are past initial background frames
//create/update background and advance frame counter
if (_frame == 0)
{
_bgModel = new SelectiveUpdateBGModel(image, 1.0f / FramesInBackground);
}
else if (updateBackground || _frame <= FramesInitialBackground)//only allow reduction of background update rate AFTER initial background has been created
{
if (currentFish == null || forceFullFrame)
{
_bgModel.UpdateBackground(image);
}
else
{
_bgModel.UpdateBackground(image, currentFish);
}
}
_frame++;
return(currentFish);
}
/// <summary>
/// Extracts for each well the most likely fish-blob or null if no suitable candidate was found
/// </summary>
/// <returns>All fish blobs found in the image</returns>
private BlobWithMoments[] ExtractAll()
{
int nMarkers = 0;
//we have fixed bounding boxes for each fish that are square and 1.5 times the typical fish-length
//with the fishes centroid centered in the box
int bbOffset = (int)(_typicalFishLength * 0.75);
int bbSize = (int)(_typicalFishLength * 1.5);
//Copy foreground to marker and label connected components
//IppHelper.IppCheckCall(ip.ippiCopy_8u_C1R(_foreground.Image, _foreground.Stride, _labelMarkers.Image, _labelMarkers.Stride, _foreground.Size));
IppHelper.IppCheckCall(cv.ippiLabelMarkers_8u_C1IR(_foreground.Image, _foreground.Stride, _foreground.Size, 1, 254, IppiNorm.ippiNormInf, &nMarkers, _markerBuffer));
if (nMarkers > 254)
{
nMarkers = 254;
}
//fish are identified by looping over all wells and computing the histogram of pixel values for each well. This will effectively tell us
//which marker is the most abundant in the given well => this would be the fish by our general detection logic
//The maximum value that ippiLabelMarkers has used is equal to nMarkers. Therefore, when we compute our histogram, we supply
//(nMarkers+2) as the nLevels parameter - this will give us (nMarkers+1) bins containing the counts from 0->nMarkers with 0 being our background
//loop over wells
for (int i = 0; i < _wellnumber; i++)
{
IppiROI well = _wells[i];
//get well-specific maxvalue
//byte maxVal = 0;
//IppHelper.IppCheckCall(ip.ippiMax_8u_C1R(_labelMarkers[well.TopLeft], _labelMarkers.Stride, well.Size, &maxVal));
//compute histogram in this well - from 0 until maxval
IppHelper.IppCheckCall(ip.ippiHistogramRange_8u_C1R(_foreground[well.TopLeft], _foreground.Stride, well.Size, _hist, _histogramLevels, nMarkers + 2));
//_hist now contains in position i the abundance of marker i => by looping over it from 1->nMarkers we can find the largest blob. If this blob is
//larger than our minimum size, we compute the moments and initialize a BlobWithMoments structure for it
int maxCount = 0;
int maxIndex = -1;
for (int j = 1; j <= nMarkers; j++)
{
if (_hist[j] > maxCount && _hist[j] > _minArea && _hist[j] <= _maxArea)
{
maxCount = _hist[j];
maxIndex = j;
}
}
if (maxIndex == -1)//no suitable fish found
{
_allFish[i] = null;
}
else
{
//compare and compute moments
//label all pixels with the current marker as 255 and others as 0
IppHelper.IppCheckCall(ip.ippiCompareC_8u_C1R(_foreground[well.TopLeft], _foreground.Stride, (byte)maxIndex, _wellCompare.Image, _wellCompare.Stride, well.Size, IppCmpOp.ippCmpEq));
//calculate image moments
IppHelper.IppCheckCall(ip.ippiMoments64s_8u_C1R(_wellCompare.Image, _wellCompare.Stride, well.Size, _momentState));
//retrieve moments
long m00 = 0;
long m10 = 0;
long m01 = 0;
long m20 = 0;
long m02 = 0;
long m11 = 0;
long m30 = 0;
long m03 = 0;
long m21 = 0;
long m12 = 0;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 0, 0, well.TopLeft, &m00, 0);
m00 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 1, 0, 0, well.TopLeft, &m10, 0);
m10 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 1, 0, well.TopLeft, &m01, 0);
m01 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 2, 0, 0, well.TopLeft, &m20, 0);
m20 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 2, 0, well.TopLeft, &m02, 0);
m02 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 1, 1, 0, well.TopLeft, &m11, 0);
m11 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 3, 0, 0, well.TopLeft, &m30, 0);
m30 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 0, 3, 0, well.TopLeft, &m03, 0);
m03 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 2, 1, 0, well.TopLeft, &m21, 0);
m21 /= 255;
ip.ippiGetSpatialMoment_64s(_momentState, 1, 2, 0, well.TopLeft, &m12, 0);
m12 /= 255;
_allFish[i] = new BlobWithMoments(m00, m10, m01, m20, m11, m02, m30, m03, m21, m12);
//assign bounding box
IppiRect bBox = new IppiRect(_allFish[i].Centroid.x - bbOffset, _allFish[i].Centroid.y - bbOffset, bbSize, bbSize);
if (bBox.x < well.X)
{
bBox.x = well.X;
}
if (bBox.y < well.Y)
{
bBox.y = well.Y;
}
if (bBox.x + bBox.width > well.X + well.Width)
{
bBox.width = well.X + well.Width - bBox.x;
}
if (bBox.y + bBox.height > well.Y + well.Height)
{
bBox.height = well.Y + well.Height - bBox.y;
}
_allFish[i].BoundingBox = bBox;
}
}//end looping over all wells
return(_allFish);
}
/// <summary>
/// Extracts a fish (candidate) from an image by performing background subtraction, noise filtering, thresholding and closing to obtain a foreground
/// followed by marker extraction.
/// </summary>
/// <param name="im">The image to extract the fish from</param>
/// <param name="region">The ROI to search</param>
/// <returns>The most likely fish blob or null if no suitable candidate was found</returns>
BlobWithMoments ExtractFish(Image8 im, IppiROI region)
{
int nMarkers = 0;
//Perform background subtraction - CASH BACKGROUND IMAGE POINTER - otherwise we actually do the whole costly
//32f conversion twice - once for accessing the actual image, and once for accessing the stride...
var bg = _bgModel.Background;
IppHelper.IppCheckCall(cv.ippiAbsDiff_8u_C1R(im[region.TopLeft], im.Stride, bg[region.TopLeft], bg.Stride, _calc[region.TopLeft], _calc.Stride, region.Size));
//remove noise via median filtering
_mFiltSize.width = region.Width - 2;
_mFiltSize.height = region.Height - 2;
IppHelper.IppCheckCall(ip.ippiFilterMedianWeightedCenter3x3_8u_C1R(_calc[region.X + 1, region.Y + 1], _calc.Stride,
_bgSubtracted[region.X + 1, region.Y + 1], _bgSubtracted.Stride, _mFiltSize, 1));
//Threshold and close
Im2Bw(_bgSubtracted, _foreground, region);
//Do as two step to get information back into _foreground
_strel3x3.Dilate(_foreground, _calc, region);
_strel3x3.Erode(_calc, _foreground, region);
//Label connected components
IppHelper.IppCheckCall(cv.ippiLabelMarkers_8u_C1IR(_foreground[region.TopLeft], _foreground.Stride, region.Size, 1, 254, IppiNorm.ippiNormInf, &nMarkers, _markerBuffer));
//loop over returned markers and use ipp to extract blobs
if (nMarkers > 0)
{
if (nMarkers > 254)
{
nMarkers = 254;
}
//create or update our intermediate blob storage to store the required number of marker representations
if (_blobsDetected == null || _blobsDetected.Length < nMarkers)
{
_blobsDetected = new BlobWithMoments[nMarkers];
}
for (int i = 1; i <= nMarkers; i++)
{
//label all pixels with the current marker as 255 and others as 0
IppHelper.IppCheckCall(ip.ippiCompareC_8u_C1R(_foreground[region.TopLeft], _foreground.Stride, (byte)i, _calc[region.TopLeft], _calc.Stride, region.Size, IppCmpOp.ippCmpEq));
//calculate image moments
IppHelper.IppCheckCall(ip.ippiMoments64f_8u_C1R(_calc[region.TopLeft], _calc.Stride, region.Size, _momentState));
//retrieve moments
double m00 = 0;
double m10 = 0;
double m01 = 0;
double m20 = 0;
double m02 = 0;
double m11 = 0;
double m30 = 0;
double m03 = 0;
double m21 = 0;
double m12 = 0;
ip.ippiGetSpatialMoment_64f(_momentState, 0, 0, 0, region.TopLeft, &m00);
//since our input image is not 0s and 1s but 0s and 255s we have to divide by 255 in order to re-normalize our moments
//System.Diagnostics.Debug.Assert(m00 % 255 == 0, "M00 was not a multiple of 255");
m00 /= 255;
//only retrieve other moments if this is a "fish candidate"
if (m00 >= MinArea)
{
ip.ippiGetSpatialMoment_64f(_momentState, 1, 0, 0, region.TopLeft, &m10);
m10 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 0, 1, 0, region.TopLeft, &m01);
m01 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 2, 0, 0, region.TopLeft, &m20);
m20 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 0, 2, 0, region.TopLeft, &m02);
m02 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 1, 1, 0, region.TopLeft, &m11);
m11 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 3, 0, 0, region.TopLeft, &m30);
m30 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 0, 3, 0, region.TopLeft, &m03);
m03 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 2, 1, 0, region.TopLeft, &m21);
m21 /= 255;
ip.ippiGetSpatialMoment_64f(_momentState, 1, 2, 0, region.TopLeft, &m12);
m12 /= 255;
if (_blobsDetected[i - 1] == null)
{
_blobsDetected[i - 1] = new BlobWithMoments((long)m00, (long)m10, (long)m01,
(long)m20, (long)m11, (long)m02, (long)m30, (long)m03, (long)m21, (long)m12);
}
else
{
_blobsDetected[i - 1].UpdateBlob((long)m00, (long)m10, (long)m01, (long)m20,
(long)m11, (long)m02, (long)m30, (long)m03, (long)m21, (long)m12);
}
//Determine bounding box of the blob. The following seems kinda retarded as Ipp must already
//have obtained that information before so maybe there is some way to actually retrieve it??
//Do linescans using ipp's sum function starting from the blobs centroid until we hit a line
//the sum of which is 0
int xStart, xEnd, yStart, yEnd;
double sum = 1;
IppiPoint centroid = _blobsDetected[i - 1].Centroid;
xStart = centroid.x - 1;
xEnd = centroid.x + 1;
yStart = centroid.y - 1;
yEnd = centroid.y + 1;
//in the following loops we PRE-increment, whence we stop the loop if we are at one coordinate short of the ends
//find xStart
_bboxScan.width = 1;
_bboxScan.height = region.Height;
while (sum > 0 && xStart > (region.X + 4))
{
xStart -= 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, region.Y], _calc.Stride, _bboxScan, &sum));
}
xStart += 1;//we have a sum of 0, so go back one line towards the centroid
//find xEnd
sum = 1;
while (sum > 0 && xEnd < region.X + region.Width - 5)
{
xEnd += 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xEnd, region.Y], _calc.Stride, _bboxScan, &sum));
}
xEnd -= 1;//we have sum of 0, so go back one line towards the centroid
//find yStart - we can limit our x-search-space as we already have those boundaries
_bboxScan.width = xEnd - xStart + 1;
_bboxScan.height = 1;
sum = 1;
while (sum > 0 && yStart > (region.Y + 4))
{
yStart -= 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, yStart], _calc.Stride, _bboxScan, &sum));
}
yStart += 1;
//find yEnd - again limit summation to x-search-space
sum = 1;
while (sum > 0 && yEnd < region.Y + region.Height - 5)
{
yEnd += 5;
IppHelper.IppCheckCall(ip.ippiSum_8u_C1R(_calc[xStart, yEnd], _calc.Stride, _bboxScan, &sum));
}
yEnd -= 1;
_blobsDetected[i - 1].UpdateBoundingBox(xStart, yStart, xEnd - xStart + 1, yEnd - yStart + 1);
}
else
{
if (_blobsDetected[i - 1] == null)
{
_blobsDetected[i - 1] = new BlobWithMoments();
}
else
{
_blobsDetected[i - 1].ResetBlob();
}
}
}
}
else
{
return(null);
}
//decide which of the detected objects is the fish
//usually we pick the larger blob - however to avoid
//tracking reflections on the wall, if the largest blob
//and second largest blob are of comparable size
//we pick the one which is closer to the center (as reflections
//are always more eccentric)
long maxArea = 0;
long secondMaxArea = 0;
int maxIndex = -1;
int secondMaxIndex = -1;
for (int i = 0; i < nMarkers; i++)
{
if (_blobsDetected[i] == null)
{
break;
}
//Note down the largest and second-largest blob - but only if those blobs aren't larger than the maxArea and if they eccentricity is at least MinEccentricity
//this comparison allows that if we find two exactly same-sized blobs to consider both but to not consider any further blobs of this size (which we hopefully never have anyways)
//Eccentricity and MaxAllowedArea checks removed at this point as they were mainly concieved to not track the laser.
if (_blobsDetected[i].Area >= maxArea && _blobsDetected[i].Area > secondMaxArea /*&& blobsDetected[i].Area<=MaxAllowedArea && blobsDetected[i].Eccentricity>=MinEccentricity*/)
{
secondMaxArea = maxArea;
maxArea = _blobsDetected[i].Area;
secondMaxIndex = maxIndex;
maxIndex = i;
}
}
if (maxArea < MinArea)
{
return(null);
}
else
{
//if our second-largest blob is at least two-thirds the size
//of the largest blob we also consider distance and swap accordingly
if ((float)secondMaxArea * 1.5 >= (float)maxArea)
{
double distMax, distSecondMax;
distMax = Distance.Euclidian(_blobsDetected[maxIndex].Centroid, DishCenter);
distSecondMax = Distance.Euclidian(_blobsDetected[secondMaxIndex].Centroid, DishCenter);
if (distMax > distSecondMax)
{
maxIndex = secondMaxIndex;
}
}
return(_blobsDetected[maxIndex]);
}
}
/// <summary>
/// Given an image identifies a fish in it and returns
/// it's properties such as position and heading
/// </summary>
/// <param name="image">The image to be tracked.</param>
/// <returns>A blob representation of the fish</returns>
public BlobWithMoments Track(Image8 image)
{
if (IsDisposed)
{
throw new ObjectDisposedException("Tracker90mmDish", "Can't track after disposal!");
}
BlobWithMoments currentFish = null;
if (RemoveCMOSISBrightLineArtefact)
{
//Do as two step to get information back into image
_strel3x3.Erode(image, _calc, _imageROI);
_strel3x3.Dilate(_calc, image, _imageROI);
}
if (_frame > FramesInitialBackground)
{
//if _previousFish is present we only check an area around it
if (_previousFish != null)
{
//compute search region
//top-left
tl_x = _previousFish.Centroid.x - _searchRegionSize;
tl_y = _previousFish.Centroid.y - _searchRegionSize;
tl_x = tl_x < 0 ? 0 : tl_x;
tl_y = tl_y < 0 ? 0 : tl_y;
//bottom-right
br_x = _previousFish.Centroid.x + _searchRegionSize;
br_y = _previousFish.Centroid.y + _searchRegionSize;
br_x = br_x >= image.Width ? image.Width - 1 : br_x;
br_y = br_y >= image.Height ? image.Height - 1 : br_y;
//update search region
_searchRegion.X = tl_x;
_searchRegion.Y = tl_y;
_searchRegion.Width = br_x - tl_x + 1;
_searchRegion.Height = br_y - tl_y + 1;
//extract fish within region
currentFish = ExtractFish(image, _searchRegion);
}
else
{
currentFish = ExtractFish(image, _imageROI);
}
}//If We are past initial background frames
//create/update background and advance frame counter
if (_frame == 0)
{
//Blank images - necessary specifically because of the median filter step
//which otherwise will leave pixels in _bgSubtracted un-initialized
ip.ippiSet_8u_C1R(0, Foreground.Image, Foreground.Stride, Foreground.Size);
ip.ippiSet_8u_C1R(0, _calc.Image, _calc.Stride, _calc.Size);
ip.ippiSet_8u_C1R(0, _bgSubtracted.Image, _bgSubtracted.Stride, _bgSubtracted.Size);
_bgModel = new SelectiveUpdateBGModel(image, 1.0f / FramesInBackground);
}
else
{
//update background every nth frame only
if (_frame % BGUpdateEvery == 0)
{
if (currentFish == null)
{
_bgModel.UpdateBackground(image);
}
else
{
_bgModel.UpdateBackground(image, currentFish);
}
}
}
//Update knowledge about previous fish
//if current fish is trustworthy
if (currentFish != null && currentFish.Area > FullTrustMinArea)
{
_previousFish = currentFish;
}
else
{
_previousFish = null;
}
_frame++;
return(currentFish);
}
/// <summary>
/// Process the next frame
/// </summary>
/// <param name="frameNumber">The frame index</param>
/// <param name="camImage">The camera image</param>
/// <param name="poi">The fish location</param>
/// <returns>Whether experiment should continue or not</returns>
public override bool ProcessNext(int frameNumber, Image8 camImage, out IppiPoint?poi)
{
base.ProcessNext(frameNumber, camImage, out poi);
_lastFrame++;
if (_scanner == null)
{
System.Diagnostics.Debug.WriteLine("Scanner was not initialized. Terminating experiment");
return(false);
}
if (_experimentPhase == ExperimentPhase.Done)
{
return(false);
}
BlobWithMoments fish = null;
fish = Tracker.Track(camImage);
if (fish != null)
{
poi = new IppiPoint(fish.Centroid.x, fish.Centroid.y);
try
{
if (_scanner != null)
{
_scanner.Hit(poi.GetValueOrDefault());
}
}
catch (ArgumentOutOfRangeException)
{
System.Diagnostics.Debug.WriteLine("Tried to hit coordinates outside scan table area.");
System.Diagnostics.Debug.WriteLine("Coordinates were: x={0}, y={1}", fish.xc, fish.yc);
System.Diagnostics.Debug.WriteLine("Terminating Experiment");
_laser.LaserPower = 0;
return(false);
}
}
//Write track information including current phase, current laser power
WriteTrackInfo(frameNumber, fish);
//Write images
ip.ippiSet_8u_C1R(0, _camRegion.Image, _camRegion.Stride, _camRegion.Size);
if (fish != null)
{
MainViewModel.CopyRegionImage(fish.Centroid, _camRegion, camImage);
}
_imageWriter.WriteFrame(_camRegion);
if (fish != null)
{
MainViewModel.CopyRegionImage(fish.Centroid, _camRegion, Tracker.Background);
}
_backgroundWriter.WriteFrame(_camRegion);
//Depending on phase call process method to control laser power and switch phase if appropriate
switch (_experimentPhase)
{
case ExperimentPhase.Habituation:
RunHabituation();
break;
case ExperimentPhase.Pre:
RunPrePhase();
break;
case ExperimentPhase.Gradient:
RunGradient(fish);
break;
}
//If we have lost the fish, disengage the laser
if (fish == null)
{
_laser.LaserPower = 0;
}
return(true);
}
