public SegmentationSolution SegmentImage(Image2D <Color> image, ObjectBackgroundColorModels colorModels)
{
if (image == null)
{
throw new ArgumentNullException("image");
}
if (colorModels == null)
{
throw new ArgumentNullException("colorModels");
}
this.ImageSegmentator = new ImageSegmentator(
image,
colorModels,
this.ColorDifferencePairwiseTermCutoff,
this.ColorDifferencePairwiseTermWeight,
this.ConstantPairwiseTermWeight,
this.ObjectColorUnaryTermWeight,
this.BackgroundColorUnaryTermWeight,
this.ObjectShapeUnaryTermWeight,
this.BackgroundShapeUnaryTermWeight);
DebugConfiguration.WriteImportantDebugText(
"Segmented image size is {0}x{1}.",
this.ImageSegmentator.ImageSize.Width,
this.ImageSegmentator.ImageSize.Height);
SegmentationSolution solution = null;
try
{
if (this.ShapeModel == null)
{
throw new InvalidOperationException("Shape model must be specified before segmenting image.");
}
DebugConfiguration.WriteImportantDebugText("Running segmentation algorithm...");
this.IsRunning = true;
solution = this.SegmentCurrentImage();
if (solution == null)
{
throw new InvalidOperationException("Segmentation solution can not be null.");
}
}
finally
{
if (this.IsStopping)
{
this.WasStopped = true;
}
this.IsRunning = false;
this.IsStopping = false;
}
DebugConfiguration.WriteImportantDebugText("Finished");
return(solution);
}
private void PreparePairwiseTerms()
{
this.scaledPairwiseTerms = new Image2D <Tuple <double, double, double> >(this.segmentedImage.Width, this.segmentedImage.Height);
double meanBrightnessDiff = CalculateMeanBrightnessDifference();
for (int x = 0; x < this.segmentedImage.Width; ++x)
{
for (int y = 0; y < this.segmentedImage.Height; ++y)
{
double weightRight = 0, weightBottom = 0, weightBottomRight = 0;
if (x < this.segmentedImage.Width - 1)
{
weightRight = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x + 1, y));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.Right, weightRight);
}
if (y < this.segmentedImage.Height - 1)
{
weightBottom = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x, y + 1));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.Bottom, weightBottom);
}
if (x < this.segmentedImage.Width - 1 && y < this.segmentedImage.Height - 1)
{
weightBottomRight = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x + 1, y + 1));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.RightBottom, weightBottomRight);
}
this.scaledPairwiseTerms[x, y] = new Tuple <double, double, double>(weightRight, weightBottom, weightBottomRight);
}
}
}
private Image2D LoadImage(string path)
{
using (var stream = File.OpenRead(path))
{
return(Image2D.FromStream(stream));
}
}
private void OnSegmentationCompleted(object sender, RunWorkerCompletedEventArgs e)
{
if (e.Error == null)
{
this.DisposeStatusImages();
SegmentationSolution result = (SegmentationSolution)e.Result;
if (result.Shape != null)
{
this.currentImage.Image = CreateStatusImage(result.Shape);
}
else
{
this.currentImage.Image = (Image)this.segmentedImage.Clone();
}
if (result.Mask != null)
{
this.segmentationMaskImage.Image = Image2D.ToRegularImage(result.Mask);
}
}
else
{
MessageBox.Show(e.Error.Message, "Error!", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
this.startGpuButton.Enabled = true;
this.startCpuButton.Enabled = true;
this.stopButton.Enabled = false;
this.pauseContinueButton.Enabled = false;
this.segmentationPropertiesGrid.Enabled = true;
}
public void DeleteImageByIndex(int Idx)
{
Image2D ImageObj = _Images[Idx];
ImageObj.Delete();
_Images.Remove(ImageObj);
}
private double CalcObjective(Shape shape, Image2D <bool> mask)
{
double shapeEnergy = this.ShapeModel.CalculateEnergy(shape);
double labelingEnergy = CalcShapeLabelingEnergy(shape, mask);
return(shapeEnergy * this.ShapeEnergyWeight + labelingEnergy);
}
private static void ExtractObjectBackgroundColorsByMask(
Image2D <Color> image,
Image2D <bool?> colorModelMask,
IList <Color> objectPixels,
IList <Color> backgroundPixels)
{
Debug.Assert(image.Width == colorModelMask.Width && image.Height == colorModelMask.Height);
for (int i = 0; i < image.Width; ++i)
{
for (int j = 0; j < image.Height; ++j)
{
if (!colorModelMask[i, j].HasValue)
{
continue;
}
Color pixelColor = image[i, j];
bool isObject = colorModelMask[i, j].Value;
if (isObject)
{
objectPixels.Add(pixelColor);
}
else
{
backgroundPixels.Add(pixelColor);
}
}
}
}
public Image2D <bool?> GetMask(int maskWidth, int maskHeight)
{
RenderTargetBitmap rtb = new RenderTargetBitmap(maskWidth, maskHeight, 96, 96, PixelFormats.Pbgra32);
rtb.Render(this.inkCanvas);
Image2D <Color> maskImage = ImageHelper.BitmapSourceToImage2D(rtb);
Image2D <bool?> result = new Image2D <bool?>(maskImage.Width, maskImage.Height);
const int colorIntensityThreshold = 50;
for (int i = 0; i < maskImage.Width; ++i)
{
for (int j = 0; j < maskImage.Height; ++j)
{
Color maskColor = maskImage[i, j];
if (maskColor.G > colorIntensityThreshold)
{
result[i, j] = true;
}
else if (maskColor.B > colorIntensityThreshold)
{
result[i, j] = false;
}
else
{
result[i, j] = null;
}
}
}
return(result);
}
public BranchAndBoundProgressEventArgs(
double lowerBound,
Image2D <bool> segmentationMask,
Image2D <ObjectBackgroundTerm> unaryTermsImage,
Image2D <ObjectBackgroundTerm> shapeTermsImage,
ShapeConstraints constraints)
{
if (segmentationMask == null)
{
throw new ArgumentNullException("segmentationMask");
}
if (unaryTermsImage == null)
{
throw new ArgumentNullException("unaryTermsImage");
}
if (shapeTermsImage == null)
{
throw new ArgumentNullException("shapeTermsImage");
}
if (constraints == null)
{
throw new ArgumentNullException("constraints");
}
this.LowerBound = lowerBound;
this.SegmentationMask = segmentationMask;
this.UnaryTermsImage = unaryTermsImage;
this.ShapeTermsImage = shapeTermsImage;
this.Constraints = constraints;
}
public BranchAndBoundCompletedEventArgs(
Image2D <bool> collapsedSolutionSegmentationMask,
Image2D <ObjectBackgroundTerm> collapsedSolutionUnaryTermsImage,
Image2D <ObjectBackgroundTerm> collapsedSolutionShapeTermsImage,
ShapeConstraints resultConstraints,
double lowerBound)
{
if (collapsedSolutionSegmentationMask == null)
{
throw new ArgumentNullException("collapsedSolutionSegmentationMask");
}
if (collapsedSolutionUnaryTermsImage == null)
{
throw new ArgumentNullException("collapsedSolutionUnaryTermsImage");
}
if (collapsedSolutionShapeTermsImage == null)
{
throw new ArgumentNullException("collapsedSolutionShapeTermsImage");
}
if (resultConstraints == null)
{
throw new ArgumentNullException("resultConstraints");
}
this.CollapsedSolutionSegmentationMask = collapsedSolutionSegmentationMask;
this.CollapsedSolutionUnaryTermsImage = collapsedSolutionUnaryTermsImage;
this.CollapsedSolutionShapeTermsImage = collapsedSolutionShapeTermsImage;
this.ResultConstraints = resultConstraints;
this.LowerBound = lowerBound;
}
public void DeleteImageBId(string Id)
{
Image2D Panel = _Images.Find(x => x.gameObject.name == Id);
Panel.Delete();
_Images.Remove(Panel);
}
private void PreparePairwiseTerms()
{
this.scaledPairwiseTerms = new Image2D<Tuple<double, double, double>>(this.segmentedImage.Width, this.segmentedImage.Height);
double meanBrightnessDiff = CalculateMeanBrightnessDifference();
for (int x = 0; x < this.segmentedImage.Width; ++x)
{
for (int y = 0; y < this.segmentedImage.Height; ++y)
{
double weightRight = 0, weightBottom = 0, weightBottomRight = 0;
if (x < this.segmentedImage.Width - 1)
{
weightRight = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x + 1, y));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.Right, weightRight);
}
if (y < this.segmentedImage.Height - 1)
{
weightBottom = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x, y + 1));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.Bottom, weightBottom);
}
if (x < this.segmentedImage.Width - 1 && y < this.segmentedImage.Height - 1)
{
weightBottomRight = CalculateScaledPairwiseTerms(meanBrightnessDiff, new Point(x, y), new Point(x + 1, y + 1));
this.graphCutCalculator.SetNeighborWeights(x, y, Neighbor.RightBottom, weightBottomRight);
}
this.scaledPairwiseTerms[x, y] = new Tuple<double, double, double>(weightRight, weightBottom, weightBottomRight);
}
}
}
/// <summary>
/// Handes the OnRemove-event from the cache.
/// </summary>
/// <param name="image"></param>
private void OnRemove(Image2D image)
{ // dispose of the image after it is removed from the cache.
_nativeImageCache.Release(image.NativeImage);
image.NativeImage = null;
OsmSharp.Logging.Log.TraceEvent("LayerMBTile", Logging.TraceEventType.Information, "OnRemove: {0}", _cache.Count);
}
public override void FromX(XElement xe)
{
Name = xe.Attribute("Name").Value;
LongName = xe.Attribute("LongName").Value;
Image = new Image2D();
Image.FromX(xe.Element("Image"));
LoadStandardInfo(xe);
LoadAttributeInfos(xe);
}
/// <summary>
/// Extracts style information.
/// </summary>
/// <param name="image"></param>
/// <returns></returns>
public static Image2DStyle From(Image2D image, int layer)
{
Image2DStyle newStyle = new Image2DStyle();
newStyle.MaxZoom = image.MaxZoom;
newStyle.MinZoom = image.MinZoom;
newStyle.Layer = layer;
return(newStyle);
}
public SegmentationSolution(Shape shape, Image2D<bool> mask, double energy)
{
if (shape == null && mask == null)
throw new ArgumentException("Segmentation solution should contain something.");
this.Shape = shape;
this.Mask = mask;
this.Energy = energy;
}
protected override SegmentationSolution SegmentCurrentImage()
{
DebugConfiguration.WriteImportantDebugText("Performing initial segmentation...");
this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => ObjectBackgroundTerm.Zero);
Image2D <bool> prevMask = this.ImageSegmentator.GetLastSegmentationMask();
Shape prevShape = this.ShapeModel.FitMeanShape(
this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
double prevEnergy = 0;
for (int iteration = 1; iteration <= this.MaxIterationCount && !this.IsStopping; ++iteration)
{
this.WaitIfPaused();
DebugConfiguration.WriteImportantDebugText("Iteration {0}", iteration);
Image2D <bool> prevMaskCopy = prevMask;
Shape currentShape = this.ShapeFitter.Run(
prevShape,
(s, t) => this.ShapeMutator.MutateShape(s, this.ShapeModel, this.ImageSegmentator.ImageSize, t / this.ShapeFitter.StartTemperature),
s => this.CalcObjective(s, prevMaskCopy));
double currentEnergy = this.ImageSegmentator.SegmentImageWithShapeTerms((x, y) => this.ShapeModel.CalculatePenalties(currentShape, new Vector(x, y)));
Image2D <bool> currentMask = this.ImageSegmentator.GetLastSegmentationMask();
int differentValues = Image2D <bool> .DifferentValueCount(prevMask, currentMask);
double changedPixelRate = (double)differentValues / (this.ImageSegmentator.ImageSize.Width * this.ImageSegmentator.ImageSize.Height);
DebugConfiguration.WriteImportantDebugText("On iteration {0}:", iteration);
DebugConfiguration.WriteImportantDebugText("Energy is {0:0.000}", currentEnergy);
DebugConfiguration.WriteImportantDebugText("Changed pixel rate is {0:0.000000}", changedPixelRate);
DebugConfiguration.WriteImportantDebugText();
if (iteration > this.MinIterationCount && changedPixelRate < this.MinChangeRate)
{
DebugConfiguration.WriteImportantDebugText("Changed pixel rate is too low, breaking...");
break;
}
prevMask = currentMask;
prevShape = currentShape;
prevEnergy = currentEnergy;
if (IterationFinished != null)
{
IterationFinished(this, new SegmentationIterationFinishedEventArgs(
iteration,
currentShape,
this.ImageSegmentator.GetLastSegmentationMask(),
this.ImageSegmentator.GetLastUnaryTerms(),
this.ImageSegmentator.GetLastShapeTerms()));
}
}
return(new SegmentationSolution(prevShape, prevMask, prevEnergy));
}
public Texture2D()
{
this.m_image = (Image2D)null;
this.m_blendColor = -1;
this.m_blending = 227;
this.m_imageFilter = 210;
this.m_levelFilter = 208;
this.m_wrappingS = 241;
this.m_wrappingT = 241;
}
public void Blit(Vector2I position, Image2D image)
{
for (int y = 0; y < image.Size.Y; y++)
{
for (int x = 0; x < image.Size.X; x++)
{
SetPixel(position + new Vector2I(x, y), image.GetPixel(new Vector2I(x, y)));
}
}
}
/// <summary>
/// Loads one tile.
/// </summary>
private void LoadTile(object state)
{
// get job.
KeyValuePair <TileRange, Tile> job = (KeyValuePair <TileRange, Tile>)state;
if (_tileRange != job.Key)
{ // only load tile if range is unchanged.
return;
}
// a tile was found to load.
Tile tile = job.Value;
Image2D image2D;
lock (_cache)
{ // check again for the tile.
if (_cache.TryGet(tile, out image2D))
{ // tile is already there!
return;
}
}
// load the tile.
string url = string.Format(_tilesURL,
tile.Zoom,
tile.X,
tile.Y);
var request = (HttpWebRequest)HttpWebRequest.Create(
url);
request.Accept = "text/html, image/png, image/jpeg, image/gif, */*";
request.Headers[HttpRequestHeader.UserAgent] = "OsmSharp/4.0";
WebResponse myResp = request.GetResponse();
Stream stream = myResp.GetResponseStream();
byte[] image = null;
if (stream != null)
{
// there is data: read it.
var memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
image = memoryStream.ToArray();
image2D = new Image2D(tile.Box.BottomLeft[0], tile.Box.TopLeft[1],
tile.Box.BottomLeft[0], tile.BottomRight[1], image,
(float)_projection.ToZoomFactor(tile.Zoom - _zoomMinOffset),
(float)_projection.ToZoomFactor(tile.Zoom + (1 - _zoomMinOffset)));
lock (_cache)
{ // add the result to the cache.
_cache.Add(tile, image2D);
}
}
}
public Sprite3D(bool scaled, Image2D image, Appearance app)
{
this.m_appearance = app;
this.m_image = image;
this.m_cropX = 0;
this.m_cropY = 0;
this.m_cropW = 0;
this.m_cropH = 0;
this.m_cropW = this.m_image.getWidth();
this.m_cropH = this.m_image.getHeight();
}
public SegmentationSolution(Shape shape, Image2D <bool> mask, double energy)
{
if (shape == null && mask == null)
{
throw new ArgumentException("Segmentation solution should contain something.");
}
this.Shape = shape;
this.Mask = mask;
this.Energy = energy;
}
public void Blit(Vector2I writePosition, Image2D image, Vector2I readPosition)
{
//!!!!slowly
for (int y = 0; y < image.Size.Y; y++)
{
for (int x = 0; x < image.Size.X; x++)
{
SetPixel(writePosition + new Vector2I(x, y), image.GetPixel(readPosition + new Vector2I(x, y)));
}
}
}
/// <summary>
/// Adds a new style and returns it's index.
/// </summary>
/// <param name="image"></param>
/// <param name="layer"></param>
/// <returns></returns>
public ushort AddStyle(Image2D image, int layer)
{
Image2DStyle newStyle = Image2DStyle.From(image, layer);
int indexOf = this.ImageStyles.IndexOf(newStyle);
if (indexOf < 0)
{ // the style is not found yet.
indexOf = this.ImageStyles.Count;
this.ImageStyles.Add(newStyle);
}
return((ushort)indexOf);
}
/// <summary>
/// Handes the OnRemove-event from the cache.
/// </summary>
/// <param name="image"></param>
private void OnRemove(Image2D image)
{ // dispose of the image after it is removed from the cache.
try
{
_nativeImageCache.Release(image.NativeImage);
image.NativeImage = null;
}
catch (Exception ex)
{ // don't worry about exceptions here.
OsmSharp.Logging.Log.TraceEvent("LayerTile", Logging.TraceEventType.Error, ex.Message);
}
}
/// <summary>
/// Notifies this layer that the current mapview has changed.
/// </summary>
/// <param name="map"></param>
/// <param name="zoomFactor"></param>
/// <param name="center"></param>
/// <param name="view"></param>
/// <param name="extraView"></param>
protected internal override void ViewChanged(Map map, float zoomFactor, GeoCoordinate center, View2D view, View2D extraView)
{
// calculate the current zoom level.
var zoomLevel = (int)System.Math.Round(map.Projection.ToZoomLevel(zoomFactor), 0);
if (zoomLevel >= _minZoomLevel && zoomLevel <= _maxZoomLevel)
{
// build the bounding box.
var viewBox = view.OuterBox;
// build the tile range.
var range = TileRange.CreateAroundBoundingBox(new GeoCoordinateBox(map.Projection.ToGeoCoordinates(viewBox.Min[0], viewBox.Min[1]),
map.Projection.ToGeoCoordinates(viewBox.Max[0], viewBox.Max[1])), zoomLevel);
OsmSharp.Logging.Log.TraceEvent("LayerMBTile", OsmSharp.Logging.TraceEventType.Verbose, string.Format("Requesting {0} tiles for view.", range.Count));
// request all missing tiles.
lock (_connection)
{ // make sure the connection is accessed synchronously.
// TODO: Investigate the SQLite multithreaded behaviour..
// TODO: this a very naive way of loading these tiles. Find a way to query SQLite more efficiently
// TODO: find a way to have some cached tiles.
foreach (var tile in range.EnumerateInCenterFirst())
{
Image2D value;
if (_cache.TryGet(tile, out value))
{
// Tile is already in cache. We used TryGet, and not TryPeek, to inform the cache
// that we intend to use the datum in question.
continue;
}
Tile invertTile = tile.InvertY();
var tiles = _connection.Query <tiles>("SELECT * FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?",
invertTile.Zoom, invertTile.X, invertTile.Y);
foreach (var mbTile in tiles)
{
var box = tile.ToBox(_projection);
var nativeImage = _nativeImageCache.Obtain(mbTile.tile_data);
var image2D = new Image2D(box.Min[0], box.Min[1], box.Max[1], box.Max[0], nativeImage);
image2D.Layer = (uint)(_maxZoomLevel - tile.Zoom);
lock (_cache)
{ // add the result to the cache.
_cache.Add(tile, image2D);
}
}
}
}
OsmSharp.Logging.Log.TraceEvent("LayerMBTile", Logging.TraceEventType.Information, "Cached tiles: {0}", _cache.Count);
}
}
public ImageSegmentator(
Image2D <Color> image,
ObjectBackgroundColorModels colorModels,
double colorDifferencePairwiseTermCutoff,
double colorDifferencePairwiseTermWeight,
double constantPairwiseTermWeight,
double objectColorUnaryTermWeight,
double backgroundColorUnaryTermWeight,
double objectShapeUnaryTermWeight,
double backgroundShapeUnaryTermWeight)
{
if (image == null)
{
throw new ArgumentNullException("image");
}
if (colorModels == null)
{
throw new ArgumentNullException("colorModels");
}
if (colorDifferencePairwiseTermCutoff <= 0)
{
throw new ArgumentOutOfRangeException("colorDifferencePairwiseTermCutoff", "Parameter value should be positive.");
}
if (colorDifferencePairwiseTermWeight < 0)
{
throw new ArgumentOutOfRangeException("colorDifferencePairwiseTermWeight", "Parameter value should not be negative.");
}
if (constantPairwiseTermWeight < 0)
{
throw new ArgumentOutOfRangeException("constantPairwiseTermWeight", "Parameter value should not be negative.");
}
this.ColorDifferencePairwiseTermCutoff = colorDifferencePairwiseTermCutoff;
this.ColorDifferencePairwiseTermWeight = colorDifferencePairwiseTermWeight;
this.ConstantPairwiseTermWeight = constantPairwiseTermWeight;
this.ObjectColorUnaryTermWeight = objectColorUnaryTermWeight;
this.BackgroundColorUnaryTermWeight = backgroundColorUnaryTermWeight;
this.ObjectShapeUnaryTermWeight = objectShapeUnaryTermWeight;
this.BackgroundShapeUnaryTermWeight = backgroundShapeUnaryTermWeight;
this.segmentedImage = image;
this.UnaryTermScaleCoeff = 1.0 / (image.Width * image.Height);
this.PairwiseTermScaleCoeff = 1.0 / Math.Sqrt(image.Width * image.Height);
this.graphCutCalculator = new GraphCutCalculator(this.segmentedImage.Width, this.segmentedImage.Height);
this.PrepareColorTerms(colorModels);
this.PreparePairwiseTerms();
this.PrepareOther();
}
/// <summary>
/// Converts into a basic entry.
/// </summary>
/// <param name="image"></param>
/// <param name="styleId"></param>
/// <param name="id"></param>
/// <returns></returns>
internal static Image2DEntry From(uint id, Image2D image, ushort styleId)
{
Image2DEntry entry = new Image2DEntry();
entry.Id = id;
entry.ImageData = image.ImageData;
entry.Bottom = image.Bottom;
entry.Left = image.Left;
entry.Right = image.Right;
entry.Top = image.Top;
entry.StyleId = styleId;
return(entry);
}
public SegmentationIterationFinishedEventArgs(
int iteration,
Shape shape,
Image2D <bool> segmentationMask,
Image2D <ObjectBackgroundTerm> unaryTermsImage,
Image2D <ObjectBackgroundTerm> shapeTermsImage)
{
this.Iteration = iteration;
this.Shape = shape;
this.SegmentationMask = segmentationMask;
this.UnaryTermsImage = unaryTermsImage;
this.ShapeTermsImage = shapeTermsImage;
}
/// <summary>
/// Converts this basic entry into a scene object.
/// </summary>
/// <param name="style"></param>
/// <returns></returns>
internal Image2D To(Image2DStyle style)
{
Image2D image = new Image2D();
image.Left = this.Left;
image.Bottom = this.Bottom;
image.ImageData = this.ImageData;
image.MaxZoom = style.MaxZoom;
image.MinZoom = style.MinZoom;
image.Right = this.Right;
image.Top = this.Top;
return(image);
}
public SegmentationIterationFinishedEventArgs(
int iteration,
Shape shape,
Image2D<bool> segmentationMask,
Image2D<ObjectBackgroundTerm> unaryTermsImage,
Image2D<ObjectBackgroundTerm> shapeTermsImage)
{
this.Iteration = iteration;
this.Shape = shape;
this.SegmentationMask = segmentationMask;
this.UnaryTermsImage = unaryTermsImage;
this.ShapeTermsImage = shapeTermsImage;
}
void OnCoordinateDescentIterationFinished(object sender, SegmentationIterationFinishedEventArgs e)
{
this.Invoke(new MethodInvoker(
delegate
{
this.DisposeStatusImages();
this.currentImage.Image = CreateStatusImage(e.Shape);
this.segmentationMaskImage.Image = Image2D.ToRegularImage(e.SegmentationMask);
this.unaryTermsImage.Image = Image2D.ToRegularImage(e.UnaryTermsImage, -5, 5);
this.shapeTermsImage.Image = Image2D.ToRegularImage(e.ShapeTermsImage, -5, 5);
}));
}
public static Image2D<Color> BitmapSourceToImage2D(BitmapSource source)
{
Image2D<Color> result = new Image2D<Color>(source.PixelWidth, source.PixelHeight);
PixelColor[,] sourcePixels = GetPixelsFromBitmapSource(source);
for (int i = 0; i < result.Width; ++i)
for (int j = 0; j < result.Height; ++j)
{
PixelColor sourceColor = sourcePixels[i, j];
result[i, j] = Color.FromArgb(sourceColor.Red, sourceColor.Green, sourceColor.Blue);
}
return result;
}
private void PrepareColorTerms(ObjectBackgroundColorModels colorModels)
{
this.colorTerms = new Image2D <ObjectBackgroundTerm>(this.ImageSize.Width, this.ImageSize.Height);
for (int x = 0; x < this.ImageSize.Width; ++x)
{
for (int y = 0; y < this.ImageSize.Height; ++y)
{
Color color = this.segmentedImage[x, y];
double objectTerm = -colorModels.ObjectColorModel.LogProb(color);
double backgroundTerm = -colorModels.BackgroundColorModel.LogProb(color);
this.colorTerms[x, y] = new ObjectBackgroundTerm(objectTerm, backgroundTerm);
}
}
}
protected override SegmentationSolution SegmentCurrentImage()
{
Shape startShape = this.StartShape;
if (startShape == null)
{
startShape = this.ShapeModel.FitMeanShape(
this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
}
this.shapeTerms = new Image2D<ObjectBackgroundTerm>(this.ImageSegmentator.ImageSize.Width, this.ImageSegmentator.ImageSize.Height);
Shape solutionShape = this.SolutionFitter.Run(startShape, this.MutateSolution, s => this.CalcObjective(s, false));
double solutionEnergy = CalcObjective(solutionShape, true);
Image2D<bool> solutionMask = this.ImageSegmentator.GetLastSegmentationMask();
return new SegmentationSolution(solutionShape, solutionMask, solutionEnergy);
}
public ImageSegmentator(
Image2D<Color> image,
ObjectBackgroundColorModels colorModels,
double colorDifferencePairwiseTermCutoff,
double colorDifferencePairwiseTermWeight,
double constantPairwiseTermWeight,
double objectColorUnaryTermWeight,
double backgroundColorUnaryTermWeight,
double objectShapeUnaryTermWeight,
double backgroundShapeUnaryTermWeight)
{
if (image == null)
throw new ArgumentNullException("image");
if (colorModels == null)
throw new ArgumentNullException("colorModels");
if (colorDifferencePairwiseTermCutoff <= 0)
throw new ArgumentOutOfRangeException("colorDifferencePairwiseTermCutoff", "Parameter value should be positive.");
if (colorDifferencePairwiseTermWeight < 0)
throw new ArgumentOutOfRangeException("colorDifferencePairwiseTermWeight", "Parameter value should not be negative.");
if (constantPairwiseTermWeight < 0)
throw new ArgumentOutOfRangeException("constantPairwiseTermWeight", "Parameter value should not be negative.");
this.ColorDifferencePairwiseTermCutoff = colorDifferencePairwiseTermCutoff;
this.ColorDifferencePairwiseTermWeight = colorDifferencePairwiseTermWeight;
this.ConstantPairwiseTermWeight = constantPairwiseTermWeight;
this.ObjectColorUnaryTermWeight = objectColorUnaryTermWeight;
this.BackgroundColorUnaryTermWeight = backgroundColorUnaryTermWeight;
this.ObjectShapeUnaryTermWeight = objectShapeUnaryTermWeight;
this.BackgroundShapeUnaryTermWeight = backgroundShapeUnaryTermWeight;
this.segmentedImage = image;
this.UnaryTermScaleCoeff = 1.0 / (image.Width * image.Height);
this.PairwiseTermScaleCoeff = 1.0 / Math.Sqrt(image.Width * image.Height);
this.graphCutCalculator = new GraphCutCalculator(this.segmentedImage.Width, this.segmentedImage.Height);
this.PrepareColorTerms(colorModels);
this.PreparePairwiseTerms();
this.PrepareOther();
}
public AllowedLengthAngleChecker(
VertexConstraints constraint1,
VertexConstraints constraint2,
GeneralizedDistanceTransform2D checkingTransform,
double lengthRatio,
double meanAngle)
{
this.lengthAngleStatus = new Image2D<byte>(checkingTransform.GridSize.Width, checkingTransform.GridSize.Height);
LengthAngleSpaceSeparatorSet separator = new LengthAngleSpaceSeparatorSet(constraint1, constraint2);
this.checkingTransform = checkingTransform;
// Initial fill
for (int i = 0; i < checkingTransform.GridSize.Width; ++i)
{
double scaledLength = checkingTransform.GridIndexToCoordX(i);
double length = scaledLength / lengthRatio;
for (int j = 0; j < checkingTransform.GridSize.Height; ++j)
{
double shiftedAngle = checkingTransform.GridIndexToCoordY(j);
double angle = shiftedAngle + meanAngle;
if (separator.IsInside(length, angle))
this.lengthAngleStatus[i, j] = 2;
else if (i == 0 || j == 0 || this.lengthAngleStatus[i - 1, j] == 1 || this.lengthAngleStatus[i, j - 1] == 1)
this.lengthAngleStatus[i, j] = 1;
}
}
// Fill holes
for (int i = 0; i < checkingTransform.GridSize.Width; ++i)
{
for (int j = 0; j < checkingTransform.GridSize.Height; ++j)
{
if (lengthAngleStatus[i, j] == 0)
lengthAngleStatus[i, j] = 2;
}
}
}
public Image2D<bool?> GetMask(int maskWidth, int maskHeight)
{
RenderTargetBitmap rtb = new RenderTargetBitmap(maskWidth, maskHeight, 96, 96, PixelFormats.Pbgra32);
rtb.Render(this.inkCanvas);
Image2D<Color> maskImage = ImageHelper.BitmapSourceToImage2D(rtb);
Image2D<bool?> result = new Image2D<bool?>(maskImage.Width, maskImage.Height);
const int colorIntensityThreshold = 50;
for (int i = 0; i < maskImage.Width; ++i)
for (int j = 0; j < maskImage.Height; ++j)
{
Color maskColor = maskImage[i, j];
if (maskColor.G > colorIntensityThreshold)
result[i, j] = true;
else if (maskColor.B > colorIntensityThreshold)
result[i, j] = false;
else
result[i, j] = null;
}
return result;
}
public BranchAndBoundCompletedEventArgs(
Image2D<bool> collapsedSolutionSegmentationMask,
Image2D<ObjectBackgroundTerm> collapsedSolutionUnaryTermsImage,
Image2D<ObjectBackgroundTerm> collapsedSolutionShapeTermsImage,
ShapeConstraints resultConstraints,
double lowerBound)
{
if (collapsedSolutionSegmentationMask == null)
throw new ArgumentNullException("collapsedSolutionSegmentationMask");
if (collapsedSolutionUnaryTermsImage == null)
throw new ArgumentNullException("collapsedSolutionUnaryTermsImage");
if (collapsedSolutionShapeTermsImage == null)
throw new ArgumentNullException("collapsedSolutionShapeTermsImage");
if (resultConstraints == null)
throw new ArgumentNullException("resultConstraints");
this.CollapsedSolutionSegmentationMask = collapsedSolutionSegmentationMask;
this.CollapsedSolutionUnaryTermsImage = collapsedSolutionUnaryTermsImage;
this.CollapsedSolutionShapeTermsImage = collapsedSolutionShapeTermsImage;
this.ResultConstraints = resultConstraints;
this.LowerBound = lowerBound;
}
public BranchAndBoundProgressEventArgs(
double lowerBound,
Image2D<bool> segmentationMask,
Image2D<ObjectBackgroundTerm> unaryTermsImage,
Image2D<ObjectBackgroundTerm> shapeTermsImage,
ShapeConstraints constraints)
{
if (segmentationMask == null)
throw new ArgumentNullException("segmentationMask");
if (unaryTermsImage == null)
throw new ArgumentNullException("unaryTermsImage");
if (shapeTermsImage == null)
throw new ArgumentNullException("shapeTermsImage");
if (constraints == null)
throw new ArgumentNullException("constraints");
this.LowerBound = lowerBound;
this.SegmentationMask = segmentationMask;
this.UnaryTermsImage = unaryTermsImage;
this.ShapeTermsImage = shapeTermsImage;
this.Constraints = constraints;
}
private static void TestShapeTermsImpl(string testName, ShapeModel shapeModel, IEnumerable<VertexConstraints> vertexConstraints, IEnumerable<EdgeConstraints> edgeConstraints, Size imageSize)
{
ShapeConstraints constraintSet = ShapeConstraints.CreateFromConstraints(shapeModel.Structure, vertexConstraints, edgeConstraints);
// Get CPU results
Image2D<ObjectBackgroundTerm> shapeTermsCpu = new Image2D<ObjectBackgroundTerm>(imageSize.Width, imageSize.Height);
CpuShapeTermsLowerBoundCalculator calculatorCpu = new CpuShapeTermsLowerBoundCalculator();
calculatorCpu.CalculateShapeTerms(shapeModel, constraintSet, shapeTermsCpu);
Image2D.SaveToFile(shapeTermsCpu, -1000, 1000, String.Format("./{0}_cpu.png", testName));
// Get GPU results
Image2D<ObjectBackgroundTerm> shapeTermsGpu = new Image2D<ObjectBackgroundTerm>(imageSize.Width, imageSize.Height);
GpuShapeTermsLowerBoundCalculator calculatorGpu = new GpuShapeTermsLowerBoundCalculator();
calculatorGpu.CalculateShapeTerms(shapeModel, constraintSet, shapeTermsGpu);
Image2D.SaveToFile(shapeTermsGpu, -1000, 1000, String.Format("./{0}_gpu.png", testName));
// Compare with CPU results
for (int x = 0; x < imageSize.Width; ++x)
for (int y = 0; y < imageSize.Height; ++y)
{
Assert.AreEqual(shapeTermsCpu[x, y].ObjectTerm, shapeTermsGpu[x, y].ObjectTerm, 1e-2f);
Assert.AreEqual(shapeTermsCpu[x, y].BackgroundTerm, shapeTermsGpu[x, y].BackgroundTerm, 1e-2f);
}
}
private static void ExtractObjectBackgroundColorsByMask(
Image2D<Color> image,
Image2D<bool?> colorModelMask,
IList<Color> objectPixels,
IList<Color> backgroundPixels)
{
Debug.Assert(image.Width == colorModelMask.Width && image.Height == colorModelMask.Height);
for (int i = 0; i < image.Width; ++i)
for (int j = 0; j < image.Height; ++j)
{
if (!colorModelMask[i, j].HasValue)
continue;
Color pixelColor = image[i, j];
bool isObject = colorModelMask[i, j].Value;
if (isObject)
objectPixels.Add(pixelColor);
else
backgroundPixels.Add(pixelColor);
}
}
private double CalcObjective(Shape shape, Image2D<bool> mask)
{
double shapeEnergy = this.ShapeModel.CalculateEnergy(shape);
double labelingEnergy = CalcShapeLabelingEnergy(shape, mask);
return shapeEnergy * this.ShapeEnergyWeight + labelingEnergy;
}
private double CalcShapeLabelingEnergy(Shape shape, Image2D<bool> mask)
{
double shapeTermSum = 0;
for (int x = 0; x < mask.Width; ++x)
{
for (int y = 0; y < mask.Height; ++y)
{
if (mask[x, y])
shapeTermSum += this.ShapeModel.CalculateObjectPenalty(shape, new Vector(x, y)) * this.ObjectShapeUnaryTermWeight;
else
shapeTermSum += this.ShapeModel.CalculateBackgroundPenalty(shape, new Vector(x, y)) * this.BackgroundShapeUnaryTermWeight;
}
}
return shapeTermSum * this.ImageSegmentator.UnaryTermScaleCoeff;
}
public SegmentationSolution SegmentImage(Image2D<Color> image, ObjectBackgroundColorModels colorModels)
{
if (image == null)
throw new ArgumentNullException("image");
if (colorModels == null)
throw new ArgumentNullException("colorModels");
this.ImageSegmentator = new ImageSegmentator(
image,
colorModels,
this.ColorDifferencePairwiseTermCutoff,
this.ColorDifferencePairwiseTermWeight,
this.ConstantPairwiseTermWeight,
this.ObjectColorUnaryTermWeight,
this.BackgroundColorUnaryTermWeight,
this.ObjectShapeUnaryTermWeight,
this.BackgroundShapeUnaryTermWeight);
DebugConfiguration.WriteImportantDebugText(
"Segmented image size is {0}x{1}.",
this.ImageSegmentator.ImageSize.Width,
this.ImageSegmentator.ImageSize.Height);
SegmentationSolution solution = null;
try
{
if (this.ShapeModel == null)
throw new InvalidOperationException("Shape model must be specified before segmenting image.");
DebugConfiguration.WriteImportantDebugText("Running segmentation algorithm...");
this.IsRunning = true;
solution = this.SegmentCurrentImage();
if (solution == null)
throw new InvalidOperationException("Segmentation solution can not be null.");
}
finally
{
if (this.IsStopping)
this.WasStopped = true;
this.IsRunning = false;
this.IsStopping = false;
}
DebugConfiguration.WriteImportantDebugText("Finished");
return solution;
}
public Image2D<double> GetHorizontalColorDifferencePairwiseTerms()
{
Image2D<double> result = new Image2D<double>(this.ImageSize.Width, this.ImageSize.Height);
for (int i = 0; i < result.Width; ++i)
{
for (int j = 0; j < result.Height; ++j)
{
double notScaledWeight = this.scaledPairwiseTerms[i, j].Item1 / this.PairwiseTermScaleCoeff;
result[i, j] = (notScaledWeight - this.ConstantPairwiseTermWeight) / this.ColorDifferencePairwiseTermWeight;
}
}
return result;
}
public ImageSegmentationFeatures ExtractSegmentationFeaturesForMask(
Image2D<bool> mask)
{
if (this.firstTime)
throw new InvalidOperationException("You should perform segmentation first.");
double objectColorUnaryTermSum = 0;
double backgroundColorUnaryTermSum = 0;
double objectShapeUnaryTermSum = 0;
double backgroundShapeUnaryTermSum = 0;
int nonZeroPairwiseTermsCount = 0;
double pairwiseTermSum = 0;
for (int x = 0; x < mask.Width; ++x)
{
for (int y = 0; y < mask.Height; ++y)
{
if (mask[x, y])
{
objectColorUnaryTermSum += this.colorTerms[x, y].ObjectTerm;
objectShapeUnaryTermSum += this.lastShapeTerms[x, y].ObjectTerm;
}
else
{
backgroundColorUnaryTermSum += this.colorTerms[x, y].BackgroundTerm;
backgroundShapeUnaryTermSum += this.lastShapeTerms[x, y].BackgroundTerm;
}
if (x < mask.Width - 1 && mask[x, y] != mask[x + 1, y])
{
pairwiseTermSum += this.scaledPairwiseTerms[x, y].Item1;
++nonZeroPairwiseTermsCount;
}
if (y < mask.Height - 1 && mask[x, y] != mask[x, y + 1])
{
pairwiseTermSum += this.scaledPairwiseTerms[x, y].Item2;
++nonZeroPairwiseTermsCount;
}
if (x < mask.Width - 1 && y < mask.Height - 1 && mask[x, y] != mask[x + 1, y + 1])
{
pairwiseTermSum += this.scaledPairwiseTerms[x, y].Item3;
++nonZeroPairwiseTermsCount;
}
}
}
objectColorUnaryTermSum *= this.ObjectColorUnaryTermWeight * this.UnaryTermScaleCoeff;
backgroundColorUnaryTermSum *= this.BackgroundColorUnaryTermWeight * this.UnaryTermScaleCoeff;
objectShapeUnaryTermSum *= this.ObjectShapeUnaryTermWeight * this.UnaryTermScaleCoeff;
backgroundShapeUnaryTermSum *= this.BackgroundShapeUnaryTermWeight * this.UnaryTermScaleCoeff;
double constantPairwiseTermSum = nonZeroPairwiseTermsCount * this.ConstantPairwiseTermWeight * this.PairwiseTermScaleCoeff;
double colorDifferencePairwiseTermSum = pairwiseTermSum - constantPairwiseTermSum;
return new ImageSegmentationFeatures(
objectColorUnaryTermSum,
backgroundColorUnaryTermSum,
objectShapeUnaryTermSum,
backgroundShapeUnaryTermSum,
constantPairwiseTermSum,
colorDifferencePairwiseTermSum);
}
private void PrepareColorTerms(ObjectBackgroundColorModels colorModels)
{
this.colorTerms = new Image2D<ObjectBackgroundTerm>(this.ImageSize.Width, this.ImageSize.Height);
for (int x = 0; x < this.ImageSize.Width; ++x)
{
for (int y = 0; y < this.ImageSize.Height; ++y)
{
Color color = this.segmentedImage[x, y];
double objectTerm = -colorModels.ObjectColorModel.LogProb(color);
double backgroundTerm = -colorModels.BackgroundColorModel.LogProb(color);
this.colorTerms[x, y] = new ObjectBackgroundTerm(objectTerm, backgroundTerm);
}
}
}
private void PrepareOther()
{
this.lastUnaryTerms = new Image2D<ObjectBackgroundTerm>(this.ImageSize.Width, this.ImageSize.Height);
this.lastShapeTerms = new Image2D<ObjectBackgroundTerm>(this.ImageSize.Width, this.ImageSize.Height);
this.lastSegmentationMask = new Image2D<bool>(this.ImageSize.Width, this.ImageSize.Height);
}
private Image2D<ObjectBackgroundTerm> AllocateImage()
{
Debug.Assert(shapeModel != null);
Image2D<ObjectBackgroundTerm> result;
if (freeTermImages.Count > 0)
{
result = freeTermImages.Last.Value;
freeTermImages.RemoveLast();
}
else
result = new Image2D<ObjectBackgroundTerm>(imageSize.Width, imageSize.Height);
return result;
}
private void DeallocateImage(Image2D<ObjectBackgroundTerm> image)
{
freeTermImages.AddLast(image);
}
public static BitmapSource MaskToBitmapSource(Image2D<bool> mask)
{
PixelColor[,] pixels = new PixelColor[mask.Width, mask.Height];
PixelColor objectColor = new PixelColor(255, 0, 0, 120);
PixelColor backgroundColor = new PixelColor(0, 0, 0, 0);
for (int i = 0; i < mask.Width; ++i)
for (int j = 0; j < mask.Height; ++j)
pixels[i, j] = mask[i, j] ? objectColor : backgroundColor;
return PixelsToBitmapSource(pixels);
}
public void CalculateShapeTerms(ShapeModel model, ShapeConstraints constraintsSet, Image2D<ObjectBackgroundTerm> result)
{
if (model == null)
throw new ArgumentNullException("model");
if (constraintsSet == null)
throw new ArgumentNullException("constraintsSet");
if (result == null)
throw new ArgumentNullException("result");
if (model.Structure != constraintsSet.ShapeStructure)
throw new ArgumentException("Shape model and shape constraints correspond to different shape structures.");
if (model != this.shapeModel || result.Rectangle.Size != this.imageSize)
this.SetTarget(model, result.Rectangle.Size);
for (int x = 0; x < imageSize.Width; ++x)
for (int y = 0; y < imageSize.Height; ++y)
result[x, y] = new ObjectBackgroundTerm(Double.PositiveInfinity, 0);
for (int edgeIndex = 0; edgeIndex < this.shapeModel.Structure.Edges.Count; ++edgeIndex)
{
ShapeEdge edge = this.shapeModel.Structure.Edges[edgeIndex];
VertexConstraints vertexConstraints1 = constraintsSet.VertexConstraints[edge.Index1];
VertexConstraints vertexConstraints2 = constraintsSet.VertexConstraints[edge.Index2];
EdgeConstraints edgeConstraints = constraintsSet.EdgeConstraints[edgeIndex];
Image2D<ObjectBackgroundTerm> edgeTerms;
EdgeDescription edgeDescription = new EdgeDescription(
vertexConstraints1, vertexConstraints2, edgeConstraints);
if (!this.cachedEdgeTerms.TryGetValue(edgeDescription, out edgeTerms))
{
edgeTerms = this.AllocateImage();
this.cachedEdgeTerms.Add(edgeDescription, edgeTerms);
Polygon convexHull = constraintsSet.GetConvexHullForVertexPair(edge.Index1, edge.Index2);
Parallel.For(
0,
imageSize.Width,
x =>
{
for (int y = 0; y < imageSize.Height; ++y)
{
Vector pointAsVec = new Vector(x, y);
double minDistanceSqr, maxDistanceSqr;
MinMaxDistanceForEdge(
pointAsVec,
convexHull,
vertexConstraints1,
vertexConstraints2,
out minDistanceSqr,
out maxDistanceSqr);
edgeTerms[x, y] = new ObjectBackgroundTerm(
this.shapeModel.CalculateObjectPenaltyForEdge(
minDistanceSqr, edgeConstraints.MaxWidth),
this.shapeModel.CalculateBackgroundPenaltyForEdge(
maxDistanceSqr, edgeConstraints.MinWidth));
}
});
}
for (int x = 0; x < imageSize.Width; ++x)
for (int y = 0; y < imageSize.Height; ++y)
result[x, y] = new ObjectBackgroundTerm(
Math.Min(result[x, y].ObjectTerm, edgeTerms[x, y].ObjectTerm),
Math.Max(result[x, y].BackgroundTerm, edgeTerms[x, y].BackgroundTerm));
}
}
