public ViewablePrototypeItem(Ptype ptype, string library, IEnumerable<ImageAnnotation> positives, IEnumerable<ImageAnnotation> negatives, Bitmap bitmap = null)
{
if (bitmap != null)
{
IsContent = true;
Image img = ToImage(bitmap);
CapturedImage = img;
}
else
{
Guid = ptype.Id;
PrototypeVisual = ptype;
}
Dictionary<string, Bitmap> screenshots = new Dictionary<string, Bitmap>();
if (positives != null)
{
PositiveExamples = new ObservableCollection<Example>();
foreach (ImageAnnotation img in positives)
{
if (!screenshots.ContainsKey(img.ImageId))
screenshots[img.ImageId] = AnnotationLibrary.GetImage(library, img.ImageId);
Bitmap screenshot = screenshots[img.ImageId];
Image exampleImage = GetImageFromExample(screenshot, img.Region);
Example example = new Example(exampleImage, library, img);
PositiveExamples.Add(example);
}
}
if (negatives != null)
{
NegativeExamples = new ObservableCollection<Example>();
foreach (ImageAnnotation img in negatives)
{
if (!screenshots.ContainsKey(img.ImageId))
screenshots[img.ImageId] = AnnotationLibrary.GetImage(library, img.ImageId);
Bitmap screenshot = screenshots[img.ImageId];
Image exampleImage = GetImageFromExample(screenshot, img.Region);
Example example = new Example(exampleImage, library, img);
NegativeExamples.Add(example);
}
}
}
/// <summary>
/// Finds any occurrences given the found features.
/// </summary>
/// <param name="features">Feature occurrences that correspond to this model.</param>
/// <param name="bitmap">Bitmap containing the feature occurrences.</param>
/// <returns>A list of hypotheses</returns>
public void FindOccurrences(Ptype ptype, Bitmap bitmap, IEnumerable<Tree> features, List<Tree> found)
{
Feature bottomleftFeature = ptype.Feature("bottomleft");
Feature toprightFeature = ptype.Feature("topright");
Feature bottomrightFeature = ptype.Feature("bottomright");
Feature topleftFeature = ptype.Feature("topleft");
Region topregion = ptype.Region("top");
Region leftregion = ptype.Region("left");
Region bottomregion = ptype.Region("bottom");
Region rightregion = ptype.Region("right");
Region interior = ptype.Region("interior");
//foreach each corresponding bottom left feature, find the corresponding bottom right and top right features
foreach (Tree bottomleft in features)
{
if(bottomleft["feature"].Equals(bottomleftFeature)){
//Find each bottom right feature corresponding to the bottom left feature
IEnumerable<Tree> bottomrights = GetBottomRightsFromBottomLeft(bottomrightFeature, bottomleft, features);
//foreach each bottom right feature, get the corresponding top right features
foreach (Tree bottomright in bottomrights)
{
//Get the top right feature corresponding to this bottom right feature
IEnumerable<Tree> toprights = GetTopRightsFromBottomRight(toprightFeature, bottomright, features);
foreach (Tree topright in toprights)
{
Tree topleft = GetTopLeft(topleftFeature, topright, bottomleft, features);
//Validate the hypothesis by matching edges. If they match, then create occurrence.
if (topleft != null &&
InteriorFits(interior, topregion, bottomregion, leftregion, rightregion, topleft, bottomleft, topright, bottomright) &&
EdgesMatch(ptype, bitmap, topleft, topright, bottomleft, bottomright))
{
int top = topleft.Top;
int left = topleft.Left;
int height = bottomleft.Top + bottomleft.Height - topleft.Top;
int width = topright.Left + topright.Width - topleft.Left;
BoundingBox bb = new BoundingBox(left, top, width, height);
Dictionary<String,Object> dict = new Dictionary<String,Object>();
dict.Add("type", "ptype");
dict.Add("ptype", ptype);
dict.Add("ptype_id", ptype.Id);
Tree prototypeOccurrence = Tree.FromBoundingBox(bb, dict);
found.Add(prototypeOccurrence);
}
}
}
}
}
}
public Visual Visualize(Ptype ptype, object parameter = null)
{
if (!ptype.Model.Name.Equals("onepart"))
{
throw new Exception("One Part Model cannot render prototypes created by other models.");
}
Feature part = ptype.Feature("part");
Image img = new Image();
img.Source = ViewablePrototypeItem.ToBitmapSource(part.Bitmap);
img.SnapsToDevicePixels = true;
img.Stretch = System.Windows.Media.Stretch.Uniform;
RenderOptions.SetBitmapScalingMode(img, BitmapScalingMode.NearestNeighbor);
return img;
}
public void FindOccurrences(Ptype ptype, Bitmap bitmap, IEnumerable<Tree> features, List<Tree> found)
{
Feature part = ptype.Feature("part");
foreach (Tree feature in features)
{
if(part.Equals(feature["feature"])){
Dictionary<string,object> dict = new Dictionary<string,object>();
dict.Add("type", "ptype");
dict.Add("ptype", ptype);
dict.Add("ptype_id", ptype.Id);
Tree occurrence = Tree.FromBoundingBox(feature, dict);
found.Add(occurrence);
}
}
}
public static Mutable ToMutable(Ptype ptype)
{
if (!(ptype is Hierarchical))
return new Hierarchical.HierarchicalMutable((Hierarchical)ptype);
return new Mutable(ptype);
}
public Mutable(Ptype ptype)
{
Features = new Dictionary<String,Bitmap>();
foreach(string fname in ptype.FeatureNames())
Features.Add(fname, ptype.Feature(fname).Bitmap);
Regions = new Dictionary<String,Region>();
foreach(string rname in ptype.RegionNames())
Regions.Add(rname, ptype.Region(rname));
Id = ptype.Id;
Model = ptype.Model.Name;
}
private IBoundingBox GetInteriorBox(IBoundingBox toProcess, Ptype ptype)
{
Region topregion = ptype.Region("top");
Region bottomregion = ptype.Region("bottom");
Region leftregion = ptype.Region("left");
Region rightregion = ptype.Region("right");
int top = toProcess.Top + topregion.Bitmap.Height;
int left = toProcess.Left + leftregion.Bitmap.Width;
int height = toProcess.Height - topregion.Bitmap.Height - bottomregion.Bitmap.Height;
int width = toProcess.Width - leftregion.Bitmap.Width - rightregion.Bitmap.Width;
return new BoundingBox(left, top, width, height);
}
private static void EraseFeaturesFromforeground(Bitmap foreground, Tree occurrence, Ptype ptype)
{
//top left
Feature topleft = ptype.Feature("topleft");
for(int row = occurrence.Top; row < occurrence.Top + topleft.Bitmap.Height; row++)
{
for(int col = occurrence.Left; col < occurrence.Left + topleft.Bitmap.Width; col++)
{
foreground[row, col] = Utils.BACKGROUND;
}
}
//top right
Feature topright = ptype.Feature("topright");
for (int row = occurrence.Top; row < occurrence.Top + topleft.Bitmap.Height; row++)
{
for(int col = occurrence.Left + occurrence.Width - topright.Bitmap.Width; col < occurrence.Left + occurrence.Width; col++)
{
foreground[row, col] = Utils.BACKGROUND;
}
}
//bottom left
Feature bottomleft = ptype.Feature("bottomleft");
for (int row = occurrence.Top + occurrence.Height - bottomleft.Bitmap.Height; row < occurrence.Top + occurrence.Height; row++)
{
for (int col = occurrence.Left; col < occurrence.Left + bottomleft.Bitmap.Width; col++)
{
foreground[row, col] = Utils.BACKGROUND;
}
}
//bottom right
Feature bottomright = ptype.Feature("bottomright");
for (int row = occurrence.Top + occurrence.Height - bottomright.Bitmap.Height; row < occurrence.Top + occurrence.Height; row++)
{
for (int col = occurrence.Left + occurrence.Width - bottomright.Bitmap.Width; col < occurrence.Left + occurrence.Width; col++)
{
foreground[row, col] = Utils.BACKGROUND;
}
}
}
/// <summary>
/// Returns true if every edge matches the given hypothesis.
/// </summary>
/// <param name="features">The features found in the bitmap.</param>
/// <param name="bitmap">The bitmap containing the hypothesis.</param>
/// <returns>Returns true if every edge matches the given hypothesis.</returns>
private bool EdgesMatch(Ptype ptype, Bitmap bitmap, IBoundingBox topleft, IBoundingBox topright, IBoundingBox bottomleft, IBoundingBox bottomright)
{
Region right = ptype.Region("right");
Region top = ptype.Region("top");
Region bottom = ptype.Region("bottom");
Region left = ptype.Region("left");
return HorizontalPatternMatcher.Instance.Matches(top.Bitmap, bitmap, topleft.Top, topleft.Left + topleft.Width, topright.Left - 1)
&& HorizontalPatternMatcher.Instance.Matches(bottom.Bitmap, bitmap, bottomleft.Top + bottomleft.Height - bottom.Bitmap.Height,
bottomleft.Left + bottomleft.Width, bottomright.Left - 1)
&& VerticalPatternMatcher.Instance.Matches(left.Bitmap, bitmap, topleft.Left, topleft.Top + topleft.Height,
bottomleft.Top - 1)
&& VerticalPatternMatcher.Instance.Matches(right.Bitmap, bitmap, topright.Left + topright.Width - right.Bitmap.Width, topright.Top + topright.Height,
bottomright.Top - 1);
}
public static List<Tree> FindPrototypeOccurrences(Bitmap bitmap, Ptype.Mutable ptype){
List<Ptype.Mutable> ptypes = new List<Ptype.Mutable>();
ptypes.Add(ptype);
List<Ptype> lib = Ptype.CreatePrototypeLibrary(ptypes);
List<Tree> foundPtypes = new List<Tree>();
try {
FeatureTree tree = FeatureTree.BuildTree(lib[0].Features());
List<Tree> foundFeatures = new List<Tree>();
tree.MultiThreadedMatch(bitmap, foundFeatures);
foreach(Ptype p in lib){
p.Model.Finder.FindOccurrences(p, bitmap,foundFeatures, foundPtypes);
}
} catch{
}
return foundPtypes;
}
public Visual Visualize(Ptype ptype, object parameter = null)
{
if (!ptype.Model.Name.Equals("ninepart"))
{
throw new Exception("Nine Part Model cannot render prototypes created by other models.");
}
Feature bottomleftFeature = ptype.Feature("bottomleft");
Feature toprightFeature = ptype.Feature("topright");
Feature bottomrightFeature = ptype.Feature("bottomright");
Feature topleftFeature = ptype.Feature("topleft");
Region topregion = ptype.Region("top");
Region leftregion = ptype.Region("left");
Region bottomregion = ptype.Region("bottom");
Region rightregion = ptype.Region("right");
Region interiorRegion = ptype.Region("interior");
Grid grid = new Grid();
////Find the biggest dimensions for each Part object so that
////we can figure out how many cells to add into the grid.
int longestTopOrBottom = (int)Math.Max(topregion.Bitmap.Width, bottomregion.Bitmap.Width);
int longestLeftOrRight = (int)Math.Max(leftregion.Bitmap.Height, rightregion.Bitmap.Height);
int widestTLOrBL = (int)Math.Max(topleftFeature.Bitmap.Width, bottomleftFeature.Bitmap.Width);
int widestTROrBR = (int)Math.Max(toprightFeature.Bitmap.Width, bottomrightFeature.Bitmap.Width);
int tallestTLOrTR = (int)Math.Max(topleftFeature.Bitmap.Height, toprightFeature.Bitmap.Height);
int tallestBLOrBR = (int)Math.Max(bottomleftFeature.Bitmap.Height, bottomrightFeature.Bitmap.Height);
int interiorWidth = 0;
int interiorHeight = 0;
if (interiorRegion != null)
{
interiorHeight = interiorRegion.Bitmap.Height;
interiorWidth = interiorRegion.Bitmap.Width;
}
//Assign the width and height of the grid.
int width = Math.Max(widestTLOrBL + longestTopOrBottom + widestTROrBR + 2, interiorWidth + leftregion.Bitmap.Width + rightregion.Bitmap.Width + 2);
int height = Math.Max(tallestTLOrTR + longestLeftOrRight + tallestBLOrBR + 2, interiorHeight + topregion.Bitmap.Height + bottomregion.Bitmap.Height + 2);
//Set the rows and columns of the grid.
for (int row = 0; row < height; row++)
{
RowDefinition rowdef = new RowDefinition();
grid.RowDefinitions.Add(rowdef);
}
for (int col = 0; col < width; col++)
{
ColumnDefinition coldef = new ColumnDefinition();
grid.ColumnDefinitions.Add(coldef);
}
//Add each Part to the grid (cells = pixels).
PtypeVisualizerHelper.AddFeatureToGrid(topleftFeature, new Prefab.Point(0, 0), grid);
PtypeVisualizerHelper.AddFeatureToGrid(toprightFeature, new Prefab.Point(width - toprightFeature.Bitmap.Width, 0), grid);
PtypeVisualizerHelper.AddFeatureToGrid(bottomleftFeature, new Prefab.Point(0, height - bottomleftFeature.Bitmap.Height), grid);
PtypeVisualizerHelper.AddFeatureToGrid(bottomrightFeature, new Prefab.Point(width - bottomrightFeature.Bitmap.Width, height - bottomrightFeature.Bitmap.Height), grid);
PtypeVisualizerHelper.AddHorizontalRegionToGrid(topregion.Bitmap, new Prefab.Point(topleftFeature.Bitmap.Width + 1, 0), grid);
PtypeVisualizerHelper.AddHorizontalRegionToGrid(bottomregion.Bitmap, new Prefab.Point(bottomleftFeature.Bitmap.Width + 1, height - bottomregion.Bitmap.Height), grid);
PtypeVisualizerHelper.AddVerticalRegionToGrid(leftregion.Bitmap, new Prefab.Point(0, topleftFeature.Bitmap.Height + 1), grid);
PtypeVisualizerHelper.AddVerticalRegionToGrid(rightregion.Bitmap, new Prefab.Point(width - rightregion.Bitmap.Width, toprightFeature.Bitmap.Height + 1), grid);
if (interiorRegion != null)
{
Prefab.Point location;
if (interiorRegion.MatchStrategy.Equals("horizontal"))
location = new Prefab.Point(topleftFeature.Bitmap.Width + 1, (height - interiorHeight) / 2);
else
location = new Prefab.Point((width - interiorWidth) / 2, topleftFeature.Bitmap.Height + 1);
PtypeVisualizerHelper.AddEachPixelOfBitmapToGrid(interiorRegion.Bitmap, location, grid);
}
return grid;
}
public Visual Visualize(Ptype ptype, object parameter)
{
return new Grid();
}
public bool UpdatePtypes(List <AnnotatedNode> annotations, List <Ptype> newLib)
{
bool anyRemoved = RemoveDeletedAnnotations(annotations, ptypeData);
bool needsUpdate = LoadDataFromAnnotations(annotations, ptypeData) || anyRemoved;
Dictionary <string, Bitmap> images = new Dictionary <string, Bitmap>();
foreach (AnnotatedNode n in annotations)
{
if (!images.ContainsKey(n.ImageId))
{
images.Add(n.ImageId, (Bitmap)n.Root["capturedpixels"]);
}
}
List <BuildPrototypeArgs> buildargs = new List <BuildPrototypeArgs>();
List <Ptype.Mutable> ptypes = new List <Ptype.Mutable>();
foreach (PtypeMetadata data in ptypeData.Values)
{
if (data.NeedsUpdate)
{
List <Bitmap> positives = new List <Bitmap>();
List <Bitmap> negatives = new List <Bitmap>();
foreach (Example e in data.Examples)
{
Bitmap example = Bitmap.Crop(images[e.ImageId], e.Region);
if (e.IsPositive)
{
positives.Add(example);
}
else
{
negatives.Add(example);
}
}
Examples examples = new Examples(positives, negatives);
BuildPrototypeArgs args = new BuildPrototypeArgs(examples, ModelInstances.Get(data.Ptype.Model), data.Ptype.Id);
buildargs.Add(args);
}
else
{
ptypes.Add(data.Ptype);
}
}
ptypes.AddRange(Ptype.BuildFromExamples(buildargs));
foreach (Ptype.Mutable ptype in ptypes)
{
PtypeMetadata data = ptypeData[ptype.Id];
data.Ptype.Features = ptype.Features;
data.Ptype.Regions = ptype.Regions;
data.Ptype.Model = ptype.Model;
}
SavePtypesToIntent(intent, ptypeData.Values);
newLib.AddRange(Ptype.CreatePrototypeLibrary(ptypes));
return(needsUpdate);
}
private bool MatchesAnyNegatives(Ptype.Mutable ptype, IEnumerable<Bitmap> negatives)
{
foreach (Bitmap negative in negatives)
{
IEnumerable<Tree> occurrences = PrototypeDetectionLayer.FindPrototypeOccurrences (negative, ptype);
foreach(Tree occurrence in occurrences){
if ( occurrence != null
&& occurrence.Left == 0
&& occurrence.Top == 0
&& occurrence.Width == negative.Width
&& occurrence.Height == negative.Height)
return true;
}
}
return false;
}
