private async void Highlight_Click(object sender, RoutedEventArgs e)
{
var OffsetX = ModelItem.GetValue <int>("OffsetX");
var OffsetY = ModelItem.GetValue <int>("OffsetY");
var Width = ModelItem.GetValue <int>("Width");
var Height = ModelItem.GetValue <int>("Height");
Rectangle match = await GetBaseRectangle();
var _hi = new ImageElement(match);
_ = _hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1));
var rect = new ImageElement(new Rectangle(_hi.X + OffsetX, _hi.Y + OffsetY, Width, Height));
await rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1));
Interfaces.GenericTools.Restore();
}
private async void Highlight_Click(object sender, RoutedEventArgs e)
{
Rectangle match = await GetBaseRectangle();
var X = ModelItem.GetValue <int>("X");
var Y = ModelItem.GetValue <int>("Y");
var Width = ModelItem.GetValue <int>("Width");
var Height = ModelItem.GetValue <int>("Height");
if (match.IsEmpty)
{
match = new Rectangle(0, 0, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Width, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Height);
}
var _hi = new ImageElement(match);
_ = _hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1));
var rect = new ImageElement(new Rectangle(_hi.X + X, _hi.Y + Y, Width, Height));
await rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1));
Interfaces.GenericTools.Restore();
}
protected override void StartLoop(NativeActivityContext context)
{
var relativelement = Element.Get(context);
var match = relativelement.Rectangle;
match.X += OffsetX.Get(context);
match.Y += OffsetY.Get(context);
match.Width = Width.Get(context);
match.Height = Height.Get(context);
var imageelement = relativelement as ImageElement;
if (imageelement != null)
{
var processname = imageelement.Processname;
if (!string.IsNullOrEmpty(processname))
{
var _element = AutomationHelper.GetFromPoint(match.X, match.Y);
if (_element.ProcessId < 1)
{
throw new ElementNotFoundException("Failed locating Image, expected " + processname + " but found nothing");
}
using (var p = System.Diagnostics.Process.GetProcessById(_element.ProcessId))
{
if (p.ProcessName != processname)
{
throw new ElementNotFoundException("Failed locating Image, expected " + processname + " but found " + p.ProcessName);
}
}
}
}
var b = Interfaces.Image.Util.Screenshot(match);
//Interfaces.Image.Util.SaveImageStamped(b, "c:\\temp", "GetImage-result");
var v = new ImageElement(match, b);
context.SetValue(Result, v);
IncIndex(context);
SetTotal(context, 1);
context.ScheduleAction(Body, v, OnBodyComplete);
}
public static ImageElement[] OcrImage2(Emgu.CV.OCR.Tesseract _ocr, Emgu.CV.Mat image, string wordlimit, bool casesensitive)
{
using (var imageColor = new Mat())
using (Mat imgGrey = new Mat())
{
if (image.NumberOfChannels == 1)
{
CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr);
}
else
{
image.CopyTo(imageColor);
}
// _ocr.SetImage(imageColor);
CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray);
_ocr.SetImage(imgGrey);
_ocr.AnalyseLayout();
if (_ocr.Recognize() != 0)
{
throw new Exception("Failed to recognizer image");
}
Emgu.CV.OCR.Tesseract.Character[] characters = _ocr.GetCharacters();
var index = 0;
var wordlimitindex = 0;
var chars = new List <Emgu.CV.OCR.Tesseract.Character>();
var result = new List <ImageElement>();
var wordresult = new List <ImageElement>();
var wordchars = new List <Emgu.CV.OCR.Tesseract.Character>();
Rectangle desktop = new Rectangle(0, 0, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Width, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Height);
Rectangle imagerect = new Rectangle(0, 0, image.Width, image.Height);
while (index < characters.Length)
{
if (!string.IsNullOrEmpty(wordlimit))
{
if ((characters[index].Text == wordlimit[wordlimitindex].ToString()) ||
(!casesensitive && characters[index].Text.ToLower() == wordlimit[wordlimitindex].ToString().ToLower()))
{
wordchars.Add(characters[index]);
wordlimitindex++;
if (wordchars.Count == wordlimit.Length)
{
var res = new ImageElement(Rectangle.Empty);
wordchars.ForEach(x => res.Text += x.Text);
res.Confidence = wordchars[0].Cost;
Rectangle rect = new Rectangle(wordchars[0].Region.X, wordchars[0].Region.Y, wordchars[0].Region.Width, wordchars[0].Region.Height);
rect.Width = (wordchars[wordchars.Count - 1].Region.X - wordchars[0].Region.X) + wordchars[wordchars.Count - 1].Region.Width;
rect.Height = (wordchars[wordchars.Count - 1].Region.Y - wordchars[0].Region.Y) + wordchars[wordchars.Count - 1].Region.Height;
res.Rectangle = rect;
wordresult.Add(res);
wordchars.Clear();
wordlimitindex = 0;
if (!desktop.Contains(rect))
{
Log.Error("Found element outside desktop !!!!!");
}
if (!imagerect.Contains(rect))
{
Log.Error("Found element outside desktop !!!!!");
}
Log.Debug("Found: " + res.Text + " at " + res.Rectangle.ToString());
}
}
else
{
wordchars.Clear();
wordlimitindex = 0;
}
}
if (characters[index].Text == " " || characters[index].Text == "\r" || characters[index].Text == "\n")
{
if (chars.Count > 0)
{
var res = new ImageElement(Rectangle.Empty);
chars.ForEach(x => res.Text += x.Text);
res.Confidence = chars[0].Cost;
Rectangle rect = new Rectangle(chars[0].Region.X, chars[0].Region.Y, chars[0].Region.Width, chars[0].Region.Height);
rect.Width = (chars[chars.Count - 1].Region.X - chars[0].Region.X) + chars[chars.Count - 1].Region.Width;
rect.Height = (chars[chars.Count - 1].Region.Y - chars[0].Region.Y) + chars[chars.Count - 1].Region.Height;
res.Rectangle = rect;
result.Add(res);
}
index++;
chars.Clear();
continue;
}
chars.Add(characters[index]);
index++;
}
if (chars.Count > 0)
{
var res = new ImageElement(Rectangle.Empty);
chars.ForEach(x => res.Text += x.Text);
res.Confidence = chars[0].Cost;
Rectangle rect = new Rectangle(chars[0].Region.X, chars[0].Region.Y, chars[0].Region.Width, chars[0].Region.Height);
rect.Width = (chars[chars.Count - 1].Region.X - chars[0].Region.X) + chars[chars.Count - 1].Region.Width;
rect.Height = (chars[chars.Count - 1].Region.Y - chars[0].Region.Y) + chars[chars.Count - 1].Region.Height;
res.Rectangle = rect;
result.Add(res);
}
if (!string.IsNullOrEmpty(wordlimit))
{
return(wordresult.ToArray());
}
return(result.ToArray());
}
}
private void Highlight_Click(object sender, RoutedEventArgs e)
{
var OffsetX = ModelItem.GetValue <int>("OffsetX");
var OffsetY = ModelItem.GetValue <int>("OffsetY");
var Width = ModelItem.GetValue <int>("Width");
var Height = ModelItem.GetValue <int>("Height");
ModelItem loadFrom = ModelItem.Parent;
string loadFromSelectorString = "";
ModelItem gettext = null;
while (loadFrom.Parent != null)
{
var p = loadFrom.Properties.Where(x => x.Name == "Image").FirstOrDefault();
if (p != null)
{
loadFromSelectorString = loadFrom.GetValue <string>("Selector");
break;
}
if (loadFrom.ItemType == typeof(GetText))
{
gettext = loadFrom;
}
loadFrom = loadFrom.Parent;
}
Interfaces.IElement element = null;
Rectangle match = Rectangle.Empty;
if (!string.IsNullOrEmpty(loadFromSelectorString))
{
var selector = new Interfaces.Selector.Selector(loadFromSelectorString);
var pluginname = selector.First().Selector;
var Plugin = Interfaces.Plugins.recordPlugins.Where(x => x.Name == pluginname).First();
var elements = Plugin.GetElementsWithSelector(selector, null, 1);
if (elements.Length > 0)
{
element = elements[0];
}
}
if (gettext != null && element != null)
{
var matches = GetText.Execute(element, gettext);
if (matches.Length > 0)
{
match = matches[0].Rectangle;
}
else
{
return;
//var tip = new Interfaces.Overlay.TooltipWindow("Mark a found item");
//match = await getrectangle.GetitAsync();
//tip.Close();
//tip = null;
}
}
else
{
var Image = loadFrom.GetValue <string>("Image");
var stream = new System.IO.MemoryStream(Convert.FromBase64String(Image));
var b = new System.Drawing.Bitmap(stream);
var Threshold = loadFrom.GetValue <double>("Threshold");
var CompareGray = loadFrom.GetValue <bool>("CompareGray");
var Processname = loadFrom.GetValue <string>("Processname");
var limit = loadFrom.GetValue <Rectangle>("Limit");
if (Threshold < 0.5)
{
Threshold = 0.8;
}
// Interfaces.GenericTools.minimize(Interfaces.GenericTools.mainWindow);
System.Threading.Thread.Sleep(100);
var matches = ImageEvent.waitFor(b, Threshold, Processname, TimeSpan.FromMilliseconds(100), CompareGray, limit);
if (matches.Count() == 0)
{
Interfaces.GenericTools.restore();
return;
}
match = matches[0];
}
var _hi = new ImageElement(match);
_hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1));
var rect = new ImageElement(new Rectangle(_hi.X + OffsetX, _hi.Y + OffsetY, Width, Height));
rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1));
}
public bool ParseMouseMoveAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (!e.Element.Equals(e.UIElement))
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
if (e.UIElement.ProcessId > 0)
{
using (var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId))
{
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
if (p.ProcessName.ToLower() == "saplogon")
{
return(false);
}
}
}
e.Element = lastelement;
lock (_lock)
{
if (_processing)
{
return(true);
}
_processing = true;
}
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out int newOffsetX, out int newOffsetY, out System.Drawing.Rectangle resultrect);
lock (_lock)
{
_processing = false;
}
if (image == null)
{
lastelement = null;
// Log.Debug("Found null");
e.Element = null;
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
// Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height));
lastelement = new ImageElement(resultrect, image);
e.Element = lastelement;
return(true);
}
public bool parseMouseMoveAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
if (e.UIElement.ProcessId > 0)
{
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
}
e.Element = lastelement;
lock (_lock)
{
if (_processing)
{
return(true);
}
_processing = true;
}
//var elementx = (int)element.BoundingRectangle.X;
//var elementy = (int)element.BoundingRectangle.Y;
//var elementw = (int)element.BoundingRectangle.Width;
//var elementh = (int)element.BoundingRectangle.Height;
// Log.Debug(string.Format("Search near {0}, {1} in ({2}, {3},{4},{5})", elementx, elementy, elementw, elementh, e.OffsetX, e.OffsetY));
int newOffsetX; int newOffsetY; System.Drawing.Rectangle resultrect;
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out newOffsetX, out newOffsetY, out resultrect);
lock (_lock)
{
_processing = false;
}
if (image == null)
{
lastelement = null;
// Log.Debug("Found null");
e.Element = null;
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
// Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height));
lastelement = new ImageElement(resultrect, image);
e.Element = lastelement;
return(true);
}
