private async void Highlight_Click(object sender, RoutedEventArgs e) { var OffsetX = ModelItem.GetValue <int>("OffsetX"); var OffsetY = ModelItem.GetValue <int>("OffsetY"); var Width = ModelItem.GetValue <int>("Width"); var Height = ModelItem.GetValue <int>("Height"); Rectangle match = await GetBaseRectangle(); var _hi = new ImageElement(match); _ = _hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1)); var rect = new ImageElement(new Rectangle(_hi.X + OffsetX, _hi.Y + OffsetY, Width, Height)); await rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1)); Interfaces.GenericTools.Restore(); }
private async void Highlight_Click(object sender, RoutedEventArgs e) { Rectangle match = await GetBaseRectangle(); var X = ModelItem.GetValue <int>("X"); var Y = ModelItem.GetValue <int>("Y"); var Width = ModelItem.GetValue <int>("Width"); var Height = ModelItem.GetValue <int>("Height"); if (match.IsEmpty) { match = new Rectangle(0, 0, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Width, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Height); } var _hi = new ImageElement(match); _ = _hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1)); var rect = new ImageElement(new Rectangle(_hi.X + X, _hi.Y + Y, Width, Height)); await rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1)); Interfaces.GenericTools.Restore(); }
protected override void StartLoop(NativeActivityContext context) { var relativelement = Element.Get(context); var match = relativelement.Rectangle; match.X += OffsetX.Get(context); match.Y += OffsetY.Get(context); match.Width = Width.Get(context); match.Height = Height.Get(context); var imageelement = relativelement as ImageElement; if (imageelement != null) { var processname = imageelement.Processname; if (!string.IsNullOrEmpty(processname)) { var _element = AutomationHelper.GetFromPoint(match.X, match.Y); if (_element.ProcessId < 1) { throw new ElementNotFoundException("Failed locating Image, expected " + processname + " but found nothing"); } using (var p = System.Diagnostics.Process.GetProcessById(_element.ProcessId)) { if (p.ProcessName != processname) { throw new ElementNotFoundException("Failed locating Image, expected " + processname + " but found " + p.ProcessName); } } } } var b = Interfaces.Image.Util.Screenshot(match); //Interfaces.Image.Util.SaveImageStamped(b, "c:\\temp", "GetImage-result"); var v = new ImageElement(match, b); context.SetValue(Result, v); IncIndex(context); SetTotal(context, 1); context.ScheduleAction(Body, v, OnBodyComplete); }
public static ImageElement[] OcrImage2(Emgu.CV.OCR.Tesseract _ocr, Emgu.CV.Mat image, string wordlimit, bool casesensitive) { using (var imageColor = new Mat()) using (Mat imgGrey = new Mat()) { if (image.NumberOfChannels == 1) { CvInvoke.CvtColor(image, imageColor, ColorConversion.Gray2Bgr); } else { image.CopyTo(imageColor); } // _ocr.SetImage(imageColor); CvInvoke.CvtColor(image, imgGrey, ColorConversion.Bgr2Gray); _ocr.SetImage(imgGrey); _ocr.AnalyseLayout(); if (_ocr.Recognize() != 0) { throw new Exception("Failed to recognizer image"); } Emgu.CV.OCR.Tesseract.Character[] characters = _ocr.GetCharacters(); var index = 0; var wordlimitindex = 0; var chars = new List <Emgu.CV.OCR.Tesseract.Character>(); var result = new List <ImageElement>(); var wordresult = new List <ImageElement>(); var wordchars = new List <Emgu.CV.OCR.Tesseract.Character>(); Rectangle desktop = new Rectangle(0, 0, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Width, System.Windows.Forms.Screen.PrimaryScreen.Bounds.Height); Rectangle imagerect = new Rectangle(0, 0, image.Width, image.Height); while (index < characters.Length) { if (!string.IsNullOrEmpty(wordlimit)) { if ((characters[index].Text == wordlimit[wordlimitindex].ToString()) || (!casesensitive && characters[index].Text.ToLower() == wordlimit[wordlimitindex].ToString().ToLower())) { wordchars.Add(characters[index]); wordlimitindex++; if (wordchars.Count == wordlimit.Length) { var res = new ImageElement(Rectangle.Empty); wordchars.ForEach(x => res.Text += x.Text); res.Confidence = wordchars[0].Cost; Rectangle rect = new Rectangle(wordchars[0].Region.X, wordchars[0].Region.Y, wordchars[0].Region.Width, wordchars[0].Region.Height); rect.Width = (wordchars[wordchars.Count - 1].Region.X - wordchars[0].Region.X) + wordchars[wordchars.Count - 1].Region.Width; rect.Height = (wordchars[wordchars.Count - 1].Region.Y - wordchars[0].Region.Y) + wordchars[wordchars.Count - 1].Region.Height; res.Rectangle = rect; wordresult.Add(res); wordchars.Clear(); wordlimitindex = 0; if (!desktop.Contains(rect)) { Log.Error("Found element outside desktop !!!!!"); } if (!imagerect.Contains(rect)) { Log.Error("Found element outside desktop !!!!!"); } Log.Debug("Found: " + res.Text + " at " + res.Rectangle.ToString()); } } else { wordchars.Clear(); wordlimitindex = 0; } } if (characters[index].Text == " " || characters[index].Text == "\r" || characters[index].Text == "\n") { if (chars.Count > 0) { var res = new ImageElement(Rectangle.Empty); chars.ForEach(x => res.Text += x.Text); res.Confidence = chars[0].Cost; Rectangle rect = new Rectangle(chars[0].Region.X, chars[0].Region.Y, chars[0].Region.Width, chars[0].Region.Height); rect.Width = (chars[chars.Count - 1].Region.X - chars[0].Region.X) + chars[chars.Count - 1].Region.Width; rect.Height = (chars[chars.Count - 1].Region.Y - chars[0].Region.Y) + chars[chars.Count - 1].Region.Height; res.Rectangle = rect; result.Add(res); } index++; chars.Clear(); continue; } chars.Add(characters[index]); index++; } if (chars.Count > 0) { var res = new ImageElement(Rectangle.Empty); chars.ForEach(x => res.Text += x.Text); res.Confidence = chars[0].Cost; Rectangle rect = new Rectangle(chars[0].Region.X, chars[0].Region.Y, chars[0].Region.Width, chars[0].Region.Height); rect.Width = (chars[chars.Count - 1].Region.X - chars[0].Region.X) + chars[chars.Count - 1].Region.Width; rect.Height = (chars[chars.Count - 1].Region.Y - chars[0].Region.Y) + chars[chars.Count - 1].Region.Height; res.Rectangle = rect; result.Add(res); } if (!string.IsNullOrEmpty(wordlimit)) { return(wordresult.ToArray()); } return(result.ToArray()); } }
private void Highlight_Click(object sender, RoutedEventArgs e) { var OffsetX = ModelItem.GetValue <int>("OffsetX"); var OffsetY = ModelItem.GetValue <int>("OffsetY"); var Width = ModelItem.GetValue <int>("Width"); var Height = ModelItem.GetValue <int>("Height"); ModelItem loadFrom = ModelItem.Parent; string loadFromSelectorString = ""; ModelItem gettext = null; while (loadFrom.Parent != null) { var p = loadFrom.Properties.Where(x => x.Name == "Image").FirstOrDefault(); if (p != null) { loadFromSelectorString = loadFrom.GetValue <string>("Selector"); break; } if (loadFrom.ItemType == typeof(GetText)) { gettext = loadFrom; } loadFrom = loadFrom.Parent; } Interfaces.IElement element = null; Rectangle match = Rectangle.Empty; if (!string.IsNullOrEmpty(loadFromSelectorString)) { var selector = new Interfaces.Selector.Selector(loadFromSelectorString); var pluginname = selector.First().Selector; var Plugin = Interfaces.Plugins.recordPlugins.Where(x => x.Name == pluginname).First(); var elements = Plugin.GetElementsWithSelector(selector, null, 1); if (elements.Length > 0) { element = elements[0]; } } if (gettext != null && element != null) { var matches = GetText.Execute(element, gettext); if (matches.Length > 0) { match = matches[0].Rectangle; } else { return; //var tip = new Interfaces.Overlay.TooltipWindow("Mark a found item"); //match = await getrectangle.GetitAsync(); //tip.Close(); //tip = null; } } else { var Image = loadFrom.GetValue <string>("Image"); var stream = new System.IO.MemoryStream(Convert.FromBase64String(Image)); var b = new System.Drawing.Bitmap(stream); var Threshold = loadFrom.GetValue <double>("Threshold"); var CompareGray = loadFrom.GetValue <bool>("CompareGray"); var Processname = loadFrom.GetValue <string>("Processname"); var limit = loadFrom.GetValue <Rectangle>("Limit"); if (Threshold < 0.5) { Threshold = 0.8; } // Interfaces.GenericTools.minimize(Interfaces.GenericTools.mainWindow); System.Threading.Thread.Sleep(100); var matches = ImageEvent.waitFor(b, Threshold, Processname, TimeSpan.FromMilliseconds(100), CompareGray, limit); if (matches.Count() == 0) { Interfaces.GenericTools.restore(); return; } match = matches[0]; } var _hi = new ImageElement(match); _hi.Highlight(false, System.Drawing.Color.Blue, TimeSpan.FromSeconds(1)); var rect = new ImageElement(new Rectangle(_hi.X + OffsetX, _hi.Y + OffsetY, Width, Height)); rect.Highlight(false, System.Drawing.Color.PaleGreen, TimeSpan.FromSeconds(1)); }
public bool ParseMouseMoveAction(ref IRecordEvent e) { if (e.UIElement == null) { return(false); } if (!e.Element.Equals(e.UIElement)) { return(false); } if (e.UIElement.Type != "Pane") { return(false); } var element = e.UIElement.RawElement; if (e.UIElement.ProcessId > 0) { using (var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId)) { if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe") { return(false); } if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox") { return(false); } if (p.ProcessName.ToLower() == "saplogon") { return(false); } } } e.Element = lastelement; lock (_lock) { if (_processing) { return(true); } _processing = true; } var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out int newOffsetX, out int newOffsetY, out System.Drawing.Rectangle resultrect); lock (_lock) { _processing = false; } if (image == null) { lastelement = null; // Log.Debug("Found null"); e.Element = null; return(true); } e.OffsetX = newOffsetX; e.OffsetY = newOffsetY; // Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height)); lastelement = new ImageElement(resultrect, image); e.Element = lastelement; return(true); }
public bool parseMouseMoveAction(ref IRecordEvent e) { if (e.UIElement == null) { return(false); } if (e.UIElement.Type != "Pane") { return(false); } var element = e.UIElement.RawElement; if (e.UIElement.ProcessId > 0) { var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId); if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe") { return(false); } if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox") { return(false); } } e.Element = lastelement; lock (_lock) { if (_processing) { return(true); } _processing = true; } //var elementx = (int)element.BoundingRectangle.X; //var elementy = (int)element.BoundingRectangle.Y; //var elementw = (int)element.BoundingRectangle.Width; //var elementh = (int)element.BoundingRectangle.Height; // Log.Debug(string.Format("Search near {0}, {1} in ({2}, {3},{4},{5})", elementx, elementy, elementw, elementh, e.OffsetX, e.OffsetY)); int newOffsetX; int newOffsetY; System.Drawing.Rectangle resultrect; var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out newOffsetX, out newOffsetY, out resultrect); lock (_lock) { _processing = false; } if (image == null) { lastelement = null; // Log.Debug("Found null"); e.Element = null; return(true); } e.OffsetX = newOffsetX; e.OffsetY = newOffsetY; // Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height)); lastelement = new ImageElement(resultrect, image); e.Element = lastelement; return(true); }