public static System.Drawing.Bitmap GuessContour(FlaUI.Core.AutomationElements.Infrastructure.AutomationElement element, int x, int y, out int OffsetX, out int OffsetY, out System.Drawing.Rectangle resultrect) { //var element = automationutil.getElementAt(x, y); var elementx = (int)element.BoundingRectangle.X; var elementy = (int)element.BoundingRectangle.Y; var elementw = (int)element.BoundingRectangle.Width; var elementh = (int)element.BoundingRectangle.Height; System.Diagnostics.Trace.WriteLine(string.Format("Snap screenshot of element at ({0}, {1},{2},{3})", elementx, elementy, elementx + elementw, elementy + elementh), "Debug"); List <System.Drawing.Rectangle> con = null; var desktopb = Interfaces.Image.Util.Screenshot(elementx, elementy, elementw, elementh); con = FindContours(desktopb); //var point = new System.Drawing.Point(x - elementx, y - elementy); var point = new System.Drawing.Point(x, y); var saveimage = new Image <Emgu.CV.Structure.Bgr, Byte>(desktopb); foreach (var match in con) { saveimage.Draw(match, new Emgu.CV.Structure.Bgr(System.Drawing.Color.Red), 2); } // rpaactivities.image.util.saveImage(saveimage, "FoundContours"); saveimage.Dispose(); //var con = FindContours(bitmap); var overlaps = new List <System.Drawing.Rectangle>(); // Make all matches a certain size var minSize = 30; for (var i = 0; i < con.Count; i++) { var match = con[i]; if (match.Width < minSize) { int dif = (minSize - match.Width); match.Width += dif; match.X -= (dif / 2); if (match.X < 0) { match.X = 0; } con[i] = match; } if (match.Height < minSize) { int dif = (minSize - match.Height); match.Height += dif; match.Y -= (dif / 2); if (match.Y < 0) { match.Y = 0; } con[i] = match; } } // Take only hits that foreach (var match in con) { if (match.Contains(point)) { overlaps.Add(match); } } //if(overlaps.Count > 0) //{ // saveimage = new Image<Emgu.CV.Structure.Bgr, Byte>(desktopb); // foreach (var match in con) // { // saveimage.Draw(match, new Bgr(System.Drawing.Color.Red), 2); // } // rpaactivities.image.util.saveImage(saveimage, "GuessContour-hits"); // saveimage.Dispose(); //} System.Diagnostics.Trace.WriteLine("Found " + con.Count + " Contours with " + overlaps.Count + " overlaps"); var rect = System.Drawing.Rectangle.Empty; //bool again = false; foreach (var match in overlaps) { if (match.Width < 500 && match.Height < 500) { //if (rect == System.Drawing.Rectangle.Empty) //{ // var testb = desktopb.Clone(match, System.Drawing.Imaging.PixelFormat.Undefined); // var test = Matches.FindMatches(desktopb, testb, 0.8, 2, false); // System.Diagnostics.Trace.WriteLine("Testing with " + match.ToString() + " yelded " + test.Count() + " results"); // if (test.Count() > 0) // { // rect = match; // } //} if (rect == System.Drawing.Rectangle.Empty) { rect = match; } else if (rect.Width < match.Width || rect.Height < match.Height) { //if (again) //{ // rect = match; // again = false; //} //rect = match; } } else { System.Diagnostics.Trace.WriteLine("skipping: " + match.ToString()); } } if (rect != System.Drawing.Rectangle.Empty) { saveimage = new Image <Emgu.CV.Structure.Bgr, Byte>(desktopb); saveimage.Draw(rect, new Emgu.CV.Structure.Bgr(System.Drawing.Color.Red), 2); // rpaactivities.image.util.saveImage(saveimage, "GuessContour-result"); saveimage.Dispose(); desktopb.Dispose(); System.Diagnostics.Trace.WriteLine("using overlaps " + rect.ToString()); if (rect.Width > elementw) { rect.Width = elementw; } if (rect.Height > elementh) { rect.Height = elementh; } OffsetX = x - rect.X; OffsetY = y - rect.Y; resultrect = new System.Drawing.Rectangle(elementx + rect.X, elementy + rect.Y, rect.Width, rect.Height); System.Diagnostics.Trace.WriteLine(string.Format("Snap screenshot found Contour at ({0}, {1},{2},{3})", elementx + rect.X, elementy + rect.Y, rect.Width, rect.Height), "Debug"); return(Interfaces.Image.Util.Screenshot(elementx + rect.X, elementy + rect.Y, rect.Width, rect.Height)); } OffsetX = x; OffsetY = y; desktopb.Dispose(); resultrect = System.Drawing.Rectangle.Empty; return(null); }
public static ProcessInfo GetProcessInfo(this FlaUI.Core.AutomationElements.Infrastructure.AutomationElement element) { if (!element.Properties.ProcessId.IsSupported) { return(null); } ProcessInfo result = new ProcessInfo(); int processId = -1; IntPtr handle = IntPtr.Zero; try { processId = element.Properties.ProcessId.Value; var p = System.Diagnostics.Process.GetProcessById(processId); handle = p.Handle; result.processname = p.ProcessName; result.filename = p.MainModule.FileName.replaceEnvironmentVariable(); } catch (Exception) { } bool _isImmersiveProcess = false; try { if (handle != IntPtr.Zero) { _isImmersiveProcess = IsImmersiveProcess(handle); } } catch (Exception) { } string ApplicationUserModelId = null; if (_isImmersiveProcess) { var automation = AutomationUtil.getAutomation(); var pc = new FlaUI.Core.Conditions.PropertyCondition(automation.PropertyLibrary.Element.ClassName, "Windows.UI.Core.CoreWindow"); var _el = element.FindFirstChild(pc); if (_el != null) { processId = _el.Properties.ProcessId.Value; IntPtr ptrProcess = OpenProcess(QueryLimitedInformation, false, processId); if (IntPtr.Zero != ptrProcess) { uint cchLen = 130; // Currently APPLICATION_USER_MODEL_ID_MAX_LENGTH = 130 StringBuilder sbName = new StringBuilder((int)cchLen); Int32 lResult = GetApplicationUserModelId(ptrProcess, ref cchLen, sbName); if (APPMODEL_ERROR_NO_APPLICATION == lResult) { _isImmersiveProcess = false; } else if (ERROR_SUCCESS == lResult) { ApplicationUserModelId = sbName.ToString(); } else if (ERROR_INSUFFICIENT_BUFFER == lResult) { sbName = new StringBuilder((int)cchLen); if (ERROR_SUCCESS == GetApplicationUserModelId(ptrProcess, ref cchLen, sbName)) { ApplicationUserModelId = sbName.ToString(); } } CloseHandle(ptrProcess); } } else { _isImmersiveProcess = false; } } var arguments = GetCommandLine(processId); var arr = parseCommandLine(arguments); if (arr.Length == 0) { } else if (arguments.Contains("\"" + arr[0] + "\"")) { result.arguments = arguments.Replace("\"" + arr[0] + "\"", ""); } else { result.arguments = arguments.Replace(arr[0], ""); } if (result.arguments != null) { result.arguments = result.arguments.replaceEnvironmentVariable(); } //if (arr.Length > 0) //{ // var resultarr = new string[arr.Length - 1]; // Array.Copy(arr, 1, resultarr, 0, arr.Length - 1); // result.arguments = string.Join(" ", resultarr).replaceEnvironmentVariable(); //} result.applicationUserModelId = ApplicationUserModelId; result.isImmersiveProcess = _isImmersiveProcess; return(result); }