private void Open_Selector(object sender, RoutedEventArgs e)
{
ModelItem loadFrom = ModelItem.Parent;
string loadFromSelectorString = "";
WindowsSelector anchor = null;
while (loadFrom.Parent != null)
{
var p = loadFrom.Properties.Where(x => x.Name == "Selector").FirstOrDefault();
if (p != null)
{
loadFromSelectorString = loadFrom.GetValue <string>("Selector");
anchor = new WindowsSelector(loadFromSelectorString);
break;
}
loadFrom = loadFrom.Parent;
}
string SelectorString = ModelItem.GetValue <string>("Selector");
int maxresults = ModelItem.GetValue <int>("MaxResults");
Interfaces.Selector.SelectorWindow selectors;
if (!string.IsNullOrEmpty(SelectorString))
{
var selector = new WindowsSelector(SelectorString);
selectors = new Interfaces.Selector.SelectorWindow("Windows", selector, anchor, maxresults);
}
else
{
var selector = new WindowsSelector("[{Selector: 'Windows'}]");
selectors = new Interfaces.Selector.SelectorWindow("Windows", selector, anchor, maxresults);
}
if (selectors.ShowDialog() == true)
{
ModelItem.Properties["Selector"].SetValue(new InArgument <string>()
{
Expression = new Literal <string>(selectors.vm.json)
});
var l = selectors.vm.Selector.Last();
if (l.Element != null)
{
ModelItem.Properties["Image"].SetValue(l.Element.ImageString());
NotifyPropertyChanged("Image");
}
if (anchor != null)
{
ModelItem.Properties["From"].SetValue(new InArgument <IElement>()
{
Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue <IElement>("item")
});
ModelItem.Properties["MinResults"].SetValue(new InArgument <int>()
{
Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue <int>("0")
});
ModelItem.Properties["Timeout"].SetValue(new InArgument <TimeSpan>()
{
Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue <TimeSpan>("TimeSpan.FromSeconds(0)")
});
}
}
}
protected override void Execute(NativeActivityContext context)
{
UIElement[] elements = null;
var selector = Selector.Get(context);
var sel = new WindowsSelector(selector);
var timeout = Timeout.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
var sw = new Stopwatch();
var from = From.Get(context);
sw.Start();
do
{
elements = OpenRPA.AutomationHelper.RunSTAThread <UIElement[]>(() =>
{
try
{
return(WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults));
}
catch (System.Threading.ThreadAbortException)
{
}
catch (Exception ex)
{
Log.Error(ex, "");
}
return(new UIElement[] { });
}, TimeSpan.FromMilliseconds(250)).Result;
if (elements == null)
{
elements = new UIElement[] { };
}
} while (elements != null && elements.Length == 0 && sw.Elapsed < timeout);
if (elements.Length > 0)
{
// Get them again, we need the COM objects to be loaded in the UI thread
elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults);
}
context.SetValue(Elements, elements);
if (elements.Count() < minresults)
{
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <UIElement> _enum = elements.ToList().GetEnumerator();
context.SetValue(_elements, _enum);
bool more = _enum.MoveNext();
if (more)
{
context.ScheduleAction <UIElement>(Body, _enum.Current, OnBodyComplete);
}
}
public static treeelement[] _GetRootElements(Selector anchor)
{
if (CurrentProcessId == 0)
{
CurrentProcessId = Process.GetCurrentProcess().Id;
}
var result = new List <treeelement>();
Task.Run(() =>
{
var automation = AutomationUtil.getAutomation();
var _rootElement = automation.GetDesktop();
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
if (anchor != null)
{
if (!(anchor is WindowsSelector Windowsselector))
{
Windowsselector = new WindowsSelector(anchor.ToString());
}
var elements = WindowsSelector.GetElementsWithuiSelector(Windowsselector, null, 5, null);
if (elements.Count() > 0)
{
foreach (var elementNode in elements)
{
result.Add(new WindowsTreeElement(null, false, automation, elementNode.RawElement, _treeWalker));
}
//_rootElement = elements[0].RawElement;
return;
}
}
if (_rootElement != null)
{
var elementNode = _treeWalker.GetFirstChild(_rootElement);
while (elementNode != null)
{
if (!elementNode.Properties.ProcessId.IsSupported)
{
result.Add(new WindowsTreeElement(null, false, automation, elementNode, _treeWalker));
}
else if (elementNode.Properties.ProcessId.ValueOrDefault != CurrentProcessId)
{
result.Add(new WindowsTreeElement(null, false, automation, elementNode, _treeWalker));
}
try
{
elementNode = _treeWalker.GetNextSibling(elementNode);
}
catch (Exception)
{
elementNode = null;
}
}
}
}).Wait(5000);
return(result.ToArray());
}
private void Highlight_Click(object sender, RoutedEventArgs e)
{
ModelItem loadFrom = ModelItem.Parent;
string loadFromSelectorString = "";
WindowsSelector anchor = null;
while (loadFrom.Parent != null)
{
var p = loadFrom.Properties.Where(x => x.Name == "Selector").FirstOrDefault();
if (p != null)
{
loadFromSelectorString = loadFrom.GetValue <string>("Selector");
anchor = new WindowsSelector(loadFromSelectorString);
break;
}
loadFrom = loadFrom.Parent;
}
HighlightImage = Interfaces.Extensions.GetImageSourceFromResource("search.png");
NotifyPropertyChanged("HighlightImage");
string SelectorString = ModelItem.GetValue <string>("Selector");
int maxresults = ModelItem.GetValue <int>("MaxResults");
var selector = new WindowsSelector(SelectorString);
Task.Run(() =>
{
var elements = new List <UIElement>();
if (anchor != null)
{
var _base = WindowsSelector.GetElementsWithuiSelector(anchor, null, 10, null);
foreach (var _e in _base)
{
var res = WindowsSelector.GetElementsWithuiSelector(selector, _e, maxresults, null);
elements.AddRange(res);
}
}
else
{
var res = WindowsSelector.GetElementsWithuiSelector(selector, null, maxresults, null);
elements.AddRange(res);
}
if (elements.Count() > 0)
{
HighlightImage = Interfaces.Extensions.GetImageSourceFromResource("check.png");
}
else
{
HighlightImage = Interfaces.Extensions.GetImageSourceFromResource(".x.png");
}
NotifyPropertyChanged("HighlightImage");
foreach (var ele in elements)
{
ele.Highlight(false, System.Drawing.Color.Red, TimeSpan.FromSeconds(1));
}
});
}
private void OnMouseUp(InputEventArgs e)
{
isMouseDown = false;
var thread = new Thread(new ThreadStart(() =>
{
Log.Debug(string.Format("Windows.Recording::OnMouseUp::begin"));
var re = new RecordEvent
{
Button = e.Button
}; var a = new GetElement {
DisplayName = e.Element.Name
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
WindowsSelector sel = null;
// sel = new WindowsSelector(e.Element.rawElement, null, true);
sel = new WindowsSelector(e.Element.RawElement, null, PluginConfig.enum_selector_properties);
if (sel.Count < 2)
{
return;
}
if (sel == null)
{
return;
}
a.Selector = sel.ToString();
a.MaxResults = 1;
a.Image = e.Element.ImageString();
re.OffsetX = e.X - e.Element.Rectangle.X;
re.OffsetY = e.Y - e.Element.Rectangle.Y;
re.UIElement = e.Element;
re.Element = e.Element;
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
if (sel.Count > 3)
{
var p1 = sel[1].Properties.Where(x => x.Name == "ClassName").FirstOrDefault();
var p2 = sel[2].Properties.Where(x => x.Name == "AutomationId").FirstOrDefault();
if (p1 != null && p2 != null)
{
if (p1.Value.StartsWith("Windows.UI") && p2.Value == "SplitViewFrameXAMLWindow")
{
re.SupportVirtualClick = false;
}
}
}
re.a = new GetElementResult(a);
re.SupportInput = e.Element.SupportInput;
re.SupportSelect = e.Element.SupportSelect;
Log.Debug(string.Format("Windows.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}));
thread.IsBackground = true;
thread.Start();
}
public IElement[] GetElementsWithSelector(Selector selector, IElement fromElement = null, int maxresults = 1)
{
if (!(selector is WindowsSelector winselector))
{
winselector = new WindowsSelector(selector.ToString());
}
var result = WindowsSelector.GetElementsWithuiSelector(winselector, fromElement, maxresults, null);
return(result);
}
public static bool mouse_over_search(this WindowsSelector selector)
{
var item = selector[0];
var e = item.Properties.Where(x => x.Name == "mouse_over_search").FirstOrDefault();
if (e == null)
{
return(PluginConfig.try_mouse_over_search);
}
return(bool.Parse(e.Value));
}
public Selector GetSelector(Selector anchor, treeelement item)
{
var windowsitem = item as WindowsTreeElement;
WindowsSelector winanchor = anchor as WindowsSelector;
if (winanchor == null && anchor != null)
{
winanchor = new WindowsSelector(anchor.ToString());
}
return(new WindowsSelector(windowsitem.RawElement, winanchor, true));
}
public Selector GetSelector(Selector anchor, treeelement item)
{
var windowsitem = item as WindowsTreeElement;
WindowsSelector winanchor = anchor as WindowsSelector;
if (winanchor == null && anchor != null)
{
winanchor = new WindowsSelector(anchor.ToString());
}
return(new WindowsSelector(windowsitem.RawElement, winanchor, PluginConfig.enum_selector_properties));
}
private void Highlight_Click(object sender, RoutedEventArgs e)
{
string SelectorString = ModelItem.GetValue <string>("Selector");
int maxresults = ModelItem.GetValue <int>("MaxResults");
var selector = new WindowsSelector(SelectorString);
var elements = WindowsSelector.GetElementsWithuiSelector(selector, null, maxresults);
foreach (var ele in elements)
{
ele.Highlight(false, System.Drawing.Color.Red, TimeSpan.FromSeconds(1));
}
}
private void OnMouseUp(InputEventArgs e)
{
var thread = new Thread(new ThreadStart(() =>
{
Log.Debug(string.Format("Windows.Recording::OnMouseUp::begin"));
var re = new RecordEvent(); re.Button = e.Button;
var a = new GetElement {
DisplayName = e.Element.Id + "-" + e.Element.Name
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
WindowsSelector sel = null;
// sel = new WindowsSelector(e.Element.rawElement, null, true);
sel = new WindowsSelector(e.Element.RawElement, null, false);
if (sel.Count < 2)
{
return;
}
if (sel == null)
{
return;
}
a.Selector = sel.ToString();
a.MaxResults = 1;
a.Image = e.Element.ImageString();
re.UIElement = e.Element;
re.Element = e.Element;
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
re.a = new GetElementResult(a);
re.SupportInput = e.Element.SupportInput;
Log.Debug(string.Format("Windows.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}));
thread.IsBackground = true;
thread.Start();
}
public WindowsSelector(AutomationElement element, WindowsSelector anchor, bool doEnum)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("windowsselector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
AutomationElement root = null;
AutomationElement baseElement = null;
var pathToRoot = new List <AutomationElement>();
while (element != null)
{
// Break on circular relationship (should not happen?)
//if (pathToRoot.Contains(element) || element.Equals(_rootElement)) { break; }
// if (pathToRoot.Contains(element)) { break; }
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
if (element.Parent == null)
{
root = element;
}
}
catch (Exception)
{
root = element;
}
try
{
//element = _treeWalker.GetParent(element);
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
Log.Selector(string.Format("windowsselector::create pathToRoot::end {0:mm\\:ss\\.fff}", sw.Elapsed));
pathToRoot.Reverse();
if (anchor != null)
{
var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
for (var i = 0; i < anchorlist.Count(); i++)
{
if (WindowsSelectorItem.Match(anchorlist[i], pathToRoot[0]))
//if (((WindowsSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
{
pathToRoot.Remove(pathToRoot[0]);
}
else
{
Log.Selector("Element does not match the anchor path");
return;
}
}
}
WindowsSelectorItem item;
var temppathToRoot = new List <AutomationElement>();
var newpathToRoot = new List <AutomationElement>();
foreach (var e in pathToRoot)
{
temppathToRoot.Add(e);
}
Log.Selector(string.Format("windowsselector::traverse back to element from root::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
using (var automation = AutomationUtil.getAutomation())
{
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
bool isDesktop = true;
var parent = automation.GetDesktop();
if (anchor != null)
{
parent = temppathToRoot[0].Parent; isDesktop = false;
}
while (temppathToRoot.Count > 0)
{
var i = temppathToRoot.First();
temppathToRoot.Remove(i);
item = new WindowsSelectorItem(i, false);
var m = item.matches(automation, parent, _treeWalker, 2, isDesktop);
if (m.Length > 0)
{
newpathToRoot.Add(i);
parent = i;
isDesktop = false;
}
if (m.Length == 0 && Config.local.log_selector)
{
var message = "needed to find " + Environment.NewLine + item.ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
var children = parent.FindAllChildren();
foreach (var c in children)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
Log.Debug(message);
//if (i.Parent != null)
//{
// var c = i.Parent.FindAllChildren();
// var IndexInParent = -1;
// for (var x = 0; x < c.Count(); x++)
// {
// if (i == c[x]) IndexInParent = x;
// }
//}
}
}
}
Log.Selector(string.Format("windowsselector::traverse back to element from root::end {0:mm\\:ss\\.fff}", sw.Elapsed));
if (newpathToRoot.Count != pathToRoot.Count)
{
Log.Information("Selector had " + pathToRoot.Count + " items to root, but traversing children inly matched " + newpathToRoot.Count);
pathToRoot = newpathToRoot;
}
if (pathToRoot.Count == 0)
{
Log.Error("Element has not parent, or is same as annchor");
return;
}
baseElement = pathToRoot.First();
element = pathToRoot.Last();
Clear();
Log.Selector(string.Format("windowsselector::remove anchor if needed::end {0:mm\\:ss\\.fff}", sw.Elapsed));
if (anchor == null)
{
item = new WindowsSelectorItem(baseElement, true);
item.Enabled = true;
//item.canDisable = false;
Items.Add(item);
}
bool isStartmenu = false;
for (var i = 0; i < pathToRoot.Count(); i++)
{
var o = pathToRoot[i];
item = new WindowsSelectorItem(o, false);
if (i == 0 || i == (pathToRoot.Count() - 1))
{
item.canDisable = false;
}
foreach (var p in item.Properties) // TODO: Ugly, ugly inzuBiz hack !!!!
{
int idx = p.Value.IndexOf(".");
if (p.Name == "ClassName" && idx > -1)
{
int idx2 = p.Value.IndexOf(".", idx + 1);
if (idx2 > idx)
{
p.Value = p.Value.Substring(0, idx2 + 1) + "*";
}
}
//if (p.Name == "ClassName" && p.Value.StartsWith("WindowsForms10")) p.Value = "WindowsForms10*";
if (p.Name == "ClassName" && p.Value.ToLower() == "shelldll_defview")
{
item.Enabled = false;
}
if (p.Name == "ClassName" && (p.Value.ToLower() == "dv2vontrolhost" || p.Value.ToLower() == "desktopprogramsmfu"))
{
isStartmenu = true;
}
if (p.Name == "ClassName" && p.Value == "#32770")
{
item.Enabled = false;
}
if (p.Name == "ControlType" && p.Value == "ListItem" && isStartmenu)
{
p.Enabled = false;
}
}
var hassyslistview32 = item.Properties.Where(p => p.Name == "ClassName" && p.Value.ToLower() == "syslistview32").ToList();
if (hassyslistview32.Count > 0)
{
var hasControlType = item.Properties.Where(p => p.Name == "ControlType").ToList();
if (hasControlType.Count > 0)
{
hasControlType[0].Enabled = false;
}
}
if (doEnum)
{
item.EnumNeededProperties(o, o.Parent);
}
Items.Add(item);
}
pathToRoot.Reverse();
Log.Selector(string.Format("windowsselector::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Count"));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Item[]"));
OnCollectionChanged(new System.Collections.Specialized.NotifyCollectionChangedEventArgs(System.Collections.Specialized.NotifyCollectionChangedAction.Reset));
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement = null, int maxresults = 1)
{
TimeSpan timeout = TimeSpan.FromMilliseconds(1000);
timeout = TimeSpan.FromMilliseconds(20000);
var midcounter = 1;
if (PluginConfig.allow_multiple_hits_mid_selector)
{
midcounter = 10;
}
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var current = new List <UIElement>();
var automation = AutomationUtil.getAutomation();
var search_descendants = false;
var p = selector[0].Properties.Where(x => x.Name == "SearchDescendants").FirstOrDefault();
if (p != null)
{
search_descendants = bool.Parse(p.Value);
}
UIElement[] result = null;
using (automation)
{
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
Log.SelectorVerbose("automation.GetDesktop");
bool isDesktop = false;
if (startfrom == null)
{
startfrom = automation.GetDesktop();
isDesktop = true;
}
current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var s = new WindowsSelectorItem(selectors[i]);
var elements = new List <UIElement>();
elements.AddRange(current);
current.Clear();
foreach (var _element in elements)
{
var count = maxresults;
//if (i == 0) count = midcounter;
//// if (i < selectors.Count) count = 500;
//if ((i + 1) < selectors.Count) count = 1;
if (i < selectors.Count)
{
count = 500;
}
var matches = (s).matches(startfrom, i, _element.RawElement, count, isDesktop, search_descendants); // (i == 0 ? 1: maxresults)
var uimatches = new List <UIElement>();
foreach (var m in matches)
{
var ui = new UIElement(m);
uimatches.Add(ui);
}
current.AddRange(uimatches.ToArray());
if (sw.Elapsed > timeout)
{
Log.Selector(string.Format("GetElementsWithuiSelector::timed out {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
}
if (current.Count > 1)
{
//if (i < selectors.Count && maxresults == 1)
//{
// Log.Warning("Selector had " + current.Count + " hits and not just one, at element " + i + " this selector will be slow!");
//}
}
if (i == (selectors.Count - 1))
{
result = current.ToArray();
}
isDesktop = false;
Log.Selector(string.Format("Found " + current.Count + " hits for selector # " + i + " {0:mm\\:ss\\.fff}", sw.Elapsed));
}
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
Log.Selector(string.Format("GetElementsWithuiSelector::ended with " + result.Length + " results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(result);
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement = null, int maxresults = 1)
{
var midcounter = 1;
if (PluginConfig.allow_multiple_hits_mid_selector)
{
midcounter = 10;
}
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var current = new List <UIElement>();
var automation = AutomationUtil.getAutomation();
// var search_descendants = PluginConfig.search_descendants;
var search_descendants = false;
var p = selector[0].Properties.Where(x => x.Name == "SearchDescendants").FirstOrDefault();
if (p != null)
{
search_descendants = bool.Parse(p.Value);
}
UIElement[] result = null;
using (automation)
{
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
Log.SelectorVerbose("automation.GetDesktop");
bool isDesktop = false;
if (startfrom == null)
{
startfrom = automation.GetDesktop();
isDesktop = true;
}
current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var s = new WindowsSelectorItem(selectors[i]);
var elements = new List <UIElement>();
elements.AddRange(current);
current.Clear();
int failcounter = 0;
do
{
foreach (var _element in elements)
{
var count = maxresults;
if (i == 0)
{
count = midcounter;
}
// if (i < selectors.Count) count = 500;
if ((i + 1) < selectors.Count)
{
count = 1;
}
var matches = ((WindowsSelectorItem)s).matches(automation, _element.RawElement, _treeWalker, count, isDesktop, TimeSpan.FromSeconds(250), search_descendants); // (i == 0 ? 1: maxresults)
var uimatches = new List <UIElement>();
foreach (var m in matches)
{
var ui = new UIElement(m);
var list = selectors.Take(i).ToList();
list.Add(new WindowsSelectorItem(m, false));
uimatches.Add(ui);
}
current.AddRange(uimatches.ToArray());
}
if (current.Count > 1)
{
if (i < selectors.Count)
{
Log.Warning("Selector had " + current.Count + " hits and not just one, at element " + i + " this selector will be slow!");
}
}
if (current.Count == 0 && PluginConfig.allow_child_searching)
{
Log.Warning("Selector found not hits at element " + i + ", Try searching children, this selector will be slow!");
if ((i + 1) < selectors.Count && i > 0)
{
i++;
s = new WindowsSelectorItem(selectors[i]);
foreach (var _element in elements)
{
var count = maxresults;
if (i == 0)
{
count = 1;
}
if (i < selectors.Count)
{
count = 500;
}
var matches = ((WindowsSelectorItem)s).matches(automation, _element.RawElement, _treeWalker, count, false, TimeSpan.FromSeconds(250), search_descendants); // (i == 0 ? 1 : maxresults)
var uimatches = new List <UIElement>();
foreach (var m in matches)
{
var ui = new UIElement(m);
var list = selectors.Take(i).ToList();
list.Add(new WindowsSelectorItem(m, false));
uimatches.Add(ui);
}
current.AddRange(uimatches.ToArray());
}
Console.WriteLine(current.Count());
}
}
if (current.Count == 0)
{
++failcounter;
}
if (current.Count == 0 && Config.local.log_selector)
{
if (isDesktop)
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
var windows = Win32WindowUtils.GetTopLevelWindows(automation);
foreach (var c in windows)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
// Log.Selector(message);
Log.Warning(message);
}
else
{
foreach (var element in elements)
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
var children = element.RawElement.FindAllChildren();
foreach (var c in children)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
// Log.Selector(message);
Log.Warning(message);
}
}
}
else
{
Log.SelectorVerbose(string.Format("Found " + current.Count + " hits for selector # " + i + " {0:mm\\:ss\\.fff}", sw.Elapsed));
}
} while (failcounter < 2 && current.Count == 0);
if (i == (selectors.Count - 1))
{
result = current.ToArray();
}
if (current.Count == 0 && Config.local.log_selector)
{
if (isDesktop)
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
var windows = Win32WindowUtils.GetTopLevelWindows(automation);
foreach (var c in windows)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
Log.Warning(message);
}
else
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
foreach (var element in elements)
{
var children = element.RawElement.FindAllChildren();
foreach (var c in children)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
}
Log.Warning(message);
}
return(new UIElement[] { });
}
isDesktop = false;
}
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
return(result);
}
public WindowsSelector(AutomationElement element, WindowsSelector anchor, bool doEnum)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("windowsselector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
AutomationElement root = null;
AutomationElement baseElement = null;
var pathToRoot = new List <AutomationElement>();
while (element != null)
{
// Break on circular relationship (should not happen?)
//if (pathToRoot.Contains(element) || element.Equals(_rootElement)) { break; }
// if (pathToRoot.Contains(element)) { break; }
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
if (element.Parent == null)
{
root = element;
}
}
catch (Exception)
{
root = element;
}
try
{
//element = _treeWalker.GetParent(element);
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
Log.Selector(string.Format("windowsselector::create pathToRoot::end {0:mm\\:ss\\.fff}", sw.Elapsed));
pathToRoot.Reverse();
if (anchor != null)
{
//var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
//for (var i = 0; i < anchorlist.Count(); i++)
//{
// if(WindowsSelectorItem.Match(anchorlist[i], pathToRoot[0]))
// //if (((WindowsSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
// {
// pathToRoot.Remove(pathToRoot[0]);
// }
// else
// {
// Log.Selector("Element does not match the anchor path");
// return;
// }
//}
var a = anchor.Last();
var idx = -1;
for (var i = 0; i < pathToRoot.Count(); i++)
{
if (WindowsSelectorItem.Match(a, pathToRoot[i]))
{
idx = i;
break;
}
}
pathToRoot.RemoveRange(0, idx);
}
WindowsSelectorItem item;
if (PluginConfig.traverse_selector_both_ways)
{
Log.Selector(string.Format("windowsselector::create traverse_selector_both_ways::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
var temppathToRoot = new List <AutomationElement>();
var newpathToRoot = new List <AutomationElement>();
foreach (var e in pathToRoot)
{
temppathToRoot.Add(e);
}
Log.Selector(string.Format("windowsselector::traverse back to element from root::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
using (var automation = AutomationUtil.getAutomation())
{
bool isDesktop = true;
AutomationElement parent = null;
if (anchor != null)
{
parent = temppathToRoot[0].Parent; isDesktop = false;
}
else
{
automation.GetDesktop();
}
int count = temppathToRoot.Count;
while (temppathToRoot.Count > 0)
{
count--;
var i = temppathToRoot.First();
temppathToRoot.Remove(i);
item = new WindowsSelectorItem(i, false);
var m = item.matches(root, count, parent, 2, isDesktop, false);
if (m.Length > 0)
{
newpathToRoot.Add(i);
parent = i;
isDesktop = false;
}
if (m.Length == 0 && Config.local.log_selector)
{
//var message = "needed to find " + Environment.NewLine + item.ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
//var children = parent.FindAllChildren();
//foreach (var c in children)
//{
// try
// {
// message += new UIElement(c).ToString() + Environment.NewLine;
// }
// catch (Exception)
// {
// }
//}
//Log.Debug(message);
}
}
}
if (newpathToRoot.Count != pathToRoot.Count)
{
Log.Information("Selector had " + pathToRoot.Count + " items to root, but traversing children only matched " + newpathToRoot.Count);
pathToRoot = newpathToRoot;
}
Log.Selector(string.Format("windowsselector::create traverse_selector_both_ways::end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
if (pathToRoot.Count == 0)
{
Log.Error("Element has not parent, or is same as annchor");
return;
}
baseElement = pathToRoot.First();
element = pathToRoot.Last();
Clear();
Log.Selector(string.Format("windowsselector::remove anchor if needed::end {0:mm\\:ss\\.fff}", sw.Elapsed));
if (anchor == null)
{
Log.Selector(string.Format("windowsselector::create root element::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
item = new WindowsSelectorItem(baseElement, true);
item.Enabled = true;
//item.canDisable = false;
Items.Add(item);
Log.Selector(string.Format("windowsselector::create root element::end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
if (PluginConfig.search_descendants)
{
Log.Selector(string.Format("windowsselector::search_descendants::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
if (anchor == null)
{
// Add window, we NEED to search from a window
item = new WindowsSelectorItem(pathToRoot[0], false, -1);
if (doEnum)
{
item.EnumNeededProperties(pathToRoot[pathToRoot.Count - 1], pathToRoot[pathToRoot.Count - 1].Parent);
}
item.canDisable = false;
Items.Add(item);
var FrameworkId = item.Properties.Where(x => x.Name == "FrameworkId").FirstOrDefault();
if (FrameworkId != null && (FrameworkId.Value == "XAML" || FrameworkId.Value == "WinForm"))
{
var itemname = item.Properties.Where(x => x.Name == "Name").FirstOrDefault();
if (itemname != null)
{
itemname.Enabled = false;
}
}
}
if (pathToRoot.Count > 2)
{
item = new WindowsSelectorItem(pathToRoot[pathToRoot.Count - 2], false, -1);
if (doEnum)
{
item.EnumNeededProperties(pathToRoot[pathToRoot.Count - 2], pathToRoot[pathToRoot.Count - 2].Parent);
}
Items.Add(item);
}
if (pathToRoot.Count > 1)
{
int IndexInParent = -1;
if (pathToRoot[pathToRoot.Count - 1].Parent != null)
{
var c = pathToRoot[pathToRoot.Count - 1].Parent.FindAllChildren();
for (var x = 0; x < c.Count(); x++)
{
if (pathToRoot[pathToRoot.Count - 1].Equals(c[x]))
{
IndexInParent = x;
}
}
}
item = new WindowsSelectorItem(pathToRoot[pathToRoot.Count - 1], false, IndexInParent);
if (doEnum)
{
item.EnumNeededProperties(pathToRoot[pathToRoot.Count - 1], pathToRoot[pathToRoot.Count - 1].Parent);
}
Items.Add(item);
}
Log.Selector(string.Format("windowsselector::search_descendants::end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
else
{
bool isStartmenu = false;
for (var i = 0; i < pathToRoot.Count(); i++)
{
Log.Selector(string.Format("windowsselector::search_descendants::loop element " + i + ":begin {0:mm\\:ss\\.fff}", sw.Elapsed));
var o = pathToRoot[i];
int IndexInParent = -1;
if (o.Parent != null && i > 0)
{
var c = o.Parent.FindAllChildren();
for (var x = 0; x < c.Count(); x++)
{
if (o.Equals(c[x]))
{
IndexInParent = x;
}
}
}
item = new WindowsSelectorItem(o, false, IndexInParent);
var _IndexInParent = item.Properties.Where(x => x.Name == "IndexInParent").FirstOrDefault();
if (_IndexInParent != null)
{
_IndexInParent.Enabled = false;
}
if (i == 0 || i == (pathToRoot.Count() - 1))
{
item.canDisable = false;
}
foreach (var p in item.Properties)
{
int idx = p.Value.IndexOf(".");
if (p.Name == "ClassName" && idx > -1)
{
var FrameworkId = item.Properties.Where(x => x.Name == "FrameworkId").FirstOrDefault();
//if (FrameworkId!=null && (FrameworkId.Value == "XAML" || FrameworkId.Value == "WinForm") && _IndexInParent != null)
//{
// item.Properties.ForEach(x => x.Enabled = false);
// _IndexInParent.Enabled = true;
// p.Enabled = true;
//}
int idx2 = p.Value.IndexOf(".", idx + 1);
// if (idx2 > idx) p.Value = p.Value.Substring(0, idx2 + 1) + "*";
if (idx2 > idx && item.Properties.Count > 1)
{
p.Enabled = false;
}
}
//if (p.Name == "ClassName" && p.Value.StartsWith("WindowsForms10")) p.Value = "WindowsForms10*";
if (p.Name == "ClassName" && p.Value.ToLower() == "shelldll_defview")
{
item.Enabled = false;
}
if (p.Name == "ClassName" && (p.Value.ToLower() == "dv2vontrolhost" || p.Value.ToLower() == "desktopprogramsmfu"))
{
isStartmenu = true;
}
//if (p.Name == "ClassName" && p.Value == "#32770")
//{
// item.Enabled = false;
//}
if (p.Name == "ControlType" && p.Value == "ListItem" && isStartmenu)
{
p.Enabled = false;
}
}
var hassyslistview32 = item.Properties.Where(p => p.Name == "ClassName" && p.Value.ToLower() == "syslistview32").ToList();
if (hassyslistview32.Count > 0)
{
var hasControlType = item.Properties.Where(p => p.Name == "ControlType").ToList();
if (hasControlType.Count > 0)
{
hasControlType[0].Enabled = false;
}
}
if (doEnum)
{
item.EnumNeededProperties(o, o.Parent);
}
Items.Add(item);
Log.Selector(string.Format("windowsselector::search_descendants::loop element " + i + ":end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
}
pathToRoot.Reverse();
if (anchor != null)
{
var p = Items[0].Properties.Where(x => x.Name == "SearchDescendants").FirstOrDefault();
if (p == null)
{
Items[0].Properties.Add(new SelectorItemProperty("SearchDescendants", PluginConfig.search_descendants.ToString()));
}
}
Log.Selector(string.Format("windowsselector::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Count"));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Item[]"));
OnCollectionChanged(new System.Collections.Specialized.NotifyCollectionChangedEventArgs(System.Collections.Specialized.NotifyCollectionChangedAction.Reset));
}
public override IElement[] GetElements(IElement fromElement = null, int maxresults = 1)
{
return(WindowsSelector.GetElementsWithuiSelector(this, fromElement, maxresults));
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement = null, int maxresults = 1)
{
TimeSpan timeout = TimeSpan.FromMilliseconds(5000);
timeout = TimeSpan.FromMilliseconds(20000);
var midcounter = 1;
if (PluginConfig.allow_multiple_hits_mid_selector)
{
midcounter = 10;
}
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var current = new List <UIElement>();
var automation = AutomationUtil.getAutomation();
var search_descendants = false;
var p = selector[0].Properties.Where(x => x.Name == "SearchDescendants").FirstOrDefault();
if (p != null)
{
search_descendants = bool.Parse(p.Value);
}
UIElement[] result = null;
using (automation)
{
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
Log.SelectorVerbose("automation.GetDesktop");
bool isDesktop = false;
if (startfrom == null)
{
startfrom = automation.GetDesktop();
isDesktop = true;
}
current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var s = new WindowsSelectorItem(selectors[i]);
var elements = new List <UIElement>();
elements.AddRange(current);
current.Clear();
int count = 0;
foreach (var _element in elements)
{
count = maxresults;
//if (i == 0) count = midcounter;
//// if (i < selectors.Count) count = 500;
//if ((i + 1) < selectors.Count) count = 1;
if (i < (selectors.Count - 1))
{
count = 500;
}
var matches = (s).matches(startfrom, i, _element.RawElement, count, isDesktop, search_descendants); // (i == 0 ? 1: maxresults)
var uimatches = new List <UIElement>();
foreach (var m in matches)
{
var ui = new UIElement(m);
uimatches.Add(ui);
}
current.AddRange(uimatches.ToArray());
if (sw.Elapsed > timeout)
{
Log.Selector(string.Format("GetElementsWithuiSelector::timed out {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
}
if (i == (selectors.Count - 1))
{
result = current.ToArray();
}
Log.Selector(string.Format("Found " + current.Count + " hits for selector # " + i + " {0:mm\\:ss\\.fff}", sw.Elapsed));
if (i > 0 && elements.Count > 0 && current.Count == 0)
{
//var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
//var children = elements[0].RawElement.FindAllChildren();
//foreach (var c in children)
//{
// try
// {
// message += new UIElement(c).ToString() + Environment.NewLine;
// }
// catch (Exception)
// {
// }
//}
//Log.Selector(message);
}
if (i == 0 && isDesktop && current.Count > 0)
{
if (current[0].RawElement.Patterns.Window.TryGetPattern(out var winPattern))
{
if (winPattern.WindowVisualState.Value == FlaUI.Core.Definitions.WindowVisualState.Minimized)
{
IntPtr handle = current[0].RawElement.Properties.NativeWindowHandle.Value;
winPattern.SetWindowVisualState(FlaUI.Core.Definitions.WindowVisualState.Normal);
}
}
}
isDesktop = false;
}
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
if (result.Count() > maxresults)
{
result = result.Take(maxresults).ToArray();
}
Log.Selector(string.Format("GetElementsWithuiSelector::ended with " + result.Length + " results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(result);
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement, int maxresults, WindowsCacheExtension ext)
{
TimeSpan timeout = TimeSpan.FromMilliseconds(5000);
timeout = TimeSpan.FromMilliseconds(20000);
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
// var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var selectors = selector.ToList();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
var _current = new List <UIElement>();
// var automation = AutomationUtil.getAutomation();
AutomationBase automation = null;
if (ext != null)
{
automation = ext.automation;
}
if (automation == null)
{
automation = AutomationUtil.getAutomation();
}
UIElement[] result = null;
// AutomationElement ele = null;
bool search_descendants = PluginConfig.search_descendants;
var v = selectors[0].Properties.Where(x => x.Name == "search_descendants").FirstOrDefault();
if (v == null)
{
selectors[0].Properties.Where(x => x.Name == "SearchDescendants").FirstOrDefault();
}
if (v != null)
{
search_descendants = bool.Parse(v.Value);
}
if (startfrom == null)
{
startfrom = automation.GetDesktop();
}
_current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var _sel = selectors[i];
var sel = new WindowsSelectorItem(_sel);
var current = _current.ToArray();
_current.Clear();
// if(i == 1 && current.Length == 1 && current.First().ControlType == sel.ControlType)
if (i == 1)
{
foreach (var e in current)
{
if (WindowsSelectorItem.Match(sel, e.RawElement))
{
_current.Add(e);
}
}
if (_current.Count > 0)
{
continue;
}
//_current = GetElementsWithuiSelectorItem(automation, sel, current, maxresults, i == (selectors.Count - 1)).ToList();
//if(_current.Count == 0) _current = current.ToList();
//_current = current.ToList();
}
_current = GetElementsWithuiSelectorItem(i, automation, sel, current, maxresults, i == (selectors.Count - 1), search_descendants).ToList();
if (i == 0 && _current.Count == 0)
{
_current = current.ToList();
}
}
Log.Debug(string.Format("GetElementsWithuiSelector::completed with " + _current.Count + " results {0:mm\\:ss\\.fff}", sw.Elapsed));
if (_current.Count > 0)
{
result = _current.ToArray();
if (result.Count() > maxresults)
{
Console.WriteLine("found " + result.Count() + " but only needed " + maxresults);
result = result.Take(maxresults).ToArray();
}
return(result);
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
if (result.Count() > maxresults)
{
result = result.Take(maxresults).ToArray();
}
Log.Selector(string.Format("GetElementsWithuiSelector::ended with " + result.Length + " results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(result);
}
public void DetectorCheck(AutomationElement element, FlaUI.Core.Definitions.StructureChangeType e, int[] arg3)
{
try
{
if (Entity == null || string.IsNullOrEmpty(Selector))
{
return;
}
TimeSpan timepassed = DateTime.Now - lastTriggered;
if (timepassed.Milliseconds < 100)
{
return;
}
lastTriggered = DateTime.Now;
// if (e != FlaUI.Core.Definitions.StructureChangeType.ChildAdded) return;
var pathToRoot = new List <AutomationElement>();
while (element != null)
{
if (pathToRoot.Contains(element))
{
break;
}
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
}
catch (Exception)
{
}
try
{
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
WindowsSelector selector = new WindowsSelector(Selector);
if (pathToRoot.Count < (selector.Count - 1))
{
return;
}
if (pathToRoot.Count > (selector.Count - 1))
{
return;
}
pathToRoot.Reverse();
for (var i = 0; i < pathToRoot.Count; i++)
{
element = pathToRoot[i];
if (selector.Count > (i + 1))
{
WindowsSelectorItem s = new WindowsSelectorItem(selector[(i + 1)]);
if (!s.Match(element))
{
return;
}
}
else
{
return;
}
}
var _e = new DetectorEvent(new UIElement(element));
OnDetector?.Invoke(this, _e, EventArgs.Empty);
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement, int maxresults, WindowsCacheExtension ext)
{
TimeSpan timeout = TimeSpan.FromMilliseconds(5000);
timeout = TimeSpan.FromMilliseconds(20000);
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
// var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var selectors = selector.ToList();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
var _current = new List <UIElement>();
// var automation = AutomationUtil.getAutomation();
AutomationBase automation = null;
if (ext != null)
{
automation = ext.automation;
}
if (automation == null)
{
automation = AutomationUtil.getAutomation();
}
UIElement[] result = null;
// AutomationElement ele = null;
bool search_descendants = selectors[0].SearchDescendants();
if (startfrom == null)
{
startfrom = automation.GetDesktop();
}
_current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var _sel = selectors[i];
var sel = new WindowsSelectorItem(_sel);
var current = _current.ToArray();
_current.Clear();
// if(i == 1 && current.Length == 1 && current.First().ControlType == sel.ControlType)
if (i == 1)
{
foreach (var e in current)
{
if (WindowsSelectorItem.Match(sel, e.RawElement))
{
_current.Add(e);
}
}
if (_current.Count > 0)
{
continue;
}
//_current = GetElementsWithuiSelectorItem(automation, sel, current, maxresults, i == (selectors.Count - 1)).ToList();
//if(_current.Count == 0) _current = current.ToList();
//_current = current.ToList();
}
_current = GetElementsWithuiSelectorItem(i, automation, sel, current, maxresults, i == (selectors.Count - 1), search_descendants).ToList();
if (i == 0 && _current.Count == 0)
{
_current = current.ToList();
}
else if (i > 0 && _current.Count == 0 && current.Length > 0 && (PluginConfig.try_mouse_over_search || selector.mouse_over_search()))
{
for (var z = 0; z < current.Length; z++)
{
current[z].Focus();
var x = current[z].Rectangle.X + 5; var y = current[z].Rectangle.Y + 5;
var screen = automation.FromPoint(new System.Drawing.Point(x, y));
Log.Selector("mouse point lookup at " + x + "," + y + " for " + sel.ToString());
var screenel = screen.Parent;
while (screenel != null)
{
var uiscreenel = new UIElement(screenel);
if (WindowsSelectorItem.Match(sel, screenel))
{
_current.Clear();
_current.Add(new UIElement(screenel));
Log.Selector("Mouse lookup found " + uiscreenel.ToString());
break;
}
else
{
screenel = screenel.Parent;
}
// only allow direct parents, or search for match in all child elements found ? ( like searching for a button inside a pane )
//else
//{
// Log.Output("Lookup in " + uiscreenel.ToString());
// _current = GetElementsWithuiSelectorItem(i, automation, sel, new UIElement[] { uiscreenel }, maxresults, i == (selectors.Count - 1), search_descendants).ToList();
// if (_current.Count != 0) break;
// screenel = screenel.Parent;
//}
}
if (_current.Count > 0)
{
break;
}
}
}
}
Log.Debug(string.Format("GetElementsWithuiSelector::completed with " + _current.Count + " results {0:mm\\:ss\\.fff}", sw.Elapsed));
if (_current.Count > 0)
{
result = _current.ToArray();
if (result.Count() > maxresults)
{
Console.WriteLine("found " + result.Count() + " but only needed " + maxresults);
result = result.Take(maxresults).ToArray();
}
return(result);
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
if (result.Count() > maxresults)
{
result = result.Take(maxresults).ToArray();
}
Log.Selector(string.Format("GetElementsWithuiSelector::ended with " + result.Length + " results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(result);
}
public WindowsSelector(AutomationElement element, WindowsSelector anchor, bool doEnum)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("windowsselector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
AutomationElement root = null;
AutomationElement baseElement = null;
var pathToRoot = new List <AutomationElement>();
while (element != null)
{
// Break on circular relationship (should not happen?)
//if (pathToRoot.Contains(element) || element.Equals(_rootElement)) { break; }
if (pathToRoot.Contains(element))
{
break;
}
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
if (element.Parent == null)
{
root = element;
}
}
catch (Exception)
{
root = element;
}
try
{
//element = _treeWalker.GetParent(element);
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
Log.Selector(string.Format("windowsselector::create pathToRoot::end {0:mm\\:ss\\.fff}", sw.Elapsed));
pathToRoot.Reverse();
if (anchor != null)
{
var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
for (var i = 0; i < anchorlist.Count(); i++)
{
if (WindowsSelectorItem.Match(anchorlist[i], pathToRoot[0]))
//if (((WindowsSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
{
pathToRoot.Remove(pathToRoot[0]);
}
else
{
Log.Selector("Element does not match the anchor path");
return;
}
}
}
if (pathToRoot.Count == 0)
{
Log.Error("Element is same as annchor");
return;
}
baseElement = pathToRoot.First();
element = pathToRoot.Last();
Clear();
Log.Selector(string.Format("windowsselector::remove anchor if needed::end {0:mm\\:ss\\.fff}", sw.Elapsed));
WindowsSelectorItem item;
if (anchor == null)
{
item = new WindowsSelectorItem(baseElement, true);
item.Enabled = true;
//item.canDisable = false;
Items.Add(item);
}
for (var i = 0; i < pathToRoot.Count(); i++)
{
var o = pathToRoot[i];
item = new WindowsSelectorItem(o, false);
if (i == 0 || i == (pathToRoot.Count() - 1))
{
item.canDisable = false;
}
foreach (var p in item.Properties) // TODO: Ugly, ugly inzuBiz hack !!!!
{
int idx = p.Value.IndexOf(".");
if (p.Name == "ClassName" && idx > -1)
{
int idx2 = p.Value.IndexOf(".", idx + 1);
if (idx2 > idx)
{
p.Value = p.Value.Substring(0, idx2 + 1) + "*";
}
}
//if (p.Name == "ClassName" && p.Value.StartsWith("WindowsForms10")) p.Value = "WindowsForms10*";
if (p.Name == "ClassName" && p.Value.ToLower() == "shelldll_defview")
{
item.Enabled = false;
}
}
if (doEnum)
{
item.EnumNeededProperties(o, o.Parent);
}
Items.Add(item);
}
pathToRoot.Reverse();
Log.Selector(string.Format("windowsselector::EnumNeededProperties::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Count"));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Item[]"));
OnCollectionChanged(new System.Collections.Specialized.NotifyCollectionChangedEventArgs(System.Collections.Specialized.NotifyCollectionChangedAction.Reset));
}
public static UIElement[] GetElementsWithuiSelector(WindowsSelector selector, IElement fromElement = null, int maxresults = 1)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("GetElementsWithuiSelector::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement _fromElement = fromElement as UIElement;
var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var current = new List <UIElement>();
var automation = AutomationUtil.getAutomation();
UIElement[] result = null;
using (automation)
{
var _treeWalker = automation.TreeWalkerFactory.GetControlViewWalker();
AutomationElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement.RawElement;
}
Log.Selector("automation.GetDesktop");
if (startfrom == null)
{
startfrom = automation.GetDesktop();
}
current.Add(new UIElement(startfrom));
for (var i = 0; i < selectors.Count; i++)
{
var s = new WindowsSelectorItem(selectors[i]);
var elements = new List <UIElement>();
elements.AddRange(current);
current.Clear();
int failcounter = 0;
do
{
foreach (var _element in elements)
{
var matches = ((WindowsSelectorItem)s).matches(automation, _element.RawElement, _treeWalker, (i == 0 ? 1: maxresults));
var uimatches = new List <UIElement>();
foreach (var m in matches)
{
var ui = new UIElement(m);
var list = selectors.Take(i).ToList();
list.Add(new WindowsSelectorItem(m, false));
uimatches.Add(ui);
}
current.AddRange(uimatches.ToArray());
}
if (current.Count == 0)
{
++failcounter;
foreach (var element in elements)
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
var children = element.RawElement.FindAllChildren();
foreach (var c in children)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
Log.Selector(message);
}
}
else
{
Log.Selector(string.Format("Found " + current.Count + " hits for selector # " + i + " {0:mm\\:ss\\.fff}", sw.Elapsed));
}
} while (failcounter < 2 && current.Count == 0);
if (i == (selectors.Count - 1))
{
result = current.ToArray();
}
if (current.Count == 0)
{
var message = "needed to find " + Environment.NewLine + selectors[i].ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
foreach (var element in elements)
{
var children = element.RawElement.FindAllChildren();
foreach (var c in children)
{
try
{
message += new UIElement(c).ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
}
Log.Warning(message);
return(new UIElement[] { });
}
}
}
if (result == null)
{
Log.Selector(string.Format("GetElementsWithuiSelector::ended with 0 results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(new UIElement[] { });
}
Log.Selector(string.Format("GetElementsWithuiSelector::ended with " + result.Count() + " results after {0:mm\\:ss\\.fff}", sw.Elapsed));
return(result);
}
protected override void Execute(NativeActivityContext context)
{
var sw = new Stopwatch();
sw.Start();
Log.Selector(string.Format("Windows.GetElement::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement[] elements = null;
var selector = Selector.Get(context);
var sel = new WindowsSelector(selector);
var timeout = Timeout.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
var from = From.Get(context);
// double _timeout = 250;
double _timeout = 5000;
if (PluginConfig.search_descendants)
{
_timeout = 5000;
}
//#if DEBUG
// _timeout = _timeout * 8;
//#endif
do
{
if (PluginConfig.get_elements_in_different_thread)
{
elements = OpenRPA.AutomationHelper.RunSTAThread <UIElement[]>(() =>
{
try
{
Log.Selector(string.Format("Windows.GetElement::GetElementsWithuiSelector in non UI thread {0:mm\\:ss\\.fff}", sw.Elapsed));
return(WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults));
}
catch (System.Threading.ThreadAbortException)
{
}
catch (Exception ex)
{
Log.Error(ex, "");
}
return(new UIElement[] { });
}, TimeSpan.FromMilliseconds(_timeout)).Result;
}
else
{
Log.Selector(string.Format("Windows.GetElement::GetElementsWithuiSelector using UI thread {0:mm\\:ss\\.fff}", sw.Elapsed));
elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults);
}
//elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults);
if (elements == null)
{
elements = new UIElement[] { };
}
if (elements.Length == 0)
{
Log.Selector(string.Format("Windows.GetElement::Found no elements {0:mm\\:ss\\.fff}", sw.Elapsed));
}
} while (elements != null && elements.Length == 0 && sw.Elapsed < timeout);
//if (PluginConfig.get_elements_in_different_thread && elements.Length > 0)
//{
// // Get them again, we need the COM objects to be loaded in the UI thread
// elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults);
//}
context.SetValue(Elements, elements);
if (elements.Count() < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <UIElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
if (more)
{
context.SetValue(_elements, _enum);
context.SetValue(_sw, sw);
Log.Selector(string.Format("Windows.GetElement::end:: call ScheduleAction: {0:mm\\:ss\\.fff}", sw.Elapsed));
context.ScheduleAction <UIElement>(Body, _enum.Current, OnBodyComplete);
}
else
{
Log.Selector(string.Format("Windows.GetElement:end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
}
protected override void StartLoop(NativeActivityContext context)
{
WindowsCacheExtension ext = context.GetExtension <WindowsCacheExtension>();
var sw = new Stopwatch();
sw.Start();
Log.Selector(string.Format("Windows.GetElement::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
UIElement[] elements = null;
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new WindowsSelector(selector);
var timeout = Timeout.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
var interactive = Interactive.Get(context);
var from = From.Get(context);
int failcounter = 0;
do
{
if (ClearCache != null && ClearCache.Get(context))
{
Log.Selector(string.Format("Windows.GetElement::Clearing windows element cache {0:mm\\:ss\\.fff}", sw.Elapsed));
WindowsSelectorItem.ClearCache();
}
if (PluginConfig.get_elements_in_different_thread)
{
elements = OpenRPA.AutomationHelper.RunSTAThread <UIElement[]>(() =>
{
try
{
Log.Selector(string.Format("Windows.GetElement::GetElementsWithuiSelector in non UI thread {0:mm\\:ss\\.fff}", sw.Elapsed));
return(WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults, ext));
}
catch (System.Threading.ThreadAbortException)
{
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
return(new UIElement[] { });
}, PluginConfig.search_timeout).Result;
}
else
{
Log.Selector(string.Format("Windows.GetElement::GetElementsWithuiSelector using UI thread {0:mm\\:ss\\.fff}", sw.Elapsed));
elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults, ext);
if (elements == null || elements.Length == 0)
{
elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults, ext);
}
}
//elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults);
if (elements == null)
{
elements = new UIElement[] { };
}
if (elements.Length == 0)
{
Log.Selector(string.Format("Windows.GetElement::Found no elements {0:mm\\:ss\\.fff}", sw.Elapsed));
failcounter++;
}
if (failcounter > 2)
{
WindowsSelectorItem.ClearCache();
}
} while (elements != null && elements.Length == 0 && sw.Elapsed < timeout);
if (PluginConfig.get_elements_in_different_thread && elements.Length > 0)
{
// Get them again, we need the COM objects to be loaded in the UI thread
elements = WindowsSelector.GetElementsWithuiSelector(sel, from, maxresults, ext);
}
context.SetValue(Elements, elements);
var lastelements = context.GetValue(_lastelements);
if (lastelements == null)
{
lastelements = new UIElement[] { }
}
;
context.SetValue(_lastelements, elements);
if ((elements.Length + lastelements.Length) < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <UIElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
if (lastelements.Length == elements.Length && lastelements.Length > 0)
{
more = !System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(lastelements, elements);
}
if (more)
{
if (interactive)
{
var testelement = _enum.Current;
Wait.UntilResponsive(testelement.RawElement, PluginConfig.search_timeout);
}
context.SetValue(_elements, _enum);
context.SetValue(_sw, sw);
Log.Selector(string.Format("Windows.GetElement::end:: call ScheduleAction: {0:mm\\:ss\\.fff}", sw.Elapsed));
IncIndex(context);
SetTotal(context, elements.Length);
context.ScheduleAction <UIElement>(Body, _enum.Current, OnBodyComplete);
}
else
{
Log.Selector(string.Format("Windows.GetElement:end {0:mm\\:ss\\.fff}", sw.Elapsed));
}
}
private void OnMouseUp(InputEventArgs e)
{
try
{
if (e.Element == null)
{
return;
}
if (Entity == null || string.IsNullOrEmpty(Selector))
{
return;
}
var pathToRoot = new List <AutomationElement>();
AutomationElement element = e.Element.RawElement;
while (element != null)
{
if (pathToRoot.Contains(element))
{
break;
}
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
}
catch (Exception)
{
}
try
{
element = element.Parent;
}
catch (Exception ex)
{
element = null;
Log.Error(ex, "");
// return;
}
}
WindowsSelector selector = new WindowsSelector(Selector);
if (pathToRoot.Count < (selector.Count - 1))
{
return;
}
if (pathToRoot.Count > (selector.Count - 1))
{
return;
}
pathToRoot.Reverse();
for (var i = 0; i < pathToRoot.Count; i++)
{
element = pathToRoot[i];
WindowsSelectorItem s = new WindowsSelectorItem(selector[(i + 1)]);
if (!s.Match(element))
{
Log.Verbose("WindowsClickDetectorPlugin: Element " + i + " does not match with selector");
return;
}
}
var _e = new DetectorEvent(e.Element);
OnDetector?.Invoke(this, _e, EventArgs.Empty);
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
