private void OnMouseUp(InputEventArgs e)
{
JavaElement foundElement = null;
foreach (var jvm in Javahook.Instance.jvms)
{
var _children = jvm.GetChildren();
if (_children.Count() > 0)
{
var firstac = _children.First() as WindowsAccessBridgeInterop.AccessibleContextNode;
var res = firstac.GetNodePathAtUsingAccessBridge(new System.Drawing.Point(e.X, e.Y));
if (res != null)
{
var Root = new JavaElement(res.Root);
var Parent = Root;
while (Parent.Parent != null)
{
Parent = Parent.Parent;
}
if (res.Count > 0)
{
foundElement = new JavaElement(res.Last());
}
}
}
}
if (foundElement == null)
{
return;
}
Hook_OnMouseClicked(foundElement.ac);
}
public JavaElement[] matches(JavaElement element)
{
int counter = 0;
do
{
try
{
var matchs = new List <JavaElement>();
JavaElement[] elements = element.Children;
foreach (JavaElement elementNode in elements)
{
if (Match(elementNode))
{
matchs.Add(elementNode);
}
}
Log.Selector("match count: " + matchs.Count);
return(matchs.ToArray());
}
catch (Exception)
{
++counter;
if (counter == 2)
{
throw;
}
}
} while (counter < 2);
return(new JavaElement[] { });
}
public JavaTreeElement(treeelement parent, bool expanded, JavaElement element) : base(parent)
{
JavaElement = element;
IsExpanded = expanded;
Element = element;
Name = element.title;
}
public JavaSelectorItem(JavaElement element, bool isRoot)
{
this.Element = element;
string n = null;
if (!string.IsNullOrEmpty(element.Name))
{
n = element.Name;
}
Properties = new ObservableCollection <SelectorItemProperty>();
if (isRoot)
{
Properties.Add(new SelectorItemProperty("Selector", "Java"));
Enabled = true;
canDisable = false;
return;
}
if (!string.IsNullOrEmpty(element.id))
{
Properties.Add(new SelectorItemProperty("id", element.id));
}
if (!string.IsNullOrEmpty(element.Name))
{
Properties.Add(new SelectorItemProperty("Name", element.Name));
}
if (!string.IsNullOrEmpty(element.role))
{
Properties.Add(new SelectorItemProperty("role", element.role));
}
if (!string.IsNullOrEmpty(element.title))
{
Properties.Add(new SelectorItemProperty("title", element.title));
}
if (element.IndexInParent > -1)
{
Properties.Add(new SelectorItemProperty("IndexInParent", element.IndexInParent.ToString()));
}
Enabled = (Properties.Count > 1);
foreach (var p in Properties)
{
p.Enabled = true;
p.canDisable = true;
}
;
foreach (var p in Properties)
{
p.PropertyChanged += (sender, e) =>
{
OnPropertyChanged("Displayname");
OnPropertyChanged("json");
}
}
;
canDisable = true;
}
public static JavaElement[] EnumRoots(WindowsAccessBridgeInterop.AccessibleJvm jvm)
{
var results = new List <JavaElement>();
var children = jvm.GetChildren();
if (children != null && children.Count() > 0)
{
var firstac = children.First() as WindowsAccessBridgeInterop.AccessibleContextNode;
var hwnd = jvm.AccessBridge.Functions.GetHWNDFromAccessibleContext(jvm.JvmId, firstac.AccessibleContextHandle);
RECT rect = new RECT();
GetWindowRect(hwnd, ref rect);
int x = rect.Left + ((rect.Right - rect.Left) / 2);
int y = rect.Top + ((rect.Bottom - rect.Top) / 2);
var res = firstac.GetNodePathAtUsingAccessBridge(new System.Drawing.Point(x, y));
if (res != null)
{
var Root = new JavaElement(res.Root);
var Parent = Root;
while (Parent.Parent != null)
{
Parent = Parent.Parent;
}
if (!results.Contains(Parent))
{
results.Add(Parent);
}
}
//for(var x= rect.Left; x < rect.Right; x += 10)
//{
// for (var y = rect.Top; y < rect.Bottom; y += 10)
// {
// var res = firstac.GetNodePathAtUsingAccessBridge(new System.Drawing.Point(x, y));
// if (res != null)
// {
// var Root = new JavaElement(res.Root);
// var Parent = Root;
// while (Parent.Parent != null) Parent = Parent.Parent;
// if(!results.Contains(Parent)) results.Add(Parent);
// }
// }
//}
}
return(results.ToArray());
}
private void Hook_OnMouseClicked(int vmID, WindowsAccessBridgeInterop.AccessibleContextNode ac)
{
lastElement = new JavaElement(ac);
lastElement.SetPath();
Log.Debug("OnMouseClicked: " + lastElement.id + " " + lastElement.role + " " + lastElement.Name);
if (lastElement == null)
{
return;
}
var re = new RecordEvent(); re.Button = MouseButton.Left;
var a = new GetElement {
DisplayName = lastElement.title
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
JavaSelector sel = null;
// sel = new JavaSelector(e.Element.rawElement, null, true);
sel = new JavaSelector(lastElement, null, true);
if (sel == null)
{
return;
}
if (sel.Count < 2)
{
return;
}
a.Selector = sel.ToString();
a.Image = lastElement.ImageString();
a.MaxResults = 1;
re.Element = lastElement;
re.Selector = sel;
re.X = lastElement.X;
re.Y = lastElement.Y;
re.a = new GetElementResult(a);
re.SupportInput = lastElement.SupportInput;
Log.Debug(string.Format("Java.Recording::OnMouseClicked::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}
public JavaSelectorItem(JavaElement element)
{
this.Element = element;
if (this.Element == null)
{
throw new Exception("Error!!!");
}
Properties = new ObservableCollection <SelectorItemProperty>();
if (!string.IsNullOrEmpty(element.Name))
{
Properties.Add(new SelectorItemProperty("Name", element.Name));
}
if (!string.IsNullOrEmpty(element.role))
{
Properties.Add(new SelectorItemProperty("role", element.role));
}
if (!string.IsNullOrEmpty(element.title))
{
Properties.Add(new SelectorItemProperty("title", element.title));
}
if (!string.IsNullOrEmpty(element.id))
{
Properties.Add(new SelectorItemProperty("id", element.id));
}
foreach (var p in Properties)
{
p.Enabled = true;
p.canDisable = (Properties.Count > 1);
}
;
foreach (var p in Properties)
{
p.PropertyChanged += (sender, e) =>
{
OnPropertyChanged("Displayname");
OnPropertyChanged("json");
}
}
;
}
public JavaSelector(JavaElement element, JavaSelector anchor, bool doEnum)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
Log.Selector(string.Format("Javaselector::AutomationElement::begin {0:mm\\:ss\\.fff}", sw.Elapsed));
Log.Selector(string.Format("Javaselector::GetControlVJavawWalker::end {0:mm\\:ss\\.fff}", sw.Elapsed));
JavaElement root = null;
JavaElement baseElement = null;
var pathToRoot = new List <JavaElement>();
while (element != null)
{
// Break on circular relationship (should not happen?)
//if (pathToRoot.Contains(element) || element.Equals(_rootElement)) { break; }
if (pathToRoot.Contains(element))
{
break;
}
if (element.Parent != null)
{
pathToRoot.Add(element);
}
if (element.Parent == null)
{
root = element;
}
try
{
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
pathToRoot.Reverse();
if (anchor != null)
{
var anchorlist = anchor.Where(x => x.Enabled && x.Selector == null).ToList();
for (var i = 0; i < anchorlist.Count; i++)
{
//if (((JavaSelectorItem)anchorlist[i]).Match(pathToRoot[0]))
if (JavaSelectorItem.Match(anchorlist[i], pathToRoot[0]))
{
pathToRoot.Remove(pathToRoot[0]);
}
else
{
Log.Warning("Element does not match the anchor path");
return;
}
}
}
if (pathToRoot.Count == 0)
{
Log.Error("Element is same as annchor");
return;
}
baseElement = pathToRoot.First();
element = pathToRoot.Last();
Clear();
JavaSelectorItem item;
if (anchor == null)
{
item = new JavaSelectorItem(root, true);
item.Enabled = true;
item.canDisable = false;
Items.Add(item);
}
for (var i = 0; i < pathToRoot.Count(); i++)
{
var o = pathToRoot[i];
item = new JavaSelectorItem(o, false);
if (i == 0 || i == (pathToRoot.Count() - 1))
{
item.canDisable = false;
}
if (doEnum)
{
item.EnumNeededProperties(o, o.Parent);
}
Items.Add(item);
}
pathToRoot.Reverse();
Log.Selector(string.Format("Javaselector::EnumNeededProperties::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Count"));
OnPropertyChanged(new System.ComponentModel.PropertyChangedEventArgs("Item[]"));
OnCollectionChanged(new System.Collections.Specialized.NotifyCollectionChangedEventArgs(System.Collections.Specialized.NotifyCollectionChangedAction.Reset));
}
private static JavaElement[] GetElementsWithuiSelector(WindowsAccessBridgeInterop.AccessibleJvm jvm, JavaSelector selector, IElement fromElement, int maxresults)
{
JavaElement[] result = null;
JavaElement _fromElement = fromElement as JavaElement;
var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
var current = new List <JavaElement>();
JavaElement startfrom = null;
if (_fromElement != null)
{
startfrom = _fromElement;
}
if (startfrom == null)
{
startfrom = new JavaElement(jvm);
}
current.Add(startfrom);
for (var i = 0; i < selectors.Count; i++)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
var s = new JavaSelectorItem(selectors[i]);
Log.Selector(string.Format("OpenRPA.Java::GetElementsWithuiSelector::Find for selector {0} {1}", i, s.ToString()));
var elements = new List <JavaElement>();
elements.AddRange(current);
current.Clear();
foreach (var _element in elements)
{
result = ((JavaSelectorItem)s).matches(_element);
current.AddRange(result);
}
if (current.Count == 0)
{
// TODO: Figure out, why this is needed when working with Java Menu's
foreach (var _e in elements)
{
if (s.Match(_e))
{
current.Add(_e);
}
}
}
if (i == (selectors.Count - 1))
{
result = current.ToArray();
}
if (current.Count == 0)
{
var _c = new JavaSelectorItem(selectors[i]);
var message = "needed to find " + Environment.NewLine + _c.ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
foreach (var element in elements)
{
var children = element.Children;
foreach (var c in children)
{
try
{
message += c.ToString() + Environment.NewLine;
}
catch (Exception)
{
}
}
}
Log.Selector(message);
return(new JavaElement[] { });
}
Log.Selector(string.Format("OpenRPA.Java::GetElement::found {1} for selector {2} in {0:mm\\:ss\\.fff}", sw.Elapsed, elements.Count(), i));
}
if (result == null)
{
return new JavaElement[] { }
}
;
return(result);
}
private void Hook_OnMouseClicked(int vmID, WindowsAccessBridgeInterop.AccessibleContextNode ac)
{
if (string.IsNullOrEmpty(Selector))
{
return;
}
var element = new JavaElement(ac);
element.SetPath();
Log.Debug("OnMouseClicked: " + element.id + " " + element.role + " " + element.Name);
var pathToRoot = new List <JavaElement>();
while (element != null)
{
if (pathToRoot.Contains(element))
{
break;
}
try
{
if (element.Parent != null)
{
pathToRoot.Add(element);
}
}
catch (Exception)
{
}
try
{
element = element.Parent;
}
catch (Exception ex)
{
Log.Error(ex, "");
return;
}
}
JavaSelector selector = new JavaSelector(Selector);
if (pathToRoot.Count < (selector.Count - 1))
{
return;
}
if (pathToRoot.Count > (selector.Count - 1))
{
return;
}
pathToRoot.Reverse();
for (var i = 0; i < pathToRoot.Count; i++)
{
element = pathToRoot[i];
if (selector.Count > (i + 1))
{
JavaSelectorItem s = new JavaSelectorItem(selector[(i + 1)]);
if (!s.Match(element))
{
return;
}
}
else
{
return;
}
}
var _e = new DetectorEvent(element);
OnDetector?.Invoke(this, _e, EventArgs.Empty);
}
public DetectorEvent(JavaElement element)
{
this.element = element;
host = Environment.MachineName.ToLower();
fqdn = System.Net.Dns.GetHostEntry(Environment.MachineName).HostName.ToLower();
}
public bool Match(SelectorItem item, IElement m)
{
var el = new JavaElement(m.RawElement as WindowsAccessBridgeInterop.AccessibleNode);
return(JavaSelectorItem.Match(item, el));
}
private void Hook_OnMouseEntered(int vmID, WindowsAccessBridgeInterop.AccessibleContextNode ac)
{
lastElement = new JavaElement(ac);
lastElement.SetPath();
Log.Verbose("MouseEntered: " + lastElement.id + " " + lastElement.role + " " + lastElement.Name);
}
public static bool Match(SelectorItem item, JavaElement m)
{
foreach (var p in item.Properties.Where(x => x.Enabled == true && x.Value != null))
{
if (p.Name == "Name")
{
if (!string.IsNullOrEmpty(m.Name))
{
var v = m.Name;
if (!PatternMatcher.FitsMask(v, p.Value))
{
Log.Selector(p.Name + " mismatch '" + v + "' / '" + p.Value + "'");
return(false);
}
}
else
{
Log.Selector(p.Name + " does not exists, but needed value '" + p.Value + "'");
return(false);
}
}
if (p.Name == "role")
{
if (!string.IsNullOrEmpty(m.role))
{
var v = m.role;
if (!PatternMatcher.FitsMask(m.role, p.Value))
{
Log.Selector(p.Name + " mismatch '" + v + "' / '" + p.Value + "'");
return(false);
}
}
else
{
Log.Selector(p.Name + " does not exists, but needed value '" + p.Value + "'");
return(false);
}
}
if (p.Name == "title")
{
if (!string.IsNullOrEmpty(m.title))
{
var v = m.title;
if (!PatternMatcher.FitsMask(m.title, p.Value))
{
Log.Selector(p.Name + " mismatch '" + v + "' / '" + p.Value + "'");
return(false);
}
}
else
{
Log.Selector(p.Name + " does not exists, but needed value '" + p.Value + "'");
return(false);
}
}
if (p.Name == "Id")
{
if (!string.IsNullOrEmpty(m.id))
{
var v = m.id;
if (!PatternMatcher.FitsMask(m.id, p.Value))
{
Log.Selector(p.Name + " mismatch '" + v + "' / '" + p.Value + "'");
return(false);
}
}
else
{
Log.Selector(p.Name + " does not exists, but needed value '" + p.Value + "'");
return(false);
}
}
if (p.Name == "IndexInParent")
{
if (m.IndexInParent != int.Parse(p.Value))
{
Log.Selector(p.Name + " mismatch '" + m.IndexInParent + "' / '" + p.Value + "'");
return(false);
}
}
}
return(true);
}
public bool Match(JavaElement m)
{
return(Match(this, m));
}
public void EnumNeededProperties(JavaElement element, JavaElement parent)
{
string name = null;
if (!string.IsNullOrEmpty(element.Name))
{
name = element.Name;
}
if (!string.IsNullOrEmpty(element.role))
{
name = element.role;
}
if (!string.IsNullOrEmpty(element.title))
{
name = element.title;
}
if (!string.IsNullOrEmpty(element.id))
{
name = element.id;
}
var props = GetProperties();
//int i = props.Length -1;
int i = 1;
int matchcounter = 0;
foreach (var p in Properties)
{
p.Enabled = false;
}
do
{
Log.Selector("#*******************************#");
Log.Selector("# " + i);
var selectedProps = props.Take(i).ToArray();
foreach (var p in Properties)
{
p.Enabled = selectedProps.Contains(p.Name);
}
JavaElement[] children = { };
if (element.Parent != null)
{
children = element.Parent.Children;
}
matchcounter = 0;
foreach (JavaElement elementNode in children)
{
Log.Selector("Match using " + i + " properties.");
if (Match(elementNode))
{
matchcounter++;
}
if (matchcounter > 1)
{
break;
}
}
if (matchcounter != 1)
{
++i;
}
//if (matchcounter > 1)
//{
// Log.Selector("EnumNeededProperties match with " + i + " gave more than 1 result");
// ++i;
// if (i >= props.Count()) break;
//}
} while (matchcounter != 1 && i < props.Count());
Log.Selector("EnumNeededProperties match with " + i + " gave " + matchcounter + " result");
Properties.ForEach((e) => e.Enabled = false);
foreach (var p in props.Take(i).ToArray())
{
Properties.Where(x => x.Name == p).First().Enabled = true;
}
}
