public IElement LaunchBySelector(Selector selector, bool CheckRunning, TimeSpan timeout)
{
var first = selector[0];
var second = selector[1];
var p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
string browser = "";
if (p != null)
{
browser = p.Value;
}
p = first.Properties.Where(x => x.Name == "url").FirstOrDefault();
string url = "";
if (p != null)
{
url = p.Value;
}
NMHook.openurl(browser, url);
if (browser == "chrome")
{
foreach (var process in System.Diagnostics.Process.GetProcesses())
{
string pname = process.ProcessName.ToLower();
if (pname.Contains("chrome"))
{
string title = process.MainWindowTitle;
Console.WriteLine(title);
if (!string.IsNullOrEmpty(title))
{
var exists = NMHook.tabs.Where(x => x.title == title).FirstOrDefault();
var automation = AutomationUtil.getAutomation();
var _ele = automation.FromHandle(process.MainWindowHandle);
return(new UIElement(_ele));
}
}
}
}
return(null);
}
public void LaunchBySelector(Selector selector, TimeSpan timeout)
{
var first = selector[0];
var second = selector[1];
var p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
string browser = "";
if (p != null)
{
browser = p.Value;
}
p = first.Properties.Where(x => x.Name == "url").FirstOrDefault();
string url = "";
if (p != null)
{
url = p.Value;
}
NMHook.openurl(browser, url);
}
public static NMElement[] getElement(int tabid, long frameId, string browser, string xPath, TimeSpan timeout)
{
var results = new List <NMElement>();
var getelement = new NativeMessagingMessage("getelement", PluginConfig.debug_console_output, PluginConfig.unique_xpath_ids);
getelement.browser = browser;
getelement.tabid = tabid;
getelement.xPath = xPath;
getelement.frameId = frameId;
NativeMessagingMessage result = null;
try
{
result = NMHook.sendMessageResult(getelement, true, timeout);
}
catch (Exception)
{
}
if (result != null && result.result != null && result.results == null)
{
result.results = new NativeMessagingMessage[] { result };
}
if (result != null && result.results != null && result.results.Count() > 0)
{
foreach (var res in result.results)
{
if (res.result != null)
{
//var html = new HtmlElement(getelement.xPath, getelement.cssPath, res.tabid, res.frameId, res.result);
res.tab = NMHook.tabs.Where(x => x.id == res.tabid && x.browser == res.browser).FirstOrDefault();
var html = new NMElement(res);
results.Add(html);
}
}
//result = result.results[0];
}
return(results.ToArray());
}
protected override void Execute(NativeActivityContext context)
{
var closeall = CloseAll.Get(context);
var browser = Browser.Get(context);
var timeout = TimeSpan.FromSeconds(3);
if (browser != "chrome" && browser != "ff" && browser != "edge")
{
browser = "chrome";
}
if (closeall)
{
NMHook.enumtabs();
foreach (var tab in NMHook.tabs.Where(x => x.browser == browser).ToList())
{
NMHook.closetab(browser, tab.id);
}
}
else
{
if (browser == "chrome")
{
if (NMHook.CurrentChromeTab != null)
{
NMHook.closetab(browser, NMHook.CurrentChromeTab.id);
}
}
if (browser == "ff")
{
if (NMHook.CurrentFFTab != null)
{
NMHook.closetab(browser, NMHook.CurrentFFTab.id);
}
}
}
}
public static treeelement[] _GetRootElements(Selector anchor)
{
var rootelements = new List <treeelement>();
var browser = "";
if (NMHook.chromeconnected)
{
browser = "chrome";
}
if (NMHook.ffconnected)
{
browser = "ff";
}
if (NMHook.edgeconnected)
{
browser = "edge";
}
if (string.IsNullOrEmpty(browser))
{
return(rootelements.ToArray());
}
//NMHook.enumtabs();
//// var tab = NMHook.tabs.Where(x => x.highlighted == true && x.browser == "chrome").FirstOrDefault();
//var tab = NMHook.tabs.Where(x => x.highlighted == true).FirstOrDefault();
//if (tab == null)
//{
// // tab = NMHook.tabs.Where(x => x.browser == "chrome").FirstOrDefault();
// tab = NMHook.tabs.FirstOrDefault();
//}
//if (NMHook.tabs.Count == 0) { return rootelements.ToArray(); }
//// getelement.data = "getdom";
var getelement = new NativeMessagingMessage("getelements", PluginConfig.debug_console_output, PluginConfig.unique_xpath_ids)
{
browser = browser,
xPath = "/html",
frameId = -1
};
var tab = NMHook.tabs.Where(x => x.browser == browser && (x.active || x.selected)).FirstOrDefault();
if (tab == null)
{
NMHook.enumwindowandtabs();
}
if (tab != null)
{
getelement.tabid = tab.id;
getelement.windowId = tab.windowId;
}
if (anchor != null && anchor.Count > 1)
{
var s = anchor[1];
var p = s.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
if (p != null)
{
getelement.xPath = p.Value;
}
}
NativeMessagingMessage result = null;
try
{
result = NMHook.sendMessageResult(getelement, true, PluginConfig.protocol_timeout);
}
catch (Exception)
{
}
if (result != null && result.result != null && result.results == null)
{
// result.results = new NativeMessagingMessage[] { result };
var html = new NMElement(result);
rootelements.Add(new NMTreeElement(null, true, html));
return(rootelements.ToArray());
}
if (result != null && result.results != null && result.results.Count() > 0)
{
foreach (var res in result.results)
{
if (res.result != null)
{
if (res.tab == null)
{
res.tab = NMHook.tabs.Where(x => x.browser == res.browser && x.id == res.tabid).FirstOrDefault();
}
var html = new NMElement(res);
rootelements.Add(new NMTreeElement(null, true, html));
//var html = new HtmlElement(getelement.xPath, getelement.cssPath, res.tabid, res.frameId, res.result);
}
}
//result = result.results[0];
}
return(rootelements.ToArray());
}
//private static NMElement[] GetElementsWithuiSelector(WindowsAccessBridgeInterop.AccessibleJvm jvm, NMSelector selector, IElement fromElement, int maxresults)
//{
// NMElement[] result = null;
// NMElement _fromElement = fromElement as NMElement;
// var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
// var current = new List<NMElement>();
// NMElement startfrom = null;
// if (_fromElement != null) startfrom = _fromElement;
// if (startfrom == null) startfrom = new NMElement(jvm);
// current.Add(startfrom);
// for (var i = 0; i < selectors.Count; i++)
// {
// var sw = new System.Diagnostics.Stopwatch();
// sw.Start();
// var s = new NMSelectorItem(selectors[i]);
// Log.Selector(string.Format("OpenRPA.NM::GetElementsWithuiSelector::Find for selector {0} {1}", i, s.ToString()));
// var elements = new List<NMElement>();
// elements.AddRange(current);
// current.Clear();
// foreach (var _element in elements)
// {
// result = ((NMSelectorItem)s).matches(_element);
// current.AddRange(result);
// }
// if (i == (selectors.Count - 1)) result = current.ToArray();
// if (current.Count == 0)
// {
// var _c = new NMSelectorItem(selectors[i]);
// var message = "needed to find " + Environment.NewLine + _c.ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
// foreach (var element in elements)
// {
// var children = element.Children;
// foreach (var c in children)
// {
// try
// {
// message += c.ToString() + Environment.NewLine;
// }
// catch (Exception)
// {
// }
// }
// }
// Log.Selector(message);
// return new NMElement[] { };
// }
// Log.Selector(string.Format("OpenRPA.NM::GetElement::found {1} for selector {2} in {0:mm\\:ss\\.fff}", sw.Elapsed, elements.Count(), i));
// }
// if (result == null) return new NMElement[] { };
// return result;
//}
public static NMElement[] GetElementsWithuiSelector(NMSelector selector, IElement fromElement = null, int maxresults = 1)
{
var results = new List <NMElement>();
SelectorItem first = null;
SelectorItem second = null;
string browser = "";
SelectorItemProperty p = null;
if (selector.Count > 1)
{
first = selector[0];
second = selector[1];
p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
if (p != null)
{
browser = p.Value;
}
}
else if (fromElement == null)
{
throw new ArgumentException("Invalid select with onlu 1 child and no anchor");
}
else
{
second = selector[0];
}
p = second.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
string xpath = "";
if (p != null)
{
xpath = p.Value;
}
p = second.Properties.Where(x => x.Name == "cssselector").FirstOrDefault();
string cssselector = "";
if (p != null)
{
cssselector = p.Value;
}
NMElement fromNMElement = fromElement as NMElement;
if (fromElement != null)
{
browser = fromNMElement.browser;
//if (browser != fromNMElement.browser) throw new ArgumentException("browser mismatch " + browser + "/" + fromNMElement.browser);
p = second.Properties.Where(x => x.Name == "cssselector").FirstOrDefault();
if (p == null)
{
throw new ArgumentException("fromElement missing cssselector");
}
// xpath += p.Value.Substring(1);
xpath = "";
cssselector = fromNMElement.cssselector + " > " + p.Value;
}
NMHook.checkForPipes(true, true);
NMHook.reloadtabs();
var tabs = NMHook.tabs;
if (!string.IsNullOrEmpty(browser))
{
tabs = NMHook.tabs.Where(x => x.browser == browser).ToList();
}
foreach (var tab in tabs)
{
NativeMessagingMessage subresult = null;
var getelement = new NativeMessagingMessage("getelements");
getelement.browser = browser;
getelement.xPath = xpath;
getelement.cssPath = cssselector;
subresult = NMHook.sendMessageResult(getelement, false, TimeSpan.FromSeconds(2));
if (subresult != null)
{
if (subresult.results != null)
{
foreach (var b in subresult.results)
{
if (b.cssPath == "true" || b.xPath == "true")
{
var data = b.result;
var arr = JArray.Parse(data);
foreach (var _e in arr)
{
if (results.Count > maxresults)
{
continue;
}
var json = _e.ToString();
var subsubresult = Newtonsoft.Json.JsonConvert.DeserializeObject <NativeMessagingMessage>(json);
subsubresult.browser = browser;
subsubresult.result = json;
subsubresult.tabid = b.tabid;
subsubresult.tab = b.tab;
results.Add(new NMElement(subsubresult));
}
}
}
}
}
}
return(results.ToArray());
//NMHook.Instance.refreshJvms();
//NMElement[] result = null;
//foreach (var jvm in NMHook.Instance.jvms)
//{
// result = GetElementsWithuiSelector(jvm, selector, fromElement, maxresults);
// if (result.Count() > 0) return result;
//}
//if (result == null) return new NMElement[] { };
//return result;
//return new NMElement[] { };
}
public static NMElement[] GetElementsWithuiSelector(NMSelector selector, IElement fromElement = null, int maxresults = 1)
{
var results = new List <NMElement>();
SelectorItem first = null;
SelectorItem second = null;
string browser = "";
SelectorItemProperty p = null;
if (selector.Count > 1)
{
first = selector[0];
second = selector[1];
p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
if (p != null)
{
browser = p.Value;
}
}
else if (fromElement == null)
{
throw new ArgumentException("Invalid select with only 1 child and no anchor");
}
else
{
second = selector[0];
}
p = second.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
string xpath = "";
if (p != null)
{
xpath = p.Value;
}
p = second.Properties.Where(x => x.Name == "cssselector").FirstOrDefault();
string cssselector = "";
if (p != null)
{
cssselector = p.Value;
}
NMElement fromNMElement = fromElement as NMElement;
string fromcssPath = "";
string fromxPath = "";
if (fromElement != null)
{
fromcssPath = fromNMElement.cssselector;
fromxPath = fromNMElement.xpath;
}
//NMHook.checkForPipes(true, true);
//NMHook.reloadtabs();
//var tabs = NMHook.tabs.ToList();
//if (!string.IsNullOrEmpty(browser)) {
// lock(NMHook.tabs)
// {
// tabs = NMHook.tabs.Where(x => x.browser == browser).ToList();
// }
//}
//foreach (var tab in tabs)
//{
//}
NativeMessagingMessage subresult = null;
var getelement = new NativeMessagingMessage("getelements", PluginConfig.debug_console_output);
getelement.browser = browser;
getelement.xPath = xpath;
getelement.cssPath = cssselector;
getelement.fromxPath = fromxPath;
getelement.fromcssPath = fromcssPath;
if (fromElement != null && fromElement is NMElement)
{
getelement.windowId = ((NMElement)fromElement).message.windowId;
getelement.tabid = ((NMElement)fromElement).message.tabid;
getelement.frameId = ((NMElement)fromElement).message.frameId;
}
subresult = NMHook.sendMessageResult(getelement, false, TimeSpan.FromSeconds(2));
if (subresult != null)
{
if (subresult.results != null)
{
foreach (var b in subresult.results)
{
if (b.cssPath == "true" || b.xPath == "true")
{
if (results.Count > maxresults)
{
continue;
}
results.Add(new NMElement(b));
//var data = b.result;
//var arr = JArray.Parse(data.ToString());
//foreach (var _e in arr)
//{
// if (results.Count > maxresults) continue;
// var json = _e.ToString();
// var subsubresult = Newtonsoft.Json.JsonConvert.DeserializeObject<NativeMessagingMessage>(json);
// subsubresult.browser = browser;
// subsubresult.result = json;
// subsubresult.tabid = b.tabid;
// subsubresult.tab = b.tab;
// results.Add(new NMElement(subsubresult));
//}
}
}
}
}
return(results.ToArray());
}
protected override async Task <object> ExecuteAsync(NativeActivityContext context)
{
IsCancel = false;
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From?.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
string browser = sel.browser;
if (WaitForReady != null && WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var allelements = context.GetValue(_allelements);
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.FindTabByURL(browser, s.url);
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
await Task.Run(() =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
Log.Selector("BEGIN:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
if (allelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
if (!allelements.Contains(element))
{
newelements.Insert(0, element);
}
}
elements = newelements.ToArray();
}
} while (elements.Count() == 0 && sw.Elapsed < timeout && !IsCancel);
if (IsCancel)
{
Console.WriteLine("was Canceled: true! DisplayName" + DisplayName);
}
//if (sw.Elapsed >= timeout)
//{
// Console.WriteLine("Timeout !");
//}
});
return(elements);
}
protected override async Task <object> ExecuteAsync(NativeActivityContext context)
{
IsCancel = false;
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From?.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
string browser = sel.browser;
if (WaitForReady != null && WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var allelements = context.GetValue(_allelements);
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
if (timeout.Minutes > 5 || timeout.Hours > 1)
{
Activity _Activity = null;
try
{
var strProperty = context.GetType().GetProperty("Activity", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
var strGetter = strProperty.GetGetMethod(nonPublic: true);
_Activity = (Activity)strGetter.Invoke(context, null);
}
catch (Exception)
{
}
if (_Activity != null)
{
Log.Warning("Timeout for Activity " + _Activity.Id + " is above 5 minutes, was this the intention ? calculated value " + timeout.ToString());
}
else
{
Log.Warning("Timeout for on of your NM.GetElementAsync is above 5 minutes, was this the intention ? calculated value " + timeout.ToString());
}
}
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.FindTabByURL(browser, s.url);
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
await Task.Run(() =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
Log.Selector("BEGIN:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
if (allelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
if (!allelements.Contains(element))
{
newelements.Insert(0, element);
}
}
elements = newelements.ToArray();
}
} while (elements.Count() == 0 && sw.Elapsed < timeout && !IsCancel);
if (IsCancel)
{
Console.WriteLine("was Canceled: true! DisplayName" + DisplayName);
}
//if (sw.Elapsed >= timeout)
//{
// Console.WriteLine("Timeout !");
//}
});
return(elements);
}
//private static NMElement[] GetElementsWithuiSelector(WindowsAccessBridgeInterop.AccessibleJvm jvm, NMSelector selector, IElement fromElement, int maxresults)
//{
// NMElement[] result = null;
// NMElement _fromElement = fromElement as NMElement;
// var selectors = selector.Where(x => x.Enabled == true && x.Selector == null).ToList();
// var current = new List<NMElement>();
// NMElement startfrom = null;
// if (_fromElement != null) startfrom = _fromElement;
// if (startfrom == null) startfrom = new NMElement(jvm);
// current.Add(startfrom);
// for (var i = 0; i < selectors.Count; i++)
// {
// var sw = new System.Diagnostics.Stopwatch();
// sw.Start();
// var s = new NMSelectorItem(selectors[i]);
// Log.Selector(string.Format("OpenRPA.NM::GetElementsWithuiSelector::Find for selector {0} {1}", i, s.ToString()));
// var elements = new List<NMElement>();
// elements.AddRange(current);
// current.Clear();
// foreach (var _element in elements)
// {
// result = ((NMSelectorItem)s).matches(_element);
// current.AddRange(result);
// }
// if (i == (selectors.Count - 1)) result = current.ToArray();
// if (current.Count == 0)
// {
// var _c = new NMSelectorItem(selectors[i]);
// var message = "needed to find " + Environment.NewLine + _c.ToString() + Environment.NewLine + "but found only: " + Environment.NewLine;
// foreach (var element in elements)
// {
// var children = element.Children;
// foreach (var c in children)
// {
// try
// {
// message += c.ToString() + Environment.NewLine;
// }
// catch (Exception)
// {
// }
// }
// }
// Log.Selector(message);
// return new NMElement[] { };
// }
// Log.Selector(string.Format("OpenRPA.NM::GetElement::found {1} for selector {2} in {0:mm\\:ss\\.fff}", sw.Elapsed, elements.Count(), i));
// }
// if (result == null) return new NMElement[] { };
// return result;
//}
public static NMElement[] GetElementsWithuiSelector(NMSelector selector, IElement fromElement = null, int maxresults = 1)
{
var results = new List <NMElement>();
var first = selector[0];
var second = selector[1];
var p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
string browser = "";
if (p != null)
{
browser = p.Value;
}
p = second.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
string xpath = "";
if (p != null)
{
xpath = p.Value;
}
p = second.Properties.Where(x => x.Name == "cssselector").FirstOrDefault();
string cssselector = "";
if (p != null)
{
cssselector = p.Value;
}
NMHook.checkForPipes(true, true);
NMHook.reloadtabs();
var tabs = NMHook.tabs;
if (!string.IsNullOrEmpty(browser))
{
tabs = NMHook.tabs.Where(x => x.browser == browser).ToList();
}
foreach (var tab in tabs)
{
NativeMessagingMessage subresult = null;
var getelement = new NativeMessagingMessage("getelements");
getelement.browser = browser;
getelement.xPath = xpath;
getelement.cssPath = cssselector;
subresult = NMHook.sendMessageResult(getelement, false, TimeSpan.FromSeconds(2));
if (subresult != null)
{
foreach (var b in subresult.results)
{
if (b.cssPath == "true" || b.xPath == "true")
{
var data = b.result;
var arr = JArray.Parse(data);
foreach (var _e in arr)
{
var json = _e.ToString();
var subsubresult = Newtonsoft.Json.JsonConvert.DeserializeObject <NativeMessagingMessage>(json);
subsubresult.browser = browser;
subsubresult.result = json;
subsubresult.tabid = b.tabid;
subsubresult.tab = b.tab;
results.Add(new NMElement(subsubresult));
}
}
}
}
}
return(results.ToArray());
//NMHook.Instance.refreshJvms();
//NMElement[] result = null;
//foreach (var jvm in NMHook.Instance.jvms)
//{
// result = GetElementsWithuiSelector(jvm, selector, fromElement, maxresults);
// if (result.Count() > 0) return result;
//}
//if (result == null) return new NMElement[] { };
//return result;
//return new NMElement[] { };
}
public bool ParseUserAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.ProcessId < 1)
{
return(false);
}
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName.ToLower() != "chrome" && p.ProcessName.ToLower() != "firefox" && p.ProcessName.ToLower() != "msedge")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" && !NMHook.chromeconnected)
{
System.Windows.MessageBox.Show("You clicked inside Chrome, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (p.ProcessName.ToLower() == "msedge" && !NMHook.edgeconnected)
{
System.Windows.MessageBox.Show("You clicked inside Edge, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (p.ProcessName.ToLower() == "firefox" && !NMHook.ffconnected)
{
System.Windows.MessageBox.Show("You clicked inside Firefix, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (LastElement == null)
{
return(false);
}
if (LastElement.message == null)
{
return(false);
}
if (LastElement.message.tab == null)
{
LastElement.message.tab = NMHook.FindTabById(LastElement.message.browser, LastElement.message.tabid);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "msedge")
{
// if (e.UIElement.FrameworkId != "chrome" && e.UIElement.FrameworkId != "Chrome") return false;
}
var selector = new NMSelector(LastElement, null, true, null);
var a = new GetElement {
DisplayName = LastElement.id + " " + LastElement.type + " " + LastElement.Name
};
a.Selector = selector.ToString();
a.Image = LastElement.ImageString();
a.MaxResults = 1;
e.Element = LastElement;
e.Selector = selector;
e.a = new GetElementResult(a);
e.SupportInput = LastElement.SupportInput;
e.SupportSelect = LastElement.tagname.ToLower() == "select";
e.OffsetX = e.X - LastElement.Rectangle.X;
e.OffsetY = e.Y - LastElement.Rectangle.Y;
e.ClickHandled = false;
//e.ClickHandled = true;
//LastElement.Click(true, e.Button, e.X, e.Y, false, false);
return(true);
}
private void OnMessage(NativeMessagingMessage message)
{
try
{
//lock(_lock)
//{
// //if (dpiscale == 0)
// //{
// // dpiscale = NativeMethods.getScalingFactor();
// //} else
// //{
// // dpiscale = NativeMethods.getScalingFactor();
// //}
// var dpi = NativeMethods.GetDpiForSystem();
// dpiscale = 1;
// if (dpi == 96) dpiscale = 1;
// if (dpi == 120) dpiscale = 1.25f;
// if (dpi == 144) dpiscale = 1.5f;
// if (dpi == 192) dpiscale = 2;
//}
if (message.uiy > 0 && message.uix > 0 && message.uiwidth > 0 && message.uiheight > 0)
{
//message.uiy = (int)(message.uiy * dpiscale);
//message.uix = (int)(message.uix * dpiscale);
//message.uiwidth = (int)(message.uiwidth * dpiscale);
//message.uiheight = (int)(message.uiheight * dpiscale);
if (dpiscale == 1.25)
{
message.uiy += 158;
}
LastElement = new NMElement(message);
}
if (message.functionName == "click")
{
if (IsRecording)
{
if (LastElement == null)
{
return;
}
var re = new RecordEvent
{
Button = Input.MouseButton.Left
}; var a = new GetElement {
DisplayName = LastElement.ToString()
};
message.tab = NMHook.FindTabById(message.browser, message.tabid);
var selector = new NMSelector(LastElement, null, true, null);
a.Selector = selector.ToString();
a.Image = LastElement.ImageString();
a.MaxResults = 1;
re.Selector = selector;
re.a = new GetElementResult(a);
re.SupportInput = LastElement.SupportInput;
re.SupportSelect = false;
re.ClickHandled = true;
OnUserAction?.Invoke(this, re);
return;
}
}
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
protected override void Execute(NativeActivityContext context)
{
var eq = new Activities.NMEqualityComparer();
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
var sw = new Stopwatch();
sw.Start();
string browser = sel.browser;
if (WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.tabs.Where(x => x.browser == browser && x.url.ToLower().StartsWith(s.url.ToLower())).FirstOrDefault();
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
} while (elements.Count() == 0 && sw.Elapsed < timeout);
if (elements.Count() > maxresults)
{
elements = elements.Take(maxresults).ToArray();
}
var lastelements = context.GetValue(_lastelements);
if (lastelements == null)
{
lastelements = new NMElement[] { }
}
;
context.SetValue(_lastelements, elements);
if (lastelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
bool exists = false;
foreach (var ele in lastelements)
{
if (eq.Equals(ele, element))
{
exists = true;
break;
}
else if (element.ToString() == ele.ToString())
{
var b = eq.Equals(ele, element);
}
}
if (!exists)
{
newelements.Insert(0, element);
}
}
if (elements.Length > 0)
{
elements = newelements.ToArray();
newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
bool exists = false;
foreach (var ele in lastelements)
{
if (eq.Equals(ele, elements[i]))
{
exists = true;
break;
}
}
if (!exists)
{
newelements.Insert(0, elements[i]);
}
}
elements = newelements.ToArray();
}
}
if ((elements.Length + lastelements.Length) < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <NMElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
//if (lastelements.Length == elements.Length && lastelements.Length > 0)
//{
// var eq = new Activities.NMEqualityComparer();
// more = !System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(lastelements, elements);
//}
if (more)
{
context.SetValue(_elements, _enum);
context.ScheduleAction(Body, _enum.Current, OnBodyComplete);
}
var allelements = context.GetValue(_allelements);
if (more)
{
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
var allelementslength = allelements.Length;
Array.Resize(ref allelements, allelements.Length + elements.Length);
Array.Copy(elements, 0, allelements, allelementslength, elements.Length);
}
context.SetValue(Elements, allelements);
context.SetValue(_allelements, allelements);
}
protected override async Task <object> ExecuteAsync(NativeActivityContext context)
{
IsCancel = false;
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From?.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
sw.Start();
string browser = sel.browser;
if (WaitForReady != null && WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var allelements = context.GetValue(_allelements);
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.tabs.Where(x => x.browser == browser && x.url.ToLower().StartsWith(s.url.ToLower())).FirstOrDefault();
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
//var id = Guid.NewGuid().ToString();
//context.CreateBookmark(id, (activityContext, bookmark, value) =>
//{
//});
//Task.Factory.StartNew(() =>
//{
// // context.RemoveBookmark(id);
//});
// await Task.Delay(100);
await Task.Run(() =>
{
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
Log.Selector("BEGIN:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
// allelements = allelements.Concat(elements).ToArray();
if (allelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
if (!allelements.Contains(element))
{
newelements.Insert(0, element);
}
}
elements = newelements.ToArray();
//if(elements.Count() > 20)
//{
// for(var i=0; i < allelements.Length && i < elements.Length; i++)
// {
// if (!eq.Equals(allelements[i], elements[i]) || allelements[i].GetHashCode() != elements[i].GetHashCode())
// {
// Log.Output(allelements[i].GetHashCode() + " / " + elements[i].GetHashCode());
// }
// }
//}
}
} while (elements.Count() == 0 && sw.Elapsed < timeout && !IsCancel);
if (IsCancel)
{
Console.WriteLine("was Canceled: true! DisplayName" + DisplayName);
}
});
return(elements);
}
protected override void Execute(NativeActivityContext context)
{
var url = Url.Get(context);
var browser = Browser.Get(context);
var timeout = TimeSpan.FromSeconds(3);
var newtab = NewTab.Get(context);
if (browser != "chrome" && browser != "ff" && browser != "edge")
{
browser = "chrome";
}
if (!string.IsNullOrEmpty(url))
{
NMHook.enumtabs();
var tab = NMHook.FindTabByURL(browser, url);
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
var userDataFolderMode = UserDataFolderMode.Get(context);
var userDataFolderPath = UserDataFolderPath.Get(context);
string profilepath = "";
string profilename = "";
if (string.IsNullOrEmpty(userDataFolderMode))
{
userDataFolderMode = "automatic";
}
userDataFolderMode = userDataFolderMode.ToLower();
if (userDataFolderMode == "automatic") // child session uses a different folder than the default mode, automatically generated if UserDataFolderPath is not set
{
if (Plugin.client.isRunningInChildSession)
{
profilepath = userDataFolderPath;
if (string.IsNullOrEmpty(profilepath))
{
profilepath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\openrpa\\ChildSession\\" + browser;
profilename = "ChildSession";
}
}
}
else if (userDataFolderMode == "customfolder")
{
profilepath = userDataFolderPath;
if (string.IsNullOrEmpty(profilepath))
{
if (Plugin.client.isRunningInChildSession)
{
profilepath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\openrpa\\ChildSession\\" + browser;
profilename = "ChildSession";
}
else
{
profilepath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\openrpa\\" + browser;
profilename = "openrpa";
}
}
}
else if (userDataFolderMode == "defaultfolder")
{
// Uses the default browser folder, no matter what
}
NMHook.openurl(browser, url, newtab, profilename, profilepath);
}
protected override void Execute(NativeActivityContext context)
{
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
var sw = new Stopwatch();
sw.Start();
if (WaitForReady.Get(context))
{
string browser = sel.browser;
if (from != null)
{
browser = from.browser;
}
if (!string.IsNullOrEmpty(browser))
{
NMHook.enumtabs();
if (browser == "chrome" && NMHook.CurrentChromeTab != null)
{
NMHook.WaitForTab(NMHook.CurrentChromeTab.id, browser, TimeSpan.FromSeconds(10));
}
if (browser == "ff" && NMHook.CurrentFFTab != null)
{
NMHook.WaitForTab(NMHook.CurrentFFTab.id, browser, TimeSpan.FromSeconds(10));
}
}
}
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
} while (elements.Count() == 0 && sw.Elapsed < timeout);
if (elements.Count() > maxresults)
{
elements = elements.Take(maxresults).ToArray();
}
context.SetValue(Elements, elements);
var lastelements = context.GetValue(_lastelements);
if (lastelements == null)
{
lastelements = new NMElement[] { }
}
;
context.SetValue(_lastelements, elements);
if ((elements.Length + lastelements.Length) < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <NMElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
if (lastelements.Length == elements.Length && lastelements.Length > 0)
{
var eq = new Activities.NMEqualityComparer();
more = !System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(lastelements, elements);
}
if (more)
{
context.SetValue(_elements, _enum);
context.ScheduleAction(Body, _enum.Current, OnBodyComplete);
}
}
protected override void Execute(NativeActivityContext context)
{
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
NMElement[] elements = { };
var sw = new Stopwatch();
sw.Start();
string browser = sel.browser;
if (WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var allelements = context.GetValue(_allelements);
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.FindTabByURL(browser, s.url);
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
Log.Selector("BEGIN:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
// allelements = allelements.Concat(elements).ToArray();
if (allelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
if (!allelements.Contains(element))
{
newelements.Insert(0, element);
}
}
elements = newelements.ToArray();
//if(elements.Count() > 20)
//{
// for(var i=0; i < allelements.Length && i < elements.Length; i++)
// {
// if (!eq.Equals(allelements[i], elements[i]) || allelements[i].GetHashCode() != elements[i].GetHashCode())
// {
// Log.Output(allelements[i].GetHashCode() + " / " + elements[i].GetHashCode());
// }
// }
//}
}
} while (elements.Count() == 0 && sw.Elapsed < timeout);
if (elements.Count() > maxresults)
{
elements = elements.Take(maxresults).ToArray();
}
if ((elements.Length + allelements.Length) < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <NMElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
//if (lastelements.Length == elements.Length && lastelements.Length > 0)
//{
// var eq = new Activities.NMEqualityComparer();
// more = !System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(lastelements, elements);
//}
if (more)
{
allelements = allelements.Concat(elements).ToArray();
var eq = new Activities.NMEqualityComparer();
allelements = allelements.Distinct(eq).ToArray();
//var allelementslength = allelements.Length;
//Array.Resize(ref allelements, allelements.Length + elements.Length);
//Array.Copy(elements, 0, allelements, allelementslength, elements.Length);
}
context.SetValue(_allelements, allelements);
context.SetValue(Elements, allelements);
Log.Selector("END:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
if (more)
{
context.SetValue(_elements, _enum);
context.ScheduleAction(Body, _enum.Current, OnBodyComplete);
}
}
protected override void StartLoop(NativeActivityContext context)
{
var selector = Selector.Get(context);
selector = OpenRPA.Interfaces.Selector.Selector.ReplaceVariables(selector, context.DataContext);
var sel = new NMSelector(selector);
var timeout = Timeout.Get(context);
var from = From.Get(context);
var maxresults = MaxResults.Get(context);
var minresults = MinResults.Get(context);
if (maxresults < 1)
{
maxresults = 1;
}
if (timeout.Minutes > 5 || timeout.Hours > 1)
{
Activity _Activity = null;
try
{
var strProperty = context.GetType().GetProperty("Activity", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
var strGetter = strProperty.GetGetMethod(nonPublic: true);
_Activity = (Activity)strGetter.Invoke(context, null);
}
catch (Exception)
{
}
if (_Activity != null)
{
Log.Warning("Timeout for Activity " + _Activity.Id + " is above 5 minutes, was this the intention ? calculated value " + timeout.ToString());
}
else
{
Log.Warning("Timeout for on of your NM.GetElements is above 5 minutes, was this the intention ? calculated value " + timeout.ToString());
}
}
NMElement[] elements = { };
var sw = new Stopwatch();
sw.Start();
string browser = sel.browser;
if (WaitForReady.Get(context))
{
if (from != null)
{
browser = from.browser;
}
DoWaitForReady(browser);
}
var allelements = context.GetValue(_allelements);
if (allelements == null)
{
allelements = new NMElement[] { }
}
;
var s = new NMSelectorItem(sel[0]);
if (!string.IsNullOrEmpty(s.url))
{
var tab = NMHook.FindTabByURL(browser, s.url);
if (tab != null)
{
if (!tab.highlighted || !tab.selected)
{
var _tab = NMHook.selecttab(browser, tab.id);
}
}
}
do
{
elements = NMSelector.GetElementsWithuiSelector(sel, from, maxresults);
Log.Selector("BEGIN:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
// allelements = allelements.Concat(elements).ToArray();
if (allelements.Length > 0)
{
var newelements = new List <NMElement>();
for (var i = elements.Length - 1; i >= 0; i--)
{
var element = elements[i];
if (!allelements.Contains(element))
{
newelements.Insert(0, element);
}
}
elements = newelements.ToArray();
//if(elements.Count() > 20)
//{
// for(var i=0; i < allelements.Length && i < elements.Length; i++)
// {
// if (!eq.Equals(allelements[i], elements[i]) || allelements[i].GetHashCode() != elements[i].GetHashCode())
// {
// Log.Output(allelements[i].GetHashCode() + " / " + elements[i].GetHashCode());
// }
// }
//}
}
} while (elements.Count() == 0 && sw.Elapsed < timeout);
if (elements.Count() > maxresults)
{
elements = elements.Take(maxresults).ToArray();
}
if ((elements.Length + allelements.Length) < minresults)
{
Log.Selector(string.Format("Windows.GetElement::Failed locating " + minresults + " item(s) {0:mm\\:ss\\.fff}", sw.Elapsed));
throw new ElementNotFoundException("Failed locating " + minresults + " item(s)");
}
IEnumerator <NMElement> _enum = elements.ToList().GetEnumerator();
bool more = _enum.MoveNext();
//if (lastelements.Length == elements.Length && lastelements.Length > 0)
//{
// var eq = new Activities.NMEqualityComparer();
// more = !System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(lastelements, elements);
//}
if (more)
{
allelements = allelements.Concat(elements).ToArray();
var eq = new Activities.NMEqualityComparer();
allelements = allelements.Distinct(eq).ToArray();
//var allelementslength = allelements.Length;
//Array.Resize(ref allelements, allelements.Length + elements.Length);
//Array.Copy(elements, 0, allelements, allelementslength, elements.Length);
}
context.SetValue(_allelements, allelements);
context.SetValue(Elements, allelements);
Log.Selector("END:: I have " + elements.Count() + " elements, and " + allelements.Count() + " in all elements");
if (more)
{
context.SetValue(_elements, _enum);
IncIndex(context);
SetTotal(context, allelements.Length);
context.ScheduleAction(Body, _enum.Current, OnBodyComplete);
}
}
public static treeelement[] _GetRootElements(Selector anchor)
{
var rootelements = new List <treeelement>();
NMHook.reloadtabs();
// var tab = NMHook.tabs.Where(x => x.highlighted == true && x.browser == "chrome").FirstOrDefault();
var tab = NMHook.tabs.Where(x => x.highlighted == true).FirstOrDefault();
if (tab == null)
{
// tab = NMHook.tabs.Where(x => x.browser == "chrome").FirstOrDefault();
tab = NMHook.tabs.FirstOrDefault();
}
if (NMHook.tabs.Count == 0)
{
return(rootelements.ToArray());
}
// getelement.data = "getdom";
var getelement = new NativeMessagingMessage("getelement");
getelement.browser = tab.browser;
getelement.tabid = tab.id;
getelement.xPath = "/html";
if (anchor != null && anchor.Count > 1)
{
var s = anchor[1];
var p = s.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
if (p != null)
{
getelement.xPath = p.Value;
}
}
NativeMessagingMessage result = null;
try
{
result = NMHook.sendMessageResult(getelement, true, TimeSpan.FromSeconds(2));
}
catch (Exception)
{
}
if (result != null && result.result != null && result.results == null)
{
result.results = new NativeMessagingMessage[] { result };
}
if (result != null && result.results != null && result.results.Count() > 0)
{
foreach (var res in result.results)
{
if (res.result != null)
{
//var html = new HtmlElement(getelement.xPath, getelement.cssPath, res.tabid, res.frameId, res.result);
var html = new NMElement(res);
rootelements.Add(new NMTreeElement(null, true, html));
}
}
//result = result.results[0];
}
return(rootelements.ToArray());
}
public static NMElement[] GetElementsWithuiSelector(NMSelector selector, IElement fromElement = null, int maxresults = 1)
{
var results = new List <NMElement>();
SelectorItem first = null;
SelectorItem second = null;
string browser = "";
SelectorItemProperty p = null;
if (selector.Count > 1)
{
first = selector[0];
second = selector[1];
p = first.Properties.Where(x => x.Name == "browser").FirstOrDefault();
if (p != null)
{
browser = p.Value;
}
}
else if (fromElement == null)
{
throw new ArgumentException("Invalid select with only 1 child and no anchor");
}
else
{
second = selector[0];
}
p = second.Properties.Where(x => x.Name == "xpath").FirstOrDefault();
string xpath = "";
if (p != null)
{
xpath = p.Value;
}
p = second.Properties.Where(x => x.Name == "cssselector").FirstOrDefault();
string cssselector = "";
if (p != null)
{
cssselector = p.Value;
}
NMElement fromNMElement = fromElement as NMElement;
string fromcssPath = "";
string fromxPath = "";
if (fromElement != null)
{
fromcssPath = fromNMElement.cssselector;
fromxPath = fromNMElement.xpath;
}
NativeMessagingMessage subresult = null;
var getelement = new NativeMessagingMessage("getelements", PluginConfig.debug_console_output, PluginConfig.unique_xpath_ids);
getelement.browser = browser;
getelement.xPath = xpath;
getelement.cssPath = cssselector;
getelement.fromxPath = fromxPath;
getelement.fromcssPath = fromcssPath;
if (fromElement != null && fromElement is NMElement)
{
getelement.windowId = ((NMElement)fromElement).message.windowId;
getelement.tabid = ((NMElement)fromElement).message.tabid;
getelement.frameId = ((NMElement)fromElement).message.frameId;
}
subresult = NMHook.sendMessageResult(getelement, PluginConfig.protocol_timeout);
if (subresult != null)
{
if (subresult.results != null)
{
foreach (var b in subresult.results)
{
if (b.cssPath == "true" || b.xPath == "true")
{
if (results.Count > maxresults)
{
continue;
}
var nme = new NMElement(b);
results.Add(nme);
}
}
}
}
return(results.ToArray());
}
