public void CloseBySelector(Selector selector, TimeSpan timeout, bool Force)
{
if (timeout == TimeSpan.Zero)
{
OnUserAction?.Invoke(null, null); // dummy use of OnUserAction to get rid of warning
}
throw new NotImplementedException();
}
private void onMessage(NativeMessagingMessage message)
{
if (message.uiy > 0 && message.uix > 0 && message.uiwidth > 0 && message.uiheight > 0)
{
// Console.WriteLine("{0} {1},{2} {3},{4} - {5} ", message.functionName, message.uix, message.uiy, message.uiwidth, message.uiheight, message.xPath);
if (!string.IsNullOrEmpty(message.data))
{
lastElement = new NMElement(message);
}
else
{
lastElement = new NMElement(message);
}
}
if (message.functionName == "click")
{
if (recording)
{
if (lastElement == null)
{
return;
}
var re = new RecordEvent(); re.Button = Input.MouseButton.Left;
var a = new GetElement {
DisplayName = lastElement.ToString()
};
var selector = new NMSelector(lastElement, null, true);
a.Selector = selector.ToString();
a.Image = lastElement.ImageString();
a.MaxResults = 1;
re.Selector = selector;
re.a = new GetElementResult(a);
re.SupportInput = lastElement.SupportInput;
re.ClickHandled = true;
OnUserAction?.Invoke(this, re);
return;
}
//var getelement = new NativeMessagingMessage("getelement");
////getelement.tabid = message.tabid;
//getelement.cssPath = message.cssPath;
//getelement.xPath = message.xPath;
//NativeMessagingMessage subresult = null;
////if (message.browser == "chrome") subresult = rpaactivities.nm.nmhook.sendMessageChromeResult(getelement, true);
////if (message.browser == "ff") subresult = rpaactivities.nm.nmhook.sendMessageFFResult(getelement, true);
//if (subresult == null)
//{
// // Console.WriteLine("getelement returned null???");
// return;
//}
//// Console.WriteLine(getelement.messageid + " " + getelement.functionName + " " + subresult.messageid + " " + subresult.functionName);
//// Console.WriteLine(subresult.cssPath + " " + subresult.xPath);
////rpaactivities.nm.nmhook.sendMessageChromeResult(getelement);
////Console.WriteLine(getelement.messageid + " " + getelement.functionName);
}
}
private void OnMouseUp(InputEventArgs e)
{
isMouseDown = false;
var thread = new Thread(new ThreadStart(() =>
{
Log.Debug(string.Format("Windows.Recording::OnMouseUp::begin"));
var re = new RecordEvent
{
Button = e.Button
}; var a = new GetElement {
DisplayName = e.Element.Name
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
WindowsSelector sel = null;
// sel = new WindowsSelector(e.Element.rawElement, null, true);
sel = new WindowsSelector(e.Element.RawElement, null, PluginConfig.enum_selector_properties);
if (sel.Count < 2)
{
return;
}
if (sel == null)
{
return;
}
a.Selector = sel.ToString();
a.MaxResults = 1;
a.Image = e.Element.ImageString();
re.OffsetX = e.X - e.Element.Rectangle.X;
re.OffsetY = e.Y - e.Element.Rectangle.Y;
re.UIElement = e.Element;
re.Element = e.Element;
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
if (sel.Count > 3)
{
var p1 = sel[1].Properties.Where(x => x.Name == "ClassName").FirstOrDefault();
var p2 = sel[2].Properties.Where(x => x.Name == "AutomationId").FirstOrDefault();
if (p1 != null && p2 != null)
{
if (p1.Value.StartsWith("Windows.UI") && p2.Value == "SplitViewFrameXAMLWindow")
{
re.SupportVirtualClick = false;
}
}
}
re.a = new GetElementResult(a);
re.SupportInput = e.Element.SupportInput;
re.SupportSelect = e.Element.SupportSelect;
Log.Debug(string.Format("Windows.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}));
thread.IsBackground = true;
thread.Start();
}
private void OnMouseUp(InputEventArgs e)
{
var thread = new Thread(new ThreadStart(() =>
{
Log.Debug(string.Format("IE.Recording::OnMouseUp::begin"));
var re = new RecordEvent(); re.Button = e.Button;
var a = new GetElement {
DisplayName = (e.Element.Name).Replace(Environment.NewLine, "").Trim()
};
var browser = new Browser(e.Element.RawElement);
var htmlelement = browser.ElementFromPoint(e.X, e.Y);
if (htmlelement == null)
{
return;
}
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
IESelector sel = null;
// sel = new IESelector(e.Element.rawElement, null, true);
GenericTools.RunUI(() =>
{
sel = new IESelector(browser, htmlelement, null, false, e.X, e.Y);
});
if (sel == null)
{
return;
}
if (sel.Count < 2)
{
return;
}
a.Selector = sel.ToString();
a.Image = sel.Last().Element.ImageString();
re.UIElement = e.Element;
re.Element = new IEElement(browser, htmlelement);
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
Log.Debug(e.Element.SupportInput + " / " + e.Element.ControlType);
re.a = new GetElementResult(a);
if (htmlelement.tagName.ToLower() == "input" && htmlelement.tagName.ToLower() == "select")
{
MSHTML.IHTMLInputElement inputelement = (MSHTML.IHTMLInputElement)htmlelement;
re.SupportInput = (inputelement.type.ToLower() == "text" || inputelement.type.ToLower() == "password");
}
Log.Debug(string.Format("IE.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}));
thread.IsBackground = true;
thread.Start();
}
private void OnMessage(NativeMessagingMessage message)
{
try
{
if (message.uiy > 0 && message.uix > 0 && message.uiwidth > 0 && message.uiheight > 0)
{
if (!string.IsNullOrEmpty(message.data))
{
LastElement = new NMElement(message);
}
else
{
LastElement = new NMElement(message);
}
}
if (message.functionName == "click")
{
if (IsRecording)
{
if (LastElement == null)
{
return;
}
var re = new RecordEvent
{
Button = Input.MouseButton.Left
}; var a = new GetElement {
DisplayName = LastElement.ToString()
};
var selector = new NMSelector(LastElement, null, true, null);
a.Selector = selector.ToString();
a.Image = LastElement.ImageString();
a.MaxResults = 1;
re.Selector = selector;
re.a = new GetElementResult(a);
re.SupportInput = LastElement.SupportInput;
re.SupportSelect = false;
re.ClickHandled = true;
OnUserAction?.Invoke(this, re);
return;
}
}
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
private void Hook_OnMouseClicked(int vmID, WindowsAccessBridgeInterop.AccessibleContextNode ac)
{
lastElement = new JavaElement(ac);
lastElement.SetPath();
Log.Debug("OnMouseClicked: " + lastElement.id + " " + lastElement.role + " " + lastElement.Name);
if (lastElement == null)
{
return;
}
var re = new RecordEvent(); re.Button = MouseButton.Left;
var a = new GetElement {
DisplayName = lastElement.title
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
JavaSelector sel = null;
// sel = new JavaSelector(e.Element.rawElement, null, true);
sel = new JavaSelector(lastElement, null, true);
if (sel == null)
{
return;
}
if (sel.Count < 2)
{
return;
}
a.Selector = sel.ToString();
a.Image = lastElement.ImageString();
a.MaxResults = 1;
re.Element = lastElement;
re.Selector = sel;
re.X = lastElement.X;
re.Y = lastElement.Y;
re.a = new GetElementResult(a);
re.SupportInput = lastElement.SupportInput;
Log.Debug(string.Format("Java.Recording::OnMouseClicked::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}
private void onMessage(NativeMessagingMessage message)
{
if (message.uiy > 0 && message.uix > 0 && message.uiwidth > 0 && message.uiheight > 0)
{
// Console.WriteLine("{0} {1},{2} {3},{4} - {5} ", message.functionName, message.uix, message.uiy, message.uiwidth, message.uiheight, message.xPath);
if (!string.IsNullOrEmpty(message.data))
{
lastElement = new NMElement(message);
}
else
{
lastElement = new NMElement(message);
}
}
if (message.functionName == "click")
{
if (recording)
{
if (lastElement == null)
{
return;
}
var re = new RecordEvent(); re.Button = Input.MouseButton.Left;
var a = new GetElement {
DisplayName = lastElement.ToString()
};
var selector = new NMSelector(lastElement, null, true);
a.Selector = selector.ToString();
a.Image = lastElement.ImageString();
a.MaxResults = 1;
re.Selector = selector;
re.a = new GetElementResult(a);
re.SupportInput = lastElement.SupportInput;
re.ClickHandled = true;
OnUserAction?.Invoke(this, re);
return;
}
}
}
private void OnMouseUp(InputEventArgs e)
{
var thread = new Thread(new ThreadStart(() =>
{
Log.Debug(string.Format("Windows.Recording::OnMouseUp::begin"));
var re = new RecordEvent(); re.Button = e.Button;
var a = new GetElement {
DisplayName = e.Element.Id + "-" + e.Element.Name
};
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
WindowsSelector sel = null;
// sel = new WindowsSelector(e.Element.rawElement, null, true);
sel = new WindowsSelector(e.Element.RawElement, null, false);
if (sel.Count < 2)
{
return;
}
if (sel == null)
{
return;
}
a.Selector = sel.ToString();
a.MaxResults = 1;
a.Image = e.Element.ImageString();
re.UIElement = e.Element;
re.Element = e.Element;
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
re.a = new GetElementResult(a);
re.SupportInput = e.Element.SupportInput;
Log.Debug(string.Format("Windows.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}));
thread.IsBackground = true;
thread.Start();
}
private void OnMessage(NativeMessagingMessage message)
{
try
{
//lock(_lock)
//{
// //if (dpiscale == 0)
// //{
// // dpiscale = NativeMethods.getScalingFactor();
// //} else
// //{
// // dpiscale = NativeMethods.getScalingFactor();
// //}
// var dpi = NativeMethods.GetDpiForSystem();
// dpiscale = 1;
// if (dpi == 96) dpiscale = 1;
// if (dpi == 120) dpiscale = 1.25f;
// if (dpi == 144) dpiscale = 1.5f;
// if (dpi == 192) dpiscale = 2;
//}
if (message.uiy > 0 && message.uix > 0 && message.uiwidth > 0 && message.uiheight > 0)
{
//message.uiy = (int)(message.uiy * dpiscale);
//message.uix = (int)(message.uix * dpiscale);
//message.uiwidth = (int)(message.uiwidth * dpiscale);
//message.uiheight = (int)(message.uiheight * dpiscale);
if (dpiscale == 1.25)
{
message.uiy += 158;
}
LastElement = new NMElement(message);
}
if (message.functionName == "click")
{
if (IsRecording)
{
if (LastElement == null)
{
return;
}
var re = new RecordEvent
{
Button = Input.MouseButton.Left
}; var a = new GetElement {
DisplayName = LastElement.ToString()
};
message.tab = NMHook.tabs.Where(x => x.id == message.tabid && x.windowId == message.windowId).FirstOrDefault();
var selector = new NMSelector(LastElement, null, true, null);
a.Selector = selector.ToString();
a.Image = LastElement.ImageString();
a.MaxResults = 1;
re.Selector = selector;
re.a = new GetElementResult(a);
re.SupportInput = LastElement.SupportInput;
re.SupportSelect = false;
re.ClickHandled = true;
OnUserAction?.Invoke(this, re);
return;
}
}
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
private void ParseMouseUp(InputEventArgs e)
{
try
{
// if(string.IsNullOrEmpty(Thread.CurrentThread.Name)) Thread.CurrentThread.Name = "IE.Plugin.OnMouseUP";
Log.Debug(string.Format("IE.Recording::OnMouseUp::begin"));
var re = new RecordEvent
{
Button = e.Button
}; var a = new GetElement {
DisplayName = (e.Element.Name).Replace(Environment.NewLine, "").Trim()
};
a.Variables.Add(new Variable <int>("Index", 0));
a.Variables.Add(new Variable <int>("Total", 0));
using (var p = System.Diagnostics.Process.GetProcessById(e.Element.ProcessId))
{
if (p.ProcessName != "iexplore" && p.ProcessName != "iexplore.exe")
{
return;
}
}
var browser = new Browser();
var htmlelement = browser.ElementFromPoint(e.X, e.Y);
if (htmlelement == null)
{
return;
}
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
IESelector sel = null;
// sel = new IESelector(e.Element.rawElement, null, true);
GenericTools.RunUI(() =>
{
try
{
sel = new IESelector(browser, htmlelement, null, false, e.X, e.Y);
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
});
if (sel == null)
{
return;
}
if (sel.Count < 2)
{
return;
}
a.Selector = sel.ToString();
a.Image = sel.Last().Element.ImageString();
re.UIElement = e.Element;
re.Element = new IEElement(browser, htmlelement);
re.Selector = sel;
re.X = e.X;
re.Y = e.Y;
Log.Debug(e.Element.SupportInput + " / " + e.Element.ControlType);
re.a = new GetElementResult(a);
//if (htmlelement.tagName.ToLower() == "input" && htmlelement.tagName.ToLower() == "select")
if (htmlelement.tagName.ToLower() == "input")
{
MSHTML.IHTMLInputElement inputelement = (MSHTML.IHTMLInputElement)htmlelement;
re.SupportInput = (inputelement.type.ToLower() == "text" || inputelement.type.ToLower() == "password");
}
re.SupportSelect = false;
Log.Debug(string.Format("IE.Recording::OnMouseUp::end {0:mm\\:ss\\.fff}", sw.Elapsed));
OnUserAction?.Invoke(this, re);
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
private void WebSocketServer_OnUserConnect(object sender, OnUserAction e)
{
Console.WriteLine($"{e.Sender.Socket.ConnectedSocket.RemoteEndPoint} Connected as a new user, giving user 5 sec for authentication");
Task.Run(() => WaitForAuthentication(e.Sender));
}
private void WebSocketServer_OnUserDisconnect(object sender, OnUserAction e)
{
}
