public bool ParseMouseMoveAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (!e.Element.Equals(e.UIElement))
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
if (e.UIElement.ProcessId > 0)
{
using (var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId))
{
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
if (p.ProcessName.ToLower() == "saplogon")
{
return(false);
}
}
}
e.Element = lastelement;
if (System.Threading.Monitor.TryEnter(_lock, Config.local.thread_lock_timeout_seconds * 1000))
{
try
{
if (_processing)
{
return(true);
}
_processing = true;
}
finally
{
System.Threading.Monitor.Exit(_lock);
}
}
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out int newOffsetX, out int newOffsetY, out System.Drawing.Rectangle resultrect);
if (System.Threading.Monitor.TryEnter(_lock, Config.local.thread_lock_timeout_seconds * 1000))
{
try
{
_processing = false;
}
finally
{
System.Threading.Monitor.Exit(_lock);
}
}
if (image == null)
{
lastelement = null;
// Log.Debug("Found null");
e.Element = null;
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
// Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height));
lastelement = new ImageElement(resultrect, image);
e.Element = lastelement;
return(true);
}
public bool ParseUserAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (!e.Element.Equals(e.UIElement))
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
string Processname = "";
if (e.UIElement.ProcessId > 0)
{
using (var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId))
{
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
if (p.ProcessName.ToLower() == "saplogon")
{
return(false);
}
Processname = p.ProcessName;
}
}
NativeMethods.SetCursorPos(e.X - 80, e.Y - 80);
System.Threading.Thread.Sleep(PluginConfig.recording_mouse_move_time);
var a = new GetElement
{
Threshold = 0.9,
Processname = Processname
};
e.SupportInput = false;
e.SupportSelect = false;
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out int newOffsetX, out int newOffsetY, out System.Drawing.Rectangle resultrect);
if (image == null)
{
var tip = new Interfaces.Overlay.TooltipWindow("Failed Guessing Contour, please select manually");
tip.SetTimeout(TimeSpan.FromSeconds(2));
e.OffsetX = 10;
e.OffsetY = 10;
e.a = new GetElementResult(a);
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
e.Element = new ImageElement(resultrect, image);
a.Image = Interfaces.Image.Util.Bitmap2Base64(image);
e.a = new GetElementResult(a);
NativeMethods.SetCursorPos(e.X, e.Y);
return(true);
}
public bool ParseUserAction(ref IRecordEvent e)
{
return(false);
}
public bool ParseUserAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.ProcessId < 1)
{
return(false);
}
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName.ToLower() != "chrome" && p.ProcessName.ToLower() != "firefox" && p.ProcessName.ToLower() != "msedge")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" && !NMHook.chromeconnected)
{
System.Windows.MessageBox.Show("You clicked inside Chrome, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (p.ProcessName.ToLower() == "msedge" && !NMHook.edgeconnected)
{
System.Windows.MessageBox.Show("You clicked inside Edge, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (p.ProcessName.ToLower() == "firefox" && !NMHook.ffconnected)
{
System.Windows.MessageBox.Show("You clicked inside Firefix, but it looks like you dont have the OpenRPA plugin installed");
return(false);
}
if (LastElement == null)
{
return(false);
}
if (LastElement.message == null)
{
return(false);
}
if (LastElement.message.tab == null)
{
LastElement.message.tab = NMHook.FindTabById(LastElement.message.browser, LastElement.message.tabid);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "msedge")
{
// if (e.UIElement.FrameworkId != "chrome" && e.UIElement.FrameworkId != "Chrome") return false;
}
var selector = new NMSelector(LastElement, null, true, null);
var a = new GetElement {
DisplayName = LastElement.id + " " + LastElement.type + " " + LastElement.Name
};
a.Selector = selector.ToString();
a.Image = LastElement.ImageString();
a.MaxResults = 1;
e.Element = LastElement;
e.Selector = selector;
e.a = new GetElementResult(a);
e.SupportInput = LastElement.SupportInput;
e.SupportSelect = LastElement.tagname.ToLower() == "select";
e.OffsetX = e.X - LastElement.Rectangle.X;
e.OffsetY = e.Y - LastElement.Rectangle.Y;
e.ClickHandled = false;
//e.ClickHandled = true;
//LastElement.Click(true, e.Button, e.X, e.Y, false, false);
return(true);
}
public bool ParseUserAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.ProcessId < 1)
{
return(false);
}
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName != "iexplore" && p.ProcessName != "iexplore.exe")
{
return(false);
}
var browser = new Browser();
var htmlelement = browser.ElementFromPoint(e.X, e.Y);
if (htmlelement == null)
{
return(false);
}
var selector = new IESelector(browser, htmlelement, null, true, e.X, e.Y);
e.Selector = selector;
e.Element = new IEElement(browser, htmlelement);
var a = new GetElement {
DisplayName = (htmlelement.id + "-" + htmlelement.tagName + "-" + htmlelement.className).Replace(Environment.NewLine, "").Trim()
};
a.Selector = selector.ToString();
var last = selector.Last() as IESelectorItem;
a.Image = last.Element.ImageString();
var tagName = htmlelement.tagName;
if (string.IsNullOrEmpty(tagName))
{
tagName = "";
}
tagName = tagName.ToLower();
e.a = new GetElementResult(a);
//if (tagName == "input")
//{
// // MSHTML.IHTMLInputElement inputelement = (MSHTML.IHTMLInputElement)htmlelement;
// e.SupportInput = (last.type.ToLower() == "text" || last.type.ToLower() == "password");
//}
// if (htmlelement.tagName.ToLower() == "input" && htmlelement.tagName.ToLower() == "select")
if (htmlelement.tagName.ToLower() == "input")
{
MSHTML.IHTMLInputElement inputelement = (MSHTML.IHTMLInputElement)htmlelement;
e.SupportInput = (inputelement.type.ToLower() == "text" || inputelement.type.ToLower() == "password");
}
e.SupportSelect = htmlelement.tagName.ToLower() == "select";
return(true);
}
public bool ParseMouseMoveAction(ref IRecordEvent e)
{
return(false);
}
public bool parseMouseMoveAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
if (e.UIElement.ProcessId > 0)
{
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
}
e.Element = lastelement;
lock (_lock)
{
if (_processing)
{
return(true);
}
_processing = true;
}
//var elementx = (int)element.BoundingRectangle.X;
//var elementy = (int)element.BoundingRectangle.Y;
//var elementw = (int)element.BoundingRectangle.Width;
//var elementh = (int)element.BoundingRectangle.Height;
// Log.Debug(string.Format("Search near {0}, {1} in ({2}, {3},{4},{5})", elementx, elementy, elementw, elementh, e.OffsetX, e.OffsetY));
int newOffsetX; int newOffsetY; System.Drawing.Rectangle resultrect;
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out newOffsetX, out newOffsetY, out resultrect);
lock (_lock)
{
_processing = false;
}
if (image == null)
{
lastelement = null;
// Log.Debug("Found null");
e.Element = null;
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
// Log.Debug(string.Format("Found element at ({0}, {1},{2},{3})", resultrect.X, resultrect.Y, resultrect.Width, resultrect.Height));
lastelement = new ImageElement(resultrect, image);
e.Element = lastelement;
return(true);
}
public bool parseUserAction(ref IRecordEvent e)
{
if (e.UIElement == null)
{
return(false);
}
if (e.UIElement.Type != "Pane")
{
return(false);
}
var element = e.UIElement.RawElement;
string Processname = "";
if (e.UIElement.ProcessId > 0)
{
var p = System.Diagnostics.Process.GetProcessById(e.UIElement.ProcessId);
if (p.ProcessName == "iexplore" || p.ProcessName == "iexplore.exe")
{
return(false);
}
if (p.ProcessName.ToLower() == "chrome" || p.ProcessName.ToLower() == "firefox")
{
return(false);
}
Processname = p.ProcessName;
}
FlaUI.Core.Shapes.Point point = new FlaUI.Core.Shapes.Point(e.X - 80, e.Y - 80);
if (point.X < 0)
{
point.X = e.X + 80;
}
if (point.Y < 0)
{
point.Y = e.Y + 80;
}
FlaUI.Core.Input.Mouse.MoveTo(point);
var a = new GetElement();
a.Threshold = 0.9;
a.Processname = Processname;
e.SupportInput = false;
e.SupportSelect = false;
int newOffsetX; int newOffsetY; System.Drawing.Rectangle resultrect;
var image = getrectangle.GuessContour(element, e.OffsetX, e.OffsetY, out newOffsetX, out newOffsetY, out resultrect);
if (image == null)
{
var tip = new Interfaces.Overlay.TooltipWindow("Failed Guessing Contour, please select manually");
tip.SetTimeout(TimeSpan.FromSeconds(2));
e.OffsetX = 10;
e.OffsetY = 10;
e.a = new GetElementResult(a);
return(true);
}
e.OffsetX = newOffsetX;
e.OffsetY = newOffsetY;
e.Element = new ImageElement(resultrect, image);
a.Image = Interfaces.Image.Util.Bitmap2Base64(image);
e.a = new GetElementResult(a);
point.X = e.X; point.Y = e.Y;
FlaUI.Core.Input.Mouse.MoveTo(point);
return(true);
}
public bool parseMouseMoveAction(ref IRecordEvent e)
{
return(true);
}
public static IValidatingAggregateCommand <TEvent, TError> BuildCommand <TEvent, TError>(this IRecordEvent <TEvent> recorder) => new AggregateCommandRoot <TEvent, TError>(recorder);
public void OnUserAction(IPlugin sender, IRecordEvent e)
{
StopRecordPlugins();
AutomationHelper.syncContext.Post(o =>
{
foreach (var p in Plugins.recordPlugins)
{
if (p.Name != sender.Name)
{
if (p.parseUserAction(ref e))
{
continue;
}
}
}
if (e.a == null)
{
StartRecordPlugins();
if (e.ClickHandled == false)
{
InputDriver.Instance.CallNext = true;
Log.Debug("MouseMove to " + e.X + "," + e.Y + " and click " + e.Button + " button");
InputDriver.Instance.MouseMove(e.X, e.Y);
// InputDriver.Instance.Click(lastInputEventArgs.Button);
InputDriver.DoMouseClick();
Log.Debug("Click done");
}
return;
}
InputDriver.Instance.CallNext = true;
if (mainTabControl.SelectedContent is Views.WFDesigner view)
{
e.a.addActivity(new Activities.ClickElement
{
Element = new System.Activities.InArgument <IElement>()
{
Expression = new Microsoft.VisualBasic.Activities.VisualBasicValue <IElement>("item")
}
}, "item");
if (e.SupportInput)
{
var win = new Views.InsertText();
win.Topmost = true;
isRecording = false;
if (win.ShowDialog() == true)
{
e.a.addActivity(new System.Activities.Statements.Assign <string>
{
// TODO: use assign
//To = new VisualBasicReference<string>("item.Text")
To = new Microsoft.VisualBasic.Activities.VisualBasicReference <string>("item.value"),
Value = win.Text
}, "item");
e.Element.Value = win.Text;
}
else
{
e.SupportInput = false;
}
isRecording = true;
}
view.lastinserted = e.a.Activity;
view.lastinsertedmodel = view.addActivity(e.a.Activity);
if (e.ClickHandled == false && e.SupportInput == false)
{
InputDriver.Instance.CallNext = true;
Log.Debug("MouseMove to " + e.X + "," + e.Y + " and click " + e.Button + " button");
InputDriver.Instance.MouseMove(e.X, e.Y);
// InputDriver.Instance.Click(lastInputEventArgs.Button);
InputDriver.DoMouseClick();
Log.Debug("Click done");
}
System.Threading.Thread.Sleep(200);
}
InputDriver.Instance.CallNext = false;
StartRecordPlugins();
}, null);
}
