private void IfConditionHelper_CellContentClick(object sender, DataGridViewCellEventArgs e) { var senderGrid = (DataGridView)sender; if (senderGrid.Columns[e.ColumnIndex] is DataGridViewButtonColumn && e.RowIndex >= 0) { var buttonSelected = senderGrid.Rows[e.RowIndex].Cells[e.ColumnIndex] as DataGridViewButtonCell; var selectedRow = v_IfConditionsTable.Rows[e.RowIndex]; if (buttonSelected.Value.ToString() == "Edit") { //launch editor var statement = selectedRow["Statement"]; var commandData = selectedRow["CommandData"].ToString(); var ifCommand = JsonConvert.DeserializeObject <BeginIfCommand>(commandData); var automationCommands = UIControlsHelper.GenerateCommandsandControls().Where(f => f.Command is BeginIfCommand).ToList(); frmCommandEditor editor = new frmCommandEditor(automationCommands, null); editor.SelectedCommand = ifCommand; editor.EditingCommand = ifCommand; editor.OriginalCommand = ifCommand; editor.CreationModeInstance = CreationMode.Edit; editor.ScriptVariables = _scriptVariables; editor.ScriptElements = _scriptElements; if (editor.ShowDialog() == DialogResult.OK) { var editedCommand = editor.EditingCommand as BeginIfCommand; var displayText = editedCommand.GetDisplayValue(); var serializedData = JsonConvert.SerializeObject(editedCommand); selectedRow["Statement"] = displayText; selectedRow["CommandData"] = serializedData; } } else if (buttonSelected.Value.ToString() == "Delete") { //delete v_IfConditionsTable.Rows.Remove(selectedRow); } else { throw new NotImplementedException("Requested Action is not implemented."); } } }
private void CreateLoopCondition(object sender, EventArgs e) { var automationCommands = UIControlsHelper.GenerateCommandsandControls().Where(f => f.Command is BeginLoopCommand).ToList(); frmCommandEditor editor = new frmCommandEditor(automationCommands, null); editor.SelectedCommand = new BeginLoopCommand(); var res = editor.ShowDialog(); if (res == DialogResult.OK) { //get data var configuredCommand = editor.SelectedCommand as BeginLoopCommand; var displayText = configuredCommand.GetDisplayValue(); var serializedData = JsonConvert.SerializeObject(configuredCommand); //add to list v_LoopConditionsTable.Rows.Add(displayText, serializedData); } }
public override void RunCommand(object sender) { var engine = (AutomationEngineInstance)sender; bool testMode = TestMode; //user image to bitmap Bitmap userImage = new Bitmap(Common.Base64ToImage(v_ImageCapture)); double accuracy; try { accuracy = double.Parse(v_MatchAccuracy.ConvertUserVariableToString(engine)); if (accuracy > 1 || accuracy < 0) { throw new ArgumentOutOfRangeException("Accuracy value is out of range (0-1)"); } } catch (Exception) { throw new InvalidDataException("Accuracy value is invalid"); } if (testMode) { FindImageElement(userImage, accuracy); return; } dynamic element = null; if (v_ImageAction == "Wait For Image To Exist") { var timeoutText = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Timeout (Seconds)" select rw.Field <string>("Parameter Value")).FirstOrDefault(); timeoutText = timeoutText.ConvertUserVariableToString(engine); int timeOut = Convert.ToInt32(timeoutText); var timeToEnd = DateTime.Now.AddSeconds(timeOut); while (timeToEnd >= DateTime.Now) { try { element = FindImageElement(userImage, accuracy); if (element == null) { throw new Exception("Image Element Not Found"); } else { break; } } catch (Exception) { engine.ReportProgress("Element Not Yet Found... " + (timeToEnd - DateTime.Now).Seconds + "s remain"); Thread.Sleep(1000); } } if (element == null) { throw new Exception("Image Element Not Found"); } return; } else { element = FindImageElement(userImage, accuracy); } try { string clickPosition; int xAdjust; int yAdjust; switch (v_ImageAction) { case "Click Image": string clickType = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Click Type" select rw.Field <string>("Parameter Value")).FirstOrDefault(); clickPosition = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Click Position" select rw.Field <string>("Parameter Value")).FirstOrDefault(); xAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "X Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); yAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Y Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); Point clickPositionPoint = GetClickPosition(clickPosition, element); //move mouse to position var mouseX = (clickPositionPoint.X + xAdjust).ToString(); var mouseY = (clickPositionPoint.Y + yAdjust).ToString(); User32Functions.SendMouseMove(mouseX, mouseY, clickType); break; case "Set Text": string textToSet = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Text To Set" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine); clickPosition = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Click Position" select rw.Field <string>("Parameter Value")).FirstOrDefault(); xAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "X Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); yAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Y Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); string encryptedData = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Encrypted Text" select rw.Field <string>("Parameter Value")).FirstOrDefault(); if (encryptedData == "Encrypted") { textToSet = EncryptionServices.DecryptString(textToSet, "OPENBOTS"); } Point setTextPositionPoint = GetClickPosition(clickPosition, element); //move mouse to position and set text var xPos = (setTextPositionPoint.X + xAdjust).ToString(); var yPos = (setTextPositionPoint.Y + yAdjust).ToString(); User32Functions.SendMouseMove(xPos, yPos, "Left Click"); var simulator = new InputSimulator(); simulator.Keyboard.TextEntry(textToSet); Thread.Sleep(100); break; case "Set Secure Text": var secureString = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Secure String Variable" select rw.Field <string>("Parameter Value")).FirstOrDefault(); clickPosition = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Click Position" select rw.Field <string>("Parameter Value")).FirstOrDefault(); xAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "X Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); yAdjust = Convert.ToInt32((from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Y Adjustment" select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertUserVariableToString(engine)); var secureStrVariable = secureString.ConvertUserVariableToObject(engine); if (secureStrVariable is SecureString) { secureString = ((SecureString)secureStrVariable).ConvertSecureStringToString(); } else { throw new ArgumentException("Provided Argument is not a 'Secure String'"); } Point setSecureTextPositionPoint = GetClickPosition(clickPosition, element); //move mouse to position and set text var xPosition = (setSecureTextPositionPoint.X + xAdjust).ToString(); var yPosition = (setSecureTextPositionPoint.Y + yAdjust).ToString(); User32Functions.SendMouseMove(xPosition, yPosition, "Left Click"); var simulator2 = new InputSimulator(); simulator2.Keyboard.TextEntry(secureString); Thread.Sleep(100); break; case "Check If Image Exists": var outputVariable = (from rw in v_ImageActionParameterTable.AsEnumerable() where rw.Field <string>("Parameter Name") == "Output Bool Variable Name" select rw.Field <string>("Parameter Value")).FirstOrDefault(); //remove brackets from variable outputVariable = outputVariable.Replace("{", "").Replace("}", ""); if (element != null) { "True".StoreInUserVariable(engine, outputVariable); } else { "False".StoreInUserVariable(engine, outputVariable); } break; default: break; } UIControlsHelper.ShowAllForms(); } catch (Exception ex) { UIControlsHelper.ShowAllForms(); if (element == null) { throw new Exception("Specified image was not found in window!"); } else { throw ex; } } }
public ImageElement FindImageElement(Bitmap smallBmp, double accuracy) { UIControlsHelper.HideAllForms(); bool testMode = TestMode; dynamic element = null; double tolerance = 1.0 - accuracy; Bitmap bigBmp = ImageMethods.Screenshot(); Bitmap smallTestBmp = new Bitmap(smallBmp); Bitmap bigTestBmp = new Bitmap(bigBmp); Graphics bigTestGraphics = Graphics.FromImage(bigTestBmp); BitmapData smallData = smallBmp.LockBits(new Rectangle(0, 0, smallBmp.Width, smallBmp.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb); BitmapData bigData = bigBmp.LockBits(new Rectangle(0, 0, bigBmp.Width, bigBmp.Height), ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb); int smallStride = smallData.Stride; int bigStride = bigData.Stride; int bigWidth = bigBmp.Width; int bigHeight = bigBmp.Height - smallBmp.Height + 1; int smallWidth = smallBmp.Width * 3; int smallHeight = smallBmp.Height; int margin = Convert.ToInt32(255.0 * tolerance); unsafe { byte *pSmall = (byte *)(void *)smallData.Scan0; byte *pBig = (byte *)(void *)bigData.Scan0; int smallOffset = smallStride - smallBmp.Width * 3; int bigOffset = bigStride - bigBmp.Width * 3; bool matchFound = true; for (int y = 0; y < bigHeight; y++) { for (int x = 0; x < bigWidth; x++) { byte *pBigBackup = pBig; byte *pSmallBackup = pSmall; //Look for the small picture. for (int i = 0; i < smallHeight; i++) { int j = 0; matchFound = true; for (j = 0; j < smallWidth; j++) { //With tolerance: pSmall value should be between margins. int inf = pBig[0] - margin; int sup = pBig[0] + margin; if (sup < pSmall[0] || inf > pSmall[0]) { matchFound = false; break; } pBig++; pSmall++; } if (!matchFound) { break; } //We restore the pointers. pSmall = pSmallBackup; pBig = pBigBackup; //Next rows of the small and big pictures. pSmall += smallStride * (1 + i); pBig += bigStride * (1 + i); } //If match found, we return. if (matchFound) { element = new ImageElement { LeftX = x, MiddleX = x + smallBmp.Width / 2, RightX = x + smallBmp.Width, TopY = y, MiddleY = y + smallBmp.Height / 2, BottomY = y + smallBmp.Height }; if (testMode) { //draw on output to demonstrate finding var Rectangle = new Rectangle(x, y, smallBmp.Width - 1, smallBmp.Height - 1); Pen pen = new Pen(Color.Red); pen.Width = 5.0F; bigTestGraphics.DrawRectangle(pen, Rectangle); frmImageCapture captureOutput = new frmImageCapture(); captureOutput.pbTaggedImage.Image = smallTestBmp; captureOutput.pbSearchResult.Image = bigTestBmp; captureOutput.TopMost = true; captureOutput.Show(); } break; } //If no match found, we restore the pointers and continue. else { pBig = pBigBackup; pSmall = pSmallBackup; pBig += 3; } } if (matchFound) { break; } pBig += bigOffset; } } bigBmp.UnlockBits(bigData); smallBmp.UnlockBits(smallData); bigTestGraphics.Dispose(); return(element); }
private void frmScriptBuilder_Load(object sender, EventArgs e) { //load all commands _automationCommands = UIControlsHelper.GenerateCommandsandControls(); //set controls double buffered foreach (Control control in Controls) { typeof(Control).InvokeMember("DoubleBuffered", BindingFlags.SetProperty | BindingFlags.Instance | BindingFlags.NonPublic, null, control, new object[] { true }); } //get app settings _appSettings = new ApplicationSettings(); _appSettings = _appSettings.GetOrCreateApplicationSettings(); string clientLoggerFilePath = Path.Combine(Folders.GetFolder(FolderType.LogFolder), "OpenBots Automation Client Logs.txt"); Logger automationClientLogger = new Logging().CreateFileLogger(clientLoggerFilePath, Serilog.RollingInterval.Day); //Core.Sockets.SocketClient.Initialize(); //Core.Sockets.SocketClient.associatedBuilder = this; //handle action bar preference //hide action panel if (_editMode) { tlpControls.RowStyles[0].SizeType = SizeType.Absolute; tlpControls.RowStyles[0].Height = 0; tlpControls.RowStyles[1].SizeType = SizeType.Absolute; tlpControls.RowStyles[1].Height = 81; } else if (_appSettings.ClientSettings.UseSlimActionBar) { tlpControls.RowStyles[1].SizeType = SizeType.Absolute; tlpControls.RowStyles[1].Height = 0; } else { tlpControls.RowStyles[0].SizeType = SizeType.Absolute; tlpControls.RowStyles[0].Height = 0; } //get scripts folder var rpaScriptsFolder = Folders.GetFolder(FolderType.ScriptsFolder); if (!Directory.Exists(rpaScriptsFolder)) { frmDialog userDialog = new frmDialog("Would you like to create a folder to save your scripts in now? " + "A script folder is required to save scripts generated with this application. " + "The new script folder path would be '" + rpaScriptsFolder + "'.", "Unable to locate Script Folder!", DialogType.YesNo, 0); if (userDialog.ShowDialog() == DialogResult.OK) { Directory.CreateDirectory(rpaScriptsFolder); } } //get latest files for recent files list on load GenerateRecentFiles(); //no height for status bar HideNotificationRow(); //instantiate for script variables if (!_editMode) { _scriptVariables = new List <ScriptVariable>(); _scriptElements = new List <ScriptElement>(); } //pnlHeader.BackColor = Color.FromArgb(255, 214, 88); //instantiate and populate display icons for commands _uiImages = UIImage.UIImageList(); //set image list _selectedTabScriptActions.SmallImageList = _uiImages; //set listview column size frmScriptBuilder_SizeChanged(null, null); var groupedCommands = _automationCommands.GroupBy(f => f.DisplayGroup); foreach (var cmd in groupedCommands) { TreeNode newGroup = new TreeNode(cmd.Key); foreach (var subcmd in cmd) { TreeNode subNode = new TreeNode(subcmd.ShortName); subNode.ToolTipText = subcmd.Description; newGroup.Nodes.Add(subNode); } tvCommands.Nodes.Add(newGroup); } tvCommands.Sort(); //tvCommands.ImageList = uiImages; _tvCommandsCopy = new TreeView(); _tvCommandsCopy.ShowNodeToolTips = true; CopyTreeView(tvCommands, _tvCommandsCopy); txtCommandSearch.Text = _txtCommandWatermark; //start attended mode if selected if (_appSettings.ClientSettings.StartupMode == "Attended Task Mode") { WindowState = FormWindowState.Minimized; var frmAttended = new frmAttendedMode(ScriptProjectPath); frmAttended.Show(); } }