public override void RunCommand(object sender)
        {
            bool testMode = TestMode;

            //user image to bitmap
            Bitmap userImage = new Bitmap(Core.Common.Base64ToImage(v_ImageCapture));

            //take screenshot
            Size     shotSize              = System.Windows.Forms.Screen.PrimaryScreen.Bounds.Size;
            Point    upperScreenPoint      = new Point(0, 0);
            Point    upperDestinationPoint = new Point(0, 0);
            Bitmap   desktopImage          = new Bitmap(shotSize.Width, shotSize.Height);
            Graphics graphics              = Graphics.FromImage(desktopImage);

            graphics.CopyFromScreen(upperScreenPoint, upperDestinationPoint, shotSize);

            //create desktopOutput file
            Bitmap desktopOutput = new Bitmap(desktopImage);

            //get graphics for drawing on output file
            Graphics screenShotUpdate = Graphics.FromImage(desktopOutput);

            //declare maximum boundaries
            int userImageMaxWidth     = userImage.Width - 1;
            int userImageMaxHeight    = userImage.Height - 1;
            int desktopImageMaxWidth  = desktopImage.Width - 1;
            int desktopImageMaxHeight = desktopImage.Height - 1;

            //newfingerprinttechnique

            //create desktopOutput file
            Bitmap sampleOut = new Bitmap(userImage);

            //get graphics for drawing on output file
            Graphics sampleUpdate = Graphics.FromImage(sampleOut);

            List <ImageRecognitionFingerPrint> uniqueFingerprint = new List <ImageRecognitionFingerPrint>();
            Color lastcolor = Color.Transparent;

            //create fingerprint
            var pixelDensity = (userImage.Width * userImage.Height);

            int    iteration = 0;
            Random random    = new Random();

            while ((uniqueFingerprint.Count() < 10) && (iteration < pixelDensity))
            {
                int   x     = random.Next(userImage.Width);
                int   y     = random.Next(userImage.Height);
                Color color = sampleOut.GetPixel(x, y);

                if ((lastcolor != color) && (!uniqueFingerprint.Any(f => f.xLocation == x && f.yLocation == y)))
                {
                    uniqueFingerprint.Add(new ImageRecognitionFingerPrint()
                    {
                        PixelColor = color, xLocation = x, yLocation = y
                    });
                    sampleUpdate.DrawRectangle(Pens.Yellow, x, y, 1, 1);
                }

                iteration++;
            }



            //begin search
            DateTime timeoutDue = DateTime.Now.AddSeconds(v_TimeoutSeconds);


            bool imageFound = false;

            //for each row on the screen
            for (int rowPixel = 0; rowPixel < desktopImage.Height - 1; rowPixel++)
            {
                if (rowPixel + uniqueFingerprint.First().yLocation >= desktopImage.Height)
                {
                    continue;
                }


                //for each column on screen
                for (int columnPixel = 0; columnPixel < desktopImage.Width - 1; columnPixel++)
                {
                    if ((v_TimeoutSeconds > 0) && (DateTime.Now > timeoutDue))
                    {
                        throw new Exception("Image recognition command ran out of time searching for image");
                    }

                    if (columnPixel + uniqueFingerprint.First().xLocation >= desktopImage.Width)
                    {
                        continue;
                    }



                    try
                    {
                        //get the current pixel from current row and column
                        // userImageFingerPrint.First() for now will always be from top left (0,0)
                        var currentPixel = desktopImage.GetPixel(columnPixel + uniqueFingerprint.First().xLocation, rowPixel + uniqueFingerprint.First().yLocation);

                        //compare to see if desktop pixel matches top left pixel from user image
                        if (currentPixel == uniqueFingerprint.First().PixelColor)
                        {
                            //look through each item in the fingerprint to see if offset pixel colors match
                            int matchCount = 0;
                            for (int item = 0; item < uniqueFingerprint.Count; item++)
                            {
                                //find pixel color from offset X,Y relative to current position of row and column
                                currentPixel = desktopImage.GetPixel(columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation);

                                //if color matches
                                if (uniqueFingerprint[item].PixelColor == currentPixel)
                                {
                                    matchCount++;

                                    //draw on output to demonstrate finding
                                    if (testMode)
                                    {
                                        screenShotUpdate.DrawRectangle(Pens.Blue, columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation, 5, 5);
                                    }
                                }
                                else
                                {
                                    //mismatch in the pixel series, not a series of matching coordinate
                                    //?add threshold %?
                                    imageFound = false;

                                    //draw on output to demonstrate finding
                                    if (testMode)
                                    {
                                        screenShotUpdate.DrawRectangle(Pens.OrangeRed, columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation, 5, 5);
                                    }
                                }
                            }

                            if (matchCount == uniqueFingerprint.Count())
                            {
                                imageFound = true;

                                var topLeftX = columnPixel;
                                var topLeftY = rowPixel;

                                if (testMode)
                                {
                                    //draw on output to demonstrate finding
                                    var   Rectangle = new Rectangle(topLeftX, topLeftY, userImageMaxWidth, userImageMaxHeight);
                                    Brush brush     = new SolidBrush(Color.ForestGreen);
                                    screenShotUpdate.FillRectangle(brush, Rectangle);
                                }

                                //move mouse to position
                                var mouseMove = new SendMouseMoveCommand
                                {
                                    v_XMousePosition = (topLeftX + (v_xOffsetAdjustment)).ToString(),
                                    v_YMousePosition = (topLeftY + (v_xOffsetAdjustment)).ToString(),
                                    v_MouseClick     = v_MouseClick
                                };

                                mouseMove.RunCommand(sender);
                            }
                        }


                        if (imageFound)
                        {
                            break;
                        }
                    }
                    catch (Exception)
                    {
                        //continue
                    }
                }


                if (imageFound)
                {
                    break;
                }
            }



            if (testMode)
            {
                //screenShotUpdate.FillRectangle(Brushes.White, 5, 20, 275, 105);
                //screenShotUpdate.DrawString("Blue = Matching Point", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 20);
                // screenShotUpdate.DrawString("OrangeRed = Mismatched Point", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 60);
                // screenShotUpdate.DrawString("Green Rectangle = Match Area", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 100);

                //screenShotUpdate.DrawImage(sampleOut, desktopOutput.Width - sampleOut.Width, 0);

                UI.Forms.Supplement_Forms.frmImageCapture captureOutput = new UI.Forms.Supplement_Forms.frmImageCapture();
                captureOutput.pbTaggedImage.Image  = sampleOut;
                captureOutput.pbSearchResult.Image = desktopOutput;
                captureOutput.Show();
                captureOutput.TopMost = true;
                //captureOutput.WindowState = System.Windows.Forms.FormWindowState.Maximized;
            }

            graphics.Dispose();
            userImage.Dispose();
            desktopImage.Dispose();
            screenShotUpdate.Dispose();

            if (!imageFound)
            {
                throw new Exception("Specified image was not found in window!");
            }
        }
Exemple #2
0
        private void RunCommandActions(IHTMLElement element, object sender, InternetExplorer browserInstance)
        {
            if (v_WebAction == "Fire onmousedown event")
            {
                ((IHTMLElement3)element).FireEvent("onmousedown");
            }
            else if (v_WebAction == "Fire onmouseover event")
            {
                ((IHTMLElement3)element).FireEvent("onmouseover");
            }
            else if (v_WebAction == "Invoke Click")
            {
                element.click();
                IEBrowserCreateCommand.WaitForReadyState(browserInstance);
            }
            else if ((v_WebAction == "Left Click") || (v_WebAction == "Middle Click") || (v_WebAction == "Right Click"))
            {
                int elementXposition = FindElementXPosition(element);
                int elementYposition = FindElementYPosition(element);

                //inputs need to be validated

                int userXAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
                                                   where rw.Field <string>("Parameter Name") == "X Adjustment"
                                                   select rw.Field <string>("Parameter Value")).FirstOrDefault());

                int userYAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
                                                   where rw.Field <string>("Parameter Name") == "Y Adjustment"
                                                   select rw.Field <string>("Parameter Value")).FirstOrDefault());

                var ieClientLocation = User32Functions.GetWindowPosition(new IntPtr(browserInstance.HWND));

                SendMouseMoveCommand newMouseMove = new SendMouseMoveCommand();

                newMouseMove.v_XMousePosition = ((elementXposition + ieClientLocation.left + 10) + userXAdjust).ToString();                                                       // + 10 gives extra padding
                newMouseMove.v_YMousePosition = ((elementYposition + ieClientLocation.top + 90 + System.Windows.Forms.SystemInformation.CaptionHeight) + userYAdjust).ToString(); // +90 accounts for title bar height
                newMouseMove.v_MouseClick     = v_WebAction;
                newMouseMove.RunCommand(sender);
            }
            else if (v_WebAction == "Set Attribute")
            {
                string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                        where rw.Field <string>("Parameter Name") == "Attribute Name"
                                        select rw.Field <string>("Parameter Value")).FirstOrDefault();

                string valueToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
                                     where rw.Field <string>("Parameter Name") == "Value To Set"
                                     select rw.Field <string>("Parameter Value")).FirstOrDefault();

                valueToSet = valueToSet.ConvertToUserVariable(sender);

                element.setAttribute(attributeName, valueToSet);
            }
            else if (v_WebAction == "Set Text")
            {
                string attributeName = "value";

                string textToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
                                    where rw.Field <string>("Parameter Name") == "Text To Set"
                                    select rw.Field <string>("Parameter Value")).FirstOrDefault();

                textToSet = textToSet.ConvertToUserVariable(sender);

                element.setAttribute(attributeName, textToSet);
            }
            else if (v_WebAction == "Get Attribute")
            {
                string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                        where rw.Field <string>("Parameter Name") == "Attribute Name"
                                        select rw.Field <string>("Parameter Value")).FirstOrDefault();

                string variableName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                       where rw.Field <string>("Parameter Name") == "Variable Name"
                                       select rw.Field <string>("Parameter Value")).FirstOrDefault();

                string convertedAttribute = Convert.ToString(element.getAttribute(attributeName));

                convertedAttribute.StoreInUserVariable(sender, variableName);
            }
        }
Exemple #3
0
        public override void RunCommand(object sender)
        {
            var engine = (Core.Automation.Engine.AutomationEngineInstance)sender;

            //create variable window name
            var variableWindowName = v_WindowName.ConvertToUserVariable(sender);

            if (variableWindowName == "Current Window")
            {
                variableWindowName = User32Functions.GetActiveWindowTitle();
            }

            var requiredHandle = SearchForGUIElement(sender, variableWindowName);


            //if element exists type
            if (v_AutomationType == "Check If Element Exists")
            {
                //apply to variable
                var applyToVariable = (from rw in v_UIAActionParameters.AsEnumerable()
                                       where rw.Field <string>("Parameter Name") == "Apply To Variable"
                                       select rw.Field <string>("Parameter Value")).FirstOrDefault();


                //remove brackets from variable
                applyToVariable = applyToVariable.Replace(engine.engineSettings.VariableStartMarker, "").Replace(engine.engineSettings.VariableEndMarker, "");

                //declare search result
                string searchResult;

                //determine search result
                if (requiredHandle == null)
                {
                    searchResult = "FALSE";
                }
                else
                {
                    searchResult = "TRUE";
                }

                //store data
                searchResult.StoreInUserVariable(sender, applyToVariable);
            }

            //determine element click type
            else if (v_AutomationType == "Click Element")
            {
                //if handle was not found
                if (requiredHandle == null)
                {
                    throw new Exception("Element was not found in window '" + variableWindowName + "'");
                }

                //create search params
                var clickType = (from rw in v_UIAActionParameters.AsEnumerable()
                                 where rw.Field <string>("Parameter Name") == "Click Type"
                                 select rw.Field <string>("Parameter Value")).FirstOrDefault();

                //get x adjust
                var xAdjust = (from rw in v_UIAActionParameters.AsEnumerable()
                               where rw.Field <string>("Parameter Name") == "X Adjustment"
                               select rw.Field <string>("Parameter Value")).FirstOrDefault();

                //get y adjust
                var yAdjust = (from rw in v_UIAActionParameters.AsEnumerable()
                               where rw.Field <string>("Parameter Name") == "Y Adjustment"
                               select rw.Field <string>("Parameter Value")).FirstOrDefault();

                //convert potential variable
                var xAdjustVariable = xAdjust.ConvertToUserVariable(sender);
                var yAdjustVariable = yAdjust.ConvertToUserVariable(sender);

                //parse to int
                var xAdjustInt = int.Parse(xAdjustVariable);
                var yAdjustInt = int.Parse(yAdjustVariable);

                //get clickable point
                var newPoint = requiredHandle.GetClickablePoint();

                //send mousemove command
                var newMouseMove = new SendMouseMoveCommand
                {
                    v_XMousePosition = (newPoint.X + xAdjustInt).ToString(),
                    v_YMousePosition = (newPoint.Y + yAdjustInt).ToString(),
                    v_MouseClick     = clickType
                };

                //run commands
                newMouseMove.RunCommand(sender);
            }
            else if (v_AutomationType == "Get Value From Element")
            {
                //if handle was not found
                if (requiredHandle == null)
                {
                    throw new Exception("Element was not found in window '" + variableWindowName + "'");
                }
                //get value from property
                var propertyName = (from rw in v_UIAActionParameters.AsEnumerable()
                                    where rw.Field <string>("Parameter Name") == "Get Value From"
                                    select rw.Field <string>("Parameter Value")).FirstOrDefault();

                //apply to variable
                var applyToVariable = (from rw in v_UIAActionParameters.AsEnumerable()
                                       where rw.Field <string>("Parameter Name") == "Apply To Variable"
                                       select rw.Field <string>("Parameter Value")).FirstOrDefault();

                //remove brackets from variable
                applyToVariable = applyToVariable.Replace(engine.engineSettings.VariableStartMarker, "").Replace(engine.engineSettings.VariableEndMarker, "");

                //get required value
                var requiredValue = requiredHandle.Current.GetType().GetRuntimeProperty(propertyName)?.GetValue(requiredHandle.Current).ToString();

                //store into variable
                requiredValue.StoreInUserVariable(sender, applyToVariable);
            }
            else
            {
                throw new NotImplementedException("Automation type '" + v_AutomationType + "' not supported.");
            }
        }
Exemple #4
0
        //private void ElementsGridViewHelper_MouseEnter(object sender, EventArgs e)
        //{
        //    seleniumAction_SelectionChangeCommitted(null, null);
        //}

        public override void RunCommand(object sender)
        {
            var engine = (Core.Automation.Engine.AutomationEngineInstance)sender;
            //convert to user variable -- https://github.com/saucepleez/taskt/issues/66
            var seleniumSearchParam = v_SeleniumSearchParameter.ConvertToUserVariable(sender);


            var vInstance = v_InstanceName.ConvertToUserVariable(engine);

            var browserObject = engine.GetAppInstance(vInstance);

            var seleniumInstance = (OpenQA.Selenium.IWebDriver)browserObject;

            dynamic element = null;

            if (v_SeleniumElementAction == "Wait For Element To Exist")
            {
                var timeoutText = (from rw in v_WebActionParameterTable.AsEnumerable()
                                   where rw.Field <string>("Parameter Name") == "Timeout (Seconds)"
                                   select rw.Field <string>("Parameter Value")).FirstOrDefault();

                timeoutText = timeoutText.ConvertToUserVariable(sender);

                int timeOut = Convert.ToInt32(timeoutText);

                var timeToEnd = DateTime.Now.AddSeconds(timeOut);

                while (timeToEnd >= DateTime.Now)
                {
                    try
                    {
                        element = FindElement(seleniumInstance, seleniumSearchParam);
                        break;
                    }
                    catch (Exception)
                    {
                        engine.ReportProgress("Element Not Yet Found... " + (timeToEnd - DateTime.Now).Seconds + "s remain");
                        System.Threading.Thread.Sleep(1000);
                    }
                }

                if (element == null)
                {
                    throw new Exception("Element Not Found");
                }

                return;
            }
            else
            {
                element = FindElement(seleniumInstance, seleniumSearchParam);
            }



            switch (v_SeleniumElementAction)
            {
            case "Invoke Click":
                element.Click();
                break;

            case "Left Click":
            case "Right Click":
            case "Middle Click":
            case "Double Left Click":


                int userXAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
                                                   where rw.Field <string>("Parameter Name") == "X Adjustment"
                                                   select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender));

                int userYAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
                                                   where rw.Field <string>("Parameter Name") == "Y Adjustment"
                                                   select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender));

                var elementLocation = element.Location;
                SendMouseMoveCommand newMouseMove = new SendMouseMoveCommand();
                var seleniumWindowPosition        = seleniumInstance.Manage().Window.Position;
                newMouseMove.v_XMousePosition = (seleniumWindowPosition.X + elementLocation.X + 30 + userXAdjust).ToString();     // added 30 for offset
                newMouseMove.v_YMousePosition = (seleniumWindowPosition.Y + elementLocation.Y + 130 + userYAdjust).ToString();    //added 130 for offset
                newMouseMove.v_MouseClick     = v_SeleniumElementAction;
                newMouseMove.RunCommand(sender);
                break;

            case "Set Text":

                string textToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
                                    where rw.Field <string>("Parameter Name") == "Text To Set"
                                    select rw.Field <string>("Parameter Value")).FirstOrDefault();


                string clearElement = (from rw in v_WebActionParameterTable.AsEnumerable()
                                       where rw.Field <string>("Parameter Name") == "Clear Element Before Setting Text"
                                       select rw.Field <string>("Parameter Value")).FirstOrDefault();

                if (clearElement == null)
                {
                    clearElement = "No";
                }

                if (clearElement.ToLower() == "yes")
                {
                    element.Clear();
                }


                string[] potentialKeyPresses = textToSet.Split('{', '}');

                Type seleniumKeys = typeof(OpenQA.Selenium.Keys);
                System.Reflection.FieldInfo[] fields = seleniumKeys.GetFields(System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public);

                //check if chunked string contains a key press command like {ENTER}
                foreach (string chunkedString in potentialKeyPresses)
                {
                    if (chunkedString == "")
                    {
                        continue;
                    }

                    if (fields.Any(f => f.Name == chunkedString))
                    {
                        string keyPress = (string)fields.Where(f => f.Name == chunkedString).FirstOrDefault().GetValue(null);
                        element.SendKeys(keyPress);
                    }
                    else
                    {
                        //convert to user variable - https://github.com/saucepleez/taskt/issues/22
                        var convertedChunk = chunkedString.ConvertToUserVariable(sender);
                        element.SendKeys(convertedChunk);
                    }
                }

                break;

            case "Get Text":
            case "Get Attribute":

                string VariableName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                       where rw.Field <string>("Parameter Name") == "Variable Name"
                                       select rw.Field <string>("Parameter Value")).FirstOrDefault();

                string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                        where rw.Field <string>("Parameter Name") == "Attribute Name"
                                        select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender);

                string elementValue;
                if (v_SeleniumElementAction == "Get Text")
                {
                    elementValue = element.Text;
                }
                else
                {
                    elementValue = element.GetAttribute(attributeName);
                }

                elementValue.StoreInUserVariable(sender, VariableName);

                break;

            case "Get Matching Elements":
                var variableName = (from rw in v_WebActionParameterTable.AsEnumerable()
                                    where rw.Field <string>("Parameter Name") == "Variable Name"
                                    select rw.Field <string>("Parameter Value")).FirstOrDefault();

                var requiredComplexVariable = engine.VariableList.Where(x => x.VariableName == variableName).FirstOrDefault();

                if (requiredComplexVariable == null)
                {
                    engine.VariableList.Add(new Script.ScriptVariable()
                    {
                        VariableName = variableName, CurrentPosition = 0
                    });
                    requiredComplexVariable = engine.VariableList.Where(x => x.VariableName == variableName).FirstOrDefault();
                }


                //set json settings
                JsonSerializerSettings settings = new JsonSerializerSettings();
                settings.Error = (serializer, err) => {
                    err.ErrorContext.Handled = true;
                };

                settings.Formatting = Formatting.Indented;

                //create json list
                List <string> jsonList = new List <string>();
                foreach (OpenQA.Selenium.IWebElement item in element)
                {
                    var json = Newtonsoft.Json.JsonConvert.SerializeObject(item, settings);
                    jsonList.Add(json);
                }

                requiredComplexVariable.VariableValue   = jsonList;
                requiredComplexVariable.CurrentPosition = 0;

                break;

            case "Clear Element":
                element.Clear();
                break;

            default:
                throw new Exception("Element Action was not found");
            }
        }