public override void RunCommand(object sender)
{
var engine = (Core.Automation.Engine.AutomationEngineInstance)sender;
//convert to user variable -- https://github.com/saucepleez/taskt/issues/66
var seleniumSearchParam = v_SeleniumSearchParameter.ConvertToUserVariable(sender);
var vInstance = v_InstanceName.ConvertToUserVariable(engine);
var browserObject = engine.GetAppInstance(vInstance);
var seleniumInstance = (OpenQA.Selenium.IWebDriver)browserObject;
dynamic element = null;
if (v_SeleniumElementAction == "Wait For Element To Exist")
{
var timeoutText = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Timeout (Seconds)"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
timeoutText = timeoutText.ConvertToUserVariable(sender);
int timeOut = Convert.ToInt32(timeoutText);
var timeToEnd = DateTime.Now.AddSeconds(timeOut);
while (timeToEnd >= DateTime.Now)
{
try
{
element = FindElement(seleniumInstance, seleniumSearchParam);
break;
}
catch (Exception)
{
engine.ReportProgress("Element Not Yet Found... " + (timeToEnd - DateTime.Now).Seconds + "s remain");
System.Threading.Thread.Sleep(1000);
}
}
if (element == null)
{
throw new Exception("Element Not Found");
}
return;
}
else
{
element = FindElement(seleniumInstance, seleniumSearchParam);
}
switch (v_SeleniumElementAction)
{
case "Invoke Click":
element.Click();
break;
case "Left Click":
case "Right Click":
case "Middle Click":
case "Double Left Click":
int userXAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "X Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender));
int userYAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Y Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender));
var elementLocation = element.Location;
SendMouseMoveCommand newMouseMove = new SendMouseMoveCommand();
var seleniumWindowPosition = seleniumInstance.Manage().Window.Position;
newMouseMove.v_XMousePosition = (seleniumWindowPosition.X + elementLocation.X + 30 + userXAdjust).ToString(); // added 30 for offset
newMouseMove.v_YMousePosition = (seleniumWindowPosition.Y + elementLocation.Y + 130 + userYAdjust).ToString(); //added 130 for offset
newMouseMove.v_MouseClick = v_SeleniumElementAction;
newMouseMove.RunCommand(sender);
break;
case "Set Text":
string textToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Text To Set"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
string clearElement = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Clear Element Before Setting Text"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
if (clearElement == null)
{
clearElement = "No";
}
if (clearElement == "Yes")
{
element.Clear();
}
string[] potentialKeyPresses = textToSet.Split('{', '}');
Type seleniumKeys = typeof(OpenQA.Selenium.Keys);
System.Reflection.FieldInfo[] fields = seleniumKeys.GetFields(System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public);
//check if chunked string contains a key press command like {ENTER}
foreach (string chunkedString in potentialKeyPresses)
{
if (chunkedString == "")
{
continue;
}
if (fields.Any(f => f.Name == chunkedString))
{
string keyPress = (string)fields.Where(f => f.Name == chunkedString).FirstOrDefault().GetValue(null);
element.SendKeys(keyPress);
}
else
{
//convert to user variable - https://github.com/saucepleez/taskt/issues/22
var convertedChunk = chunkedString.ConvertToUserVariable(sender);
element.SendKeys(convertedChunk);
}
}
break;
case "Get Text":
case "Get Attribute":
string VariableName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Variable Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Attribute Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault().ConvertToUserVariable(sender);
string elementValue;
if (v_SeleniumElementAction == "Get Text")
{
elementValue = element.Text;
}
else
{
elementValue = element.GetAttribute(attributeName);
}
elementValue.StoreInUserVariable(sender, VariableName);
break;
case "Get Matching Elements":
var variableName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Variable Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
var requiredComplexVariable = engine.VariableList.Where(x => x.VariableName == variableName).FirstOrDefault();
if (requiredComplexVariable == null)
{
engine.VariableList.Add(new Script.ScriptVariable()
{
VariableName = variableName, CurrentPosition = 0
});
requiredComplexVariable = engine.VariableList.Where(x => x.VariableName == variableName).FirstOrDefault();
}
//set json settings
JsonSerializerSettings settings = new JsonSerializerSettings();
settings.Error = (serializer, err) => {
err.ErrorContext.Handled = true;
};
settings.Formatting = Formatting.Indented;
//create json list
List <string> jsonList = new List <string>();
foreach (OpenQA.Selenium.IWebElement item in element)
{
var json = Newtonsoft.Json.JsonConvert.SerializeObject(item, settings);
jsonList.Add(json);
}
requiredComplexVariable.VariableValue = jsonList;
break;
case "Clear Element":
element.Clear();
break;
default:
throw new Exception("Element Action was not found");
}
}
public override void RunCommand(object sender)
{
var engine = (Core.Automation.Engine.AutomationEngineInstance)sender;
//create variable window name
var variableWindowName = v_WindowName.ConvertToUserVariable(sender);
if (variableWindowName == "Current Window")
{
variableWindowName = User32Functions.GetActiveWindowTitle();
}
var requiredHandle = SearchForGUIElement(sender, variableWindowName);
//if element exists type
if (v_AutomationType == "Check If Element Exists")
{
//apply to variable
var applyToVariable = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Apply To Variable"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//remove brackets from variable
applyToVariable = applyToVariable.Replace(engine.engineSettings.VariableStartMarker, "").Replace(engine.engineSettings.VariableEndMarker, "");
//declare search result
string searchResult;
//determine search result
if (requiredHandle == null)
{
searchResult = "FALSE";
}
else
{
searchResult = "TRUE";
}
//store data
searchResult.StoreInUserVariable(sender, applyToVariable);
}
//determine element click type
else if (v_AutomationType == "Click Element")
{
//if handle was not found
if (requiredHandle == null)
{
throw new Exception("Element was not found in window '" + variableWindowName + "'");
}
//create search params
var clickType = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Click Type"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//get x adjust
var xAdjust = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "X Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//get y adjust
var yAdjust = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Y Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//convert potential variable
var xAdjustVariable = xAdjust.ConvertToUserVariable(sender);
var yAdjustVariable = yAdjust.ConvertToUserVariable(sender);
//parse to int
var xAdjustInt = int.Parse(xAdjustVariable);
var yAdjustInt = int.Parse(yAdjustVariable);
//get clickable point
var newPoint = requiredHandle.GetClickablePoint();
//send mousemove command
var newMouseMove = new SendMouseMoveCommand
{
v_XMousePosition = (newPoint.X + xAdjustInt).ToString(),
v_YMousePosition = (newPoint.Y + yAdjustInt).ToString(),
v_MouseClick = clickType
};
//run commands
newMouseMove.RunCommand(sender);
}
else if (v_AutomationType == "Get Value From Element")
{
//if handle was not found
if (requiredHandle == null)
{
throw new Exception("Element was not found in window '" + variableWindowName + "'");
}
//get value from property
var propertyName = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Get Value From"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//apply to variable
var applyToVariable = (from rw in v_UIAActionParameters.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Apply To Variable"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
//remove brackets from variable
applyToVariable = applyToVariable.Replace(engine.engineSettings.VariableStartMarker, "").Replace(engine.engineSettings.VariableEndMarker, "");
//get required value
var requiredValue = requiredHandle.Current.GetType().GetRuntimeProperty(propertyName)?.GetValue(requiredHandle.Current).ToString();
//store into variable
requiredValue.StoreInUserVariable(sender, applyToVariable);
}
else
{
throw new NotImplementedException("Automation type '" + v_AutomationType + "' not supported.");
}
}
public override void RunCommand(object sender)
{
bool testMode = TestMode;
//user image to bitmap
Bitmap userImage = new Bitmap(Core.Common.Base64ToImage(v_ImageCapture));
//take screenshot
Size shotSize = System.Windows.Forms.Screen.PrimaryScreen.Bounds.Size;
Point upperScreenPoint = new Point(0, 0);
Point upperDestinationPoint = new Point(0, 0);
Bitmap desktopImage = new Bitmap(shotSize.Width, shotSize.Height);
Graphics graphics = Graphics.FromImage(desktopImage);
graphics.CopyFromScreen(upperScreenPoint, upperDestinationPoint, shotSize);
//create desktopOutput file
Bitmap desktopOutput = new Bitmap(desktopImage);
//get graphics for drawing on output file
Graphics screenShotUpdate = Graphics.FromImage(desktopOutput);
//declare maximum boundaries
int userImageMaxWidth = userImage.Width - 1;
int userImageMaxHeight = userImage.Height - 1;
int desktopImageMaxWidth = desktopImage.Width - 1;
int desktopImageMaxHeight = desktopImage.Height - 1;
//newfingerprinttechnique
//create desktopOutput file
Bitmap sampleOut = new Bitmap(userImage);
//get graphics for drawing on output file
Graphics sampleUpdate = Graphics.FromImage(sampleOut);
List <ImageRecognitionFingerPrint> uniqueFingerprint = new List <ImageRecognitionFingerPrint>();
Color lastcolor = Color.Transparent;
//create fingerprint
var pixelDensity = (userImage.Width * userImage.Height);
int iteration = 0;
Random random = new Random();
while ((uniqueFingerprint.Count() < 10) && (iteration < pixelDensity))
{
int x = random.Next(userImage.Width);
int y = random.Next(userImage.Height);
Color color = sampleOut.GetPixel(x, y);
if ((lastcolor != color) && (!uniqueFingerprint.Any(f => f.xLocation == x && f.yLocation == y)))
{
uniqueFingerprint.Add(new ImageRecognitionFingerPrint()
{
PixelColor = color, xLocation = x, yLocation = y
});
sampleUpdate.DrawRectangle(Pens.Yellow, x, y, 1, 1);
}
iteration++;
}
//begin search
DateTime timeoutDue = DateTime.Now.AddSeconds(v_TimeoutSeconds);
bool imageFound = false;
//for each row on the screen
for (int rowPixel = 0; rowPixel < desktopImage.Height - 1; rowPixel++)
{
if (rowPixel + uniqueFingerprint.First().yLocation >= desktopImage.Height)
{
continue;
}
//for each column on screen
for (int columnPixel = 0; columnPixel < desktopImage.Width - 1; columnPixel++)
{
if ((v_TimeoutSeconds > 0) && (DateTime.Now > timeoutDue))
{
throw new Exception("Image recognition command ran out of time searching for image");
}
if (columnPixel + uniqueFingerprint.First().xLocation >= desktopImage.Width)
{
continue;
}
try
{
//get the current pixel from current row and column
// userImageFingerPrint.First() for now will always be from top left (0,0)
var currentPixel = desktopImage.GetPixel(columnPixel + uniqueFingerprint.First().xLocation, rowPixel + uniqueFingerprint.First().yLocation);
//compare to see if desktop pixel matches top left pixel from user image
if (currentPixel == uniqueFingerprint.First().PixelColor)
{
//look through each item in the fingerprint to see if offset pixel colors match
int matchCount = 0;
for (int item = 0; item < uniqueFingerprint.Count; item++)
{
//find pixel color from offset X,Y relative to current position of row and column
currentPixel = desktopImage.GetPixel(columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation);
//if color matches
if (uniqueFingerprint[item].PixelColor == currentPixel)
{
matchCount++;
//draw on output to demonstrate finding
if (testMode)
{
screenShotUpdate.DrawRectangle(Pens.Blue, columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation, 5, 5);
}
}
else
{
//mismatch in the pixel series, not a series of matching coordinate
//?add threshold %?
imageFound = false;
//draw on output to demonstrate finding
if (testMode)
{
screenShotUpdate.DrawRectangle(Pens.OrangeRed, columnPixel + uniqueFingerprint[item].xLocation, rowPixel + uniqueFingerprint[item].yLocation, 5, 5);
}
}
}
if (matchCount == uniqueFingerprint.Count())
{
imageFound = true;
var topLeftX = columnPixel;
var topLeftY = rowPixel;
if (testMode)
{
//draw on output to demonstrate finding
var Rectangle = new Rectangle(topLeftX, topLeftY, userImageMaxWidth, userImageMaxHeight);
Brush brush = new SolidBrush(Color.ForestGreen);
screenShotUpdate.FillRectangle(brush, Rectangle);
}
//move mouse to position
var mouseMove = new SendMouseMoveCommand
{
v_XMousePosition = (topLeftX + (v_xOffsetAdjustment)).ToString(),
v_YMousePosition = (topLeftY + (v_xOffsetAdjustment)).ToString(),
v_MouseClick = v_MouseClick
};
mouseMove.RunCommand(sender);
}
}
if (imageFound)
{
break;
}
}
catch (Exception)
{
//continue
}
}
if (imageFound)
{
break;
}
}
if (testMode)
{
//screenShotUpdate.FillRectangle(Brushes.White, 5, 20, 275, 105);
//screenShotUpdate.DrawString("Blue = Matching Point", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 20);
// screenShotUpdate.DrawString("OrangeRed = Mismatched Point", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 60);
// screenShotUpdate.DrawString("Green Rectangle = Match Area", new Font("Arial", 12, FontStyle.Bold), Brushes.SteelBlue, 5, 100);
//screenShotUpdate.DrawImage(sampleOut, desktopOutput.Width - sampleOut.Width, 0);
UI.Forms.Supplement_Forms.frmImageCapture captureOutput = new UI.Forms.Supplement_Forms.frmImageCapture();
captureOutput.pbTaggedImage.Image = sampleOut;
captureOutput.pbSearchResult.Image = desktopOutput;
captureOutput.Show();
captureOutput.TopMost = true;
//captureOutput.WindowState = System.Windows.Forms.FormWindowState.Maximized;
}
graphics.Dispose();
userImage.Dispose();
desktopImage.Dispose();
screenShotUpdate.Dispose();
if (!imageFound)
{
throw new Exception("Specified image was not found in window!");
}
}
private void RunCommandActions(IHTMLElement element, object sender, InternetExplorer browserInstance)
{
if (v_WebAction == "Fire onmousedown event")
{
((IHTMLElement3)element).FireEvent("onmousedown");
}
else if (v_WebAction == "Fire onmouseover event")
{
((IHTMLElement3)element).FireEvent("onmouseover");
}
else if (v_WebAction == "Invoke Click")
{
element.click();
IEBrowserCreateCommand.WaitForReadyState(browserInstance);
}
else if ((v_WebAction == "Left Click") || (v_WebAction == "Middle Click") || (v_WebAction == "Right Click"))
{
int elementXposition = FindElementXPosition(element);
int elementYposition = FindElementYPosition(element);
//inputs need to be validated
int userXAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "X Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault());
int userYAdjust = Convert.ToInt32((from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Y Adjustment"
select rw.Field <string>("Parameter Value")).FirstOrDefault());
var ieClientLocation = User32Functions.GetWindowPosition(new IntPtr(browserInstance.HWND));
SendMouseMoveCommand newMouseMove = new SendMouseMoveCommand();
newMouseMove.v_XMousePosition = ((elementXposition + ieClientLocation.left + 10) + userXAdjust).ToString(); // + 10 gives extra padding
newMouseMove.v_YMousePosition = ((elementYposition + ieClientLocation.top + 90 + System.Windows.Forms.SystemInformation.CaptionHeight) + userYAdjust).ToString(); // +90 accounts for title bar height
newMouseMove.v_MouseClick = v_WebAction;
newMouseMove.RunCommand(sender);
}
else if (v_WebAction == "Set Attribute")
{
string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Attribute Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
string valueToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Value To Set"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
valueToSet = valueToSet.ConvertToUserVariable(sender);
element.setAttribute(attributeName, valueToSet);
}
else if (v_WebAction == "Set Text")
{
string attributeName = "value";
string textToSet = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Text To Set"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
textToSet = textToSet.ConvertToUserVariable(sender);
element.setAttribute(attributeName, textToSet);
}
else if (v_WebAction == "Get Attribute")
{
string attributeName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Attribute Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
string variableName = (from rw in v_WebActionParameterTable.AsEnumerable()
where rw.Field <string>("Parameter Name") == "Variable Name"
select rw.Field <string>("Parameter Value")).FirstOrDefault();
string convertedAttribute = Convert.ToString(element.getAttribute(attributeName));
convertedAttribute.StoreInUserVariable(sender, variableName);
}
}