public static string RightClick(RecordedUiTask uiTask, string elemName) { string codeLine = $" desktopSession.DesktopSessionElement.Mouse.MouseMove(winElem_{elemName}.Coordinates);\n" + $" desktopSession.DesktopSessionElement.Mouse.ContextClick(null);\n"; return(GetCodeBlock(uiTask, elemName, codeLine)); }
public static string DoubleClick(RecordedUiTask uiTask, string elemName) { string codeLine = $" action = ActionChains(self.driver)\n" + $" action.double_click(winElem_{elemName}).perform()\n"; return(GetCodeBlock(uiTask, elemName, codeLine)); }
private void DeleteRecordedUiTask(RecordedUiTask uiTask) { if (uiTask == null) { return; } lock (RecordedUiTask.s_lockRecordedUi) { if (uiTask.UiTaskName != EnumUiTaskName.KeyboardInput) { UiTreeNode.RemoveUiTreeNode(uiTask); } RecordedUiTask.s_listRecordedUi.Remove(uiTask); if (RecordedUiTask.s_listRecordedUi.Count > 0) { comboBoxRecordedUi.SelectedIndex = 0; comboBoxRecordedUi.SelectedValue = RecordedUiTask.s_listRecordedUi[comboBoxRecordedUi.SelectedIndex]; } else { TextRange tr = new TextRange(rtbXPath.Document.ContentStart, rtbXPath.Document.ContentEnd); tr.Text = ""; textBoxPyCode.Text = ""; } comboBoxRecordedUi.Items.Refresh(); treeUiPath.Items.Refresh(); } }
public static void AddInspectUi(RecordedUiTask uiTask) { s_inspectUi = null; lock (RecordedUiTask.s_lockRecordedUi) { foreach (var recordedUi in RecordedUiTask.s_listRecordedUi) { if (recordedUi.UiTaskName == EnumUiTaskName.Inspect) { s_inspectUi = recordedUi; break; } } if (uiTask != null) { RecordedUiTask.s_listRecordedUi.Add(uiTask); if (uiTask.UiTaskName != EnumUiTaskName.KeyboardInput) { UiTreeNode.AddToUiTree(uiTask); } } } NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)UiThreadTask.RemoveInspectNode, 0, 0); NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)UiThreadTask.UpdateTreeView, 0, 0); if (uiTask != null) { NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)UiThreadTask.XPathReady, 0, 0); } }
public static string RightClick(RecordedUiTask uiTask, string elemName) { string codeLine = $" action = ActionChains(self.driver)\n" + $" action.move_to_element(winElem_{elemName})\n" + $" action.context_click().perform()\n"; return(GetCodeBlock(uiTask, elemName, codeLine)); }
public static void AddToUiTree(RecordedUiTask recordedUi) { if (recordedUi.GetUiTreeNode() == null) { if (string.IsNullOrEmpty(recordedUi.GetXPath(false))) { return; } } var uiNode = recordedUi.GetUiTreeNode(); if (uiNode == null || uiNode.Items.Count == 0) { //Log error return; } // Empty root if (s_uiTreeNodes.Count == 0) { s_uiTreeNodes.Add(uiNode); return; } // Add uiNode to s_uiNode UiTreeNode node = uiNode; UiTreeNode nodeCached = null; List <UiTreeNode> listCached = s_uiTreeNodes; while (node != null) { var nodeAddTo = FindInCachedUiTree(node, listCached); if (node.Items.Count > 0 && nodeAddTo != null) { node = node.Items.First(); listCached = nodeAddTo.Items; nodeCached = nodeAddTo; } else { if (nodeCached != null) { node.Parent = nodeCached; nodeCached.Items.Add(node); } break; } } }
public static void AddRecordedUi(RecordedUiTask uiTask) { lock (RecordedUiTask.s_lockRecordedUi) { RecordedUiTask.s_listRecordedUi.Add(uiTask); if (uiTask.UiTaskName != EnumUiTaskName.KeyboardInput) { UiTreeNode.AddToUiTree(uiTask); } } NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)UiThreadTask.UpdateTreeView, 0, 0); }
private void EditAttribute_Click(object sender, RoutedEventArgs e) { AppInsights.LogEvent("EditAttribute_Click"); var uiTreeNode = treeUiPath.SelectedItem as UiTreeNode; if (uiTreeNode == null) { return; } RecordedUiTask leafUiTask = uiTreeNode.UiTask; if (leafUiTask == null) { if (treeUiPath.Items != null && treeUiPath.Items.Count > 0) { var x = (TreeViewItem)treeUiPath.ItemContainerGenerator.ContainerFromItem(treeUiPath.Items[0]); if (x != null) { TreeViewItem tvi = UiTreeNode2TreeViewItem(x, UiTreeNode.s_uiTreeNodes.First(), uiTreeNode); if (tvi != null) { leafUiTask = GetExpandedLeafNode(tvi, uiTreeNode, selectedTreeViewItem); if (leafUiTask == null) { leafUiTask = GetExpandedLeafNode(tvi, uiTreeNode, null); selectedTreeViewItem = leafUiTask; } } } } } if (leafUiTask == null) { return; } string oldNodePath = uiTreeNode.NodePath; var winEditAttr = new WindowEditNodeAttribute(uiTreeNode); if (true == winEditAttr.ShowDialog()) { var xpath = leafUiTask.GetXPath(false).Replace(oldNodePath, uiTreeNode.NodePath); var tr = new TextRange(rtbXPath.Document.ContentStart, rtbXPath.Document.ContentEnd); tr.Text = leafUiTask.UpdateXPath(xpath); } }
public static string GetCodeBlock(RecordedUiTask uiTask, string elemName, string uiActionLine) { var xpath = "xpath_" + elemName; elemName = "winElem_" + elemName; string codeBlock = $"{xpath} = {uiTask.GetXPath(true)}\n" + $"{elemName} = desktop_session.find_element_by_xpath({xpath})\n" + $"if {elemName} != None:\n" + "CODEBLOCK" + "else:\n" + " print(f\"Failed to find element using xpath: {" + $"{xpath}" + "}\")\n"; return(codeBlock.Replace("CODEBLOCK", uiActionLine)); }
public static void RemoveUiTreeNode(RecordedUiTask uiTaskNode) { if (s_uiTreeNodes.Count == 0) { return; } UiTreeNode uiNode = FindUiTaskNode(s_uiTreeNodes.First(), uiTaskNode); if (uiNode == null) { s_uiTreeNodes.Clear(); return; } UiTreeNode uiParent = uiNode.Parent; if (uiParent == null) { s_uiTreeNodes.Clear(); return; } while (uiParent != null) { uiNode.UiTask = null; if (uiNode.Items.Count == 0) { uiParent.Items.Remove(uiNode); } if (uiParent.Items.Count > 0) { break; } else { uiNode = uiParent; uiParent = uiNode.Parent; } } if (uiParent == null && (s_uiTreeNodes.Count == 1 && s_uiTreeNodes.First().Items.Count == 0)) { s_uiTreeNodes.Clear(); } }
public static string SendKeys(RecordedUiTask uiTask, string focusedElemeName) { List <string> lines = GetDecodedKeyboardInput(uiTask.Base64Text, uiTask.CapsLock, uiTask.NumLock, uiTask.ScrollLock); StringBuilder sb = new StringBuilder(); focusedElemeName = "winElem_" + focusedElemeName; sb.AppendLine($"time.sleep(100)"); foreach (string line in lines) { sb.AppendLine($"{focusedElemeName}.send_keys({line})"); } return(sb.ToString()); }
static UiTreeNode FindUiTaskNode(UiTreeNode node, RecordedUiTask uiTaskNode) { if (node.UiTask == uiTaskNode) { return(node); } for (int i = 0; i < node.Items.Count; i++) { var ret = FindUiTaskNode(node.Items[i], uiTaskNode); if (ret != null) { return(ret); } } return(null); }
public static string GetCodeBlock(RecordedUiTask uiTask, string elemName, string uiActionLine) { var xpath = "xpath_" + elemName; elemName = "winElem_" + elemName; string codeBlock = $"string {xpath} = {uiTask.GetXPath(true)};\n" + $"var {elemName} = desktopSession.FindElementByAbsoluteXPath({xpath});\n" + $"if ({elemName} != null)\n" + "{\n" + "CODEBLOCK" + "}\n" + "else\n" + "{\n" + " Console.WriteLine($\"Failed to find element using xpath: {" + $"{xpath}" + "}\");\n" + " return;\n" + "}\n"; return(codeBlock.Replace("CODEBLOCK", uiActionLine)); }
public static void AddKeyboardInputTask(ref string strBase64, bool bCapsLock, bool bNumLock, bool bScrollLock) { if (string.IsNullOrEmpty(strBase64)) { AppInsights.LogException("AddKeyboardInputTask", "strBase64 is null"); return; } var keyboardTaskDescription = GenerateCSCode.GetDecodedKeyboardInput(strBase64, bCapsLock, bNumLock, bScrollLock); StringBuilder sb = new StringBuilder(); foreach (var strLine in keyboardTaskDescription) { sb.Append(GenerateXPath.XmlEncode(strLine)); } RecordedUiTask lastRecordedUi = null; lock (RecordedUiTask.s_lockRecordedUi) { if (RecordedUiTask.s_listRecordedUi.Count > 0) { lastRecordedUi = RecordedUiTask.s_listRecordedUi.Last(); } } if (lastRecordedUi != null && lastRecordedUi.UiTaskName == EnumUiTaskName.KeyboardInput) { lastRecordedUi.AppendKeyboardInput(strBase64); } else { var keyboarTask = new RecordedUiTask(EnumUiTaskName.KeyboardInput, strBase64, bCapsLock, bScrollLock, bNumLock); MainWindow.AddRecordedUi(keyboarTask); } strBase64 = null; }
public static string GenerateXPathToUiElement(RecordedUiTask recordedUiTask, List <string> pathNodes, ref UiTreeNode rootRet) { rootRet = null; string tag, ClassName, Name, AutomationId, Pos; string xPath = ""; UiTreeNode parent = null; for (int i = 0; i < pathNodes.Count; i++) { var nodePath = pathNodes[i]; bool bStartsWithName = false; bool bStartsWithClass = false; bool bStartsWithAutoId = false; var tagAttrs = GetTagAttributes(nodePath); tag = tagAttrs.ContainsKey("Tag") ? tagAttrs["Tag"] : "Unknown"; AutomationId = tagAttrs.ContainsKey("AutomationId") ? tagAttrs["AutomationId"] : null; Name = tagAttrs.ContainsKey("Name") ? tagAttrs["Name"] : null; ClassName = tagAttrs.ContainsKey("ClassName") ? tagAttrs["ClassName"] : null; ClassName = CheckAndFixNoneStaticValue(ClassName); Pos = tagAttrs.ContainsKey("position()") ? tagAttrs["position()"] : null; string xPathNode = $"/{tag}"; // Set AutomationId to null if it is a GUID which is very likely generated at runtime AutomationId = CheckAndFixNoneStaticValue(AutomationId); // AutomationId (like UIs on Cortana search result list) created at runtime may end with digits if (!string.IsNullOrEmpty(AutomationId) && !AutomationId.StartsWith("starts-with:")) { string patAutoIdEndsWithDigits = @"^([^\d]*)[_\.\-\d]+$"; System.Text.RegularExpressions.Regex regAutoId = new System.Text.RegularExpressions.Regex(patAutoIdEndsWithDigits, System.Text.RegularExpressions.RegexOptions.IgnoreCase); if (regAutoId != null) { System.Text.RegularExpressions.Match matchAutoId = regAutoId.Match(AutomationId); if (matchAutoId.Success && matchAutoId.Groups.Count > 1) { if (matchAutoId.Groups[1].Length > 0) { AutomationId = "starts-with:" + matchAutoId.Groups[1].ToString(); bStartsWithAutoId = true; } else { AutomationId = null; } } } } if (!string.IsNullOrEmpty(ClassName) && string.IsNullOrEmpty(AutomationId)) { if (ClassName.StartsWith("starts-with:")) { ClassName = ClassName.Remove(0, "starts-with:".Length); xPathNode += string.Format(sNameStartsWithValue, "ClassName", ClassName); bStartsWithClass = true; } else { xPathNode += string.Format(sNameValue, "ClassName", ClassName); } } if (!string.IsNullOrEmpty(Name)) { if (Name.StartsWith("starts-with:")) { Name = Name.Remove(0, "starts-with:".Length); xPathNode += string.Format(sNameStartsWithValue, "Name", Name); bStartsWithName = true; } else { xPathNode += string.Format(sNameValue, "Name", Name); } } if (!string.IsNullOrEmpty(AutomationId)) { if (AutomationId.StartsWith("starts-with:")) { AutomationId = AutomationId.Remove(0, "starts-with:".Length); xPathNode += string.Format(sNameStartsWithValue, "AutomationId", AutomationId); bStartsWithAutoId = true; } else { xPathNode += string.Format(sNameValue, "AutomationId", AutomationId); } } if (!string.IsNullOrEmpty(Pos) && string.IsNullOrEmpty(AutomationId) && string.IsNullOrEmpty(Name) && string.IsNullOrEmpty(ClassName)) { xPathNode += $"[position()={Pos}]"; } // UiTreeNode var left = tagAttrs.ContainsKey("x") ? tagAttrs["x"] : null; var top = tagAttrs.ContainsKey("y") ? tagAttrs["y"] : null; var leftLocal = tagAttrs.ContainsKey("lx") ? tagAttrs["lx"] : null; var topLocal = tagAttrs.ContainsKey("ly") ? tagAttrs["ly"] : null; var width = tagAttrs.ContainsKey("width") ? tagAttrs["width"] : null; var height = tagAttrs.ContainsKey("height") ? tagAttrs["height"] : null; var runtimeId = tagAttrs.ContainsKey("RuntimeId") ? tagAttrs["RuntimeId"] : null; xPath += xPathNode; var uiTreeNode = new UiTreeNode(parent) { Title = $"{tag}, \"{Name}\", {ClassName}" }; uiTreeNode.NodePath = xPathNode; uiTreeNode.Tag = tag; uiTreeNode.ClassName = ClassName; uiTreeNode.Name = Name; uiTreeNode.AutomationId = AutomationId; uiTreeNode.Left = left; uiTreeNode.Top = left; uiTreeNode.Width = width; uiTreeNode.Height = height; uiTreeNode.RuntimeId = runtimeId; uiTreeNode.Position = Pos; uiTreeNode.NameCompareMethod = bStartsWithName ? UiTreeNode.CompareMethod.StartsWith : UiTreeNode.CompareMethod.Equal; uiTreeNode.ClassNameCompareMethod = bStartsWithClass ? UiTreeNode.CompareMethod.StartsWith : UiTreeNode.CompareMethod.Equal; uiTreeNode.AutomationIdCompareMethod = bStartsWithAutoId ? UiTreeNode.CompareMethod.StartsWith : UiTreeNode.CompareMethod.Equal; if (i == pathNodes.Count - 1) { uiTreeNode.UiTask = recordedUiTask; recordedUiTask.Left = left; recordedUiTask.Top = top; recordedUiTask.LeftLocal = leftLocal; recordedUiTask.TopLocal = topLocal; recordedUiTask.Name = Name; recordedUiTask.Tag = tag; } if (rootRet == null) { rootRet = uiTreeNode; } if (parent != null) { parent.Items.Add(uiTreeNode); } parent = uiTreeNode; } return("\"" + xPath + "\""); }
private TreeViewItem ExpandSelectedTreeNode(UiTreeNode uiTreeNode, TreeViewItem tvi, RecordedUiTask recordedUiTask) { if (tvi == null || uiTreeNode == null) { return(null); } if (uiTreeNode.Items.Count == 0) { if (uiTreeNode.UiTask == recordedUiTask) { tvi.IsSelected = true; return(tvi); } else { return(null); } } // Collapse all foreach (var c in uiTreeNode.Items) { var subContainer = (TreeViewItem)tvi.ItemContainerGenerator.ContainerFromItem(c); if (subContainer != null) { subContainer.IsExpanded = false; } } foreach (var c in uiTreeNode.Items) { var subContainer = (TreeViewItem)tvi.ItemContainerGenerator.ContainerFromItem(c); var ret = ExpandSelectedTreeNode(c, subContainer, recordedUiTask); if (ret != null) { subContainer.IsExpanded = true; return(ret); } } return(null); }
private void treeUiPath_SelectedItemChanged(object sender, RoutedPropertyChangedEventArgs <object> e) { if (SelectChangedSource == UiSelectChangedSource.TreeViewItemSelected) { return; } var selectedItem = treeUiPath.SelectedItem as UiTreeNode; if (selectedItem == null) { return; } if (SelectChangedSource != UiSelectChangedSource.ComboBoxSelected) { SelectChangedSource = UiSelectChangedSource.TreeViewItemSelected; } RecordedUiTask leafUiTask = selectedItem.UiTask; if (leafUiTask == null) { if (treeUiPath.Items != null && treeUiPath.Items.Count > 0) { var x = (TreeViewItem)treeUiPath.ItemContainerGenerator.ContainerFromItem(treeUiPath.Items[0]); if (x != null) { TreeViewItem tvi = UiTreeNode2TreeViewItem(x, UiTreeNode.s_uiTreeNodes.First(), selectedItem); if (tvi != null) { // sync xpath with last selected tree view leaf if a parent node is selected // and the selected parent has more than 1 expanded leaf children leafUiTask = GetExpandedLeafNode(tvi, selectedItem, selectedTreeViewItem); { leafUiTask = GetExpandedLeafNode(tvi, selectedItem, null); selectedTreeViewItem = leafUiTask; } } } } } if (leafUiTask == null) { if (SelectChangedSource != UiSelectChangedSource.ComboBoxSelected) { SelectChangedSource = UiSelectChangedSource.SelectChangeFinished; } return; } else { selectedTreeViewItem = leafUiTask; } if (leafUiTask != comboBoxRecordedUi.SelectedItem) { comboBoxRecordedUi.SelectedItem = leafUiTask; } int childCount = selectedItem.Items.Count; if (childCount > 0) { treeUiPath.ContextMenu = treeUiPath.Resources["UiNode"] as System.Windows.Controls.ContextMenu; } else { treeUiPath.ContextMenu = treeUiPath.Resources["LeafNode"] as System.Windows.Controls.ContextMenu; } int nNodeCount = 1; var pParent = selectedItem.Parent; while (pParent != null) { nNodeCount++; pParent = pParent.Parent; } HighlightPath(nNodeCount, true); if (SelectChangedSource != UiSelectChangedSource.ComboBoxSelected) { SelectChangedSource = UiSelectChangedSource.SelectChangeFinished; } AppInsights.LogEvent("treeUiPath_SelectedItemChanged"); }
private RecordedUiTask GetExpandedLeafNode(TreeViewItem tvi, UiTreeNode node, RecordedUiTask recordedUiTask) { if (tvi == null || node == null) { return(null); } if (recordedUiTask != null) { if (node.UiTask == recordedUiTask) { return(node.UiTask); } } else if (node.Items.Count == 0) { return(node.UiTask); } foreach (var c in node.Items) { var subContainer = (TreeViewItem)tvi.ItemContainerGenerator.ContainerFromItem(c); if (subContainer != null) { if (subContainer.IsExpanded == true) { var retTemp = GetExpandedLeafNode(subContainer, c, recordedUiTask); if (retTemp != null) { return(retTemp); } } } } return(null); }
public static string LeftClick(RecordedUiTask uiTask, string elemName) { string codeLine = $" winElem_{elemName}.click()\n"; return(GetCodeBlock(uiTask, elemName, codeLine)); }
public static string Wheel(RecordedUiTask uiTask, string elemName) { string codeLine = $" #TODO: Wheel at ({uiTask.Left},{uiTask.Top}) on winElem_{elemName}, Count:{uiTask.DeltaX}, Total Amount:{uiTask.DeltaY}\n"; return(GetCodeBlock(uiTask, elemName, codeLine)); }
public static void UpdateLastUi(RecordedUiTask uiTask) { NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)UiThreadTask.UpdateTreeView, 0, 0); }
public static void HandleUiEvent(ref string strXml, EnumUiTaskName uiTaskName, int deltaX, int deltaY) { List <string> nodesRootToLeaf = null; if (uiTaskName == EnumUiTaskName.Inspect) { nodesRootToLeaf = GetRootToLeafNodes(strXml); if (nodesRootToLeaf != null && nodesRootToLeaf.Count > 0) { MainWindow.AddInspectUi(new RecordedUiTask(nodesRootToLeaf, uiTaskName)); } return; } if (uiTaskName != EnumUiTaskName.KeyboardInput) { nodesRootToLeaf = GetRootToLeafNodes(strXml); strXml = null; } RecordedUiTask lastRecordedUi = null; lock (RecordedUiTask.s_lockRecordedUi) { if (RecordedUiTask.s_listRecordedUi.Count > 0) { lastRecordedUi = RecordedUiTask.s_listRecordedUi.Last(); } } bool bAddNewTask = true; // Completing last UI if (uiTaskName == EnumUiTaskName.LeftDblClick && lastRecordedUi != null) { lastRecordedUi.ChangeClickToDoubleClick(); bAddNewTask = false; } else if (uiTaskName == EnumUiTaskName.MouseWheel) { if (lastRecordedUi == null || lastRecordedUi.UiTaskName != EnumUiTaskName.MouseWheel) { if (nodesRootToLeaf != null && nodesRootToLeaf.Count > 0) { lastRecordedUi = new RecordedUiTask(nodesRootToLeaf, uiTaskName); MainWindow.AddRecordedUi(lastRecordedUi); } } if (lastRecordedUi != null && lastRecordedUi.UiTaskName == EnumUiTaskName.MouseWheel) { lastRecordedUi.UpdateWheelData(deltaX); } bAddNewTask = false; } if (bAddNewTask) { if (nodesRootToLeaf != null && nodesRootToLeaf.Count > 0) { MainWindow.AddRecordedUi(new RecordedUiTask(nodesRootToLeaf, uiTaskName)); } } else if (lastRecordedUi != null) { //MouseWheel, DoubleClick MainWindow.UpdateLastUi(lastRecordedUi); } NativeMethods.PostMessage(MainWindow.s_windowHandle, (int)MainWindow.UiThreadTask.ActionAdded, 0, 0); AppInsights.LogEvent("HandleUiEvent", uiTaskName.ToString()); }