public List <Queue> GetQueues() { List <Queue> qs = new List <Queue>(); foreach (XmlNode xq in xd.SelectNodes("Route/Queue")) { List <InputItem> iis = new List <InputItem>(); { XmlNode xi = xq.SelectSingleNode(QueueNodes.InputItems.ToString()); if (xi != null) { foreach (XmlNode x in xi.SelectNodes("*")) { switch (x.Name) { case "Url": iis.Add(new InputItem.Url { Value = x.Attributes["value"].Value }); break; case "Element": iis.Add(new InputItem.Element { Value = x.Attributes["value"].Value }); break; default: throw new Exception("Unknown option!"); } } } } List <Action> as_ = new List <Action>(); { XmlNode xa = xq.SelectSingleNode(QueueNodes.Actions.ToString()); if (xa != null) { foreach (XmlNode x in xa.SelectNodes("*")) { switch (x.Name) { case "Set": as_.Add(new Action.Set { Xpath = x.Attributes["xpath"].Value, Attribute = x.Attributes["attribute"].Value, Value = x.Attributes["value"].Value }); break; case "Click": as_.Add(new Action.Click { Xpath = x.Attributes["xpath"].Value }); break; case "WaitDocumentLoaded": as_.Add(new Action.WaitDocumentLoaded { MinimalSleepMss = int.Parse(x.Attributes["minimal_sleep_mss"].Value) }); break; default: throw new Exception("Unknown option!"); } } } } List <Output> os = new List <Output>(); { XmlNode xo = xq.SelectSingleNode(QueueNodes.Outputs.ToString()); if (xo != null) { foreach (XmlNode x in xo.ChildNodes) { Output o; if (x.Name == "UrlCollection") { o = new Output.UrlCollection { Queue = x.Attributes["queue"].Value, Xpath = x.Attributes["xpath"].Value, QueuingManner = (Output.UrlCollection.QueuingManners)Enum.Parse(typeof(Output.UrlCollection.QueuingManners), x.Attributes["queuing_manner"].Value) } } ; else if (x.Name == "ElementCollection") { o = new Output.ElementCollection { Queue = x.Attributes["queue"].Value, Xpath = x.Attributes["xpath"].Value } } ; else if (x.Name == "Field") { o = new Output.Field { Attribute = x.Attributes["attribute"].Value, Xpath = x.Attributes["xpath"].Value, Name = x.Attributes["name"].Value } } ; else { throw new Exception("Unknown type: " + x.Name); } os.Add(o); } } } Queue q = new Queue { Name = xq.Attributes["name"].Value, InputItems = iis, Actions = as_, Outputs = os }; qs.Add(q); } return(qs); }
public void SetOutput(string queue_name, Output output) { XmlNode xo = get_queue_child_node(queue_name, QueueNodes.Outputs); if (output is Output.UrlCollection) { Output.UrlCollection ucs = (Output.UrlCollection)output; XmlNode xu = xo.SelectSingleNode(ucs.Tag + "[@queue='" + ucs.Queue + "']"); if (xu == null) { xu = xd.CreateElement(ucs.Tag); xo.AppendChild(xu); XmlAttribute a = xd.CreateAttribute("queue"); a.Value = ucs.Queue; xu.Attributes.Append(a); get_queue_node(ucs.Queue);//create the accepting queue } { XmlAttribute a = xd.CreateAttribute("queuing_manner"); a.Value = ucs.QueuingManner.ToString(); xu.Attributes.Append(a); a = xd.CreateAttribute("xpath"); a.Value = ucs.Xpath; xu.Attributes.Append(a); } } else if (output is Output.ElementCollection) { Output.ElementCollection ucs = (Output.ElementCollection)output; XmlNode xu = xo.SelectSingleNode(ucs.Tag + "[@queue='" + ucs.Queue + "']"); if (xu == null) { xu = xd.CreateElement(ucs.Tag); xo.AppendChild(xu); XmlAttribute a = xd.CreateAttribute("queue"); a.Value = ucs.Queue; xu.Attributes.Append(a); get_queue_node(ucs.Queue);//create the accepting queue } { XmlAttribute a = xd.CreateAttribute("xpath"); a.Value = ucs.Xpath; xu.Attributes.Append(a); } } else if (output is Output.Field) { Output.Field f = (Output.Field)output; XmlNode xn = xo.SelectSingleNode(f.Tag + "[@name='" + f.Name + "']"); if (xn == null) { xn = xd.CreateElement(f.Tag); xo.AppendChild(xn); XmlAttribute a = xd.CreateAttribute("name"); a.Value = f.Name; xn.Attributes.Append(a); } { XmlAttribute a = xd.CreateAttribute("attribute"); a.Value = f.Attribute; xn.Attributes.Append(a); a = xd.CreateAttribute("xpath"); a.Value = f.Xpath; xn.Attributes.Append(a); } } else { throw new Exception("Unknown type: " + output.GetType()); } Changed?.Invoke(this); }
public void ProcessItem(InputItem ii) { string base_xpath = ""; if (ii is InputItem.Url) { MainWindow.This.Browser.Load(ii.Value, true); } else if (ii is InputItem.Element) { base_xpath = ii.Value; } else { throw new Exception("Unknown type: " + ii.GetType()); } foreach (Action a in Actions) { a.Perform(); } string url = MainWindow.This.Browser.Url; foreach (Output o in Outputs) { if (o is Output.UrlCollection) { Output.UrlCollection uc = (Output.UrlCollection)o; foreach (string l in get_links(base_xpath + uc.Xpath)) { var i = new Controller.Queue.InputItem.Url { Value = l, Queue = uc.Queue, ParentItem = ii }; switch (uc.QueuingManner) { case Route.Output.UrlCollection.QueuingManners.FIFO: uc.Queue.InputItems.Add(i); break; case Route.Output.UrlCollection.QueuingManners.LIFO: uc.Queue.InputItems.Insert(0, i); break; default: throw new Exception("Unknown option: " + uc.QueuingManner); } Log.Main.Write("Added url to " + uc.Queue.Name + ": " + l); } } if (o is Output.ElementCollection) { Output.ElementCollection ec = (Output.ElementCollection)o; foreach (string x in get_single_element_xpaths(base_xpath + ec.Xpath)) { ec.Queue.InputItems.Add(new Controller.Queue.InputItem.Element { Value = x, Queue = ec.Queue, ParentItem = ii }); Log.Main.Write("Added element to " + ec.Queue.Name + ": " + x); } } if (o is Output.Field) { Output.Field f = (Output.Field)o; string v = null; if (f.Attribute == Route.Output.Field.INNER_TEXT) { v = get_value(base_xpath + f.Xpath, Route.Output.Field.INNER_HTML); v = FieldPreparation.Html.Normalize(v); } else { v = get_value(base_xpath + f.Xpath, f.Attribute); } ii.OutputValues.Add(v); } } }