public CollectionCrawlingQueueProxy(InitializationLink initializationLink, ICollection <ResourceLink> resourceLinks, params CollectionCrawlingQueueProxy[] dependencies) : base(dependencies) { if (initializationLink != null) { InitializationLink = initializationLink; } foreach (var resourceLink in resourceLinks) { QueueItems.Add(new CrawlingQueueItem(resourceLink)); } }
private static InitializationLink ReadInitializationDocumentSection(XmlReader reader, WebsiteConfig config, WebsiteJob job) { var url = reader.GetAttribute("url"); var httpMethod = reader.GetAttribute <string>("method", System.Net.WebRequestMethods.Http.Get); IDictionary <string, string> parameters = null; IDictionary <string, string> headers = null; IDictionary <string, ExtractionItem> extractionItemsOverride = new Dictionary <string, ExtractionItem>(); while (!(reader.Name == "initialization" && reader.NodeType == XmlNodeType.EndElement) && reader.Read()) { if (!reader.IsStartElement()) { continue; } switch (reader.Name) { case "parameters": parameters = ReadExtractionLinkParametersSection(reader).ToDictionary(pred => pred.Key, pred => pred.Value.FormatString); break; case "headers": headers = ReadHttpHeadersSection(reader).ToDictionary(pred => pred.Key, pred => pred.Value.FormatString); break; case "extraction": extractionItemsOverride = ReadExtractionItemsSection(reader, config); break; } //switch (reader.Name) //{ // case "frame": // var extractionFrame = ReadExtractionFrameSection(reader, config); // result.ExtractionItemsOverride.Add(extractionFrame.Name, extractionFrame); // break; // case "link": // var extractionLink = ReadExtractionLinkSection(reader, config); // result.ExtractionItemsOverride.Add(extractionLink.Name, extractionLink); // break; // case "item": // var extractionItem = ReadExtractionItemSection(reader, config); // result.ExtractionItemsOverride.Add(extractionItem.Name, extractionItem); // break; // default: // throw new ArgumentException("Unrecognized element", reader.Name); //} } var result = new InitializationLink( url, httpMethod, parameters, headers, true, true, config, job ); result.ExtractionItemsOverride = extractionItemsOverride; return(result); }