public void Run(string startUrl) { logger.LogInformation($"WebBot Run : {startUrl}"); pageDownloader.Download(startUrl).Do(content => { var doc = new HtmlDocument(); doc.LoadHtml(content); var nodes = doc.DocumentNode.SelectNodes(@"//*[@id=""board_list""]/div/div[2]/table/tbody/tr[*]"); //File.WriteAllText(@"d:/test.txt", driver.PageSource); var items = new List <string>(); foreach (var node in nodes) { var s = node.SelectSingleNode(@".//td[@class=""subject""]/div/a"); if (s == null) { continue; } items.Add(s.InnerText); } items.ForEach(x => logger.LogInformation(x)); }).Wait(); }
private void CreateJavascripts() { var baseUri = new Uri(baseUrl); foreach (string script in scripts) { try { generatedFiles.Add(downloader.Download(new Uri(baseUri, script), script, false)); } catch (DownloadException ex) { downloader.Errors.Add(ex); } } generatedFiles.Add(downloader.Download(new Uri(baseUri, "images/search_16.gif"), "KnowledgeBase/images/search_16.gif", false)); }
public static void Main(string[] args) { Console.WriteLine("Launch at " + DateTime.Now.ToString()); Console.WriteLine("Version " + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString()); // 実施判定 if (!hasToCheckUpdate()) { return; } // URL管理 UrlManager manager = new UrlManager(); // URL展開 UrlExtractor extractor = new UrlExtractor(); List <string> urls = extractor.ExtractUrls(); // ご新規さんを抽出 List <string> newUrls = manager.selectNonExists(urls); Console.WriteLine(" total: " + urls.Count + ", new: " + newUrls.Count); // ダウンロード PageDownloader downloader = new PageDownloader(); foreach (string url in newUrls) { System.Threading.Thread.Sleep(1000); Console.WriteLine(" new address: " + url); string fileName = downloader.Download(url); if (fileName != null) { manager.addUrl(url); } else { Console.WriteLine(" ...fail!!"); } } Console.WriteLine("Finish at " + DateTime.Now.ToString()); }