public void Run(string startUrl)
{
logger.LogInformation($"WebBot Run : {startUrl}");
pageDownloader.Download(startUrl).Do(content =>
{
var doc = new HtmlDocument();
doc.LoadHtml(content);
var nodes = doc.DocumentNode.SelectNodes(@"//*[@id=""board_list""]/div/div[2]/table/tbody/tr[*]");
//File.WriteAllText(@"d:/test.txt", driver.PageSource);
var items = new List <string>();
foreach (var node in nodes)
{
var s = node.SelectSingleNode(@".//td[@class=""subject""]/div/a");
if (s == null)
{
continue;
}
items.Add(s.InnerText);
}
items.ForEach(x => logger.LogInformation(x));
}).Wait();
}
private void CreateJavascripts()
{
var baseUri = new Uri(baseUrl);
foreach (string script in scripts)
{
try
{
generatedFiles.Add(downloader.Download(new Uri(baseUri, script), script, false));
}
catch (DownloadException ex)
{
downloader.Errors.Add(ex);
}
}
generatedFiles.Add(downloader.Download(new Uri(baseUri, "images/search_16.gif"),
"KnowledgeBase/images/search_16.gif",
false));
}
public static void Main(string[] args)
{
Console.WriteLine("Launch at " + DateTime.Now.ToString());
Console.WriteLine("Version " + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString());
// 実施判定
if (!hasToCheckUpdate())
{
return;
}
// URL管理
UrlManager manager = new UrlManager();
// URL展開
UrlExtractor extractor = new UrlExtractor();
List <string> urls = extractor.ExtractUrls();
// ご新規さんを抽出
List <string> newUrls = manager.selectNonExists(urls);
Console.WriteLine(" total: " + urls.Count + ", new: " + newUrls.Count);
// ダウンロード
PageDownloader downloader = new PageDownloader();
foreach (string url in newUrls)
{
System.Threading.Thread.Sleep(1000);
Console.WriteLine(" new address: " + url);
string fileName = downloader.Download(url);
if (fileName != null)
{
manager.addUrl(url);
}
else
{
Console.WriteLine(" ...fail!!");
}
}
Console.WriteLine("Finish at " + DateTime.Now.ToString());
}