public override bool ProcessCrawler(SQLiteConnection conn) { // http://m.cafe.naver.com/ArticleSearchList.nhn?search.clubid=10050146&search.sortBy=date&search.searchBy=1&search.query=%EB%B0%B0%ED%8A%B8%EB%A7%A8 // cafe.naver.com/ArticleSearchList.nhn?search.clubid=10050146&search.searchtype=1&search.query= CreateMarketTable(conn); Console.WriteLine("Accessing {0} ...", WebUrl); var request = WebRequest.Create(WebUrl); request.Credentials = CredentialCache.DefaultCredentials; var response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode != HttpStatusCode.OK) { return true; } var dataStream = response.GetResponseStream(); if (dataStream == null) { return true; } var encode = response.CharacterSet != null && response.CharacterSet.ToLower() == "utf-8" ? Encoding.UTF8 : Encoding.Default; var reader = new StreamReader(dataStream, encode); var responseFromServer = reader.ReadToEnd(); reader.Close(); dataStream.Close(); response.Close(); var doc = new HtmlDocument(); doc.LoadHtml(responseFromServer); var client = new SlackClientWebhooks(Program.SlackWebHookUrl); foreach (var node in doc.DocumentNode.SelectNodes(@"//div[@class ='article-board m-tcol-c']//tr[@align ='center']")) { int marketNum; if (IsFindNew(node, conn, @".//span[@class ='m-tcol-c list-count']", out marketNum) == false) { continue; } // ������ db�� ����� ��, slack�� ������. var marketSubjectNode = node.SelectSingleNode(@".//span[@class ='aaa']/a"); var marketSubject = marketSubjectNode.InnerText.Trim(); var marketHyperReference = marketSubjectNode.GetAttributeValue(@"href", null); Console.WriteLine(marketNum); Console.WriteLine(marketSubject); InsertIntoTable(marketNum, marketSubject, marketHyperReference, conn); PostToSlack(marketNum, marketSubjectNode, client); } return false; }
public override bool ProcessCrawler(SQLiteConnection conn) { CreateMarketTable(conn); Console.WriteLine("Accessing {0} ...", WebUrl); var request = WebRequest.Create(WebUrl); request.Credentials = CredentialCache.DefaultCredentials; var response = (HttpWebResponse) request.GetResponse(); if (response.StatusCode != HttpStatusCode.OK) { return true; } var dataStream = response.GetResponseStream(); if (dataStream == null) { return true; } var reader = new StreamReader(dataStream, Encoding.UTF8); var responseFromServer = reader.ReadToEnd(); reader.Close(); dataStream.Close(); response.Close(); var doc = new HtmlDocument(); doc.LoadHtml(responseFromServer); var client = new SlackClientWebhooks(Program.SlackWebHookUrl); foreach (var node in doc.DocumentNode.SelectNodes(@"//div[@id ='ruliboard_list']")) { int marketNum; if (IsFindNew(node, conn, @".//span[@class ='market_num']", out marketNum) == false) { continue; } // ������ db�� ����� ��, slack�� ������. var marketSubjectNode = node.SelectSingleNode(@".//span[@class ='market_subject']"); var marketSubject = marketSubjectNode.InnerText.Trim(); var anchor = marketSubjectNode.SelectSingleNode(@".//a"); var marketHyperReference = anchor.GetAttributeValue(@"href", null); Console.WriteLine(marketNum); Console.WriteLine(marketSubjectNode.InnerText.Trim()); // Console.WriteLine(Weburl + marketHyperReference); InsertIntoTable(marketNum, marketSubject, marketHyperReference, conn); PostToSlack(marketNum, marketSubjectNode, client); } return false; }
/// <summary> /// ������ ����Ʈ �Ѵ�. /// </summary> protected void PostToSlack(int marketNum, HtmlNode marketSubjectNode, SlackClientWebhooks slackClient) { var arguments = new Arguments { UnfurlLinks = "true", // Text = Weburl + href + "\n>" + marketNum + " : " + marketSubjectNode.InnerText Text = WebUrl + "\n>" + marketNum + " : " + marketSubjectNode.InnerText }; slackClient.PostMessage(arguments); }