public override bool ProcessCrawler(SQLiteConnection conn)
{
// http://m.cafe.naver.com/ArticleSearchList.nhn?search.clubid=10050146&search.sortBy=date&search.searchBy=1&search.query=%EB%B0%B0%ED%8A%B8%EB%A7%A8
// cafe.naver.com/ArticleSearchList.nhn?search.clubid=10050146&search.searchtype=1&search.query=
CreateMarketTable(conn);
Console.WriteLine("Accessing {0} ...", WebUrl);
var request = WebRequest.Create(WebUrl);
request.Credentials = CredentialCache.DefaultCredentials;
var response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode != HttpStatusCode.OK)
{
return true;
}
var dataStream = response.GetResponseStream();
if (dataStream == null)
{
return true;
}
var encode = response.CharacterSet != null && response.CharacterSet.ToLower() == "utf-8" ? Encoding.UTF8 : Encoding.Default;
var reader = new StreamReader(dataStream, encode);
var responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
var doc = new HtmlDocument();
doc.LoadHtml(responseFromServer);
var client = new SlackClientWebhooks(Program.SlackWebHookUrl);
foreach (var node in doc.DocumentNode.SelectNodes(@"//div[@class ='article-board m-tcol-c']//tr[@align ='center']"))
{
int marketNum;
if (IsFindNew(node, conn, @".//span[@class ='m-tcol-c list-count']", out marketNum) == false)
{
continue;
}
// ������ db�� ����� ��, slack�� ������.
var marketSubjectNode = node.SelectSingleNode(@".//span[@class ='aaa']/a");
var marketSubject = marketSubjectNode.InnerText.Trim();
var marketHyperReference = marketSubjectNode.GetAttributeValue(@"href", null);
Console.WriteLine(marketNum);
Console.WriteLine(marketSubject);
InsertIntoTable(marketNum, marketSubject, marketHyperReference, conn);
PostToSlack(marketNum, marketSubjectNode, client);
}
return false;
}
public override bool ProcessCrawler(SQLiteConnection conn)
{
CreateMarketTable(conn);
Console.WriteLine("Accessing {0} ...", WebUrl);
var request = WebRequest.Create(WebUrl);
request.Credentials = CredentialCache.DefaultCredentials;
var response = (HttpWebResponse) request.GetResponse();
if (response.StatusCode != HttpStatusCode.OK)
{
return true;
}
var dataStream = response.GetResponseStream();
if (dataStream == null)
{
return true;
}
var reader = new StreamReader(dataStream, Encoding.UTF8);
var responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
var doc = new HtmlDocument();
doc.LoadHtml(responseFromServer);
var client = new SlackClientWebhooks(Program.SlackWebHookUrl);
foreach (var node in doc.DocumentNode.SelectNodes(@"//div[@id ='ruliboard_list']"))
{
int marketNum;
if (IsFindNew(node, conn, @".//span[@class ='market_num']", out marketNum) == false)
{
continue;
}
// ������ db�� ����� ��, slack�� ������.
var marketSubjectNode = node.SelectSingleNode(@".//span[@class ='market_subject']");
var marketSubject = marketSubjectNode.InnerText.Trim();
var anchor = marketSubjectNode.SelectSingleNode(@".//a");
var marketHyperReference = anchor.GetAttributeValue(@"href", null);
Console.WriteLine(marketNum);
Console.WriteLine(marketSubjectNode.InnerText.Trim());
// Console.WriteLine(Weburl + marketHyperReference);
InsertIntoTable(marketNum, marketSubject, marketHyperReference, conn);
PostToSlack(marketNum, marketSubjectNode, client);
}
return false;
}
/// <summary>
/// ������ ����Ʈ �Ѵ�.
/// </summary>
protected void PostToSlack(int marketNum, HtmlNode marketSubjectNode, SlackClientWebhooks slackClient)
{
var arguments = new Arguments
{
UnfurlLinks = "true",
// Text = Weburl + href + "\n>" + marketNum + " : " + marketSubjectNode.InnerText
Text = WebUrl + "\n>" + marketNum + " : " + marketSubjectNode.InnerText
};
slackClient.PostMessage(arguments);
}
