private string MakeGoogleRequest(string googleSearchQuery)
{
var webClient = new AdvancedWebClient();
var result = webClient.Get(googleSearchQuery);
return(result.HtmlContent);
}
private static List <ISINTemp> getISINListFromSinglePage(string uri, string postData)
{
List <ISINTemp> isinList = new List <ISINTemp>();
AdvancedWebClient wc = new AdvancedWebClient();
HtmlDocument htc = new HtmlDocument();
string pageSource = WebClientUtil.GetPageSource(wc, uri, 300000, postData);
isinList = getISINListFromSinglePage(pageSource);
return(isinList);
}
private ELWExtractEntity GetELWExtractEntityStepOne(string strStartDate, string strEndDate, string strRic)
{
string strUrl = @"http://isin.krx.co.kr/srch/srch.do?method=srchList";
string strPostData = string.Format("std_cd_grnt_start_dd={0}&std_cd_grnt_end_dd={1}&std_cd_grnt=1&searchRadio2=11&searchRadio1=11&searchRadio=11&list_start_dd=&list_end_dd=&listRadio=1&isu_start_dd=&isu_end_dd=&isuRadio=1&isur_cd=&isur_nm={2}&com_nm=", strStartDate, strEndDate, strRic);
string strPageSource = string.Empty;
ELWExtractEntity tmp = new ELWExtractEntity();
try
{
AdvancedWebClient wc = new AdvancedWebClient();
HtmlDocument htc = new HtmlDocument();
strPageSource = WebClientUtil.GetPageSource(wc, strUrl, 300000, strPostData);
if (string.IsNullOrEmpty(strPageSource))
{
Logger.Log(string.Format("return response is null,when query ric:{0}", strRic));
if (!listRicNoResponse.Contains(strRic))
{
listRicNoResponse.Add(strRic);
}
return(null);
}
htc.LoadHtml(strPageSource);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
HtmlNode table = tables[1];
HtmlNodeCollection trs = table.SelectNodes(".//tr");
if (trs.Count == 1)
{
Logger.Log(string.Format("can't get ric in strPageSource,when query ric:{0}", strRic));
if (!listRicNoResponse.Contains(strRic))
{
listRicNoResponse.Add(strRic);
}
return(null);
}
else if (trs.Count >= 2)
{
string std_cd = trs[1].SelectNodes(".//td")[1].InnerText.Replace(" ", "").Replace("\n", "").Replace("\r", "").Replace("\t", "").Trim();
string isu_nm = trs[1].SelectNodes(".//td")[3].InnerText.Replace(" ", "").Replace("\n", "").Replace("\r", "").Replace("\t", "").Trim();
tmp = GetGetELWExtractEntityStepTwo(std_cd, isu_nm);
}
return(tmp);
}
catch (Exception ex)
{
if (!listRicError.Contains(strRic))
{
listRicError.Add(strRic);
}
Logger.Log(string.Format("Error found in function: {0}. Exception message: {1}", "GetELWExtractEntityStepOne", ex.Message));
return(null);
}
}
//baseUrl ="http://isin.krx.co.kr/jsp/realBoard07.jsp";
public static List <ISINTemp> getISINListFromISINWebPage(ISINQuery query)
{
List <ISINTemp> isinList = new List <ISINTemp>();
//string startDate = "";
//string endDate = "";
//if (query.StartDate != null)
//{
// startDate = query.StartDate.ToString("yyyyMMdd", new System.Globalization.CultureInfo("en-US"));
//}
//if(query.EndDate!=null)
//{
// endDate = query.EndDate.ToString("yyyyMMdd", new System.Globalization.CultureInfo("en-US"));
//}
try
{
int pageCount = 0;
String issuername = HttpUtility.UrlEncode(query.IssueCompany, Encoding.GetEncoding("euc-kr"));
String num = HttpUtility.UrlEncode(query.Code.EndsWith("호") ? query.Code : string.Format(query.Code, "호"), Encoding.GetEncoding("euc-kr"));
string postData = string.Format("kind=&ef_std_cd_grnt_dt_from={0}&ef_std_cd_grnt_dt_to={1}&secuGubun={2}&lst_yn_all=on&lst_yn1=Y&lst_yn2=N&lst_yn3=D&els_dls_all=on&els_dls1=els&els_dls2=dls&so_gb_all=on&so_gb1=s&so_gb2=o&jp_gb_all=on&jp_gb1=c&jp_gb2=t&jp_gb3=r&jp_gb4=i&hg_gb_all=on&hg_gb1=h&hg_gb2=g&tg_gb_all=on&tg_gb1=x&tg_gb2=z&df_gb_all=on&df_gb1=df1&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word={3}&ef_iss_inst_cd=&ef_isu_nm={4}&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=",
query.StartDate, query.EndDate, query.Category, issuername, num);
//String uri = "http://isin.krx.co.kr/jsp/BA_LT113.jsp"; Website Change
string uri = string.Format("http://isin.krx.co.kr/jsp/realBoard{0}.jsp", query.Category);
postData = "kind=&ef_std_cd_grnt_dt_from=&ef_std_cd_grnt_dt_to=&secuGubun=06&lst_yn_all=on&lst_yn1=Y&lst_yn2=N&lst_yn3=D&els_dls_all=on&els_dls1=els&els_dls2=dls&so_gb_all=on&so_gb1=s&so_gb2=o&jp_gb_all=on&jp_gb1=c&jp_gb2=t&jp_gb3=r&jp_gb4=i&hg_gb_all=on&hg_gb1=h&hg_gb2=g&tg_gb_all=on&tg_gb1=x&tg_gb2=z&df_gb_all=on&df_gb1=df1&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word=&ef_iss_inst_cd=&ef_isu_nm=%B4%EB%BE%E7%B1%DD%BC%D3+2WR&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=";
AdvancedWebClient wc = new AdvancedWebClient();
string pageSource = WebClientUtil.GetPageSource(wc, uri, 300000, postData);
pageCount = GetTotalPageCount(postData);
isinList.AddRange(getISINListFromSinglePage(pageSource));
if (pageCount == 0)
{
pageCount += 1;
}
for (var i = 2; i <= pageCount; i++)
{
postData = string.Format("pg_no={0}&lst_yn1=Y&lst_yn2=N&lst_yn3=D&df_gb1=df1&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word={1}&ef_isu_nm={2}&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=&ef_std_cd_grnt_dt_from={3}&ef_std_cd_grnt_dt_to={4}", i.ToString("D2"), issuername, num, query.StartDate, query.EndDate);
HtmlDocument htc = new HtmlDocument();
pageSource = WebClientUtil.GetPageSource(uri, 300000, postData);
isinList.AddRange(getISINListFromSinglePage(pageSource));
}
}
catch (Exception ex)
{
String msg = "Error found in GrabDataFromWebpage() : \r\n" + ex.ToString();
}
return(isinList);
}
//Get all the newly generated ISIN for goverment bond
//http://www.chinaclear.cn/isin/user/userApplyLogin.do?m=enter
//http://www.chinaclear.cn/isin/user/userApplyLogin.do?m=queryFast
public List <RicISINInfo> GetAllISINForGoverBond()
{
AdvancedWebClient wc = new AdvancedWebClient();
string postData = "loginName=&password=&securitiesName=%BC%C7%D5%CB%CA%BD%B8%BD%CF%A2&securitiesCode=";
string url = "http://www.chinaclear.cn/isin/user/userApplyLogin.do?m=enter";
string pageSource = WebClientUtil.GetPageSource(wc, url, 18000, postData);
pageSource = WebClientUtil.GetPageSource(wc, "http://www.chinaclear.cn/isin/user/userApplyLogin.do?m=queryFast", 18000, "", Encoding.GetEncoding("gb2312"));
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(pageSource);
var nodeList = htmlDoc.DocumentNode.SelectNodes("//tr[@class='td5']");
return(nodeList.Select(node => new RicISINInfo
{
ISIN = MiscUtil.GetCleanTextFromHtml(node.ChildNodes[1 * 2 + 1].InnerText), Name = MiscUtil.GetCleanTextFromHtml(node.ChildNodes[2 * 2 + 1].InnerText)
}).ToList());
}
public static HtmlNodeCollection SearchISIN(string companyName, bool onlyEquity, bool onlyNonListing)
{
string securityScope = onlyEquity ? "01" : "99";
onlyNonListing = onlyEquity && onlyNonListing; // no choice for all security scope
string listScope = onlyNonListing ? "lst_yn2=N" : "lst_yn_all=on&lst_yn1=Y&lst_yn2=N&lst_yn3=D";
companyName = HttpUtility.UrlEncode(companyName, Encoding.GetEncoding("euc-kr"));
string postData = string.Format("kind=&ef_std_cd_grnt_dt_from=&ef_std_cd_grnt_dt_to=&secuGubun={0}"
+ "&{1}&els_dls_all=on&els_dls1=els&els_dls2=dls&so_gb_all=on&so_gb1=s&so_gb2=o&jp_gb_all=on"
+ "&jp_gb1=c&jp_gb2=t&jp_gb3=r&jp_gb4=i&hg_gb_all=on&hg_gb1=h&hg_gb2=g&tg_gb_all=on&tg_gb1=x&tg_gb2=z&df_gb_all=on&df_gb1=df1"
+ "&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word={2}"
+ "&ef_iss_inst_cd=&ef_isu_nm=&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=",
securityScope,
listScope,
companyName);
AdvancedWebClient wc = new AdvancedWebClient();
string pageSource = WebClientUtil.GetPageSource(wc, queryURL, 300000, postData, Encoding.GetEncoding("euc-kr"));
if (string.IsNullOrEmpty(pageSource))
{
return(null);
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(pageSource);
if (doc.DocumentNode.SelectNodes("//table").Count < 2)
{
return(null);
}
HtmlNodeCollection records = doc.DocumentNode.SelectNodes("//table")[1].SelectNodes(".//tr");
if (records.Count == 1)
{
return(null);
}
records.RemoveAt(0);
return(records);
}
/// <summary>
/// ISIN Website. Get target url by isin.
/// </summary>
/// <param name="isin">isin</param>
/// <returns>url</returns>
private static string GetUrlByIsin(string isin)
{
string postData = string.Format("kind=&ef_std_cd_grnt_dt_from=&ef_std_cd_grnt_dt_to=&secuGubun=99&lst_yn_all=on&lst_yn1=Y&lst_yn2=N&lst_yn3=D&els_dls_all=on&els_dls1=els&els_dls2=dls&so_gb_all=on&so_gb1=s&so_gb2=o&jp_gb_all=on&jp_gb1=c&jp_gb2=t&jp_gb3=r&jp_gb4=i&hg_gb_all=on&hg_gb1=h&hg_gb2=g&tg_gb_all=on&tg_gb1=x&tg_gb2=z&df_gb_all=on&df_gb1=df1&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word=&ef_iss_inst_cd=&ef_isu_nm={0}&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=", isin);
string uri = "http://isin.krx.co.kr/jsp/realBoard99.jsp";
string pageSource = null;
int retries = 3;
while (pageSource == null && retries-- > 0)
{
try
{
AdvancedWebClient wc = new AdvancedWebClient();
pageSource = WebClientUtil.GetPageSource(wc, uri, 180000, postData, Encoding.GetEncoding("EUC-KR"));
}
catch
{
System.Threading.Thread.Sleep(5000);
}
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(pageSource);
HtmlNodeCollection trs = null;
try
{
trs = doc.DocumentNode.SelectNodes("/html[1]/body[1]/table[1]/tr[2]/td[1]/table[1]/tr");
}
catch
{
return(null);
}
HtmlNode tr = trs[1];
HtmlNode td = tr.SelectNodes("./td")[1];
HtmlNode aNode = td.SelectSingleNode(".//a");
string checkMod = aNode.Attributes["href"].Value.Trim();
string type = checkMod.Split(',')[2].Trim(new char[] { '\'', ')', ';' });
string targetPage = GetTargetPageCode(type);
string url = string.Format("http://isin.krx.co.kr/jsp/{0}?isu_cd={1}&modi=f&req_no=", targetPage, isin);
return(url);
}
private void GetISINFromWebpage()
{
try
{
// string uri = "http://isin.krx.co.kr/jsp/BA_LT113.jsp";
string uri = "http://isin.krx.co.kr/jsp/realBoard07.jsp";
foreach (var item in dropList)
{
string postData = string.Empty;
string issuername = HttpUtility.UrlEncode(item.Issuername, Encoding.GetEncoding("euc-kr"));
string num = HttpUtility.UrlEncode(item.Num, Encoding.GetEncoding("euc-kr"));
// %c7%f6%b4%eb %c1%a61730%c8%a3
//postData = string.Format("kind=W&pg_no=1&cb_search_column=co_nm&ef_key_word={0}&ef_isu_nm={1}&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst_dt_from=&ef_lst_dt_to=&ef_std_cd_grnt_dt_from=&ef_std_cd_grnt_dt_to=&chk_bs410=W", issuername, num);
postData = string.Format("kind=&ef_std_cd_grnt_dt_from=&ef_std_cd_grnt_dt_to=&secuGubun=07&lst_yn_all=on&lst_yn1=Y&lst_yn2=N&lst_yn3=D&els_dls_all=on&els_dls1=els&els_dls2=dls&so_gb_all=on&so_gb1=s&so_gb2=o&jp_gb_all=on&jp_gb1=c&jp_gb2=t&jp_gb3=r&jp_gb4=i&hg_gb_all=on&hg_gb1=h&hg_gb2=g&tg_gb_all=on&tg_gb1=x&tg_gb2=z&df_gb_all=on&df_gb1=df1&df_gb2=df2&df_gb3=df3&df_gb4=df4&df_gb5=df5&df_gb6=df6&df_gb7=df7&cb_search_column=co_nm&ef_key_word={0}&ef_iss_inst_cd=&ef_isu_nm={1}&ef_iss_dt_from=&ef_iss_dt_to=&ef_lst", issuername, num);
AdvancedWebClient wc = new AdvancedWebClient();
string pageSource = WebClientUtil.GetPageSource(wc, uri, 300000, postData);
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
if (!string.IsNullOrEmpty(pageSource))
{
doc.LoadHtml(pageSource);
}
if (doc != null)
{
HtmlNode node = doc.DocumentNode.SelectNodes("//table/tr/td/table/tr")[1].SelectNodes("td")[1];
string isin = string.Empty;
if (node != null)
{
isin = node.InnerText.Trim().ToString();
}
if (!string.IsNullOrEmpty(isin))
{
item.ISIN = isin;
}
}
}
}
catch (Exception ex)
{
string msg = "Error found in GetISINFromWebpage() : \r\n" + ex.ToString();
Logger.Log(msg, Logger.LogType.Error);
}
}
private List <FID> getFidList(string url, DateTime startDate, DateTime endDate, int tier2GroupNum, int tier2Num, string type)
{
List <FID> FIDList = new List <FID>();
AdvancedWebClient wc = new AdvancedWebClient();
string postData = string.Format("__VIEWSTATEENCRYPTED=&ctl00%24txt_today=20130221&ctl00%24hfStatus=AEM&ctl00%24hfAlert=&ctl00%24txt_stock_code=&ctl00%24txt_stock_name=&ctl00%24rdo_SelectDocType=rbAfter2006&ctl00%24sel_tier_1=5&ctl00%24sel_DocTypePrior2006=-1&ctl00%24sel_tier_2_group={0}&ctl00%24sel_tier_2={1}&ctl00%24ddlTierTwo=23%2C1%2C3&ctl00%24ddlTierTwoGroup=10%2C2&ctl00%24txtKeyWord=&ctl00%24rdo_SelectDateOfRelease=rbManualRange&ctl00%24sel_DateOfReleaseFrom_d={2}&ctl00%24sel_DateOfReleaseFrom_m={3}&ctl00%24sel_DateOfReleaseFrom_y={4}&ctl00%24sel_DateOfReleaseTo_d={5}&ctl00%24sel_DateOfReleaseTo_m={6}&ctl00%24sel_DateOfReleaseTo_y={7}&ctl00%24sel_defaultDateRange=SevenDays&ctl00%24rdo_SelectSortBy=rbDateTime", tier2GroupNum, tier2Num, startDate.ToString("dd"), startDate.ToString("MM"), startDate.ToString("yyyy"), endDate.ToString("dd"), endDate.ToString("MM"), endDate.ToString("yyyy"));
string viewState = getViewStateValue(WebClientUtil.GetPageSource(wc, url, 1800000, postData));
postData = string.Format("__VIEWSTATE={0}&{1}", viewState, postData);
string pageSource = WebClientUtil.GetPageSource(wc, url, 1800000, postData);
FIDList.AddRange(getPerPageFIDList(pageSource, type));
while (true)
{
if (pageSource.Contains("ctl00_gvMain_ctl24_btnNext"))
{
viewState = getViewStateValue(pageSource);
postData = "__VIEWSTATEENCRYPTED=&ctl00%24gvMain%24ctl24%24btnNext.x=15&ctl00%24gvMain%24ctl24%24btnNext.y=12";
postData = string.Format("__VIEWSTATE={0}&{1}", viewState, postData);
pageSource = WebClientUtil.GetPageSource(wc, url, 1800000, postData);
FIDList.AddRange(getPerPageFIDList(pageSource, type));
}
break;
}
if (type == "CBBC")
{
AddResult("CBBC_" + downloadTime, Path.Combine(configObj.CBBC_DOWNLOAD_FOLDER, ("CBBC_" + downloadTime)), "Have Finished Download CBBC Files");
}
if (type == "Warrant")
{
AddResult("Warrant_" + downloadTime, Path.Combine(configObj.WARRANT_DOWNLOAD_FOLDER, ("Warrant_" + downloadTime)), "Have Finished Download Warrant Files");
}
return(FIDList);
}
static void Main(string[] args)
{
var serpClient = new GoogleSerpClient();
var serpResult = serpClient.Get("kuchen", 100);
var websites = new List <HtmlPage>();
foreach (var link in serpResult.Links)
{
var webClient = new AdvancedWebClient();
var content = webClient.Get(link.UrlAsString);
var htmlAnalyzer = new HtmlAnalyzer();
var website = htmlAnalyzer.AnalyzeHtmlPage(content.HtmlContent);
websites.Add(website);
}
var textAnalyzer = new TextAnalyzer();
foreach (var website in websites)
{
var words = textAnalyzer.GetWords(website.Paragraphs, true);
}
}
private ELWExtractEntity GetGetELWExtractEntityStepTwo(string std_cd, string isu_nm)
{
string strUrl = @"http://isin.krx.co.kr/srch/srch.do?method=srchPopup11";
string stdcd_type = "11";
string mod_del_cd = "";
string pershr_isu_prc = "";
string isu_shrs = "";
string strPostData = string.Format("stdcd_type={0}&std_cd={1}&mod_del_cd={2}&isu_nm={3}&pershr_isu_prc={4}&isu_shrs={5}", stdcd_type, std_cd, mod_del_cd, isu_nm, pershr_isu_prc, isu_shrs);
string strPageSource = string.Empty;
ELWExtractEntity tmp = new ELWExtractEntity();
try
{
AdvancedWebClient wc = new AdvancedWebClient();
HtmlDocument htc = new HtmlDocument();
strPageSource = WebClientUtil.GetPageSource(wc, strUrl, 300000, strPostData);
if (string.IsNullOrEmpty(strPageSource))
{
Logger.Log(string.Format("return response is null,when query ric:{0}", std_cd));
if (!listRicNoResponse.Contains(std_cd))
{
listRicNoResponse.Add(std_cd);
}
return(null);
}
htc.LoadHtml(strPageSource);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
if (tables.Count < 5)
{
Logger.Log(string.Format("tables.count<5,so missing data in the pageSource ,current ric:", isu_nm));
if (!listRicNoResponse.Contains(isu_nm))
{
listRicNoResponse.Add(isu_nm);
}
return(null);
}
HtmlNode table1 = tables[1];
HtmlNodeCollection trs1 = table1.SelectNodes(".//tr"); //11*2=22
HtmlNode table2 = tables[2];
HtmlNodeCollection trs2 = table2.SelectNodes(".//tr"); //0*2+1*1+2*1=4
HtmlNode table3 = tables[3];
HtmlNodeCollection trs3 = table3.SelectNodes(".//tr"); //3*1=3
HtmlNode table4 = tables[4];
HtmlNodeCollection trs4 = table4.SelectNodes(".//tr"); //1*1=1
if (trs1.Count < 11 || trs2.Count < 3 || trs3.Count < 3 || trs4.Count < 1)
{
Logger.Log(string.Format("trs1.count is too small,so missing data in the pageSource ,current ric:", isu_nm));
if (!listRicNoResponse.Contains(isu_nm))
{
listRicNoResponse.Add(isu_nm);
}
return(null);
}
//first table left on ELW website
tmp.IssuingAuthorityCongenial = FormatInnerText(trs1[0].SelectNodes(".//td")[0].InnerText);
tmp.StandardCongenial = FormatInnerText(trs1[1].SelectNodes(".//td")[0].InnerText);
tmp.KoreanProjectName = FormatInnerText(trs1[2].SelectNodes(".//td")[0].InnerText);
tmp.FinancialProducts = FormatInnerText(trs1[3].SelectNodes(".//td")[0].InnerText);
tmp.ListedOrNot = FormatInnerText(trs1[4].SelectNodes(".//td")[0].InnerText);
tmp.Listed = FormatInnerText(trs1[5].SelectNodes(".//td")[0].InnerText);
tmp.IssueNumber = FormatInnerText(trs1[6].SelectNodes(".//td")[0].InnerText);
tmp.ReleaseTheUnitPrice = FormatInnerText(trs1[7].SelectNodes(".//td")[0].InnerText);
tmp.Money = FormatInnerText(trs1[8].SelectNodes(".//td")[0].InnerText);
tmp.ReleaseForm = FormatInnerText(trs1[9].SelectNodes(".//td")[0].InnerText);
tmp.StandardNonStandard = FormatInnerText(trs1[10].SelectNodes(".//td")[0].InnerText);
//first table right on ELW website
tmp.IssuingAuthority = FormatInnerText(trs1[0].SelectNodes(".//td")[1].InnerText);
tmp.ShortenTheCongenial = FormatInnerText(trs1[1].SelectNodes(".//td")[1].InnerText);
tmp.TheProjectNameEnglish = FormatInnerText(trs1[2].SelectNodes(".//td")[1].InnerText);
tmp.ToIssue = FormatInnerText(trs1[3].SelectNodes(".//td")[1].InnerText);
tmp.WhetherTheActivity = FormatInnerText(trs1[4].SelectNodes(".//td")[1].InnerText);
tmp.ListedUntil = FormatInnerText(trs1[5].SelectNodes(".//td")[1].InnerText);
tmp.TheDate = FormatInnerText(trs1[6].SelectNodes(".//td")[1].InnerText);
tmp.TheExpirationOfThe = FormatInnerText(trs1[7].SelectNodes(".//td")[1].InnerText);
tmp.IssueToDistinguish = FormatInnerText(trs1[8].SelectNodes(".//td")[1].InnerText);
tmp.ConversionRatio = FormatInnerText(trs1[9].SelectNodes(".//td")[1].InnerText);
tmp.ThePowerForm = FormatInnerText(trs1[10].SelectNodes(".//td")[1].InnerText);
//second table on ELW website
tmp.UnderlyingAssetTypes = FormatInnerText(trs2[0].SelectNodes(".//td")[0].InnerText);
tmp.SovereignIssuingAuthority = FormatInnerText(trs2[0].SelectNodes(".//td")[1].InnerText);
tmp.StockIndexTypes = FormatInnerText(trs2[1].SelectNodes(".//td")[0].InnerText);
tmp.UnderlyingAssetGuitar = FormatInnerText(trs2[2].SelectNodes(".//td")[0].InnerText);
//third table on ELW website
tmp.TheRightType = FormatInnerText(trs3[0].SelectNodes(".//td")[0].InnerText);
tmp.TheIssueOfTheSpecialConditions = FormatInnerText(trs3[0].SelectNodes(".//td")[1].InnerText);
tmp.TheExerciseOfTheRightWay = FormatInnerText(trs3[1].SelectNodes(".//td")[0].InnerText);
//fourth table on ELW website
tmp.CFI = FormatInnerText(trs4[0].SelectNodes(".//td")[0].InnerText);
return(tmp);
}
catch (Exception ex)
{
if (!listRicError.Contains(std_cd))
{
listRicError.Add(std_cd);
}
Logger.Log(string.Format("Error found in function: {0}. Exception message: {1}", "GetELWExtractEntityStepOne", ex.Message));
return(null);
}
}
private void GrabDataFromWebpage()
{
String startDate = configObj.Korea_SPCRAdjustment_StartDate.Trim().ToString();
String endDate = configObj.Korea_SPCRAdjustment_EndDate.Trim().ToString();
//method=searchDisclosureByStockTypeSub¤tPageSize=15&pageIndex=1&menuIndex=3&orderIndex=1&forward=disclosurebystocktype_sub&elwIsuCd=&elwUly=&lpMbr=&corpNameList=&marketType=&fromData=2011-06-01&toData=2011-12-01&reportNm=&elwRsnClss=09003
//method=searchDisclosureByStockTypeSub¤tPageSize=100&pageIndex=1&menuIndex=3&orderIndex=1&forward=disclosurebystocktype_sub&elwIsuCd=&elwUly=&lpMbr=&corpNameList=&marketType=&fromData=2011-06-01&toData=2011-12-01&reportNm=&elwRsnClss=09003
//method=searchDisclosureByStockTypeSub¤tPageSize=100&pageIndex=1&menuIndex=3&orderIndex=1&forward=disclosurebystocktype_sub&elwIsuCd=&elwUly=&lpMbr=&corpNameList=&marketType=&fromData=2011-06-02&toData=2011-12-02&reportNm=&elwRsnClss=09003
String postData = String.Format("method=searchDisclosureByStockTypeSub¤tPageSize=300&pageIndex=1&menuIndex=3&orderIndex=1&forward=disclosurebystocktype_sub&elwIsuCd=&elwUly=&lpMbr=&corpNameList=&marketType=&fromData={0}&toData={1}&reportNm=&elwRsnClss=09003", startDate, endDate);
String uri = "http://kind.krx.co.kr/disclosure/disclosurebystocktype.do";
AdvancedWebClient wc = new AdvancedWebClient();
String pageSource = String.Empty;
pageSource = WebClientUtil.GetDynamicPageSource(uri, 300000, postData);
//pageSource = WebClientUtil.GetPageSource(wc, uri, 300000, postData);
HtmlDocument htc = new HtmlDocument();
if (!String.IsNullOrEmpty(pageSource))
{
htc.LoadHtml(pageSource);
}
if (htc != null)
{
HtmlNodeCollection nodeCollections = htc.DocumentNode.SelectNodes(".//div[@id='menu1']/table/tbody/tr");
int count = nodeCollections.Count;
if (count > 0)
{
for (var i = 0; i < count; i++)
{
HtmlNode node = nodeCollections[i].SelectSingleNode(".//td[4]/a");
String attribute = String.Empty;
if (node != null)
{
attribute = node.Attributes["onclick"].Value.Trim().ToString();
}
if (!String.IsNullOrEmpty(attribute))
{
attribute = attribute.Split('(')[1].Split(',')[0].Trim(new Char[] { ' ', '\'' }).ToString();
//http://kind.krx.co.kr/common/disclsviewer.do?method=search&acptno=20111201000283&docno=&viewerhost=&viewerport=
uri = String.Format("http://kind.krx.co.kr/common/disclsviewer.do?method=search&acptno={0}&docno=&viewerhost=&viewerport=", attribute /*"20111201000283"*/);
HtmlDocument doc = new HtmlDocument();
pageSource = WebClientUtil.GetDynamicPageSource(uri, 300000, null);
if (!String.IsNullOrEmpty(pageSource))
{
doc.LoadHtml(pageSource);
}
String parameter = String.Empty;
if (doc != null)
{
//doc.DocumentNode.SelectSingleNode(".//div[@id='pWrapper']/div[@id='pContArea']/form[@name='frm']/select[@id='mainDocId']/option[2]")
parameter = doc.DocumentNode.SelectSingleNode(".//select[@id='mainDocId']/option[2]").Attributes["value"].Value.Trim().ToString();
}
attribute = attribute.Insert(4, "/").Insert(7, "/").Insert(10, "/");
//http://kind.krx.co.kr/external/2011/12/01/000283/20111201000641/99858.htm
uri = String.Format("http://kind.krx.co.kr/external/{0}/{1}/99858.htm", attribute, parameter);
doc = null;
doc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
if (doc != null)
{
String judgeStr = doc.DocumentNode.SelectSingleNode("//table[@id='XFormD1_Form0_RepeatTable0']/tbody/tr[1]/td[2]/span").InnerText;
if (!String.IsNullOrEmpty(judgeStr))
{
if (!judgeStr.Equals("확정"))
{
continue;
}
else
{
HtmlNodeCollection trNodes = doc.DocumentNode.SelectNodes("//tr");
int amount = trNodes.Count;
HtmlNode eNode = trNodes[(amount - 1)];
String edate = String.Empty;
if (eNode != null)
{
edate = eNode.SelectSingleNode(".//td[2]").InnerText.Trim().ToString().Replace(" ", "");
int pos = edate.IndexOf("부터");
edate = edate.Substring((pos - 11), 11);
Char[] arr = edate.ToCharArray();
if (arr[0] < 47 && arr[0] > 58)
{
edate = edate.Substring(1);
arr = edate.ToCharArray();
if (arr[0] < 47 && arr[0] > 58)
{
edate = edate.Substring(1);
}
}
edate = Convert.ToDateTime(edate).ToString("yyyy-MMM-dd", new CultureInfo("en-US")).ToUpper();
}
for (var x = 6; x < (amount - 3); x++)
{
SPCRAdjustmentTemplate spcr = new SPCRAdjustmentTemplate();
HtmlNode itemNode = trNodes[x];
if (itemNode != null)
{
String isin = itemNode.SelectSingleNode(".//td[1]").InnerText.Trim().ToString();
String kname = itemNode.SelectSingleNode(".//td[2]").InnerText.Trim().ToString();
String sprice = itemNode.SelectSingleNode(".//td[7]").InnerText.Trim().ToString();
sprice = sprice.Contains(",") ? sprice.Replace(",", "") : sprice;
String cratio = itemNode.SelectSingleNode(".//td[9]").InnerText.Trim().ToString();
spcr.EffectiveDate = edate;
spcr.ISIN = isin;
spcr.KoreaName = kname;
spcr.StrikePrice = sprice;
spcr.ConversionRatio = cratio;
spcrList.Add(spcr);
}
}
}
}
}
}
}
}
}
}
private void ExtractDataStepTwo(FMELWEntity fm)
{
string isu_nm = fm.IssuingAuthority.Trim();
string std_cd = fm.ISIN.Trim();
string strUrl = @"http://isin.krx.co.kr/srch/srch.do?method=srchPopup11";
string stdcd_type = "11";
string mod_del_cd = "";
string pershr_isu_prc = "";
string isu_shrs = "";
string strPostData = string.Format("stdcd_type={0}&std_cd={1}&mod_del_cd={2}&isu_nm={3}&pershr_isu_prc={4}&isu_shrs={5}", stdcd_type, std_cd, mod_del_cd, isu_nm, pershr_isu_prc, isu_shrs);
string strPageSource = string.Empty;
//string strReleaseForm = string.Empty;
try
{
AdvancedWebClient wc = new AdvancedWebClient();
HtmlDocument htc = new HtmlDocument();
strPageSource = WebClientUtil.GetPageSource(wc, strUrl, 300000, strPostData);
if (string.IsNullOrEmpty(strPageSource))
{
Logger.Log(string.Format("return response is null,when query ric:{0}", std_cd));
if (!listNoResponse.Contains(std_cd))
{
listNoResponse.Add(std_cd);
}
return;
}
htc.LoadHtml(strPageSource);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
if (tables.Count < 5)
{
Logger.Log(string.Format("tables.count<5,so missing data in the pageSource ,current ric:", isu_nm));
if (!listNoResponse.Contains(isu_nm))
{
listNoResponse.Add(isu_nm);
}
return;
}
HtmlNode table1 = tables[1];
HtmlNodeCollection trs1 = table1.SelectNodes(".//tr"); //11*2=22
HtmlNode table2 = tables[2];
HtmlNodeCollection trs2 = table2.SelectNodes(".//tr"); //0*2+1*1+2*1=4
HtmlNode table3 = tables[3];
HtmlNodeCollection trs3 = table3.SelectNodes(".//tr"); //3*1=3
if (trs1.Count < 11 || trs2.Count < 3 || trs3.Count < 3)
{
Logger.Log(string.Format("trs1.count is too small,so missing data in the pageSource ,current ric:", isu_nm));
if (!listNoResponse.Contains(isu_nm))
{
listNoResponse.Add(isu_nm);
}
return;
}
fm.ReleaseForm = FormatInnerText(trs1[10].SelectNodes(".//td")[0].InnerText);
//strReleaseForm = FormatInnerText(trs1[9].SelectNodes(".//td")[0].InnerText);
//if (!strReleaseForm.Equals("공모"))
//{
// return;
//}
//first table left on ELW website
fm.QuanityOfWarrants = FormatInnerText(trs1[6].SelectNodes(".//td")[0].InnerText); //quanity of warrants
fm.IssuePrice = FormatInnerText(trs1[7].SelectNodes(".//td")[0].InnerText); //issue price
//first table right on ELW website
fm.Ticker = FormatInnerText(trs1[1].SelectNodes(".//td")[1].InnerText); //ticker
fm.Issuer = FormatInnerText(trs1[2].SelectNodes(".//td")[1].InnerText); //issuer
fm.IssueDate = FormatInnerText(trs1[6].SelectNodes(".//td")[1].InnerText); //issuer date
fm.MatDate = FormatInnerText(trs1[7].SelectNodes(".//td")[1].InnerText); //mat date
fm.ConversionRatio = FormatInnerText(trs1[9].SelectNodes(".//td")[1].InnerText); //conversion ratio
//second table on ELW website
fm.XSovereignIssuingAuthority = FormatInnerText(trs2[0].SelectNodes(".//td")[1].InnerText); //X
fm.YStockIndexTypes = FormatInnerText(trs2[1].SelectNodes(".//td")[0].InnerText); //Y
//third table on ELW website
fm.KoreaWarrantName = FormatInnerText(trs3[0].SelectNodes(".//td")[0].InnerText);//korea warrant name
}
catch (Exception ex)
{
if (!listError.Contains(std_cd))
{
listError.Add(std_cd);
}
Logger.Log(string.Format("Error found in function: {0}. Exception message: {1}", "GetELWExtractEntityStepOne", ex.Message));
return;
}
}