private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteFive()
{
List <ETFinsertTaskTemplate> partOne = new List <ETFinsertTaskTemplate>();
HtmlDocument root = WebClientUtil.GetHtmlDocument("https://www.kgifund.com.tw/50_05.asp", 6000);
HtmlNodeCollection trs = root.DocumentNode.SelectNodes("/html[1]/body[1]/table[1]/tr[1]/td[2]/table[1]/tr[3]/td[2]/table[1]/tr[2]/td[1]/table[1]/tr[2]/td[2]/div[1]/table[1]/tr[3]/td[1]/table[1]/tr[2]/td[1]/table[1]/tr");
HtmlNode codeNode = root.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]/tr[1]/td[2]/table[1]/tr[3]/td[2]/table[1]/tr[2]/td[1]/table[1]/tr[2]/td[2]/div[1]/table[1]/tr[3]/td[1]/table[1]/tr[1]/td[1]/table[1]/tr");
ETFinsertTaskTemplate oneRes = new ETFinsertTaskTemplate();
Regex regex = new Regex("\\d{1,}");
Match match = regex.Match(codeNode.InnerText.TrimStart().TrimEnd());
oneRes.Code = match.Value;
foreach (var tr in trs)
{
if (tr.InnerText.Contains("昨日收盤淨值(台幣)"))
{
regex = new Regex("\\d{1,}.\\d{1,}");
match = regex.Match(tr.InnerText.TrimStart().TrimEnd());
oneRes.ReferenceValue = match.Value;
}
if (tr.InnerText.Contains("盤中預估淨值(台幣)"))
{
regex = new Regex("\\d{1,}.\\d{1,}");
match = regex.Match(tr.InnerText.TrimStart().TrimEnd());
oneRes.EstimatedValue = match.Value;
}
}
partOne.Add(oneRes);
return(partOne);
}
/// <summary>
/// Get "(Main Board) up to day "links from uri http://www.hkex.com.hk/eng/market/sec_tradinfo/tradnews/today/news.htm .
/// </summary>
/// <param name="uri">the uri of the data source which is a web html.</param>
/// <returns>a list which consists of the uris of the links</returns>
public List <string> GetUrlLinksFromMainPage(string uri)
{
List <string> linkUrlList = new List <string>();
//uri = MiscUtil.UrlCombine(configObj.BASE_URI, uri);
var htmlDoc = WebClientUtil.GetHtmlDocument(uri, 180000);
var linkNodeList = htmlDoc.DocumentNode.SelectNodes("//span[@id='Content']/table/tbody/tr//td//a");
foreach (var linkNode in linkNodeList)
{
if (linkNode.Attributes["href"] != null)
{
string linkText = linkNode.InnerText;
string linkUrl = linkNode.Attributes["href"].Value;
if (!MiscUtil.IsAbsUrl(linkUrl))
{
linkUrl = MiscUtil.UrlCombine(uri, linkUrl);
}
if (linkText.Trim().ToLower().StartsWith(newsTitlePrefix) && !(linkUrlList.Contains(linkUrl)))
{
linkUrlList.Add(linkUrl);
}
}
}
return(linkUrlList);
}
public static string GetTickerByISIN(string isin, int type)
{
string url = null;
if (type == 1)
{
url = string.Format(detailURLFormat, isin);
}
else if (type == 2)
{
url = string.Format(detailURLFormatKDR, isin);
}
HtmlDocument detailDoc = WebClientUtil.GetHtmlDocument(url, 180000, null, Encoding.GetEncoding("EUC-KR"));
if (detailDoc == null)
{
return(null);
}
if (detailDoc.DocumentNode.SelectNodes("//table").Count < 2)
{
return(null);
}
HtmlNode tickerNode = detailDoc.DocumentNode.SelectNodes("//table")[2].SelectSingleNode(".//tr[3]/td[4]");
return(tickerNode == null ? null : tickerNode.InnerText.Trim().Substring(1).Trim());
}
private void GetChainCode(string pageUrl, string[] chains)
{
HtmlDocument doc = WebClientUtil.GetHtmlDocument(pageUrl, 180000);
List <HtmlNode> codeTables = new List <HtmlNode>();
HtmlNodeCollection tables = doc.DocumentNode.SelectNodes("//table");
foreach (HtmlNode table in tables)
{
if (table.Attributes["summary"] == null)
{
break;
}
string startTitle = table.SelectSingleNode(".//tr[1]/th[1]").InnerText.Trim();
if (startTitle.Equals("Issue Name"))
{
codeTables.Add(table);
}
}
if (codeTables.Count != 2)
{
string msg = string.Format("Error! Can not get 2 tables for Underlying Basket Bonds from web page: {0}", pageUrl);
throw new Exception(msg);
}
for (int i = 0; i < 2; i++)
{
GetBasketBondsDetail(codeTables[i], chains[i]);
}
}
/// <summary>
/// Get abbreviation array from website
/// </summary>
private void GetAbbreviation()
{
HtmlDocument htc = new HtmlDocument();
string uri = String.Format("http://dataops.datastream.com/cgi-bin/readfile.pl?filename=H:/Production/Loaders/Global/DataStream/Tools/Abbreviation/Mload/abbrev.txt&warnold=1");
htc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
HtmlNode fullFile = htc.DocumentNode.SelectSingleNode(".//xmp[1]");
string abbrFile = fullFile.InnerText;
Regex rgxSpace = new Regex(@"\s+");
string[] stringSeparators = new string[] { "\r\n" };
char[] stringSeparators2 = new char[] { ' ' };
string[] lines = abbrFile.Split(stringSeparators, StringSplitOptions.None);
foreach (string line in lines)
{
string formattedLine = rgxSpace.Replace(line.Trim(), " ");
string[] lineTab = formattedLine.Split(stringSeparators2);
if (lineTab.Length >= 2)
{
string key = "";
for (int i = 0; i < lineTab.Length - 1; i++)
{
key += lineTab[i] + " ";
}
key = key.Trim();
if (!abbr.ContainsKey(key))
{
abbr[key] = lineTab[lineTab.Length - 1];
}
}
}
}
private void GrabDataByISIN()
{
List <String> lists = configObj.ISINList.ToList();
if (lists.Count > 0)
{
foreach (var item in lists)
{
if (String.IsNullOrEmpty(item))
{
continue;
}
String uri = String.Format("http://isin.krx.co.kr/jsp/BA_VW021.jsp?isu_cd={0}&modi=f&req_no=", item);
HtmlDocument htc = WebClientUtil.GetHtmlDocument(uri, 300000);
if (htc != null)
{
HtmlNode table = htc.DocumentNode.SelectNodes("//table")[2];
if (table != null)
{
String tr5_td2 = table.SelectSingleNode(".//tr[5]/td[2]").InnerText.Trim().ToString();
//String tr3_td2 = table.SelectSingleNode(".//tr[3]/td[2]").InnerText.Trim().ToString();
if (tr5_td2.Equals("사모"))
{
continue;
}
WarrantTemplate wt = new WarrantTemplate();
wt.ISIN = item.Trim().ToString();
DataAnalysisForSingleSearch(table, wt);
koreaList.Add(wt);
}
}
}
}
}
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteFour()
{
List <ETFinsertTaskTemplate> partOne = new List <ETFinsertTaskTemplate>();
HtmlDocument root = WebClientUtil.GetHtmlDocument("http://sitc.sinopac.com/web/etf/Ajax/GetTradeinfo_nav.aspx", 6000, "fundcode=37");
HtmlNodeCollection trs = root.DocumentNode.SelectNodes("//table/tbody/tr");
foreach (var tr in trs)
{
HtmlNodeCollection tds = tr.SelectNodes("./td");
if (tds == null || tds.Count < 4)
{
continue;
}
Regex regex = new Regex("\\d{1,}.\\d{1,}");
Match match = regex.Match(tds[2].InnerText.TrimStart().TrimEnd());
if (!match.Success)
{
continue;
}
ETFinsertTaskTemplate oneRes = new ETFinsertTaskTemplate
{
ReferenceValue = match.Value.TrimStart().TrimEnd()
};
match = regex.Match(tds[0].InnerText.TrimStart().TrimEnd());
oneRes.Code = match.Value.TrimStart().TrimEnd();
match = regex.Match(tds[3].InnerText.TrimStart().TrimEnd());
oneRes.EstimatedValue = match.Value.TrimStart().TrimEnd();
partOne.Add(oneRes);
}
return(partOne);
}
private IEnumerable <string> GetPages()
{
ServicePointManager.Expect100Continue = false;
var results = new List <string>();
//foreach (string url in Enumerable.Range(0, 10).Select(i => string.Format("http://www.set.or.th/set/newslist.do?to={0}%2F{1}%2F{2}&headline=adds+new+listed&submit=Search&symbol=¤tpage={3}&from={0}%2F{1}%2F{2}&newsType=&country=US&exchangeSymbols=&company=false&exchangeNews=on&language=en&exchange=true"
// , DateTime.Now.AddDays(-13).ToString("dd"), DateTime.Now.AddDays(-13).ToString("MM"), DateTime.Now.Year, i)))
foreach (string url in Enumerable.Range(0, 10).Select(i => string.Format("http://www.set.or.th/set/newslist.do?to={0}%2F{1}%2F{2}&headline=adds+new+listed&submit=Search&symbol=¤tpage={3}&from={0}%2F{1}%2F{2}&newsType=&country=US&exchangeSymbols=&company=false&exchangeNews=on&language=en&exchange=true"
, DateTime.Now.AddDays(0).ToString("dd"), DateTime.Now.AddDays(0).ToString("MM"), DateTime.Now.Year, i)))
{
try
{
var htc = WebClientUtil.GetHtmlDocument(url, 300000);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
HtmlNodeCollection links = tables[15].SelectNodes(".//a");
results.AddRange(from link in links
select link.Attributes["href"].Value.Trim() into rawlink
select rawlink.Substring(rawlink.IndexOf("?")) into rawlink
select rawlink.Replace("&", "&") into rawlink
select "http://www.set.or.th/set/newsdetails.do" + rawlink);
}
catch
{
break;
}
}
return(results);
}
private void GrabDataFromInnerHtm(string url)
{
count++;
HtmlAgilityPack.HtmlDocument htmlDoc = WebClientUtil.GetHtmlDocument(url, 18000);
string txtFile = htmlDoc.DocumentNode.SelectSingleNode("//pre").InnerText;
DataAnalysis(txtFile);
}
private void getSZSEIndexList()
{
string pageSource = WebClientUtil.GetPageSource(configObj.SzseBaseUri, 180000, "");
var doc = WebClientUtil.GetHtmlDocument(string.Format("{0}/main/marketdata/hqcx/zsybg/", configObj.SzseBaseUri), 180000, "", Encoding.GetEncoding("gb2312"));
string szseIndexSourceFileUrl = MiscUtil.GetCleanTextFromHtml(doc.DocumentNode.SelectNodes("//td[@align='right']/a")[0].Attributes["href"].Value);
downloadAndParseIndexFile(string.Format("{0}{1}", configObj.SzseBaseUri, szseIndexSourceFileUrl));
updateSZSEIndexListWithRic();
}
private HtmlAgilityPack.HtmlNodeCollection GetNewlyWarrantRicNode(string uri)
{
HtmlAgilityPack.HtmlDocument htmlDoc = WebClientUtil.GetHtmlDocument(uri, 180000);
var node = htmlDoc.DocumentNode.SelectNodes("html/body/font/table/tbody/tr/td/table/tbody")[0].ChildNodes[1];
HtmlAgilityPack.HtmlNodeCollection ricNodeList = node.SelectNodes("//td/table/tr/td/table/tbody/tr[@class='tr_normal']");
if (ricNodeList == null || ricNodeList.Count == 0)
{
logger.Log("There's no newly launched Warrant. Please go to " + uri + "to have a check.");
}
return(ricNodeList);
}
//Get the target csv file source address
public string GetCSVFileSrc()
{
HtmlAgilityPack.HtmlDocument htmlDoc = null;
htmlDoc = WebClientUtil.GetHtmlDocument(BASE_URI, 180000);
string csvFileSrc = htmlDoc.DocumentNode.SelectSingleNode("//table/tbody/tr/td/a/img[@name='btn_download']").ParentNode.Attributes["href"].Value;
csvFileSrc = csvFileSrc.Remove(0, csvFileSrc.IndexOf("eng", 0) + 3);
string baseUri = BASE_URI.Remove(BASE_URI.IndexOf("/plw", 0));
csvFileSrc = baseUri + csvFileSrc;
return(csvFileSrc);
}
public static void GetTickerAndLegalNameByISIN(KoreaEquityInfo item)
{
string isin = item.ISIN;
string type = item.Type;
if (string.IsNullOrEmpty(isin) || string.IsNullOrEmpty(type))
{
return;
}
string url = null;
if (type.Equals("ORD") || type.Equals("PRF"))
{
url = string.Format(detailURLFormat, isin);
}
else if (type.Equals("KDR"))
{
url = string.Format(detailURLFormatKDR, isin);
}
HtmlDocument detailDoc = WebClientUtil.GetHtmlDocument(url, 180000, null, Encoding.GetEncoding("EUC-KR"));
if (detailDoc == null)
{
return;
}
if (detailDoc.DocumentNode.SelectNodes("//table").Count < 2)
{
return;
}
HtmlNode tickerNode = detailDoc.DocumentNode.SelectNodes("//table")[2].SelectSingleNode(".//tr[3]/td[4]");
HtmlNode legalNameNode;
if (type.Equals("ORD") || type.Equals("PRF"))
{
legalNameNode = detailDoc.DocumentNode.SelectNodes("//table")[2].SelectSingleNode(".//tr[11]/td[4]");
}
else
{
legalNameNode = detailDoc.DocumentNode.SelectNodes("//table")[2].SelectSingleNode(".//tr[5]/td[4]");
}
if (tickerNode == null)
{
return;
}
item.Ticker = tickerNode.InnerText.Trim().Substring(1).Trim();
item.LegalName = legalNameNode.InnerText.Trim();
}
private void updateSSEIndexListWithRic()
{
foreach (ChinaIndex index in sseIndexList)
{
var htmlDoc = WebClientUtil.GetHtmlDocument(index.SourceUrl, 180000, "", Encoding.GetEncoding("gb2312"));
var nodeList = htmlDoc.DocumentNode.SelectNodes("//td[@class='table3']//a");
foreach (HtmlNode node in nodeList)
{
string officialCode = MiscUtil.GetCleanTextFromHtml(node.InnerText);
officialCode = officialCode.Substring(officialCode.IndexOf("(") + 1, (officialCode.IndexOf(")") - officialCode.IndexOf("(") - 1));
index.RicList.Add(generateRic(officialCode));
}
}
}
private string GetIsin(string code)
{
try
{
string url = String.Format("http://isin.twse.com.tw/isin/single_main.jsp?owncode={0}", code);
var htc = WebClientUtil.GetHtmlDocument(url, 300000, null);
return(htc.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[2]/tr[2]/td[2]").InnerText.Trim());
}
catch (Exception)
{
throw new Exception("Isin did not came out on the website yet, try again later");
}
}
/// <summary>
/// Get company infomation from page: companysummary.do
/// </summary>
/// <param name="attribute">onclick attribute of node</param>
/// <returns>company infomation</returns>
public static KoreaCompany GetCompanyInfoBak(string attribute)
{
KoreaCompany company = null;
try
{
//http://kind.krx.co.kr/common/companysummary.do?method=searchCompanySummary&strIsurCd=00347&lstCd=undefined
string url = string.Format("http://kind.krx.co.kr/common/companysummary.do?method=searchCompanySummary&strIsurCd={0}&lstCd=undefined", attribute);
HtmlDocument doc = WebClientUtil.GetHtmlDocument(url, 300000, null);
if (doc != null)
{
company = new KoreaCompany();
HtmlNode table = doc.DocumentNode.SelectNodes("//table")[0];
string koreaName = table.SelectSingleNode(".//tr[1]/td[1]").InnerText.Trim();
string legalName = table.SelectSingleNode(".//tr[1]/td[2]").InnerText.Trim();
string isin = table.SelectSingleNode(".//tr[2]/td[1]").InnerText.Trim();;
string market = table.SelectSingleNode(".//tr[2]/td[2]").InnerText.Trim();
market = GetMarketCode(market);
string listingDate = table.SelectSingleNode(".//tr[3]/td[2]").InnerText.Trim();
if (!string.IsNullOrEmpty(market))
{
company.Market = market;
}
if (!string.IsNullOrEmpty(isin))
{
company.ISIN = isin;
}
if (!string.IsNullOrEmpty(legalName))
{
company.LegalName = legalName;
}
if (!string.IsNullOrEmpty(koreaName))
{
company.KoreaName = koreaName;
}
if (!string.IsNullOrEmpty(listingDate))
{
company.ListingDate = listingDate;
}
}
}
catch
{
return(null);
}
return(company);
}
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteOne()
{
HtmlDocument root = WebClientUtil.GetHtmlDocument("http://www.p-shares.com/fundindex_i_now.asp", 6000);
HtmlNodeCollection trs = root.DocumentNode.SelectNodes("//tr");
return((from tr in trs
select tr.SelectNodes("./td")
into tds
let regex = new Regex("\\d{1,}")
let match = regex.Match(tds[0].InnerText.TrimStart().TrimEnd())
where match.Success
select new ETFinsertTaskTemplate
{
Code = tds[0].InnerText.TrimStart().TrimEnd(), ReferenceValue = tds[2].InnerText.TrimStart().TrimEnd(), EstimatedValue = tds[3].InnerText.TrimStart().TrimEnd()
}).ToList());
}
//http://www.chinaclear.cn/isin/isinbiz/queryCommon.do?m=querycommon
//http://www.chinaclear.cn/isin/isinbiz/queryCommon.do
public string GetISINFromCode(string officialCode)
{
string isin = string.Empty;
string postData = string.Format("m=querycommon¤tPage=0&securitiesCode={0}&isin=&securitiesNameCL=&query=%B2%E9%D1%AF", officialCode);
var htmlDoc = WebClientUtil.GetHtmlDocument(configObj.QueryUrl, 18000, postData);
htmlDoc.DocumentNode.SelectNodes("//body//table//tr//td//table//tr//td//form//table//tr//td[@class='isnr']");
var htmlDocumentNodes = htmlDoc.DocumentNode.SelectNodes("//body//table//tr//td//table//tr//td//table//tr//td[@class='isnr']");
//TODO: To Find a Better Way to Get the ISIN
if (htmlDocumentNodes.Count == 11)
{
isin = MiscUtil.GetCleanTextFromHtml(htmlDocumentNodes[8].InnerText);
}
return(isin);
}
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteTwo()
{
HtmlDocument root = WebClientUtil.GetHtmlDocument("http://www.assetmanagement.hsbc.com.tw/HSBC_WCTS_FE/feEvents/ETFSite/etf_message.asp?ETFOK=Y", 6000);
HtmlNodeCollection trs = root.DocumentNode.SelectNodes("//table[@class=\"data\"]/tr");
return((from tr in trs
select tr.SelectNodes("./td")
into tds
where tds != null && tds.Count >= 4
let regex = new Regex("\\d{1,}.\\d{1,}")
let match = regex.Match(tds[2].InnerText.TrimStart().TrimEnd())
where match.Success
select new ETFinsertTaskTemplate
{
Code = tds[0].InnerText.TrimStart().TrimEnd(), ReferenceValue = match.Value.TrimStart().TrimEnd(), EstimatedValue = tds[3].InnerText.TrimStart().TrimEnd()
}).ToList());
}
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteThree()
{
HtmlDocument root = WebClientUtil.GetHtmlDocument("http://etrade.fsit.com.tw/ETF/etf/realtime_fund_nav_content.aspx", 6000);
HtmlNodeCollection trs = root.DocumentNode.SelectNodes("//table/tr");
return((from tr in trs
select tr.SelectNodes("./td")
into tds
where tds != null && tds.Count >= 4
let regex = new Regex("\\d{1,}.\\d{1,}")
let match = regex.Match(tds[2].InnerText.TrimStart().TrimEnd())
where match.Success
select new ETFinsertTaskTemplate
{
Code = tds[0].InnerText.TrimStart().TrimEnd(), ReferenceValue = match.Value.TrimStart().TrimEnd(), EstimatedValue = tds[3].InnerText.TrimStart().TrimEnd()
}).ToList());
}
/// <summary>
/// Read the page and find the corresponding excel link
/// </summary>
/// <param name="page">Link of the page</param>
/// <returns>The excel link</returns>
private string FindExcelLink(string page)
{
HtmlDocument htc = new HtmlDocument();
try
{
Thread.Sleep(1000);
htc = WebClientUtil.GetHtmlDocument(page, 300000, null);
if (page.Contains("cnindex"))
{
HtmlNodeCollection uls = htc.DocumentNode.SelectNodes(".//ul");
HtmlNode excelLinkNode = uls[6];
if (excelLinkNode == null)
{
excelLinkNode = uls[5];
}
return(cnIndexUrl + excelLinkNode.SelectSingleNode(".//li[3]/a").Attributes["href"].Value.Trim().ToString().Replace("../", ""));
}
else
{
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
foreach (HtmlNode tr in tables[1].SelectNodes(".//tr"))
{
string excelUrl = tr.SelectSingleNode(".//td[2]/a").Attributes["href"].Value.Trim().ToString();
if (excelUrl.EndsWith(".xls"))
{
if (excelUrl.StartsWith("http"))
{
return(excelUrl);
}
else
{
return(csIndexUrl + excelUrl);
}
}
}
}
}
catch (Exception ex)
{
string msg = "Error :" + ex.ToString();
Logger.Log(msg, Logger.LogType.Error);
}
return(null);
}
//Get Ration info from http://www.hkex.com.hk/eng/cbbc/newissue/newlaunch.htm
public List <BullBearInfo> GetBullBearInfo()
{
HtmlAgilityPack.HtmlDocument htmlDoc = null;
string uri = "http://www.hkex.com.hk/eng/cbbc/newissue/newlaunch.htm";
htmlDoc = WebClientUtil.GetHtmlDocument(uri, 180000);
HtmlAgilityPack.HtmlNodeCollection normalNodeList = htmlDoc.DocumentNode.SelectNodes("//table/tbody/tr[@class='tr_normal']");
return((from normalNode in normalNodeList
select normalNode.ChildNodes
into tableblankNodeList
where DateEqualLastBusinessDay(tableblankNodeList[2 * 11 + 1].InnerText)
select new BullBearInfo
{
Code = tableblankNodeList[2 * 1 + 1].ChildNodes[0].InnerText,
Date = tableblankNodeList[2 * 11 + 1].InnerText,
Ratio = tableblankNodeList[2 * 9 + 1].InnerText
}).ToList());
}
/// <summary>
/// Download Excel for update
/// </summary>
private void DownloadSecondFile()
{
try
{
HtmlDocument htc = new HtmlDocument();
string uri = String.Format("http://www.tse.or.jp/market/data/margin/index.html");
htc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
int trNb = tables[4].SelectNodes(".//tr").Count;
string source2Url = tables[4].SelectSingleNode(String.Format(".//tr[{0}]/td[2]/a", trNb)).Attributes["href"].Value.Trim();
source2Url = baseUrl + source2Url;
WebClientUtil.DownloadFile(source2Url, 100000, Path.GetDirectoryName(configObj.ResultFilePath) + @"\test2.xls");
}
catch (Exception ex)
{
string msg = "Error :" + ex;
Logger.Log(msg, Logger.LogType.Error);
}
}
/// <summary>
/// From a given ric, find in TWSE website informations about it
/// </summary>
/// <param name="code"></param>
/// <returns></returns>
private Dictionary <string, string> GetInfos(string code)
{
const string url = "http://mops.twse.com.tw/mops/web/ajax_t05st03";
bool skipped = false;
bool skipped2 = false;
var datas = new Dictionary <string, string>();
string postData = String.Format("encodeURIComponent=1&step=1&firstin=1&off=1&keyword4={0}&code1=&TYPEK2=&checkbtn=1&queryName=co_id&TYPEK=all&co_id={0}"
, code);
var htc = WebClientUtil.GetHtmlDocument(url, 300000, postData);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
foreach (var tr in tables[1].SelectNodes(".//tr"))
{
var ths = tr.SelectNodes(".//th");
var tds = tr.SelectNodes(".//td");
int i = 0;
foreach (var th in ths)
{
if (i == 31 && skipped == false)
{
skipped = true;
continue;
}
if (i == 32 && skipped2 == false)
{
skipped2 = true;
continue;
}
if (!datas.ContainsKey(th.InnerText.Replace(" ", "").Replace(" ", "").Trim()) && i < tds.Count && i != 29)
{
datas.Add(th.InnerText.Replace(" ", "").Replace(" ", "").Trim(),
tds[i].InnerText.Replace(" ", "").Replace(" ", "").Trim());
}
i++;
}
}
datas.Add("chinesename", GetChineseName(htc));
return(datas);
}
private HtmlDocument GetDocRetryMechanism(string url, string postData, int retryTimes, int waitSecond)
{
HtmlDocument doc = new HtmlDocument();
try
{
for (int i = 0; i < retryTimes; i++)
{
try
{
doc = WebClientUtil.GetHtmlDocument(url, 300000, postData);
if (doc != null)
{
break;
}
}
catch (Exception ex)
{
Thread.Sleep(waitSecond);
if (i == retryTimes - 1)
{
string msg = string.Format("url:{0} retryTimes:{1} msg:{3}", url, retryTimes.ToString(), ex.Message);
Logger.Log(msg, Logger.LogType.Error);
throw;
}
}
}
}
catch (Exception ex)
{
string msg = string.Format("\r\n ClassName: {0}\r\n MethodName: {1}\r\n Message: {2}",
System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.ToString(),
System.Reflection.MethodBase.GetCurrentMethod().Name,
ex.Message);
Logger.Log(msg, Logger.LogType.Error);
}
return(doc);
}
/// <summary>
/// From informations create a FmInfo object
/// </summary>
/// <param name="results"></param>
/// <returns></returns>
private FmInfo InfosToFm(List <string> results)
{
FmInfo fm = new FmInfo();
Dictionary <string, string> infos = new Dictionary <string, string>();
char[] stringSeparators = new char[] { '：' };
foreach (string entry in results)
{
string[] parts = entry.Split(stringSeparators, StringSplitOptions.None);
if (parts.Length == 2)
{
infos.Add(parts[0].Trim(), parts[1].Trim());
}
}
fm.Isin = infos["國際編碼"];
fm.MatureDate = ConvertDate(infos["到期日期"]);
fm.EffectiveDate = ConvertDate(infos["發行日期"]);
fm.ChineseName = infos["債券簡稱"];
fm.Strikes = infos["最新轉(交)換價格"];
fm.Shares = infos["實際發行總額"];
fm.Ric = infos["債券代碼"];
string[] nameParts = infos["債券英文名稱"].Split(fm.Ric.Substring(4).ToCharArray());
fm.EnglishName = nameParts[0];
string url = "http://mops.twse.com.tw/mops/web/ajax_t05st03";
string postData = String.Format("encodeURIComponent=1&step=1&firstin=1&off=1&keyword4={0}&code1=&TYPEK2=&checkbtn=1&queryName=co_id&TYPEK=all&co_id={1}", fm.Ric.Substring(0, 4), fm.Ric.Substring(0, 4));
var htc = WebClientUtil.GetHtmlDocument(url, 300000, postData);
string fullHtml = htc.DocumentNode.InnerText;
if (fullHtml.Contains("(上市公司)"))
{
fm.Type = ".TW";
}
else
{
fm.Type = ".TWO";
}
return(fm);
}
private void downloadSourceFiles()
{
string fileUrl = null;
try
{
HtmlDocument htc = new HtmlDocument();
string uri = String.Format("http://www.ose.or.jp/market/trading_data/open_interest_by_participant");
htc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
HtmlNodeCollection tr = htc.DocumentNode.SelectNodes("//tr/td/a");
foreach (HtmlNode node in tr)
{
if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_futures_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_futures_OI_by_participant.xls");
}
else if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_options_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_options_OI_by_participant.xls");
}
else if (node.Attributes["href"].Value.Contains(configObj.Date + "_individual_options_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_individual_options_OI_by_participant.xls");
}
}
}
catch (Exception ex)
{
Logger.Log("Error when downloading files from website" + ex.Message);
LogMessage("Error when downloading files from website");
}
finally
{
}
}
public List <StockOption> GetStockOptionList()
{
List <StockOption> stockOptionList = new List <StockOption>();
List <string> linkUrlList = GetUrlLinksFromMainPage();
foreach (string uri in linkUrlList)
{
//string uri = "http://www.hkex.com.hk/eng/market/sec_tradinfo/tradnews/today/enew.htm";
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc = WebClientUtil.GetHtmlDocument(uri, 270000);
string stockOptionInfo = htmlDoc.DocumentNode.SelectSingleNode("//tbody/tr[@valign='top']/td[@valign='top']/pre").InnerText;
using (StringReader sr = new StringReader(stockOptionInfo))
{
string line1;
while (((line1 = sr.ReadLine()) != null))
{
//Parse the stock info
Regex stockInfoRegex = new Regex("(?s)(?<className>\\w{3})\\s{2,}(?<month>\\w{3})\\s{1,}(?<year>\\d{2})\\s{2,}(?<price>.*\\d)");
Match stockInfoMatch = stockInfoRegex.Match(line1);
if (stockInfoMatch.Success)
{
StockOption stockInfo = new StockOption();
stockInfo.ClassName = stockInfoMatch.Groups["className"].Value;
stockInfo.ExpiryMonth = stockInfoMatch.Groups["month"].Value;
stockInfo.ExpiryYear = stockInfoMatch.Groups["year"].Value;
stockInfo.Price = stockInfoMatch.Groups["price"].Value;
if (line1.EndsWith("/"))
{
stockInfo.Price += sr.ReadLine();
}
stockInfo.GetRicList(underlyingCodeMap.UnderlyCodeMap, monthCodeMap.ContractCodeMap);
stockOptionList.Add(stockInfo);
continue;
}
}
}
}
return(stockOptionList);
}
private void getSSEIndexList()
{
var doc = WebClientUtil.GetHtmlDocument(string.Format("{0}/sseportal/index/cn/common/index_list_new.shtml", configObj.SseBaseUri), 180000, "", Encoding.GetEncoding("gb2312"));
var nodeList = doc.DocumentNode.SelectNodes("//td[@class='table2']//a");
foreach (HtmlNode node in nodeList)
{
//http://www.sse.com.cn/sseportal/index/cn/i000010/intro.
if (node.ParentNode.Attributes["align"] == null)
{
ChinaIndex index = new ChinaIndex();
index.ChineseName = MiscUtil.GetCleanTextFromHtml(node.InnerText);
index.SourceUrl = string.Format("{0}{1}", configObj.SseBaseUri, node.Attributes["href"].Value);
//http://www.sse.com.cn/sseportal/index/cn/i000010/intro.shtml
//http://www.sse.com.cn/sseportal/index/cn/i000010/const_list.shtml
index.SourceUrl = index.SourceUrl.Replace("intro.shtml", "const_list.shtml");
sseIndexList.Add(index);
}
}
updateSSEIndexListWithRic();
}
//Get information from http://www.hkex.com.hk/eng/Warrant/Warrantsummary.asp?id={0}"
private string[] GetUnderlyingForStockAndIssuePrice(string id)
{
string[] arr = { "", "" };
string uri = string.Format("http://www.hkex.com.hk/eng/dwrc/dwsummary.asp?id={0}", id);
HtmlAgilityPack.HtmlDocument htmlDoc = WebClientUtil.GetHtmlDocument(uri, 180000);
var nodes = htmlDoc.DocumentNode.SelectNodes("//body/font/table/tbody/tr/td//table/tbody/tr/td/table/tbody/tr");
foreach (HtmlAgilityPack.HtmlNode node in nodes)
{
if (node.ChildNodes[2 * 0 + 1].InnerText == "Issue Price (Initial Issue):")
{
arr[1] = MiscUtil.GetCleanTextFromHtml(node.ChildNodes[2 * 1 + 1].InnerText).Substring(4);
}
else if (node.ChildNodes[2 * 0 + 1].InnerText == "Underlying:")
{
arr[0] = MiscUtil.GetCleanTextFromHtml(node.ChildNodes[2 * 1 + 1].InnerText).Substring(8);
}
}
return(arr);
}
