public static PaymentStatusResponse PaymentStatus(string transactionId) { PaymentStatusResponse paymentStatusResponse = new PaymentStatusResponse(); ErrorResponse errorResponse = validateCredentials(credentials); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { paymentStatusResponse.responseCode = errorResponse.responseCode; paymentStatusResponse.responseMsg = errorResponse.responseMsg; return(paymentStatusResponse); } if (String.IsNullOrEmpty(transactionId)) { paymentStatusResponse.responseCode = "22"; paymentStatusResponse.responseMsg = "Missing transactionId parameter"; return(paymentStatusResponse); } try { var response = WebClientUtil.GetResponse(credentials.url, RemitaBillerUrl.PaymentStatus(transactionId), _header); paymentStatusResponse = JsonConvert.DeserializeObject <PaymentStatusResponse>(response); } catch (Exception) { throw; } return(paymentStatusResponse); }
public static GetBillersResponse GetBillers() { GetBillersResponse billers = new GetBillersResponse(); //Console.WriteLine("Getting Error response. . ."); ErrorResponse errorResponse = validateCredentials(credentials); //Console.WriteLine("Error response. . ."+ errorResponse.responseMsg); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { billers.responseCode = errorResponse.responseCode; billers.responseMsg = errorResponse.responseMsg; return(billers); } try { var response = WebClientUtil.GetResponse(credentials.url, RemitaBillerUrl.GetBillers(), _header); billers = JsonConvert.DeserializeObject <GetBillersResponse>(response); } catch (Exception) { throw; } return(billers); }
public static GenerateRRRResponse GenerateRRR(GenerateRRRRequest generateRRRRequest) { GenerateRRRResponse generateRRRResponse = new GenerateRRRResponse(); ErrorResponse errorResponse = validateCredentials(credentials); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { generateRRRResponse.responseCode = errorResponse.responseCode; generateRRRResponse.responseMsg = errorResponse.responseMsg; return(generateRRRResponse); } if (generateRRRRequest == null) { generateRRRResponse.responseCode = "18"; generateRRRResponse.responseMsg = "Missing Generate RRR Request object"; return(generateRRRResponse); } String jsonGenerateRRRRequest = JsonConvert.SerializeObject(generateRRRRequest); try { var response = WebClientUtil.PostResponse(credentials.url, RemitaBillerUrl.Generate(), jsonGenerateRRRRequest, _header); generateRRRResponse = JsonConvert.DeserializeObject <GenerateRRRResponse>(response); } catch (Exception) { throw; } return(generateRRRResponse); }
public static GetRRRDetailsResponse GetRRRDetails(string rrr) { GetRRRDetailsResponse rrrDetails = new GetRRRDetailsResponse(); ErrorResponse errorResponse = validateCredentials(credentials); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { rrrDetails.responseCode = errorResponse.responseCode; rrrDetails.responseMsg = errorResponse.responseMsg; return(rrrDetails); } if (String.IsNullOrEmpty(rrr)) { rrrDetails.responseCode = "20"; rrrDetails.responseMsg = "Missing rrr parameter"; return(rrrDetails); } try { var response = WebClientUtil.GetResponse(credentials.url, RemitaBillerUrl.GetRRRDetails(rrr), _header); rrrDetails = JsonConvert.DeserializeObject <GetRRRDetailsResponse>(response); } catch (Exception) { throw; } return(rrrDetails); }
public static ValidateResponse Validate(ValidateRequest validateRequest) { ValidateResponse validateResponse = new ValidateResponse(); ErrorResponse errorResponse = validateCredentials(credentials); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { validateResponse.responseCode = errorResponse.responseCode; validateResponse.responseMsg = errorResponse.responseMsg; return(validateResponse); } if (validateRequest == null) { validateResponse.responseCode = "14"; validateResponse.responseMsg = "Missing Validate Request object"; return(validateResponse); } String jsonValidateRequest = JsonConvert.SerializeObject(validateRequest); try { var response = WebClientUtil.PostResponse(credentials.url, RemitaBillerUrl.Validate(), jsonValidateRequest, _header); validateResponse = JsonConvert.DeserializeObject <ValidateResponse>(response); } catch (Exception) { throw; } return(validateResponse); }
/// <summary> /// Get "(Main Board) up to day "links from uri http://www.hkex.com.hk/eng/market/sec_tradinfo/tradnews/today/news.htm . /// </summary> /// <param name="uri">the uri of the data source which is a web html.</param> /// <returns>a list which consists of the uris of the links</returns> public List <string> GetUrlLinksFromMainPage(string uri) { List <string> linkUrlList = new List <string>(); //uri = MiscUtil.UrlCombine(configObj.BASE_URI, uri); var htmlDoc = WebClientUtil.GetHtmlDocument(uri, 180000); var linkNodeList = htmlDoc.DocumentNode.SelectNodes("//span[@id='Content']/table/tbody/tr//td//a"); foreach (var linkNode in linkNodeList) { if (linkNode.Attributes["href"] != null) { string linkText = linkNode.InnerText; string linkUrl = linkNode.Attributes["href"].Value; if (!MiscUtil.IsAbsUrl(linkUrl)) { linkUrl = MiscUtil.UrlCombine(uri, linkUrl); } if (linkText.Trim().ToLower().StartsWith(newsTitlePrefix) && !(linkUrlList.Contains(linkUrl))) { linkUrlList.Add(linkUrl); } } } return(linkUrlList); }
private void GetChainCode(string pageUrl, string[] chains) { HtmlDocument doc = WebClientUtil.GetHtmlDocument(pageUrl, 180000); List <HtmlNode> codeTables = new List <HtmlNode>(); HtmlNodeCollection tables = doc.DocumentNode.SelectNodes("//table"); foreach (HtmlNode table in tables) { if (table.Attributes["summary"] == null) { break; } string startTitle = table.SelectSingleNode(".//tr[1]/th[1]").InnerText.Trim(); if (startTitle.Equals("Issue Name")) { codeTables.Add(table); } } if (codeTables.Count != 2) { string msg = string.Format("Error! Can not get 2 tables for Underlying Basket Bonds from web page: {0}", pageUrl); throw new Exception(msg); } for (int i = 0; i < 2; i++) { GetBasketBondsDetail(codeTables[i], chains[i]); } }
public static GetCustomFieldsResponse GetCustomFields(string billId) { GetCustomFieldsResponse servicesTypes = new GetCustomFieldsResponse(); ErrorResponse errorResponse = validateCredentials(credentials); string responseCode = errorResponse.responseCode; if (errorResponse.responseCode != "00") { servicesTypes.responseCode = errorResponse.responseCode; servicesTypes.responseMsg = errorResponse.responseMsg; return(servicesTypes); } if (String.IsNullOrEmpty(billId)) { servicesTypes.responseCode = "14"; servicesTypes.responseMsg = "Missing billId parameter"; return(servicesTypes); } try { var response = WebClientUtil.GetResponse(credentials.url, RemitaBillerUrl.GetServiceTypes(billId), _header); servicesTypes = JsonConvert.DeserializeObject <GetCustomFieldsResponse>(response); } catch (Exception) { throw; } return(servicesTypes); }
public static BatchGetContact GetContactList() { var reqCookies = new CookieContainer(); CookieContainer resCookies = null; reqCookies = new CookieContainer(); reqCookies.Add(new Cookie("MM_WX_NOTIFY_STATE", "1", "/", "wx.qq.com")); reqCookies.Add(new Cookie("MM_WX_SOUND_STATE", "1", "/", "wx.qq.com")); reqCookies.Add(new Cookie("last_wxuin", LoginHelper.LoginPageCookie.wxuin, "/", "wx.qq.com")); reqCookies.Add(new Cookie("login_frequency", "1", "/", "wx.qq.com")); reqCookies.Add(new Cookie("mm_lang", "zh_CN", "/", "wx.qq.com")); reqCookies.Add(new Cookie("pgv_pvi", LoginHelper.pgv_pvi, "/", "wx.qq.com")); reqCookies.Add(new Cookie("pgv_si", LoginHelper.pgv_si, "/", "wx.qq.com")); reqCookies.Add(new Cookie("webwx_auth_ticket", LoginHelper.LoginPageCookie.webwx_auth_ticket, "/", "wx.qq.com")); reqCookies.Add(new Cookie("webwx_data_ticket", LoginHelper.LoginPageCookie.webwx_data_ticket, "/", "wx.qq.com")); reqCookies.Add(new Cookie("webwxuvid", LoginHelper.LoginPageCookie.webwxuvid, "/", "wx.qq.com")); reqCookies.Add(new Cookie("wxloadtime", CalcTimeUtil.GetUnixDateTime(new TimeSpan(365, 0, 0, 0)), "/", "wx.qq.com")); reqCookies.Add(new Cookie("wxsid", LoginHelper.LoginPageCookie.wxsid, "/", "wx.qq.com")); reqCookies.Add(new Cookie("wxuin", LoginHelper.LoginPageCookie.wxuin, "/", "wx.qq.com")); var webwxgetcontactUrl = string.Format(webwxgetcontactUrlTemple, RandomUtil.GetRandom(10), LoginHelper.LoginPageXml.skey); string json = WebClientUtil.GetResponseOnCookie(webwxgetcontactUrl, "get", reqCookies, out resCookies); InitHelper.BatchGetContact = SerializeUtil.Deserialize <BatchGetContact>(json); return(InitHelper.BatchGetContact); }
private IEnumerable <string> GetPages() { ServicePointManager.Expect100Continue = false; var results = new List <string>(); //foreach (string url in Enumerable.Range(0, 10).Select(i => string.Format("http://www.set.or.th/set/newslist.do?to={0}%2F{1}%2F{2}&headline=adds+new+listed&submit=Search&symbol=¤tpage={3}&from={0}%2F{1}%2F{2}&newsType=&country=US&exchangeSymbols=&company=false&exchangeNews=on&language=en&exchange=true" // , DateTime.Now.AddDays(-13).ToString("dd"), DateTime.Now.AddDays(-13).ToString("MM"), DateTime.Now.Year, i))) foreach (string url in Enumerable.Range(0, 10).Select(i => string.Format("http://www.set.or.th/set/newslist.do?to={0}%2F{1}%2F{2}&headline=adds+new+listed&submit=Search&symbol=¤tpage={3}&from={0}%2F{1}%2F{2}&newsType=&country=US&exchangeSymbols=&company=false&exchangeNews=on&language=en&exchange=true" , DateTime.Now.AddDays(0).ToString("dd"), DateTime.Now.AddDays(0).ToString("MM"), DateTime.Now.Year, i))) { try { var htc = WebClientUtil.GetHtmlDocument(url, 300000); HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table"); HtmlNodeCollection links = tables[15].SelectNodes(".//a"); results.AddRange(from link in links select link.Attributes["href"].Value.Trim() into rawlink select rawlink.Substring(rawlink.IndexOf("?")) into rawlink select rawlink.Replace("&", "&") into rawlink select "http://www.set.or.th/set/newsdetails.do" + rawlink); } catch { break; } } return(results); }
public static void LoginPage() { var loginPageUrl = string.Format(loginPageUrlTemple, LoginHelper.Ticket, LoginHelper.UUID, LoginHelper.Scan); if (WX2) { loginPageUrl = loginPageUrl.Replace("//wx.", "//wx2."); } CookieContainer resCookies = null; CookieContainer reqCookies = null; reqCookies = new CookieContainer(); reqCookies.Add(new Cookie("MM_WX_NOTIFY_STATE", "1", "/", "wx.qq.com")); reqCookies.Add(new Cookie("MM_WX_SOUND_STATE", "zh_CN", "/", "wx.qq.com")); reqCookies.Add(new Cookie("mm_lang", "1", "/", "wx.qq.com")); reqCookies.Add(new Cookie("pgv_pvi", LoginHelper.pgv_pvi, "/", "wx.qq.com")); reqCookies.Add(new Cookie("pgv_si", LoginHelper.pgv_si, "/", "wx.qq.com")); reqCookies.Add(new Cookie("refreshTimes", "2", "/", "wx.qq.com")); LoginPageXml = LoginPageXml.Parse(WebClientUtil.GetResponseOnCookie(loginPageUrl, "get", reqCookies, out resCookies, null)); LoginPageCookie = LoginPageCookie.Parse(resCookies); DataUtil.WriteData(LoginPageCookie, LoginPageXml); }
public static string GetTickerByISIN(string isin, int type) { string url = null; if (type == 1) { url = string.Format(detailURLFormat, isin); } else if (type == 2) { url = string.Format(detailURLFormatKDR, isin); } HtmlDocument detailDoc = WebClientUtil.GetHtmlDocument(url, 180000, null, Encoding.GetEncoding("EUC-KR")); if (detailDoc == null) { return(null); } if (detailDoc.DocumentNode.SelectNodes("//table").Count < 2) { return(null); } HtmlNode tickerNode = detailDoc.DocumentNode.SelectNodes("//table")[2].SelectSingleNode(".//tr[3]/td[4]"); return(tickerNode == null ? null : tickerNode.InnerText.Trim().Substring(1).Trim()); }
/// <summary> /// Get abbreviation array from website /// </summary> private void GetAbbreviation() { HtmlDocument htc = new HtmlDocument(); string uri = String.Format("http://dataops.datastream.com/cgi-bin/readfile.pl?filename=H:/Production/Loaders/Global/DataStream/Tools/Abbreviation/Mload/abbrev.txt&warnold=1"); htc = WebClientUtil.GetHtmlDocument(uri, 300000, null); HtmlNode fullFile = htc.DocumentNode.SelectSingleNode(".//xmp[1]"); string abbrFile = fullFile.InnerText; Regex rgxSpace = new Regex(@"\s+"); string[] stringSeparators = new string[] { "\r\n" }; char[] stringSeparators2 = new char[] { ' ' }; string[] lines = abbrFile.Split(stringSeparators, StringSplitOptions.None); foreach (string line in lines) { string formattedLine = rgxSpace.Replace(line.Trim(), " "); string[] lineTab = formattedLine.Split(stringSeparators2); if (lineTab.Length >= 2) { string key = ""; for (int i = 0; i < lineTab.Length - 1; i++) { key += lineTab[i] + " "; } key = key.Trim(); if (!abbr.ContainsKey(key)) { abbr[key] = lineTab[lineTab.Length - 1]; } } } }
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteFour() { List <ETFinsertTaskTemplate> partOne = new List <ETFinsertTaskTemplate>(); HtmlDocument root = WebClientUtil.GetHtmlDocument("http://sitc.sinopac.com/web/etf/Ajax/GetTradeinfo_nav.aspx", 6000, "fundcode=37"); HtmlNodeCollection trs = root.DocumentNode.SelectNodes("//table/tbody/tr"); foreach (var tr in trs) { HtmlNodeCollection tds = tr.SelectNodes("./td"); if (tds == null || tds.Count < 4) { continue; } Regex regex = new Regex("\\d{1,}.\\d{1,}"); Match match = regex.Match(tds[2].InnerText.TrimStart().TrimEnd()); if (!match.Success) { continue; } ETFinsertTaskTemplate oneRes = new ETFinsertTaskTemplate { ReferenceValue = match.Value.TrimStart().TrimEnd() }; match = regex.Match(tds[0].InnerText.TrimStart().TrimEnd()); oneRes.Code = match.Value.TrimStart().TrimEnd(); match = regex.Match(tds[3].InnerText.TrimStart().TrimEnd()); oneRes.EstimatedValue = match.Value.TrimStart().TrimEnd(); partOne.Add(oneRes); } return(partOne); }
private void updateSZSEIndexListWithRic() { foreach (ChinaIndex index in szseIndexList) { //http://www.szse.cn/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM=399328 string url = string.Format("{0}/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM={1}", configObj.SzseBaseUri, index.Chain); //string pageSource = WebClientUtil.GetPageSource(null, url, 180000, "", Encoding.GetEncoding("gb2312")); string ricFilePath = targetDownloadFileDir() + "\\SZSE\\"; ricFilePath += string.Format("{0}.xls", index.Chain); WebClientUtil.DownloadFile(url, 180000, ricFilePath); using (ExcelApp app = new ExcelApp(false, false)) { var workbook = ExcelUtil.CreateOrOpenExcelFile(app, ricFilePath); var worksheet = workbook.Worksheets[1] as Worksheet; int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1; for (int i = 2; i <= lastUsedRow; i++) { Range aRange = ExcelUtil.GetRange(i, 1, worksheet); if (aRange != null && !string.IsNullOrEmpty(aRange.Text.ToString().Trim())) { index.RicList.Add(generateRic(aRange.Text.ToString().Trim())); } } } } }
private List <string> GetSourceFileLinks(string baseUrl) { holidayList = HolidayManager.SelectHoliday(5); List <string> linkList = new List <string>(); string pageSource = WebClientUtil.GetPageSource(baseUrl, 1800000); HtmlDocument htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(pageSource); var tableList = htmlDoc.DocumentNode.SelectNodes("//table[@class = 'styleShiryo']"); var nodeDailyList = tableList[1].SelectNodes("tr/td"); string dailyExcelLink = "http://www.tse.or.jp" + MiscUtil.GetCleanTextFromHtml(nodeDailyList[2].SelectSingleNode("a").Attributes["href"].Value); linkList.Add(dailyExcelLink); if (DateTime.Today == MiscUtil.GetNextWeeklyTradingDay(DateTime.Today, holidayList, 2) || specialCase.Equals("yes")) { var nodeWeeklyList = tableList[2].SelectNodes("tr/td"); string titleWeekly = MiscUtil.GetCleanTextFromHtml(nodeWeeklyList[0].InnerText); if (IsLastWeekInfo(titleWeekly) || specialCase.Equals("yes")) { string weeklyExcelLink = "http://www.tse.or.jp" + MiscUtil.GetCleanTextFromHtml(nodeWeeklyList[2].SelectSingleNode("a").Attributes["href"].Value); linkList.Add(weeklyExcelLink); } } return(linkList); }
public string GetPDFUrl(string id) { string pdfUrl = string.Empty; string postData = getPostData(id); string Uri = "http://www.hkexnews.hk/listedco/listconews/advancedsearch/search_active_main.aspx"; try { string pageSource = WebClientUtil.GetPageSource(Uri, 24000, postData); HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument(); htmlDoc.LoadHtml(pageSource); HtmlAgilityPack.HtmlNode pdfLinkNode = htmlDoc.DocumentNode.SelectSingleNode("//a[contains(@href, '.pdf')]"); if (pdfLinkNode == null) { Logger.Log("There's no PDF file for ric " + id); return(null); } else { pdfUrl = "http://www.hkexnews.hk"; pdfUrl += pdfLinkNode.Attributes["href"].Value; } } catch (Exception ex) { string errInfo = ex.ToString(); } return(pdfUrl); }
private String GetMatDate(String isin) { String matdate = String.Empty; try { // http://isin.krx.co.kr/jsp/BA_VW021.jsp?isu_cd=KRA631193158&modi=f&req_no=201105240094 String uri = String.Format("http://isin.krx.co.kr/jsp/BA_VW021.jsp?isu_cd={0}&modi=f&req_no=", isin); String pageSource = WebClientUtil.GetPageSource(uri, 300000); if (!String.IsNullOrEmpty(pageSource)) { HtmlDocument htc = new HtmlDocument(); htc.LoadHtml(pageSource); if (htc != null) { HtmlNode node = htc.DocumentNode.SelectSingleNode(".//tr[6]/td[4]"); if (node != null) { matdate = node.InnerText.Trim().ToString(); } } } } catch (Exception ex) { String msg = "Error found in GetMatDate() : \r\n" + ex.ToString(); Logger.Log(msg, Logger.LogType.Error); } return(matdate); }
private void RetryMechanism(string downloadPath, string savePath, int retryTimes, int waitSecond) { for (int i = 0; i < retryTimes; i++) { try { if (File.Exists(savePath)) { break; } WebClientUtil.DownloadFile(downloadPath, 5000, savePath); if (File.Exists(savePath)) { break; } } catch (Exception ex) { Thread.Sleep(waitSecond); if (i == retryTimes - 1) { string msg = string.Format("url:{0} retryTimes:{1} msg:{2}", downloadPath, retryTimes.ToString(), ex.Message); Logger.Log(msg, Logger.LogType.Error); } } } }
public string GetPDFToTxtFilePath(string codeId, FMType fmType) { string txtFilePath = string.Empty; string pdfUrl = GetPDFUrl(codeId); string pdfFilePath = string.Empty; if (pdfUrl == null) { return(null); } if (fmType == FMType.Cbbc) { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "CBBC" + "\\" + codeId + ".pdf"; } else { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "Warrant" + "\\" + codeId + ".pdf"; } if (!Directory.Exists(Path.GetDirectoryName(pdfFilePath))) { Directory.CreateDirectory(Path.GetDirectoryName(pdfFilePath)); } WebClientUtil.DownloadFile(pdfUrl, 24000, pdfFilePath); txtFilePath = PDFToTxt(codeId, pdfFilePath, fmType); return(txtFilePath); }
private void downloadAndParseIndexFile(string url) { string szseIndexFilePath = targetDownloadFileDir(); szseIndexFilePath += "\\SZSE"; if (!Directory.Exists(szseIndexFilePath)) { Directory.CreateDirectory(szseIndexFilePath); } szseIndexFilePath += "\\Index.xls"; WebClientUtil.DownloadFile(url, 180000, szseIndexFilePath); using (ExcelApp app = new ExcelApp(false, false)) { var workbook = ExcelUtil.CreateOrOpenExcelFile(app, szseIndexFilePath); var worksheet = workbook.Worksheets[1] as Worksheet; int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1; for (int i = 2; i <= lastUsedRow; i++) { Range aRange = ExcelUtil.GetRange(i, 1, worksheet); if (aRange != null && aRange.Text.ToString().Trim() != string.Empty) { ChinaIndex index = new ChinaIndex(); index.Chain = aRange.Text.ToString().Trim(); index.ChineseName = ExcelUtil.GetRange(i, 2, worksheet).Text.ToString(); szseIndexList.Add(index); } } } }
private void GrabDataByISIN() { List <String> lists = configObj.ISINList.ToList(); if (lists.Count > 0) { foreach (var item in lists) { if (String.IsNullOrEmpty(item)) { continue; } String uri = String.Format("http://isin.krx.co.kr/jsp/BA_VW021.jsp?isu_cd={0}&modi=f&req_no=", item); HtmlDocument htc = WebClientUtil.GetHtmlDocument(uri, 300000); if (htc != null) { HtmlNode table = htc.DocumentNode.SelectNodes("//table")[2]; if (table != null) { String tr5_td2 = table.SelectSingleNode(".//tr[5]/td[2]").InnerText.Trim().ToString(); //String tr3_td2 = table.SelectSingleNode(".//tr[3]/td[2]").InnerText.Trim().ToString(); if (tr5_td2.Equals("사모")) { continue; } WarrantTemplate wt = new WarrantTemplate(); wt.ISIN = item.Trim().ToString(); DataAnalysisForSingleSearch(table, wt); koreaList.Add(wt); } } } } }
private IEnumerable <ETFinsertTaskTemplate> GrabDataFromSiteFive() { List <ETFinsertTaskTemplate> partOne = new List <ETFinsertTaskTemplate>(); HtmlDocument root = WebClientUtil.GetHtmlDocument("https://www.kgifund.com.tw/50_05.asp", 6000); HtmlNodeCollection trs = root.DocumentNode.SelectNodes("/html[1]/body[1]/table[1]/tr[1]/td[2]/table[1]/tr[3]/td[2]/table[1]/tr[2]/td[1]/table[1]/tr[2]/td[2]/div[1]/table[1]/tr[3]/td[1]/table[1]/tr[2]/td[1]/table[1]/tr"); HtmlNode codeNode = root.DocumentNode.SelectSingleNode("/html[1]/body[1]/table[1]/tr[1]/td[2]/table[1]/tr[3]/td[2]/table[1]/tr[2]/td[1]/table[1]/tr[2]/td[2]/div[1]/table[1]/tr[3]/td[1]/table[1]/tr[1]/td[1]/table[1]/tr"); ETFinsertTaskTemplate oneRes = new ETFinsertTaskTemplate(); Regex regex = new Regex("\\d{1,}"); Match match = regex.Match(codeNode.InnerText.TrimStart().TrimEnd()); oneRes.Code = match.Value; foreach (var tr in trs) { if (tr.InnerText.Contains("昨日收盤淨值(台幣)")) { regex = new Regex("\\d{1,}.\\d{1,}"); match = regex.Match(tr.InnerText.TrimStart().TrimEnd()); oneRes.ReferenceValue = match.Value; } if (tr.InnerText.Contains("盤中預估淨值(台幣)")) { regex = new Regex("\\d{1,}.\\d{1,}"); match = regex.Match(tr.InnerText.TrimStart().TrimEnd()); oneRes.EstimatedValue = match.Value; } } partOne.Add(oneRes); return(partOne); }
private string DownloadFile(string linkUrl) { string[] part = linkUrl.Split('/'); string filePath = Path.Combine(sourceFileDir, part[part.Length - 1]); WebClientUtil.DownloadFile(linkUrl, 180000, filePath); return(filePath); }
private void GrabDataFromInnerHtm(string url) { count++; HtmlAgilityPack.HtmlDocument htmlDoc = WebClientUtil.GetHtmlDocument(url, 18000); string txtFile = htmlDoc.DocumentNode.SelectSingleNode("//pre").InnerText; DataAnalysis(txtFile); }
private List <List <string> > GrabTodayData() { //string postData = string.Format("method=searchPubofrProgComSub¤tPageSize=50&pageIndex=1&orderMode=1&orderStat=D&searchMode=1&searchCodeType=&isurCd=&repIsuSrtCd=&bzProcsNo=&detailMarket=&forward=pubofrprogcom_sub&marketType=&searchCorpName=&fromDate={0}&toDate={1}", // DateTime.Now.AddMonths(-6).ToString("yyyy-MM-dd"), DateTime.Now.ToString("yyyy-MM-dd")); ////string source = WebClientUtil.GetPageSource(null, "http://kind.krx.co.kr/listinvstg/pubofrprogcom.do", 40000, postData, Encoding.UTF8); //string source = WebClientUtil.GetPageSource(null, "http://kind.krx.co.kr/listinvstg/pubofrprogcom.do?method=searchPubofrProgComMain", 40000, postData, Encoding.UTF8); //HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument(); //document.LoadHtml(source); string url = @"http://kind.krx.co.kr/listinvstg/pubofrprogcom.do?method=searchPubofrProgComMain"; HtmlDocument document = new HtmlDocument(); HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; request.Timeout = 300000; request.Method = "GET"; request.CookieContainer = cookies; request.Host = @"kind.krx.co.kr"; request.Accept = @"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"; request.Headers["Accept-Encoding"] = @"gzip,deflate,sdch"; request.Headers["Accept-Language"] = @"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4"; request.Headers["Cache-Control"] = @"max-age=0"; request.KeepAlive = true; request.UserAgent = @"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36"; HttpWebResponse response = (HttpWebResponse)request.GetResponse(); StreamReader sr = new StreamReader(response.GetResponseStream()); string st = sr.ReadToEnd(); url = @"http://203.235.1.91/tracker.jsp?u=4&XU=&EXEN=5&dr=&XDR=&dt=%EB%8C%80%ED%95%9C%EB%AF%BC%EA%B5%AD%20%EB%8C%80%ED%91%9C%20%EA%B8%B0%EC%97%85%EA%B3%B5%EC%8B%9C%EC%B1%84%EB%84%90%20KIND&du=http%3A%2F%2Fkind.krx.co.kr%2Flistinvstg%2Fpubofrprogcom.do%3Fmethod=searchPubofrProgComMain&SX=M&js=Y&ss=1280x1024&cd=32&ce=Y&je=Y&tzo=-480&tye=2014&tmo=7&tda=8&tho=16&tmi=4&tse=34"; request = WebRequest.Create(url) as HttpWebRequest; request.Timeout = 300000; request.Method = "GET"; request.CookieContainer = cookies; request.Headers["Accept-Encoding"] = @"gzip,deflate,sdch"; request.Headers["Accept-Language"] = @"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4"; request.Headers["Cache-Control"] = @"max-age=0"; request.Host = @"203.235.1.91"; request.Referer = @"http://kind.krx.co.kr/listinvstg/pubofrprogcom.do?method=searchPubofrProgComMain"; request.UserAgent = @"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36"; request.Accept = @"image/webp,*/*;q=0.8"; response = (HttpWebResponse)request.GetResponse(); sr = new StreamReader(response.GetResponseStream()); st = sr.ReadToEnd(); string postData = string.Format("method=searchPubofrProgComSub¤tPageSize=50&pageIndex=1&orderMode=1&orderStat=D&searchMode=1&searchCodeType=&isurCd=&repIsuSrtCd=&bzProcsNo=&detailMarket=&forward=pubofrprogcom_sub&marketType=&searchCorpName=&fromDate={0}&toDate={1}", DateTime.Now.AddMonths(-6).ToString("yyyy-MM-dd"), DateTime.Now.ToString("yyyy-MM-dd")); string source = WebClientUtil.GetPageSource(null, "http://kind.krx.co.kr/listinvstg/pubofrprogcom.do", 40000, postData, Encoding.UTF8); document.LoadHtml(source); List <List <string> > records = GetHtmlDataNeeded(document.DocumentNode); return(records); }
private void getSZSEIndexList() { string pageSource = WebClientUtil.GetPageSource(configObj.SzseBaseUri, 180000, ""); var doc = WebClientUtil.GetHtmlDocument(string.Format("{0}/main/marketdata/hqcx/zsybg/", configObj.SzseBaseUri), 180000, "", Encoding.GetEncoding("gb2312")); string szseIndexSourceFileUrl = MiscUtil.GetCleanTextFromHtml(doc.DocumentNode.SelectNodes("//td[@align='right']/a")[0].Attributes["href"].Value); downloadAndParseIndexFile(string.Format("{0}{1}", configObj.SzseBaseUri, szseIndexSourceFileUrl)); updateSZSEIndexListWithRic(); }
private HtmlNode GetHtmlSource() { string source = WebClientUtil.GetPageSource(null, "http://isin.twse.com.tw/isin/C_public.jsp?strMode=6", 180000, "strMode=6", Encoding.GetEncoding("big5")); var document = new HtmlDocument(); document.LoadHtml(source); return(document.DocumentNode); }
private static List <ISINTemp> getISINListFromSinglePage(string uri, string postData) { List <ISINTemp> isinList = new List <ISINTemp>(); AdvancedWebClient wc = new AdvancedWebClient(); HtmlDocument htc = new HtmlDocument(); string pageSource = WebClientUtil.GetPageSource(wc, uri, 300000, postData); isinList = getISINListFromSinglePage(pageSource); return(isinList); }
public MyBlogDialog(NewMainWindow suc, Shortcut shortcut, String url) { InitializeComponent(); Title = "ThirdPartyPage"; this.shortcut = shortcut; WebClientUtil.WebToModel(url, ref blogPageModel); InitData(); }