public string GetPDFToTxtFilePath(string codeId, FMType fmType) { string txtFilePath = string.Empty; string pdfUrl = GetPDFUrl(codeId); string pdfFilePath = string.Empty; if (pdfUrl == null) { return(null); } if (fmType == FMType.Cbbc) { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "CBBC" + "\\" + codeId + ".pdf"; } else { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "Warrant" + "\\" + codeId + ".pdf"; } if (!Directory.Exists(Path.GetDirectoryName(pdfFilePath))) { Directory.CreateDirectory(Path.GetDirectoryName(pdfFilePath)); } WebClientUtil.DownloadFile(pdfUrl, 24000, pdfFilePath); txtFilePath = PDFToTxt(codeId, pdfFilePath, fmType); return(txtFilePath); }
private void downloadAndParseIndexFile(string url) { string szseIndexFilePath = targetDownloadFileDir(); szseIndexFilePath += "\\SZSE"; if (!Directory.Exists(szseIndexFilePath)) { Directory.CreateDirectory(szseIndexFilePath); } szseIndexFilePath += "\\Index.xls"; WebClientUtil.DownloadFile(url, 180000, szseIndexFilePath); using (ExcelApp app = new ExcelApp(false, false)) { var workbook = ExcelUtil.CreateOrOpenExcelFile(app, szseIndexFilePath); var worksheet = workbook.Worksheets[1] as Worksheet; int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1; for (int i = 2; i <= lastUsedRow; i++) { Range aRange = ExcelUtil.GetRange(i, 1, worksheet); if (aRange != null && aRange.Text.ToString().Trim() != string.Empty) { ChinaIndex index = new ChinaIndex(); index.Chain = aRange.Text.ToString().Trim(); index.ChineseName = ExcelUtil.GetRange(i, 2, worksheet).Text.ToString(); szseIndexList.Add(index); } } } }
private void RetryMechanism(string downloadPath, string savePath, int retryTimes, int waitSecond) { for (int i = 0; i < retryTimes; i++) { try { if (File.Exists(savePath)) { break; } WebClientUtil.DownloadFile(downloadPath, 5000, savePath); if (File.Exists(savePath)) { break; } } catch (Exception ex) { Thread.Sleep(waitSecond); if (i == retryTimes - 1) { string msg = string.Format("url:{0} retryTimes:{1} msg:{2}", downloadPath, retryTimes.ToString(), ex.Message); Logger.Log(msg, Logger.LogType.Error); } } } }
private void updateSZSEIndexListWithRic() { foreach (ChinaIndex index in szseIndexList) { //http://www.szse.cn/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM=399328 string url = string.Format("{0}/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM={1}", configObj.SzseBaseUri, index.Chain); //string pageSource = WebClientUtil.GetPageSource(null, url, 180000, "", Encoding.GetEncoding("gb2312")); string ricFilePath = targetDownloadFileDir() + "\\SZSE\\"; ricFilePath += string.Format("{0}.xls", index.Chain); WebClientUtil.DownloadFile(url, 180000, ricFilePath); using (ExcelApp app = new ExcelApp(false, false)) { var workbook = ExcelUtil.CreateOrOpenExcelFile(app, ricFilePath); var worksheet = workbook.Worksheets[1] as Worksheet; int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1; for (int i = 2; i <= lastUsedRow; i++) { Range aRange = ExcelUtil.GetRange(i, 1, worksheet); if (aRange != null && !string.IsNullOrEmpty(aRange.Text.ToString().Trim())) { index.RicList.Add(generateRic(aRange.Text.ToString().Trim())); } } } } }
private string DownloadFile(string linkUrl) { string[] part = linkUrl.Split('/'); string filePath = Path.Combine(sourceFileDir, part[part.Length - 1]); WebClientUtil.DownloadFile(linkUrl, 180000, filePath); return(filePath); }
//Download csv file from page http://www.hkex.com.hk/eng/plw/plw_search.asp public void DownLoadSCVFile() { string uri = GetCSVFileSrc(); Thread.Sleep(2000); if (File.Exists(configObj.CURRENT_SEHKFILE_PATH)) { MiscUtil.BackUpFile(configObj.CURRENT_SEHKFILE_PATH); File.Delete(configObj.CURRENT_SEHKFILE_PATH); } WebClientUtil.DownloadFile(uri, 180000, configObj.CURRENT_SEHKFILE_PATH); }
//To Do: private List <FID> getPerPageFIDList(string pageSource, string type) { List <FID> FIDList = new List <FID>(); if (pageSource == null) { return(FIDList); } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(pageSource); HtmlNodeCollection node = doc.DocumentNode.SelectNodes("//div")[3].SelectNodes(".//table")[0].SelectNodes(".//tr"); if (node.Count > 5) { for (int i = 3; i < node.Count - 3; i++) { FID fid = new FID(); HtmlNode trNode = node[i]; var tdNodeList = trNode.SelectNodes("td"); string[] date = GetCleanTextFromHtml(tdNodeList[0].InnerText.ToString()).Split('/'); string releaseYear = date[2].Substring(0, 4); string time = date[2].Substring(4).Replace(":", ""); fid.WebLink = "http://www.hkexnews.hk" + GetCleanTextFromHtml(tdNodeList[3].SelectSingleNode("a").Attributes["href"].Value); fid.FileName = GetCleanTextFromHtml(tdNodeList[3].InnerText.ToString().Split(']')[1].Split('(')[0]) + "_" + releaseYear + date[1] + date[0] + "_" + time + ".xls"; while (fid.FileName.Contains("/")) { fid.FileName = fid.FileName.Replace("/", "-"); } if (type == "CBBC") { fid.LocalPath = Path.Combine(string.Format("{0}\\{1}", configObj.CBBC_DOWNLOAD_FOLDER, ("CBBC_" + downloadTime)), fid.FileName); } if (type == "Warrant") { fid.LocalPath = Path.Combine(string.Format("{0}\\{1}", configObj.WARRANT_DOWNLOAD_FOLDER, ("Warrant_" + downloadTime)), fid.FileName); } WebClientUtil.DownloadFile(fid.WebLink, 180000, fid.LocalPath); FIDList.Add(fid); } } return(FIDList); }
/// <summary> /// Download Excel for update /// </summary> private void DownloadSecondFile() { try { HtmlDocument htc = new HtmlDocument(); string uri = String.Format("http://www.tse.or.jp/market/data/margin/index.html"); htc = WebClientUtil.GetHtmlDocument(uri, 300000, null); HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table"); int trNb = tables[4].SelectNodes(".//tr").Count; string source2Url = tables[4].SelectSingleNode(String.Format(".//tr[{0}]/td[2]/a", trNb)).Attributes["href"].Value.Trim(); source2Url = baseUrl + source2Url; WebClientUtil.DownloadFile(source2Url, 100000, Path.GetDirectoryName(configObj.ResultFilePath) + @"\test2.xls"); } catch (Exception ex) { string msg = "Error :" + ex; Logger.Log(msg, Logger.LogType.Error); } }
private void downloadSourceFiles() { string fileUrl = null; try { HtmlDocument htc = new HtmlDocument(); string uri = String.Format("http://www.ose.or.jp/market/trading_data/open_interest_by_participant"); htc = WebClientUtil.GetHtmlDocument(uri, 300000, null); HtmlNodeCollection tr = htc.DocumentNode.SelectNodes("//tr/td/a"); foreach (HtmlNode node in tr) { if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_futures_OI_by_participant.xls")) { fileUrl = node.Attributes["href"].Value; WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_futures_OI_by_participant.xls"); } else if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_options_OI_by_participant.xls")) { fileUrl = node.Attributes["href"].Value; WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_options_OI_by_participant.xls"); } else if (node.Attributes["href"].Value.Contains(configObj.Date + "_individual_options_OI_by_participant.xls")) { fileUrl = node.Attributes["href"].Value; WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_individual_options_OI_by_participant.xls"); } } } catch (Exception ex) { Logger.Log("Error when downloading files from website" + ex.Message); LogMessage("Error when downloading files from website"); } finally { } }
private List <string> getRicsForCSI(string xlsFileUrl) { List <string> ricList = new List <string>(); string xlsFilePath = targetDownloadFileDir(); xlsFilePath += "\\CSI"; if (!Directory.Exists(xlsFilePath)) { Directory.CreateDirectory(xlsFilePath); } string[] subLinkNode = xlsFileUrl.Split('/'); xlsFilePath += "\\" + subLinkNode[subLinkNode.Length - 1]; WebClientUtil.DownloadFile(xlsFileUrl, 180000, xlsFilePath); using (ExcelApp app = new ExcelApp(false, false)) { var workbook = ExcelUtil.CreateOrOpenExcelFile(app, xlsFilePath); var worksheet = workbook.Worksheets[1] as Worksheet; int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1; for (int i = 2; i <= lastUsedRow; i++) { Range aRange = ExcelUtil.GetRange(i, 1, worksheet); if (aRange != null && aRange.Text.ToString().Trim() != string.Empty) { string officialCode = aRange.Text.ToString(); int temp = -1; if (int.TryParse(officialCode, out temp)) { officialCode = temp.ToString("D6"); } ricList.Add(generateRic(officialCode)); } } } return(ricList); }
private List <string> GetDownloadFilePath() { List <string> result = new List <string>(); HtmlDocument htc = null; string baseUrl = @"http://www.tfex.co.th"; string downloadFolder = Path.Combine(configObj.FolderPath, "Download"); try { RetryUtil.Retry(5, TimeSpan.FromSeconds(2), true, delegate { htc = WebClientUtil.GetHtmlDocument(sourceUrl, 3000); }); if (htc == null) { throw new Exception(string.Format("open website: {0} error.", sourceUrl)); } HtmlNodeCollection trs = htc.DocumentNode.SelectNodes(".//table")[7].SelectNodes(".//tr"); List <string> urlFoundAll = GetUrlFoundAll(trs); if (!Directory.Exists(downloadFolder)) { Directory.CreateDirectory(downloadFolder); } string tdNewSeries = GetUrlFoundOne(urlFoundAll); if ((tdNewSeries + "").Trim().Length == 0) { return(null); } string dexsrp = Path.Combine(downloadFolder, string.Format("dexsrp{0}.txt", DateTimeConvert(configObj.DateOfSource, "yyyy-MM-dd", "yyyyMMdd"))); if (File.Exists(dexsrp)) { File.Delete(dexsrp); } RetryUtil.Retry(5, TimeSpan.FromSeconds(2), true, delegate { WebClientUtil.DownloadFile(string.Format("{0}{1}", baseUrl, tdNewSeries), 30000, dexsrp); }); if (File.Exists(dexsrp)) { result.Add(dexsrp); } } catch (Exception ex) { string msg = string.Format("\r\n ClassName: {0}\r\n MethodName: {1}\r\n Message: {2}", System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.ToString(), System.Reflection.MethodBase.GetCurrentMethod().Name, ex.Message); Logger.Log(msg, Logger.LogType.Error); } return(result); }
public string GetPDFToTxtFilePath(string codeId, FMType fmType) { string txtFilePath = string.Empty; string pdfUrl = null; for (int i = 0; i < 10; i++) //as website problem, if get pdf url error retry 10 times { try { pdfUrl = GetPDFUrl(codeId); if (!string.IsNullOrEmpty(pdfUrl)) { break; } } catch (Exception ex) { Thread.Sleep(2000); if (i == 9) { string msg = string.Format("get pdf url :{0} error,retry get 9 times.\r\nmsg:{0}", pdfUrl, ex.Message); Logger.Log(msg, Logger.LogType.Error); } } } string pdfFilePath = string.Empty; if (pdfUrl == null) { return(null); } if (fmType == FMType.Cbbc) { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "CBBC" + "\\" + codeId + ".pdf"; } else { pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "Warrant" + "\\" + codeId + ".pdf"; } if (!Directory.Exists(Path.GetDirectoryName(pdfFilePath))) { Directory.CreateDirectory(Path.GetDirectoryName(pdfFilePath)); } for (int i = 0; i < 5; i++) //as website problem, if download pdf error retry 5 times { try { WebClientUtil.DownloadFile(pdfUrl, 24000, pdfFilePath); if (File.Exists(pdfFilePath)) { break; } } catch (Exception ex) { Thread.Sleep(2000); if (i == 4) { string msg = string.Format("Download pdf:{0} error,retry download 5 times.\r\nmsg:{0}", pdfUrl, ex.Message); Logger.Log(msg, Logger.LogType.Error); } } } return(pdfFilePath); }
private void DownloadFurtherFile() { WebClientUtil.DownloadFile("http://www.hkex.com.hk/eng/cbbc/furtherissue/fissue.xls", 1000, futherCBBCFilePath); WebClientUtil.DownloadFile("http://www.hkex.com.hk/eng/dwrc/furtherissue/fissue.xls", 1000, futherDWRCFilePath); }