public string GetPDFToTxtFilePath(string codeId, FMType fmType)
{
string txtFilePath = string.Empty;
string pdfUrl = GetPDFUrl(codeId);
string pdfFilePath = string.Empty;
if (pdfUrl == null)
{
return(null);
}
if (fmType == FMType.Cbbc)
{
pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "CBBC" + "\\" + codeId + ".pdf";
}
else
{
pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "Warrant" + "\\" + codeId + ".pdf";
}
if (!Directory.Exists(Path.GetDirectoryName(pdfFilePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(pdfFilePath));
}
WebClientUtil.DownloadFile(pdfUrl, 24000, pdfFilePath);
txtFilePath = PDFToTxt(codeId, pdfFilePath, fmType);
return(txtFilePath);
}
private void downloadAndParseIndexFile(string url)
{
string szseIndexFilePath = targetDownloadFileDir();
szseIndexFilePath += "\\SZSE";
if (!Directory.Exists(szseIndexFilePath))
{
Directory.CreateDirectory(szseIndexFilePath);
}
szseIndexFilePath += "\\Index.xls";
WebClientUtil.DownloadFile(url, 180000, szseIndexFilePath);
using (ExcelApp app = new ExcelApp(false, false))
{
var workbook = ExcelUtil.CreateOrOpenExcelFile(app, szseIndexFilePath);
var worksheet = workbook.Worksheets[1] as Worksheet;
int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1;
for (int i = 2; i <= lastUsedRow; i++)
{
Range aRange = ExcelUtil.GetRange(i, 1, worksheet);
if (aRange != null && aRange.Text.ToString().Trim() != string.Empty)
{
ChinaIndex index = new ChinaIndex();
index.Chain = aRange.Text.ToString().Trim();
index.ChineseName = ExcelUtil.GetRange(i, 2, worksheet).Text.ToString();
szseIndexList.Add(index);
}
}
}
}
private void RetryMechanism(string downloadPath, string savePath, int retryTimes, int waitSecond)
{
for (int i = 0; i < retryTimes; i++)
{
try
{
if (File.Exists(savePath))
{
break;
}
WebClientUtil.DownloadFile(downloadPath, 5000, savePath);
if (File.Exists(savePath))
{
break;
}
}
catch (Exception ex)
{
Thread.Sleep(waitSecond);
if (i == retryTimes - 1)
{
string msg = string.Format("url:{0} retryTimes:{1} msg:{2}", downloadPath, retryTimes.ToString(), ex.Message);
Logger.Log(msg, Logger.LogType.Error);
}
}
}
}
private void updateSZSEIndexListWithRic()
{
foreach (ChinaIndex index in szseIndexList)
{
//http://www.szse.cn/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM=399328
string url = string.Format("{0}/szseWeb/FrontController.szse?ACTIONID=8&CATALOGID=1747&TABKEY=tab1&ENCODE=1&ZSDM={1}", configObj.SzseBaseUri, index.Chain);
//string pageSource = WebClientUtil.GetPageSource(null, url, 180000, "", Encoding.GetEncoding("gb2312"));
string ricFilePath = targetDownloadFileDir() + "\\SZSE\\";
ricFilePath += string.Format("{0}.xls", index.Chain);
WebClientUtil.DownloadFile(url, 180000, ricFilePath);
using (ExcelApp app = new ExcelApp(false, false))
{
var workbook = ExcelUtil.CreateOrOpenExcelFile(app, ricFilePath);
var worksheet = workbook.Worksheets[1] as Worksheet;
int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1;
for (int i = 2; i <= lastUsedRow; i++)
{
Range aRange = ExcelUtil.GetRange(i, 1, worksheet);
if (aRange != null && !string.IsNullOrEmpty(aRange.Text.ToString().Trim()))
{
index.RicList.Add(generateRic(aRange.Text.ToString().Trim()));
}
}
}
}
}
private string DownloadFile(string linkUrl)
{
string[] part = linkUrl.Split('/');
string filePath = Path.Combine(sourceFileDir, part[part.Length - 1]);
WebClientUtil.DownloadFile(linkUrl, 180000, filePath);
return(filePath);
}
//Download csv file from page http://www.hkex.com.hk/eng/plw/plw_search.asp
public void DownLoadSCVFile()
{
string uri = GetCSVFileSrc();
Thread.Sleep(2000);
if (File.Exists(configObj.CURRENT_SEHKFILE_PATH))
{
MiscUtil.BackUpFile(configObj.CURRENT_SEHKFILE_PATH);
File.Delete(configObj.CURRENT_SEHKFILE_PATH);
}
WebClientUtil.DownloadFile(uri, 180000, configObj.CURRENT_SEHKFILE_PATH);
}
//To Do:
private List <FID> getPerPageFIDList(string pageSource, string type)
{
List <FID> FIDList = new List <FID>();
if (pageSource == null)
{
return(FIDList);
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(pageSource);
HtmlNodeCollection node = doc.DocumentNode.SelectNodes("//div")[3].SelectNodes(".//table")[0].SelectNodes(".//tr");
if (node.Count > 5)
{
for (int i = 3; i < node.Count - 3; i++)
{
FID fid = new FID();
HtmlNode trNode = node[i];
var tdNodeList = trNode.SelectNodes("td");
string[] date = GetCleanTextFromHtml(tdNodeList[0].InnerText.ToString()).Split('/');
string releaseYear = date[2].Substring(0, 4);
string time = date[2].Substring(4).Replace(":", "");
fid.WebLink = "http://www.hkexnews.hk" + GetCleanTextFromHtml(tdNodeList[3].SelectSingleNode("a").Attributes["href"].Value);
fid.FileName = GetCleanTextFromHtml(tdNodeList[3].InnerText.ToString().Split(']')[1].Split('(')[0]) + "_" + releaseYear + date[1] + date[0] + "_" + time + ".xls";
while (fid.FileName.Contains("/"))
{
fid.FileName = fid.FileName.Replace("/", "-");
}
if (type == "CBBC")
{
fid.LocalPath = Path.Combine(string.Format("{0}\\{1}", configObj.CBBC_DOWNLOAD_FOLDER, ("CBBC_" + downloadTime)), fid.FileName);
}
if (type == "Warrant")
{
fid.LocalPath = Path.Combine(string.Format("{0}\\{1}", configObj.WARRANT_DOWNLOAD_FOLDER, ("Warrant_" + downloadTime)), fid.FileName);
}
WebClientUtil.DownloadFile(fid.WebLink, 180000, fid.LocalPath);
FIDList.Add(fid);
}
}
return(FIDList);
}
/// <summary>
/// Download Excel for update
/// </summary>
private void DownloadSecondFile()
{
try
{
HtmlDocument htc = new HtmlDocument();
string uri = String.Format("http://www.tse.or.jp/market/data/margin/index.html");
htc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
HtmlNodeCollection tables = htc.DocumentNode.SelectNodes(".//table");
int trNb = tables[4].SelectNodes(".//tr").Count;
string source2Url = tables[4].SelectSingleNode(String.Format(".//tr[{0}]/td[2]/a", trNb)).Attributes["href"].Value.Trim();
source2Url = baseUrl + source2Url;
WebClientUtil.DownloadFile(source2Url, 100000, Path.GetDirectoryName(configObj.ResultFilePath) + @"\test2.xls");
}
catch (Exception ex)
{
string msg = "Error :" + ex;
Logger.Log(msg, Logger.LogType.Error);
}
}
private void downloadSourceFiles()
{
string fileUrl = null;
try
{
HtmlDocument htc = new HtmlDocument();
string uri = String.Format("http://www.ose.or.jp/market/trading_data/open_interest_by_participant");
htc = WebClientUtil.GetHtmlDocument(uri, 300000, null);
HtmlNodeCollection tr = htc.DocumentNode.SelectNodes("//tr/td/a");
foreach (HtmlNode node in tr)
{
if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_futures_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_futures_OI_by_participant.xls");
}
else if (node.Attributes["href"].Value.Contains(configObj.Date + "_index_options_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_index_options_OI_by_participant.xls");
}
else if (node.Attributes["href"].Value.Contains(configObj.Date + "_individual_options_OI_by_participant.xls"))
{
fileUrl = node.Attributes["href"].Value;
WebClientUtil.DownloadFile(@"http://www.ose.or.jp/" + fileUrl, 18000, configObj.DownloadFilePath + "\\" + configObj.Date + "_individual_options_OI_by_participant.xls");
}
}
}
catch (Exception ex)
{
Logger.Log("Error when downloading files from website" + ex.Message);
LogMessage("Error when downloading files from website");
}
finally
{
}
}
private List <string> getRicsForCSI(string xlsFileUrl)
{
List <string> ricList = new List <string>();
string xlsFilePath = targetDownloadFileDir();
xlsFilePath += "\\CSI";
if (!Directory.Exists(xlsFilePath))
{
Directory.CreateDirectory(xlsFilePath);
}
string[] subLinkNode = xlsFileUrl.Split('/');
xlsFilePath += "\\" + subLinkNode[subLinkNode.Length - 1];
WebClientUtil.DownloadFile(xlsFileUrl, 180000, xlsFilePath);
using (ExcelApp app = new ExcelApp(false, false))
{
var workbook = ExcelUtil.CreateOrOpenExcelFile(app, xlsFilePath);
var worksheet = workbook.Worksheets[1] as Worksheet;
int lastUsedRow = worksheet.UsedRange.Row + worksheet.UsedRange.Rows.Count - 1;
for (int i = 2; i <= lastUsedRow; i++)
{
Range aRange = ExcelUtil.GetRange(i, 1, worksheet);
if (aRange != null && aRange.Text.ToString().Trim() != string.Empty)
{
string officialCode = aRange.Text.ToString();
int temp = -1;
if (int.TryParse(officialCode, out temp))
{
officialCode = temp.ToString("D6");
}
ricList.Add(generateRic(officialCode));
}
}
}
return(ricList);
}
private List <string> GetDownloadFilePath()
{
List <string> result = new List <string>();
HtmlDocument htc = null;
string baseUrl = @"http://www.tfex.co.th";
string downloadFolder = Path.Combine(configObj.FolderPath, "Download");
try
{
RetryUtil.Retry(5, TimeSpan.FromSeconds(2), true, delegate
{
htc = WebClientUtil.GetHtmlDocument(sourceUrl, 3000);
});
if (htc == null)
{
throw new Exception(string.Format("open website: {0} error.", sourceUrl));
}
HtmlNodeCollection trs = htc.DocumentNode.SelectNodes(".//table")[7].SelectNodes(".//tr");
List <string> urlFoundAll = GetUrlFoundAll(trs);
if (!Directory.Exists(downloadFolder))
{
Directory.CreateDirectory(downloadFolder);
}
string tdNewSeries = GetUrlFoundOne(urlFoundAll);
if ((tdNewSeries + "").Trim().Length == 0)
{
return(null);
}
string dexsrp = Path.Combine(downloadFolder, string.Format("dexsrp{0}.txt", DateTimeConvert(configObj.DateOfSource, "yyyy-MM-dd", "yyyyMMdd")));
if (File.Exists(dexsrp))
{
File.Delete(dexsrp);
}
RetryUtil.Retry(5, TimeSpan.FromSeconds(2), true, delegate
{
WebClientUtil.DownloadFile(string.Format("{0}{1}", baseUrl, tdNewSeries), 30000, dexsrp);
});
if (File.Exists(dexsrp))
{
result.Add(dexsrp);
}
}
catch (Exception ex)
{
string msg = string.Format("\r\n ClassName: {0}\r\n MethodName: {1}\r\n Message: {2}",
System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.ToString(),
System.Reflection.MethodBase.GetCurrentMethod().Name,
ex.Message);
Logger.Log(msg, Logger.LogType.Error);
}
return(result);
}
public string GetPDFToTxtFilePath(string codeId, FMType fmType)
{
string txtFilePath = string.Empty;
string pdfUrl = null;
for (int i = 0; i < 10; i++) //as website problem, if get pdf url error retry 10 times
{
try
{
pdfUrl = GetPDFUrl(codeId);
if (!string.IsNullOrEmpty(pdfUrl))
{
break;
}
}
catch (Exception ex)
{
Thread.Sleep(2000);
if (i == 9)
{
string msg = string.Format("get pdf url :{0} error,retry get 9 times.\r\nmsg:{0}", pdfUrl, ex.Message);
Logger.Log(msg, Logger.LogType.Error);
}
}
}
string pdfFilePath = string.Empty;
if (pdfUrl == null)
{
return(null);
}
if (fmType == FMType.Cbbc)
{
pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "CBBC" + "\\" + codeId + ".pdf";
}
else
{
pdfFilePath = configObj.OutPutDir + "\\" + pdfFileDir + "\\" + "Warrant" + "\\" + codeId + ".pdf";
}
if (!Directory.Exists(Path.GetDirectoryName(pdfFilePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(pdfFilePath));
}
for (int i = 0; i < 5; i++) //as website problem, if download pdf error retry 5 times
{
try
{
WebClientUtil.DownloadFile(pdfUrl, 24000, pdfFilePath);
if (File.Exists(pdfFilePath))
{
break;
}
}
catch (Exception ex)
{
Thread.Sleep(2000);
if (i == 4)
{
string msg = string.Format("Download pdf:{0} error,retry download 5 times.\r\nmsg:{0}", pdfUrl, ex.Message);
Logger.Log(msg, Logger.LogType.Error);
}
}
}
return(pdfFilePath);
}
private void DownloadFurtherFile()
{
WebClientUtil.DownloadFile("http://www.hkex.com.hk/eng/cbbc/furtherissue/fissue.xls", 1000, futherCBBCFilePath);
WebClientUtil.DownloadFile("http://www.hkex.com.hk/eng/dwrc/furtherissue/fissue.xls", 1000, futherDWRCFilePath);
}
