private static void DoDownloadHistoricalData(IEnumerable <string> stockList, int numYears, CSVOutputSettings outputSettings, Action <string> BeforeDownload, Action <string> AfterDownload, Action <Exception> ExceptionCallback) { // initialize parameters HistoricalDownloadParams downloadParams = new HistoricalDownloadParams(); downloadParams.stockList = stockList; downloadParams.NumYears = numYears; downloadParams.OutputSettings = outputSettings.Clone() as CSVOutputSettings; downloadParams.BeforeStockDataDownloadCallback = BeforeDownload; downloadParams.AfterStockDataDownloadCallback = AfterDownload; downloadParams.OnExceptionCallback = ExceptionCallback; // start the thread DownloadWorker worker = new DownloadWorker(); Thread t = new Thread(worker.DownloadStockHistoryData); Variables.threadList.Add(t); t.Start(downloadParams); //worker.DownloadStockHistoryData(downloadParams); }
public void DownloadStockHistoryData(object downloadParams) { HistoricalDownloadParams _downloadParams = downloadParams as HistoricalDownloadParams; IEnumerable <string> stocks = _downloadParams.stockList as IEnumerable <string>; string downloadStr; int numYears = _downloadParams.NumYears; CSVOutputSettings outputSettings = _downloadParams.OutputSettings; Dictionary <string, string> indexDict = new Dictionary <string, string>(); indexDict.Add("^PSEi", "PSE"); indexDict.Add("^ALLSHARES", "ALL"); indexDict.Add("^FINANCIAL", "FIN"); indexDict.Add("^INDUSTRIAL", "IND"); indexDict.Add("^HOLDING", "HDG"); indexDict.Add("^PROPERTY", "PRO"); indexDict.Add("^SERVICE", "SVC"); indexDict.Add("^MINING-OIL", "M-O"); try { foreach (string symbol in stocks) { string tmpSymbol = symbol; if (tmpSymbol.Contains("^")) { tmpSymbol = indexDict[symbol]; downloadStr = "http://www2.pse.com.ph/servlet/ChartForPhisixServlet?indexID=%s&years=%f"; } else { downloadStr = "http://www2.pse.com.ph/servlet/PSEChartServlet?securitySymbol=%s&years=%f"; } downloadStr = downloadStr.Replace("%s", tmpSymbol).Replace("%f", numYears.ToString()); outputSettings.Filename = symbol + ".csv"; // before download callback if (_downloadParams.BeforeStockDataDownloadCallback != null) { _downloadParams.BeforeStockDataDownloadCallback(symbol); } HistoricalDataDownloader downloader = new HistoricalDataDownloader(new Uri(downloadStr)); downloader.Download(); HistoricalDataReader reader = downloader.GetReader(symbol); lock (DownloadWorker.threadLock) { reader.ToCSV(outputSettings); Variables.DownloadedCount++; // after download if (_downloadParams.AfterStockDataDownloadCallback != null) { _downloadParams.AfterStockDataDownloadCallback(symbol); } } } } catch (Exception e) { //throw new Exception(e.Message); if (_downloadParams.OnExceptionCallback != null) { _downloadParams.OnExceptionCallback(e); } // abort threads foreach (Thread t in Variables.threadList) { if (t.IsAlive) { t.Abort(); } } } }
public static void DownloadAndConvertHistoricalData(object downloadParams) { Variables.DownloadedCount = 0; // track downloads Variables.threadList.Clear(); HistoricalDownloadParams _downloadParams = downloadParams as HistoricalDownloadParams; if (_downloadParams.StartDownloadProcess != null) { _downloadParams.StartDownloadProcess(); } List <string> tmpList = new List <string>(); tmpList.AddRange(_downloadParams.stockList.ToArray()); const int NUM_THREADS = 5; int stockCount = tmpList.Count; int threadCount = stockCount / NUM_THREADS; if (threadCount == 0) { threadCount = 1; } int startIndex = 0; int adder = 0; for (int i = 0; i < threadCount; i++) { // add the remainder of the stocks in the list if (threadCount == i + 1) { adder = stockCount % NUM_THREADS; } int downloadItemCount = (stockCount >= NUM_THREADS) ? NUM_THREADS + adder : adder; // take the items List <string> l = new List <string>(); l.AddRange(tmpList.Take(downloadItemCount).ToArray()); DoDownloadHistoricalData(l, _downloadParams.NumYears, _downloadParams.OutputSettings, _downloadParams.BeforeStockDataDownloadCallback, _downloadParams.AfterStockDataDownloadCallback, _downloadParams.OnExceptionCallback); // remove added items tmpList.RemoveRange(startIndex, downloadItemCount); } // wait for everything to complete while (true) { int count = (from t in Variables.threadList where t.IsAlive == true select t).Count(); if (count == 0) { if (_downloadParams.DownloadAllCompleteCallback != null) { _downloadParams.DownloadAllCompleteCallback(); } break; } } }