/// <summary> /// Fetch and set the location of the data from the user's BaseData factory: /// </summary> /// <param name="date">Date of the source file.</param> /// <returns>Boolean true on successfully retrieving the data</returns> public bool RefreshSource(DateTime date) { //Update the source from the getSource method: _date = date; var newSource = ""; //If this is an equity, apply the backward price scaling (backtesting only). if (_isQCEquity && (_feedEndpoint == DataFeedEndpoint.Backtesting || _feedEndpoint == DataFeedEndpoint.FileSystem)) { UpdateScaleFactors(date); } //Make sure this particular security is trading today: if (!_security.Exchange.DateIsOpen(date)) { _endOfStream = true; return(false); } //Choose the new source file, hide the QC source file locations if (_isQCData) { newSource = GetQuantConnectSource(date); } else { //If its not a QC bar, load the source from the user function. newSource = GetSource(date); } //When stream over stop looping on this data. if (newSource == "") { _endOfStream = true; return(false); } //Log.Debug("SubscriptionDataReader.MoveNext(): Source Refresh: " + newSource); if (_source != newSource && newSource != "") { //If a new file, reset the EOS flag: _endOfStream = false; //Set the new source. _source = newSource; //Close out the last source file. Dispose(); //Load the source: try { //Log.Debug("SubscriptionDataReader.RefreshSource(): Created new reader for source: " + source); _reader = GetReader(_source, (_isQCData && !Engine.IsLocal)); } catch (Exception err) { Log.Error("SubscriptionDataReader.RefreshSource(): Failed to get reader: " + err.Message); //Engine.ResultHandler.DebugMessage("Failed to get a reader for the data source. There may be an error in your custom data source reader. Skipping date (" + date.ToShortDateString() + "). Err: " + err.Message); return(false); } if (_reader == null) { Log.Error("Failed to get StreamReader for data source(" + _source + "), symbol(" + _mappedSymbol + "). Skipping date(" + date.ToShortDateString() + "). Reader is null."); //Engine.ResultHandler.DebugMessage("We could not find the requested data. This may be an invalid data request, failed download of custom data, or a public holiday. Skipping date (" + date.ToShortDateString() + ")."); if (!_isQCData) { Engine.ResultHandler.ErrorMessage("We could not fetch the requested data. This may not be valid data, or a failed download of custom data. Skipping source (" + _source + ")."); } return(false); } //Reset the public properties so we can explicitly set them with lastBar data. Current = null; Previous = null; //99% of time, populate the first "Current". 1% of of time no source file (getReader fails), so // method sets the Subscription properties as if no data. try { MoveNext(); } catch (Exception err) { throw new Exception("SubscriptionDataReader.RefreshSource(): Could not MoveNext to init stream: " + _source + " " + err.Message + " >> " + err.StackTrace); } } //Success: return(true); }
/// <summary> /// Using this source URL, download it to our cache and open a local reader. /// </summary> /// <param name="source">Source URL for the data:</param> /// <param name="qcFile">Boolean if this is a QC managed data file, if so can look in QC store</param> /// <returns>StreamReader for the data source</returns> private SubscriptionStreamReader GetReader(string source, bool qcFile = false) { //Prepare local folders: const string cache = "./cache/data"; SubscriptionStreamReader reader = null; if (!Directory.Exists(cache)) { Directory.CreateDirectory(cache); } foreach (var file in Directory.EnumerateFiles(cache)) { File.Delete(file); } //1. Download this source file as fast as possible: //1.1 Create filename from source: var filename = source.ToMD5() + source.GetExtension(); var location = cache + @"/" + filename; //1.2 Based on Endpoint, Download File (Backtest) or directly open SR of source: switch (_feedEndpoint) { case DataFeedEndpoint.FileSystem: case DataFeedEndpoint.Backtesting: //1.2 Download from source to location if (qcFile) { if (!File.Exists(source)) { Log.Trace("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: " + source); Engine.ResultHandler.SamplePerformance(_date.Date, 0); return(reader); } //Data source is also location of raw file; root /data directory. location = source; } else { //If the file already exists a try { using (var client = new WebClient()) { client.DownloadFile(source, location); } } catch (Exception err) { Engine.ResultHandler.ErrorMessage("Error downloading custom data source file, skipped: " + source + " Err: " + err.Message, err.StackTrace); if (OS.IsWindows) { Engine.ResultHandler.SamplePerformance(_date.Date, 0); } return(reader); } } //2. File downloaded. Open Stream: if (File.Exists(location)) { if (source.GetExtension() == ".zip") { //Extracting zip returns stream reader: var sr = Compression.Unzip(location); if (sr == null) { return(null); } reader = new SubscriptionStreamReader(sr, _feedEndpoint); } else { //Custom file stream: open from disk reader = new SubscriptionStreamReader(location, _feedEndpoint); } } break; //Directly open for REST Requests: case DataFeedEndpoint.Tradier: case DataFeedEndpoint.LiveTrading: reader = new SubscriptionStreamReader(source, _feedEndpoint); break; } return(reader); }
/// <summary> /// Using this source URL, download it to our cache and open a local reader. /// </summary> /// <param name="source">Source URL for the data:</param> /// <returns>StreamReader for the data source</returns> private SubscriptionStreamReader GetReader(string source) { //Prepare local folders: const string cache = "./cache/data"; SubscriptionStreamReader reader = null; if (!Directory.Exists(cache)) { Directory.CreateDirectory(cache); } foreach (var file in Directory.EnumerateFiles(cache)) { if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24)) { File.Delete(file); } } //1. Download this source file as fast as possible: //1.1 Create filename from source: var filename = source.ToMD5() + source.GetExtension(); var location = cache + @"/" + filename; //1.2 Based on Endpoint, Download File (Backtest) or directly open SR of source: switch (_feedEndpoint) { case DataFeedEndpoint.FileSystem: case DataFeedEndpoint.Backtesting: var uri = new Uri(source, UriKind.RelativeOrAbsolute); // check if this is not a local uri then download it to the local cache if (uri.IsAbsoluteUri && !uri.IsLoopback) { try { using (var client = new WebClient()) { client.Proxy = WebRequest.GetSystemWebProxy(); client.DownloadFile(source, location); // reassign source since it's now on local disk source = location; } } catch (Exception err) { Engine.ResultHandler.ErrorMessage("Error downloading custom data source file, skipped: " + source + " Err: " + err.Message, err.StackTrace); Engine.ResultHandler.SamplePerformance(_date.Date, 0); return(null); } } //2. File downloaded. Open Stream: if (File.Exists(source)) { if (source.GetExtension() == ".zip") { //Extracting zip returns stream reader: var sr = Compression.Unzip(source); if (sr == null) { return(null); } reader = new SubscriptionStreamReader(sr, _feedEndpoint); } else { //Custom file stream: open from disk reader = new SubscriptionStreamReader(source, _feedEndpoint); } } else { Log.Trace("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: " + source); Engine.ResultHandler.SamplePerformance(_date.Date, 0); return(null); } break; //Directly open for REST Requests: case DataFeedEndpoint.LiveTrading: reader = new SubscriptionStreamReader(source, _feedEndpoint); break; } return(reader); }