/// <summary>
/// Fetch and set the location of the data from the user's BaseData factory:
/// </summary>
/// <param name="date">Date of the source file.</param>
/// <returns>Boolean true on successfully retrieving the data</returns>
public bool RefreshSource(DateTime date)
{
//Update the source from the getSource method:
_date = date;
var newSource = "";
//If this is an equity, apply the backward price scaling (backtesting only).
if (_isQCEquity && (_feedEndpoint == DataFeedEndpoint.Backtesting || _feedEndpoint == DataFeedEndpoint.FileSystem))
{
UpdateScaleFactors(date);
}
//Make sure this particular security is trading today:
if (!_security.Exchange.DateIsOpen(date))
{
_endOfStream = true;
return(false);
}
//Choose the new source file, hide the QC source file locations
if (_isQCData)
{
newSource = GetQuantConnectSource(date);
}
else
{
//If its not a QC bar, load the source from the user function.
newSource = GetSource(date);
}
//When stream over stop looping on this data.
if (newSource == "")
{
_endOfStream = true;
return(false);
}
//Log.Debug("SubscriptionDataReader.MoveNext(): Source Refresh: " + newSource);
if (_source != newSource && newSource != "")
{
//If a new file, reset the EOS flag:
_endOfStream = false;
//Set the new source.
_source = newSource;
//Close out the last source file.
Dispose();
//Load the source:
try
{
//Log.Debug("SubscriptionDataReader.RefreshSource(): Created new reader for source: " + source);
_reader = GetReader(_source, (_isQCData && !Engine.IsLocal));
}
catch (Exception err)
{
Log.Error("SubscriptionDataReader.RefreshSource(): Failed to get reader: " + err.Message);
//Engine.ResultHandler.DebugMessage("Failed to get a reader for the data source. There may be an error in your custom data source reader. Skipping date (" + date.ToShortDateString() + "). Err: " + err.Message);
return(false);
}
if (_reader == null)
{
Log.Error("Failed to get StreamReader for data source(" + _source + "), symbol(" + _mappedSymbol + "). Skipping date(" + date.ToShortDateString() + "). Reader is null.");
//Engine.ResultHandler.DebugMessage("We could not find the requested data. This may be an invalid data request, failed download of custom data, or a public holiday. Skipping date (" + date.ToShortDateString() + ").");
if (!_isQCData)
{
Engine.ResultHandler.ErrorMessage("We could not fetch the requested data. This may not be valid data, or a failed download of custom data. Skipping source (" + _source + ").");
}
return(false);
}
//Reset the public properties so we can explicitly set them with lastBar data.
Current = null;
Previous = null;
//99% of time, populate the first "Current". 1% of of time no source file (getReader fails), so
// method sets the Subscription properties as if no data.
try
{
MoveNext();
}
catch (Exception err)
{
throw new Exception("SubscriptionDataReader.RefreshSource(): Could not MoveNext to init stream: " + _source + " " + err.Message + " >> " + err.StackTrace);
}
}
//Success:
return(true);
}
/// <summary>
/// Using this source URL, download it to our cache and open a local reader.
/// </summary>
/// <param name="source">Source URL for the data:</param>
/// <param name="qcFile">Boolean if this is a QC managed data file, if so can look in QC store</param>
/// <returns>StreamReader for the data source</returns>
private SubscriptionStreamReader GetReader(string source, bool qcFile = false)
{
//Prepare local folders:
const string cache = "./cache/data";
SubscriptionStreamReader reader = null;
if (!Directory.Exists(cache))
{
Directory.CreateDirectory(cache);
}
foreach (var file in Directory.EnumerateFiles(cache))
{
File.Delete(file);
}
//1. Download this source file as fast as possible:
//1.1 Create filename from source:
var filename = source.ToMD5() + source.GetExtension();
var location = cache + @"/" + filename;
//1.2 Based on Endpoint, Download File (Backtest) or directly open SR of source:
switch (_feedEndpoint)
{
case DataFeedEndpoint.FileSystem:
case DataFeedEndpoint.Backtesting:
//1.2 Download from source to location
if (qcFile)
{
if (!File.Exists(source))
{
Log.Trace("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: " + source);
Engine.ResultHandler.SamplePerformance(_date.Date, 0);
return(reader);
}
//Data source is also location of raw file; root /data directory.
location = source;
}
else
{
//If the file already exists a
try
{
using (var client = new WebClient())
{
client.DownloadFile(source, location);
}
}
catch (Exception err)
{
Engine.ResultHandler.ErrorMessage("Error downloading custom data source file, skipped: " + source + " Err: " + err.Message, err.StackTrace);
if (OS.IsWindows)
{
Engine.ResultHandler.SamplePerformance(_date.Date, 0);
}
return(reader);
}
}
//2. File downloaded. Open Stream:
if (File.Exists(location))
{
if (source.GetExtension() == ".zip")
{
//Extracting zip returns stream reader:
var sr = Compression.Unzip(location);
if (sr == null)
{
return(null);
}
reader = new SubscriptionStreamReader(sr, _feedEndpoint);
}
else
{
//Custom file stream: open from disk
reader = new SubscriptionStreamReader(location, _feedEndpoint);
}
}
break;
//Directly open for REST Requests:
case DataFeedEndpoint.Tradier:
case DataFeedEndpoint.LiveTrading:
reader = new SubscriptionStreamReader(source, _feedEndpoint);
break;
}
return(reader);
}
/// <summary>
/// Using this source URL, download it to our cache and open a local reader.
/// </summary>
/// <param name="source">Source URL for the data:</param>
/// <returns>StreamReader for the data source</returns>
private SubscriptionStreamReader GetReader(string source)
{
//Prepare local folders:
const string cache = "./cache/data";
SubscriptionStreamReader reader = null;
if (!Directory.Exists(cache))
{
Directory.CreateDirectory(cache);
}
foreach (var file in Directory.EnumerateFiles(cache))
{
if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24))
{
File.Delete(file);
}
}
//1. Download this source file as fast as possible:
//1.1 Create filename from source:
var filename = source.ToMD5() + source.GetExtension();
var location = cache + @"/" + filename;
//1.2 Based on Endpoint, Download File (Backtest) or directly open SR of source:
switch (_feedEndpoint)
{
case DataFeedEndpoint.FileSystem:
case DataFeedEndpoint.Backtesting:
var uri = new Uri(source, UriKind.RelativeOrAbsolute);
// check if this is not a local uri then download it to the local cache
if (uri.IsAbsoluteUri && !uri.IsLoopback)
{
try
{
using (var client = new WebClient())
{
client.Proxy = WebRequest.GetSystemWebProxy();
client.DownloadFile(source, location);
// reassign source since it's now on local disk
source = location;
}
}
catch (Exception err)
{
Engine.ResultHandler.ErrorMessage("Error downloading custom data source file, skipped: " + source + " Err: " + err.Message, err.StackTrace);
Engine.ResultHandler.SamplePerformance(_date.Date, 0);
return(null);
}
}
//2. File downloaded. Open Stream:
if (File.Exists(source))
{
if (source.GetExtension() == ".zip")
{
//Extracting zip returns stream reader:
var sr = Compression.Unzip(source);
if (sr == null)
{
return(null);
}
reader = new SubscriptionStreamReader(sr, _feedEndpoint);
}
else
{
//Custom file stream: open from disk
reader = new SubscriptionStreamReader(source, _feedEndpoint);
}
}
else
{
Log.Trace("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: " + source);
Engine.ResultHandler.SamplePerformance(_date.Date, 0);
return(null);
}
break;
//Directly open for REST Requests:
case DataFeedEndpoint.LiveTrading:
reader = new SubscriptionStreamReader(source, _feedEndpoint);
break;
}
return(reader);
}
