public void ZipDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral)
{
var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), isDataEphemeral: isDataEphemeral);
var remoteReader = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
var remoteReader2 = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
remoteReader.Dispose();
remoteReader2.Dispose();
cacheProvider.DisposeSafely();
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache);
break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
continue;
}
yield return(instances);
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
public void SetUp()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new TestDownloadProvider());
TestDownloadProvider.DownloadCount = 0;
// create cache directory if not existing
if (!Directory.Exists(Globals.Cache))
{
Directory.CreateDirectory(Globals.Cache);
}
else
{
// clean old files out of the cache
Directory.Delete(Globals.Cache, true);
Directory.CreateDirectory(Globals.Cache);
}
}
public void QuandlDownloadDoesNotThrow()
{
Quandl.SetAuthCode("WyAazVXnq7ATy_fefTqm");
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var data = new HistoryAlgorithm.QuandlFuture();
const string ticker = "CHRIS/CME_SP1";
var date = new DateTime(2018, 8, 31);
var config = new SubscriptionDataConfig(typeof(HistoryAlgorithm.QuandlFuture), Symbol.Create(ticker, SecurityType.Base, QuantConnect.Market.USA), Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true);
var source = data.GetSource(config, date, false);
var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider());
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data);
var rows = factory.Read(source).ToList();
Assert.IsTrue(rows.Count > 0);
}
public void InvalidDataSource()
{
var remoteReader = new RemoteFileSubscriptionStreamReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
@"http://helloworld.com",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
// Fails to get helloworld.com, missing http://
Assert.Throws <WebException>(() => new RemoteFileSubscriptionStreamReader(
new SingleEntryDataCacheProvider(new DefaultDataProvider()),
@"helloworld.com",
Globals.Cache,
null),
"Api.Download(): Failed to download data from helloworld.com. Please verify the source for missing http:// or https://"
);
}
public void EmitsDailyQuandlFutureDataOverWeekends()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
var startDate = new DateTime(2018, 4, 1);
var endDate = new DateTime(2018, 4, 20);
// delete temp files
foreach (var ticker in tickers)
{
var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
File.Delete(fileName);
}
var algorithm = new QCAlgorithm();
CreateDataFeed();
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataPointsEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
var cancellationTokenSource = new CancellationTokenSource();
var lastFileWriteDate = DateTime.MinValue;
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(20);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}");
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
if (currentTime.Date > lastFileWriteDate.Date)
{
foreach (var ticker in tickers)
{
var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");
// write new local file including only rows up to current date
var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
var sb = new StringBuilder();
{
using (var reader = new StreamReader(source))
{
var firstLine = true;
string line;
while ((line = reader.ReadLine()) != null)
{
if (firstLine)
{
sb.AppendLine(line);
firstLine = false;
continue;
}
var csv = line.Split(',');
var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd");
if (time.Date >= currentTime.Date)
{
break;
}
sb.AppendLine(line);
}
}
}
if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
{
var fileContent = sb.ToString();
try
{
File.WriteAllText(outputFileName, fileContent);
}
catch (IOException)
{
Log.Error("IOException: will sleep 200ms and retry once more");
// lets sleep 200ms and retry once more, consumer could be reading the file
// this exception happens in travis intermittently, GH issue 3273
Thread.Sleep(200);
File.WriteAllText(outputFileName, fileContent);
}
Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
}
}
lastFileWriteDate = currentTime;
}
// 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
timeProvider.Advance(TimeSpan.FromMinutes(20));
//Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");
// restart the timer
timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite);
}
catch (Exception exception)
{
Log.Error(exception);
_feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, timerInterval.Milliseconds, Timeout.Infinite);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
foreach (var dataPoint in timeSlice.Slice.Values)
{
Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " +
$"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " +
$"{dataPoint.EndTime.ToStringInvariant()}"
);
dataPointsEmitted++;
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
}
/// <summary>
/// Sets the IAlgorithm instance in the ILeanManager
/// </summary>
/// <param name="algorithm">The IAlgorithm instance being run</param>
public void SetAlgorithm(IAlgorithm algorithm)
{
algorithm.SetApi(_systemHandlers.Api);
RemoteFileSubscriptionStreamReader.SetDownloadProvider((Api.Api)_systemHandlers.Api);
}
public void RemoteDataDoesNotIncreaseNumberOfSlices()
{
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var startDate = new DateTime(2017, 4, 2);
var endDate = new DateTime(2017, 4, 23);
var algorithm = new QCAlgorithm();
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
{
TickType = TickType.Trade
};
return(new[] { tick, tick2 });
}, timeProvider);
CreateDataFeed(dataQueueHandler);
var dataManager = new DataManagerStub(algorithm, _feed);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var symbols = new List <Symbol>
{
algorithm.AddData <TestableRemoteCustomData>("FB", Resolution.Daily).Symbol,
algorithm.AddData <TestableRemoteCustomData>("IBM", Resolution.Daily).Symbol,
algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
};
var cancellationTokenSource = new CancellationTokenSource();
var dataPointsEmitted = 0;
var slicesEmitted = 0;
RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time
// create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
var timerInterval = TimeSpan.FromMilliseconds(100);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
_feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(3));
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}, null, TimeSpan.FromSeconds(2), timerInterval);
try
{
foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Slice.HasData)
{
slicesEmitted++;
dataPointsEmitted += timeSlice.Slice.Values.Count;
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
}
}
}
catch (Exception exception)
{
Log.Trace($"Error: {exception}");
}
timer.Value.Dispose();
dataManager.RemoveAllSubscriptions();
dataQueueHandler.DisposeSafely();
Assert.AreEqual(14, slicesEmitted);
Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source, source.Headers, _isLiveMode);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache, source.Headers);
break;
}
}
catch (Exception e)
{
OnInvalidSource(source, e);
yield break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances = null;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null && !reader.ShouldBeRateLimited)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
if (!reader.ShouldBeRateLimited)
{
continue;
}
}
if (_isLiveMode
// this shouldn't happen, rest reader is the only one to be rate limited
// and in live mode, but just in case...
|| instances == null && reader.ShouldBeRateLimited)
{
yield return(instances);
}
else
{
foreach (var instance in instances.Data)
{
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
