/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
foreach (var date in tradableDays)
{
var fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date);
var fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataCacheProvider, fineFundamentalConfiguration, date, _isLiveMode);
var fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault();
yield return(new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios(),
AssetClassification = fineFundamentalForDate != null ? fineFundamentalForDate.AssetClassification : new AssetClassification(),
CompanyProfile = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyProfile : new CompanyProfile()
});
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
// We decide to use the ZipDataCacheProvider instead of the SingleEntryDataCacheProvider here
// for resiliency and as a fix for an issue preventing us from reading non-equity options data.
// It has the added benefit of caching any zip files that we request from the filesystem, and reading
// files contained within the zip file, which the SingleEntryDataCacheProvider does not support.
var sourceFactory = request.Configuration.GetBaseDataInstance();
using (var dataCacheProvider = new ZipDataCacheProvider(dataProvider))
{
foreach (var date in _tradableDaysProvider(request))
{
if (sourceFactory.RequiresMapping())
{
request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
}
var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var configuration = request.Configuration;
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
// we want the first selection to happen on the start time
// so we need the previous tradable day time, since coarse
// files are for each tradable date but emitted with next day time
var previousTradableDay = Time.GetStartTimeForTradeBars(
request.Security.Exchange.Hours,
request.StartTimeLocal,
Time.OneDay,
1,
false);
var tradableDays = new[] { previousTradableDay }.Concat(request.TradableDays);
// Behaves in the same way as in live trading
// (i.e. only emit coarse data on dates following a trading day)
// The shifting of dates is needed to ensure we never emit coarse data on the same date,
// because it would enable look-ahead bias.
foreach (var date in tradableDays)
{
var source = sourceFactory.GetSource(configuration, date, false);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false);
var coarseFundamentalForDate = factory.Read(source);
// shift all date of emitting the file forward one day to model emitting coarse midnight the next day.
yield return(new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate));
}
}
}
public void EstimizeConsensusReaderTest()
{
var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider());
var config = new SubscriptionDataConfig(
typeof(EstimizeConsensus),
Symbol.Create("AAPL.C", SecurityType.Base, QuantConnect.Market.USA),
Resolution.Daily,
DateTimeZone.Utc,
DateTimeZone.Utc,
false,
false,
false,
true
);
var data = new EstimizeConsensus();
var date = new DateTime(2019, 6, 10);
var source = data.GetSource(config, date, false);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data);
var rows = factory.Read(source).ToList();
Assert.IsTrue(rows.Count > 0);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request)
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
return((
from date in tradableDays
let fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date)
let fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, fineFundamentalConfiguration, date, false)
let fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault()
select new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios()
}
).GetEnumerator());
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var configuration = request.Configuration;
var tradableDays = _tradableDaysProvider(request);
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
// Note: this enumerator factory is currently only used in backtesting with coarse data
// and has been updated to behave in the same way as in live trading
// (i.e. only emit coarse data on dates following a trading day)
// The shifting of dates is needed to ensure we never emit coarse data on the same date,
// because it would enable look-ahead bias.
// shift all tradeable dates forward one day
foreach (var date in tradableDays.Select(x => x.AddDays(1)))
{
// request the file for the previous date, which is a tradeable day
var source = sourceFactory.GetSource(configuration, date.AddDays(-1), false);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date.AddDays(-1), false);
var coarseFundamentalForDate = factory.Read(source);
// Coarse data has a period of one day (EndTime == Time + OneDay) but BaseDataCollection has no period (EndTime == Time),
// so we need to add one more day here.
yield return(new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate));
}
}
}
/// <summary>
/// Gets the <see cref="ISubscriptionDataSourceReader"/> for the specified source
/// </summary>
protected virtual ISubscriptionDataSourceReader GetSubscriptionDataSourceReader(SubscriptionDataSource source,
IDataCacheProvider dataCacheProvider,
SubscriptionDataConfig config,
DateTime date
)
{
return(SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, true));
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var configuration = request.Configuration;
var tradableDays = _tradableDaysProvider(request);
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
return((
from date in tradableDays
let source = sourceFactory.GetSource(configuration, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, configuration, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate)
).GetEnumerator());
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request)
{
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
return((
from date in _tradableDaysProvider(request)
let source = sourceFactory.GetSource(request.Configuration, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, request.Configuration, date, false)
let entriesForDate = factory.Read(source)
from entry in entriesForDate
select entry
)
.GetEnumerator());
}
public void QuandlDownloadDoesNotThrow()
{
Quandl.SetAuthCode("WyAazVXnq7ATy_fefTqm");
RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
var data = new HistoryAlgorithm.QuandlFuture();
const string ticker = "CHRIS/CME_SP1";
var date = new DateTime(2018, 8, 31);
var config = new SubscriptionDataConfig(typeof(HistoryAlgorithm.QuandlFuture), Symbol.Create(ticker, SecurityType.Base, QuantConnect.Market.USA), Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true);
var source = data.GetSource(config, date, false);
var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider());
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data);
var rows = factory.Read(source).ToList();
Assert.IsTrue(rows.Count > 0);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider)
{
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
foreach (var date in _tradableDaysProvider(request))
{
var currentSymbol = request.Configuration.MappedSymbol;
request.Configuration.MappedSymbol = GetMappedSymbol(request, date);
var source = sourceFactory.GetSource(request.Configuration, date, false);
request.Configuration.MappedSymbol = currentSymbol;
var factory = SubscriptionDataSourceReader.ForSource(source, dataFileProvider, request.Configuration, date, false);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var configuration = request.Configuration;
var tradableDays = _tradableDaysProvider(request);
var sourceFactory = (BaseData)Activator.CreateInstance(request.Configuration.Type);
foreach (var date in tradableDays)
{
var source = sourceFactory.GetSource(configuration, date, false);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, configuration, date, false);
var coarseFundamentalForDate = factory.Read(source);
yield return(new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate));
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var sourceFactory = (BaseData)ObjectActivator.GetActivator(request.Configuration.Type).Invoke(new object[] { request.Configuration.Type });
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
foreach (var date in _tradableDaysProvider(request))
{
request.Configuration.MappedSymbol = GetMappedSymbol(request, date);
var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var sourceFactory = request.Configuration.GetBaseDataInstance();
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
foreach (var date in _tradableDaysProvider(request))
{
if (sourceFactory.RequiresMapping())
{
request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
}
var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory);
var entriesForDate = factory.Read(source);
foreach (var entry in entriesForDate)
{
yield return(entry);
}
}
}
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
foreach (var date in tradableDays)
{
var fineFundamentalSource = GetSource(FineFundamental, fineFundamentalConfiguration, date);
var fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataCacheProvider, fineFundamentalConfiguration, date, _isLiveMode, FineFundamental, dataProvider);
var fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault();
// directly do not emit null points. Null points won't happen when used with Coarse data since we are pre filtering based on Coarse.HasFundamentalData
// but could happen when fine filtering custom universes
if (fineFundamentalForDate != null)
{
yield return(new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate.CompanyReference,
SecurityReference = fineFundamentalForDate.SecurityReference,
FinancialStatements = fineFundamentalForDate.FinancialStatements,
EarningReports = fineFundamentalForDate.EarningReports,
OperationRatios = fineFundamentalForDate.OperationRatios,
EarningRatios = fineFundamentalForDate.EarningRatios,
ValuationRatios = fineFundamentalForDate.ValuationRatios,
AssetClassification = fineFundamentalForDate.AssetClassification,
CompanyProfile = fineFundamentalForDate.CompanyProfile
});
}
}
}
}
