/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
var enumerator = CorporateEventEnumeratorFactory.CreateEnumerators(
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
_includeAuxiliaryData);
// has to be initialized after adding all the enumerators since it will execute a MoveNext
dataReader.Initialize();
return(new SynchronizingEnumerator(dataReader, enumerator));
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Create a new backtesting data feed.
/// </summary>
/// <param name="algorithm">Instance of the algorithm</param>
/// <param name="job">Algorithm work task</param>
public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
{
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
DataFeed = DataFeedEndpoint.FileSystem;
IsActive = true;
Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
EndOfBridge = new bool[_subscriptions];
SubscriptionReaders = new SubscriptionDataReader[_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
//Class Privates:
_job = job;
_algorithm = algorithm;
_endOfStreams = false;
_bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
for (var i = 0; i < _subscriptions; i++)
{
//Create a new instance in the dictionary:
Bridge[i] = new ConcurrentQueue<List<BaseData>>();
EndOfBridge[i] = false;
SubscriptionReaders[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish);
FillForwardFrontiers[i] = new DateTime();
}
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.TickerShouldBeMapped()
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.StartDateLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
var result = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
_includeAuxiliaryData,
request.StartTimeLocal,
_enablePriceScaling);
return(result);
}
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution)
{
var start = new DateTime(2019, 12, 9);
var end = new DateTime(2019, 12, 12);
var symbol = Symbols.SPY;
var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
var config = new SubscriptionDataConfig(typeof(TradeBar),
symbol,
dataResolution,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false);
var dataReader = new SubscriptionDataReader(config,
new HistoryRequest(config, entry.ExchangeHours, start, end),
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
new TestDataCacheProvider
{
Data = data
},
TestGlobals.DataProvider
);
Assert.IsTrue(dataReader.MoveNext());
Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext());
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider,
_includeAuxiliaryData
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
dataReader.Initialize();
return(dataReader);
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Create an instance of the base datafeed.
/// </summary>
public BaseDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
{
//Save the data subscriptions
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
DataFeed = DataFeedEndpoint.FileSystem;
IsActive = true;
Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
EndOfBridge = new bool[_subscriptions];
SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
_frontierTime = new DateTime[_subscriptions];
//Class Privates:
_job = job;
_algorithm = algorithm;
_endOfStreams = false;
_bridgeMax = _bridgeMax / _subscriptions;
//Initialize arrays:
for (var i = 0; i < _subscriptions; i++)
{
_frontierTime[i] = job.PeriodStart;
EndOfBridge[i] = false;
Bridge[i] = new ConcurrentQueue<List<BaseData>>();
SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, job.PeriodStart, job.PeriodFinish);
}
}
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution)
{
var start = new DateTime(2019, 12, 9);
var end = new DateTime(2019, 12, 12);
var mapFileProvider = new LocalDiskMapFileProvider();
var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA));
var dataReader = new SubscriptionDataReader(
new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
dataResolution,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false),
start,
end,
mapFileResolver,
new LocalDiskFactorFileProvider(mapFileProvider),
LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)),
false,
new TestDataCacheProvider
{
Data = data
}
);
Assert.IsTrue(dataReader.MoveNext());
Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext());
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.SecurityType,
request.Symbol,
request.Resolution,
request.Market,
request.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, 1.0m);
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
reader = new FillForwardEnumerator(reader, security.Exchange, request.FillForwardResolution.Value.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
return(new Subscription(security, reader, start, end, false, false));
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.TickerShouldBeMapped()
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider,
dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.StartDateLimited += (sender, args) =>
{
// Queue this warning into our dictionary to report on dispose
if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount)
{
_startDateLimitedWarnings.TryAdd(args.Symbol, args.Message);
}
};
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
dataReader.NumericalPrecisionLimited += (sender, args) =>
{
// Set a hard limit to keep this warning list from getting unnecessarily large
if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount)
{
_numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message);
}
};
var result = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
request.StartTimeLocal,
_enablePriceScaling);
return(result);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket) job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue, resultHandler);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
null,
true,
request.DataNormalizationMode
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
_dataProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider,
false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
{
var config = security.SubscriptionDataConfig;
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
if (userDefined)
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
}
return null;
}
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
return subscription;
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
_cancellationTokenSource = new CancellationTokenSource();
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
IsActive = true;
_endOfBridge = new bool[_subscriptions];
SubscriptionReaders = new IEnumerator<BaseData>[_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
//Class Privates:
_algorithm = algorithm;
// find the minimum resolution, ignoring ticks
var fillForwardResolution = Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution.ToTimeSpan())
.DefaultIfEmpty(TimeSpan.FromSeconds(1))
.Min();
// figure out how many subscriptions are at the minimum resolution
var subscriptionsAtMinimumResolution =
(from sub in Subscriptions
where sub.Resolution == Subscriptions.Min(x => x.Resolution)
select sub).Count();
Bridge = new BlockingCollection<TimeSlice>(Math.Min(1000, 50000/subscriptionsAtMinimumResolution));
for (var i = 0; i < _subscriptions; i++)
{
_endOfBridge[i] = false;
var config = Subscriptions[i];
var start = algorithm.StartDate;
var end = algorithm.EndDate;
var security = _algorithm.Securities[Subscriptions[i].Symbol];
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security,
start, end,
resultHandler,
tradeableDates,
false,
// we want a backtest to use the same symbols regardless of start/end dates, so we set the resolution
// to null which means give me the current/most recent symbol mapping for the requested symbol
symbolResolutionDate: null
);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
FillForwardFrontiers[i] = new DateTime();
// prime the pump for iteration in Run
_endOfBridge[i] = !SubscriptionReaders[i].MoveNext();
if (_endOfBridge[i])
{
Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol);
}
}
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="security">The security to create a subscription for</param>
/// <param name="utcStartTime">The start time of the subscription in UTC</param>
/// <param name="utcEndTime">The end time of the subscription in UTC</param>
/// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
{
Subscription subscription = null;
try
{
var config = security.SubscriptionDataConfig;
var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);
IEnumerator<BaseData> enumerator;
if (config.IsCustomData)
{
// custom data uses backtest readers
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);
// apply fast forwarding, this is especially important for RemoteFile types that
// can send in large chunks of old, irrelevant data
var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);
// apply rate limits (1x per increment, max 30 minutes between calls)
// TODO : Pull limits from config file?
var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
// add the enumerator to the exchange
_customExchange.AddEnumerator(rateLimit);
// this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (config.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
_exchange.SetHandler(config.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (config.FillDataForward)
{
// TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
}
// define market hours and user filters to incoming data
enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
// finally, make our subscriptions aware of the frontier of the data feed, this will help
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Initialize activators to invoke types in the algorithm
/// </summary>
private void ResetActivators()
{
for (var i = 0; i < _subscriptions; i++)
{
//Create a new instance in the dictionary:
Bridge[i] = new ConcurrentQueue<List<BaseData>>();
EndOfBridge[i] = false;
SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish);
FillForwardFrontiers[i] = new DateTime();
}
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
/// <param name="algorithm">Algorithm requesting data</param>
protected LiveTradingDataFeed(IAlgorithm algorithm)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_dataFeed = DataFeedEndpoint.LiveTrading;
_isActive = true;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Class Privates:
_algorithm = algorithm;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(CashBook.AccountCurrency, 0, 1m),
SymbolProperties.GetDefault(CashBook.AccountCurrency),
ErrorCurrencyConverter.Instance
);
var dataReader = new SubscriptionDataReader(config,
start,
end,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider,
false
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
dataReader.Initialize();
IEnumerator <BaseData> reader = dataReader;
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator<BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable<DateTime> tradeableDates,
bool applySubscripterFilterEnumerator = true)
{
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (applySubscripterFilterEnumerator)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return enumerator;
}
/// <summary>
/// Prepare and create the new MySQL Database connection datafeed.
/// </summary>
/// <param name="algorithm"></param>
/// <param name="job"></param>
/// <param name="resultHandler"></param>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
Bridge = new BlockingCollection<TimeSlice>();
//Save the data subscriptions
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
IsActive = true;
EndOfBridge = new bool[_subscriptions];
_subscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
_mySQLBridgeTime = new DateTime[_subscriptions];
//Class Privates:
_endTime = algorithm.EndDate;
//Initialize arrays:
for (var i = 0; i < _subscriptions; i++)
{
_mySQLBridgeTime[i] = algorithm.StartDate;
EndOfBridge[i] = false;
_subscriptionReaderManagers[i] = new SubscriptionDataReader(
Subscriptions[i],
algorithm.Securities[Subscriptions[i].Symbol],
algorithm.StartDate,
algorithm.EndDate,
resultHandler,
Time.EachTradeableDay(algorithm.Securities, algorithm.StartDate, algorithm.EndDate),
job is LiveNodePacket
);
}
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
_cancellationTokenSource = new CancellationTokenSource();
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
Bridge = new BlockingCollection<TimeSlice>();
//Set Properties:
_isActive = true;
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new IEnumerator<BaseData>[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
//Class Privates:
_algorithm = algorithm;
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket) job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
//This is quantconnect data source, store here for speed/ease of access
var security = algorithm.Securities[_subscriptions[i].Symbol];
_isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData);
// only make readers for custom data, live data will come through data queue handler
if (_isDynamicallyLoadedData[i])
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var periodStart = DateTime.Today.AddDays(-7);
var subscriptionDataReader = new SubscriptionDataReader(
_subscriptions[i],
security,
periodStart,
DateTime.MaxValue,
resultHandler,
Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue),
true,
DateTime.Today
);
// wrap the subscription data reader with a filter enumerator
_subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue);
}
_realtimePrices.Add(0);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
private void ProcessMissingDayFillForward(SubscriptionDataReader manager, int i, TimeSpan increment, DateTime dateToFill)
{
// we'll copy the current into the next day
var subscription = Subscriptions[i];
if (!subscription.FillDataForward || manager.Current == null) return;
var start = dateToFill.Date + manager.Exchange.MarketOpen;
if (subscription.ExtendedMarketHours)
{
// we need to loop to find the extended market open
start = dateToFill.Date;
while (!manager.Exchange.DateTimeIsExtendedOpen(start))
{
start += increment;
}
}
var current = manager.Current;
for (var date = start; date.Date == dateToFill.Date; date = date + increment)
{
if (manager.IsMarketOpen(date) || (subscription.ExtendedMarketHours && manager.IsExtendedMarketOpen(date)))
{
var cache = new List<BaseData>(1);
var fillforward = current.Clone(true);
fillforward.Time = date;
FillForwardFrontiers[i] = date;
cache.Add(fillforward);
Bridge[i].Enqueue(cache);
}
else
{
// stop fill forwarding when we're no longer open
break;
}
}
}
private static Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined)
{
var config = security.SubscriptionDataConfig;
var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date);
var symbolResolutionDate = userDefined ? (DateTime?)null : start;
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(),
security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end);
var subscription = new Subscription(security, enumerator, start, end, userDefined, false);
return subscription;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
.Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
.Select(dt => new Tick {Time = dt}).GetEnumerator();
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false);
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
// only message the user if it's one of their universe types
var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
PrimeSubscriptionPump(subscription, messageUser);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource)
{
//Subscription Count:
_subscriptions = algorithm.SubscriptionManager.Subscriptions;
//Set Properties:
_isActive = true;
_dataFeed = DataFeedEndpoint.LiveTrading;
_bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count];
_endOfBridge = new bool[Subscriptions.Count];
_subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count];
_realtimePrices = new List<decimal>();
//Set the source of the live data:
_dataQueue = dataSource;
//Class Privates:
_algorithm = algorithm;
_job = job;
//Setup the arrays:
for (var i = 0; i < Subscriptions.Count; i++)
{
_endOfBridge[i] = false;
_bridge[i] = new ConcurrentQueue<List<BaseData>>();
//This is quantconnect data source, store here for speed/ease of access
_isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData);
//Subscription managers for downloading user data:
_subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue);
//Set up the source file for today:
_subscriptionManagers[i].RefreshSource(DateTime.Now.Date);
_realtimePrices.Add(0);
}
// request for data from these symbols
_dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Prepare and create the new MySQL Database connection datafeed.
/// </summary>
/// <param name="algorithm"></param>
/// <param name="job"></param>
/// <param name="resultHandler"></param>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
//Save the data subscriptions
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
IsActive = true;
DataFeed = DataFeedEndpoint.FileSystem;
Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
EndOfBridge = new bool[_subscriptions];
SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
_mySQLBridgeTime = new DateTime[_subscriptions];
//Class Privates:
_endOfStreams = false;
_bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
_endTime = algorithm.EndDate;
//Initialize arrays:
for (var i = 0; i < _subscriptions; i++)
{
_mySQLBridgeTime[i] = algorithm.StartDate;
EndOfBridge[i] = false;
Bridge[i] = new ConcurrentQueue<List<BaseData>>();
SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, algorithm.StartDate, algorithm.EndDate, resultHandler);
}
}
/// <summary>
/// If this is a fillforward subscription, look at the previous time, and current time, and add new
/// objects to queue until current time to fill up the gaps.
/// </summary>
/// <param name="manager">Subscription to process</param>
/// <param name="i">Subscription position in the bridge ( which queue are we pushing data to )</param>
/// <param name="increment">Timespan increment to jump the fillforward results</param>
void ProcessFillForward(SubscriptionDataReader manager, int i, TimeSpan increment)
{
// If previous == null cannot fill forward nothing there to move forward (e.g. cases where file not found on first file).
if (!Subscriptions[i].FillDataForward || manager.Previous == null) return;
//Last tradebar and the current one we're about to add to queue:
var previous = manager.Previous;
var current = manager.Current;
//Initialize the frontier:
if (FillForwardFrontiers[i].Ticks == 0) FillForwardFrontiers[i] = previous.Time;
//Data ended before the market closed: premature ending flag - continue filling forward until market close.
if (manager.EndOfStream && manager.MarketOpen(current.Time))
{
//Premature end of stream: fill manually until market closed.
for (var date = FillForwardFrontiers[i] + increment; manager.MarketOpen(date); date = date + increment)
{
var cache = new List<BaseData>(1);
var fillforward = current.Clone(true);
fillforward.Time = date;
FillForwardFrontiers[i] = date;
cache.Add(fillforward);
Bridge[i].Enqueue(cache);
}
return;
}
//Once per increment, add a new cache to the Bridge:
//If the current.Time is before market close, (e.g. suspended trading at 2pm) the date is always greater than currentTime and fillforward never runs.
//In this circumstance we need to keep looping till market/extended hours close even if no data.
for (var date = FillForwardFrontiers[i] + increment; (date < current.Time); date = date + increment)
{
//If we don't want aftermarket data, rewind it backwards until the market closes.
if (!Subscriptions[i].ExtendedMarketHours)
{
if (!manager.MarketOpen(date))
{
// Move fill forward so we don't waste time in this tight loop.
//Question is where to shuffle the date?
// --> If BEFORE market open, shuffle forward.
// --> If AFTER market close, and current.Time after market close, quit loop.
date = current.Time;
do
{
date = date - increment;
} while (manager.MarketOpen(date));
continue;
}
}
else
{
//If we've asked for extended hours, and the security is no longer inside extended market hours, skip:
if (!manager.ExtendedMarketOpen(date))
{
continue;
}
}
var cache = new List<BaseData>(1);
var fillforward = previous.Clone(true);
fillforward.Time = date;
FillForwardFrontiers[i] = date;
cache.Add(fillforward);
Bridge[i].Enqueue(cache);
}
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config);
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator<BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof (CoarseFundamental))
{
var cf = new CoarseFundamental();
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// load coarse data day by day
var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
let source = cf.GetSource(config, dateInDataTimeZone, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);
ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/********************************************************
* CLASS CONSTRUCTOR
*********************************************************/
/// <summary>
/// Create a new backtesting data feed.
/// </summary>
/// <param name="algorithm">Instance of the algorithm</param>
/// <param name="job">Algorithm work task</param>
public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job)
{
Subscriptions = algorithm.SubscriptionManager.Subscriptions;
_subscriptions = Subscriptions.Count;
//Public Properties:
DataFeed = DataFeedEndpoint.FileSystem;
IsActive = true;
Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions];
EndOfBridge = new bool[_subscriptions];
SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions];
FillForwardFrontiers = new DateTime[_subscriptions];
RealtimePrices = new List<decimal>(_subscriptions);
//Class Privates:
_job = job;
_algorithm = algorithm;
_endOfStreams = false;
_bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count:
}
private static LiveSubscription CreateSubscription(IAlgorithm algorithm,
IResultHandler resultHandler,
Security security,
DateTime periodStart,
DateTime periodEnd)
{
IEnumerator<BaseData> enumerator = null;
if (security.IsDynamicallyLoadedData)
{
//Subscription managers for downloading user data:
// TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today
var subscriptionDataReader = new SubscriptionDataReader(
security.SubscriptionDataConfig,
security,
periodStart, Time.EndOfTime,
resultHandler,
Time.EachTradeableDay(algorithm.Securities.Values, periodStart, periodEnd),
true,
DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).Date
);
// wrap the subscription data reader with a filter enumerator
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, periodEnd);
}
return new LiveSubscription(security, enumerator, periodStart, periodEnd, true, false);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
protected virtual Subscription CreateUniverseSubscription(
IUniverse universe,
DateTime startTimeUtc,
DateTime endTimeUtc
)
{
// grab the relevant exchange hours
var config = universe.Configuration;
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
.GetExchangeHours(config.Market, null, config.SecurityType);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
IEnumerator<BaseData> enumerator;
if (config.Type == typeof (CoarseFundamental))
{
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>>
{
{config.SecurityType, new List<string>{config.Symbol}}
});
var enqueable = new EnqueableEnumerator<BaseData>();
_exchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
});
enumerator = enqueable;
}
else
{
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
// define our data enumerator
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true);
_customExchange.AddEnumerator(reader);
var enqueable = new EnqueableEnumerator<BaseData>();
_customExchange.SetHandler(config.Symbol, data =>
{
var universeData = data as BaseDataCollection;
if (universeData != null)
{
enqueable.EnqueueRange(universeData.Data);
}
else
{
enqueable.Enqueue(data);
}
});
enumerator = enqueable;
}
// create the subscription
var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc);
return subscription;
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(
IUniverse universe,
DateTime startTimeUtc,
DateTime endTimeUtc
)
{
// grab the relevant exchange hours
SubscriptionDataConfig config = universe.Configuration;
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
.GetExchangeHours(config.Market, null, config.SecurityType);
// create a canonical security object
var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);
var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);
// define our data enumerator
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false);
// create the subscription
var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc);
// only message the user if it's one of their universe types
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return null;
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var enqueueable = new EnqueueableEnumerator<BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return subscription;
}
