/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity || request.Configuration.SecurityType == SecurityType.Option ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; var enumerator = CorporateEventEnumeratorFactory.CreateEnumerators( request.Configuration, _factorFileProvider, dataReader, mapFileResolver, _includeAuxiliaryData); // has to be initialized after adding all the enumerators since it will execute a MoveNext dataReader.Initialize(); return(new SynchronizingEnumerator(dataReader, enumerator)); }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Create a new backtesting data feed. /// </summary> /// <param name="algorithm">Instance of the algorithm</param> /// <param name="job">Algorithm work task</param> public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job) { Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: DataFeed = DataFeedEndpoint.FileSystem; IsActive = true; Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions]; EndOfBridge = new bool[_subscriptions]; SubscriptionReaders = new SubscriptionDataReader[_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); //Class Privates: _job = job; _algorithm = algorithm; _endOfStreams = false; _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count: for (var i = 0; i < _subscriptions; i++) { //Create a new instance in the dictionary: Bridge[i] = new ConcurrentQueue<List<BaseData>>(); EndOfBridge[i] = false; SubscriptionReaders[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish); FillForwardFrontiers[i] = new DateTime(); } }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.TickerShouldBeMapped() ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.StartDateLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; var result = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, request.Configuration, _factorFileProvider, dataReader, mapFileResolver, _includeAuxiliaryData, request.StartTimeLocal, _enablePriceScaling); return(result); }
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution) { var start = new DateTime(2019, 12, 9); var end = new DateTime(2019, 12, 12); var symbol = Symbols.SPY; var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType); var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, dataResolution, TimeZones.NewYork, TimeZones.NewYork, false, false, false); var dataReader = new SubscriptionDataReader(config, new HistoryRequest(config, entry.ExchangeHours, start, end), TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, new TestDataCacheProvider { Data = data }, TestGlobals.DataProvider ); Assert.IsTrue(dataReader.MoveNext()); Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext()); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity || request.Configuration.SecurityType == SecurityType.Option ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider, _includeAuxiliaryData ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; dataReader.Initialize(); return(dataReader); }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Create an instance of the base datafeed. /// </summary> public BaseDataFeed(IAlgorithm algorithm, BacktestNodePacket job) { //Save the data subscriptions Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: DataFeed = DataFeedEndpoint.FileSystem; IsActive = true; Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions]; EndOfBridge = new bool[_subscriptions]; SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); _frontierTime = new DateTime[_subscriptions]; //Class Privates: _job = job; _algorithm = algorithm; _endOfStreams = false; _bridgeMax = _bridgeMax / _subscriptions; //Initialize arrays: for (var i = 0; i < _subscriptions; i++) { _frontierTime[i] = job.PeriodStart; EndOfBridge[i] = false; Bridge[i] = new ConcurrentQueue<List<BaseData>>(); SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, job.PeriodStart, job.PeriodFinish); } }
public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecondDataPoint, Resolution dataResolution) { var start = new DateTime(2019, 12, 9); var end = new DateTime(2019, 12, 12); var mapFileProvider = new LocalDiskMapFileProvider(); var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA)); var dataReader = new SubscriptionDataReader( new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, dataResolution, TimeZones.NewYork, TimeZones.NewYork, false, false, false), start, end, mapFileResolver, new LocalDiskFactorFileProvider(mapFileProvider), LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)), false, new TestDataCacheProvider { Data = data } ); Assert.IsTrue(dataReader.MoveNext()); Assert.AreEqual(shouldEmitSecondDataPoint, dataReader.MoveNext()); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.SecurityType, request.Symbol, request.Resolution, request.Market, request.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData ); var security = new Security(request.ExchangeHours, config, 1.0m); IEnumerator <BaseData> reader = new SubscriptionDataReader(config, start, end, ResultHandlerStub.Instance, Time.EachTradeableDay(request.ExchangeHours, start, end), false, includeAuxilliaryData: false ); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { reader = new FillForwardEnumerator(reader, security.Exchange, request.FillForwardResolution.Value.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Increment); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); return(new Subscription(security, reader, start, end, false, false)); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.TickerShouldBeMapped() ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider, dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.StartDateLimited += (sender, args) => { // Queue this warning into our dictionary to report on dispose if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount) { _startDateLimitedWarnings.TryAdd(args.Symbol, args.Message); } }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; dataReader.NumericalPrecisionLimited += (sender, args) => { // Set a hard limit to keep this warning list from getting unnecessarily large if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount) { _numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message); } }; var result = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, request.Configuration, _factorFileProvider, dataReader, mapFileResolver, request.StartTimeLocal, _enablePriceScaling); return(result); }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { //Subscription Count: _subscriptions = algorithm.SubscriptionManager.Subscriptions; //Set Properties: _isActive = true; _dataFeed = DataFeedEndpoint.LiveTrading; _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count]; _endOfBridge = new bool[Subscriptions.Count]; _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count]; _realtimePrices = new List<decimal>(); //Set the source of the live data: _dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue")); //Class Privates: _algorithm = algorithm; if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _job = (LiveNodePacket) job; //Setup the arrays: for (var i = 0; i < Subscriptions.Count; i++) { _endOfBridge[i] = false; _bridge[i] = new ConcurrentQueue<List<BaseData>>(); //This is quantconnect data source, store here for speed/ease of access _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData); //Subscription managers for downloading user data: _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue, resultHandler); //Set up the source file for today: _subscriptionManagers[i].RefreshSource(DateTime.Now.Date); _realtimePrices.Add(0); } // request for data from these symbols var symbols = BuildTypeSymbolList(algorithm); if (symbols.Any()) { // don't subscribe if there's nothing there, this allows custom data to // work without an IDataQueueHandler implementation by specifying LiveDataQueue // in the configuration, that implementation throws on every method, but we actually // don't need it if we're only doing custom data _dataQueue.Subscribe(_job, symbols); } }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.TimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, null, true, request.DataNormalizationMode ); var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency)); IEnumerator <BaseData> reader = new SubscriptionDataReader(config, start, end, ResultHandlerStub.Instance, config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty, _factorFileProvider, _dataProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, _dataCacheProvider, false ); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false)); }
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined) { var config = security.SubscriptionDataConfig; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { if (userDefined) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); } return null; } // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); var subscription = new Subscription(security, enumerator, start, end, userDefined, false); return subscription; }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { _cancellationTokenSource = new CancellationTokenSource(); Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: IsActive = true; _endOfBridge = new bool[_subscriptions]; SubscriptionReaders = new IEnumerator<BaseData>[_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); //Class Privates: _algorithm = algorithm; // find the minimum resolution, ignoring ticks var fillForwardResolution = Subscriptions .Where(x => x.Resolution != Resolution.Tick) .Select(x => x.Resolution.ToTimeSpan()) .DefaultIfEmpty(TimeSpan.FromSeconds(1)) .Min(); // figure out how many subscriptions are at the minimum resolution var subscriptionsAtMinimumResolution = (from sub in Subscriptions where sub.Resolution == Subscriptions.Min(x => x.Resolution) select sub).Count(); Bridge = new BlockingCollection<TimeSlice>(Math.Min(1000, 50000/subscriptionsAtMinimumResolution)); for (var i = 0; i < _subscriptions; i++) { _endOfBridge[i] = false; var config = Subscriptions[i]; var start = algorithm.StartDate; var end = algorithm.EndDate; var security = _algorithm.Securities[Subscriptions[i].Symbol]; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, // we want a backtest to use the same symbols regardless of start/end dates, so we set the resolution // to null which means give me the current/most recent symbol mapping for the requested symbol symbolResolutionDate: null ); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); FillForwardFrontiers[i] = new DateTime(); // prime the pump for iteration in Run _endOfBridge[i] = !SubscriptionReaders[i].MoveNext(); if (_endOfBridge[i]) { Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol); } } }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // custom data uses backtest readers var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false); // apply fast forwarding, this is especially important for RemoteFile types that // can send in large chunks of old, irrelevant data var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment); // apply rate limits (1x per increment, max 30 minutes between calls) // TODO : Pull limits from config file? var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); // add the enumerator to the exchange _customExchange.AddEnumerator(rateLimit); // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription); } catch (Exception err) { Log.Error(err); } return subscription; }
/******************************************************** * CLASS METHODS *********************************************************/ /// <summary> /// Initialize activators to invoke types in the algorithm /// </summary> private void ResetActivators() { for (var i = 0; i < _subscriptions; i++) { //Create a new instance in the dictionary: Bridge[i] = new ConcurrentQueue<List<BaseData>>(); EndOfBridge[i] = false; SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], _algorithm.Securities[Subscriptions[i].Symbol], DataFeed, _job.PeriodStart, _job.PeriodFinish); FillForwardFrontiers[i] = new DateTime(); } }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> /// <param name="algorithm">Algorithm requesting data</param> protected LiveTradingDataFeed(IAlgorithm algorithm) { //Subscription Count: _subscriptions = algorithm.SubscriptionManager.Subscriptions; //Set Properties: _dataFeed = DataFeedEndpoint.LiveTrading; _isActive = true; _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count]; _endOfBridge = new bool[Subscriptions.Count]; _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count]; _realtimePrices = new List<decimal>(); //Class Privates: _algorithm = algorithm; //Setup the arrays: for (var i = 0; i < Subscriptions.Count; i++) { _endOfBridge[i] = false; _bridge[i] = new ConcurrentQueue<List<BaseData>>(); //This is quantconnect data source, store here for speed/ease of access _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData); //Subscription managers for downloading user data: _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue); //Set up the source file for today: _subscriptionManagers[i].RefreshSource(DateTime.Now.Date); _realtimePrices.Add(0); } }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security( request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency), ErrorCurrencyConverter.Instance ); var dataReader = new SubscriptionDataReader(config, start, end, config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty, _factorFileProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, _dataCacheProvider, false ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); }; dataReader.Initialize(); IEnumerator <BaseData> reader = dataReader; // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator<BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable<DateTime> tradeableDates, bool applySubscripterFilterEnumerator = true) { IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (applySubscripterFilterEnumerator) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return enumerator; }
/// <summary> /// Prepare and create the new MySQL Database connection datafeed. /// </summary> /// <param name="algorithm"></param> /// <param name="job"></param> /// <param name="resultHandler"></param> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { Bridge = new BlockingCollection<TimeSlice>(); //Save the data subscriptions Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: IsActive = true; EndOfBridge = new bool[_subscriptions]; _subscriptionReaderManagers = new SubscriptionDataReader[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); _mySQLBridgeTime = new DateTime[_subscriptions]; //Class Privates: _endTime = algorithm.EndDate; //Initialize arrays: for (var i = 0; i < _subscriptions; i++) { _mySQLBridgeTime[i] = algorithm.StartDate; EndOfBridge[i] = false; _subscriptionReaderManagers[i] = new SubscriptionDataReader( Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], algorithm.StartDate, algorithm.EndDate, resultHandler, Time.EachTradeableDay(algorithm.Securities, algorithm.StartDate, algorithm.EndDate), job is LiveNodePacket ); } }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { _cancellationTokenSource = new CancellationTokenSource(); //Subscription Count: _subscriptions = algorithm.SubscriptionManager.Subscriptions; Bridge = new BlockingCollection<TimeSlice>(); //Set Properties: _isActive = true; _endOfBridge = new bool[Subscriptions.Count]; _subscriptionManagers = new IEnumerator<BaseData>[Subscriptions.Count]; _realtimePrices = new List<decimal>(); //Set the source of the live data: _dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue")); //Class Privates: _algorithm = algorithm; if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _job = (LiveNodePacket) job; //Setup the arrays: for (var i = 0; i < Subscriptions.Count; i++) { _endOfBridge[i] = false; //This is quantconnect data source, store here for speed/ease of access var security = algorithm.Securities[_subscriptions[i].Symbol]; _isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData); // only make readers for custom data, live data will come through data queue handler if (_isDynamicallyLoadedData[i]) { //Subscription managers for downloading user data: // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today var periodStart = DateTime.Today.AddDays(-7); var subscriptionDataReader = new SubscriptionDataReader( _subscriptions[i], security, periodStart, DateTime.MaxValue, resultHandler, Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue), true, DateTime.Today ); // wrap the subscription data reader with a filter enumerator _subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue); } _realtimePrices.Add(0); } // request for data from these symbols var symbols = BuildTypeSymbolList(algorithm); if (symbols.Any()) { // don't subscribe if there's nothing there, this allows custom data to // work without an IDataQueueHandler implementation by specifying LiveDataQueue // in the configuration, that implementation throws on every method, but we actually // don't need it if we're only doing custom data _dataQueue.Subscribe(_job, symbols); } }
private void ProcessMissingDayFillForward(SubscriptionDataReader manager, int i, TimeSpan increment, DateTime dateToFill) { // we'll copy the current into the next day var subscription = Subscriptions[i]; if (!subscription.FillDataForward || manager.Current == null) return; var start = dateToFill.Date + manager.Exchange.MarketOpen; if (subscription.ExtendedMarketHours) { // we need to loop to find the extended market open start = dateToFill.Date; while (!manager.Exchange.DateTimeIsExtendedOpen(start)) { start += increment; } } var current = manager.Current; for (var date = start; date.Date == dateToFill.Date; date = date + increment) { if (manager.IsMarketOpen(date) || (subscription.ExtendedMarketHours && manager.IsExtendedMarketOpen(date))) { var cache = new List<BaseData>(1); var fillforward = current.Clone(true); fillforward.Time = date; FillForwardFrontiers[i] = date; cache.Add(fillforward); Bridge[i].Enqueue(cache); } else { // stop fill forwarding when we're no longer open break; } } }
private static Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined) { var config = security.SubscriptionDataConfig; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); var symbolResolutionDate = userDefined ? (DateTime?)null : start; IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); var subscription = new Subscription(security, enumerator, start, end, userDefined, false); return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick {Time = dt}).GetEnumerator(); } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false); } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); // only message the user if it's one of their universe types var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental); PrimeSubscriptionPump(subscription, messageUser); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public LiveTradingDataFeed(IAlgorithm algorithm, LiveNodePacket job, IDataQueueHandler dataSource) { //Subscription Count: _subscriptions = algorithm.SubscriptionManager.Subscriptions; //Set Properties: _isActive = true; _dataFeed = DataFeedEndpoint.LiveTrading; _bridge = new ConcurrentQueue<List<BaseData>>[Subscriptions.Count]; _endOfBridge = new bool[Subscriptions.Count]; _subscriptionManagers = new SubscriptionDataReader[Subscriptions.Count]; _realtimePrices = new List<decimal>(); //Set the source of the live data: _dataQueue = dataSource; //Class Privates: _algorithm = algorithm; _job = job; //Setup the arrays: for (var i = 0; i < Subscriptions.Count; i++) { _endOfBridge[i] = false; _bridge[i] = new ConcurrentQueue<List<BaseData>>(); //This is quantconnect data source, store here for speed/ease of access _isDynamicallyLoadedData.Add(algorithm.Securities[_subscriptions[i].Symbol].IsDynamicallyLoadedData); //Subscription managers for downloading user data: _subscriptionManagers[i] = new SubscriptionDataReader(_subscriptions[i], algorithm.Securities[_subscriptions[i].Symbol], DataFeedEndpoint.LiveTrading, DateTime.MinValue, DateTime.MaxValue); //Set up the source file for today: _subscriptionManagers[i].RefreshSource(DateTime.Now.Date); _realtimePrices.Add(0); } // request for data from these symbols _dataQueue.Subscribe(job, BuildTypeSymbolList(algorithm)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request) { var config = request.ToSubscriptionDataConfig(); DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var dataReader = new SubscriptionDataReader(config, request, _mapFileProvider, _factorFileProvider, _dataCacheProvider, _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, _mapFileProvider, request.StartTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out all aux data if (data.DataType == MarketDataType.Auxiliary) { return(false); } // filter out future data if (data.EndTime > request.EndTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > request.StartTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Prepare and create the new MySQL Database connection datafeed. /// </summary> /// <param name="algorithm"></param> /// <param name="job"></param> /// <param name="resultHandler"></param> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { //Save the data subscriptions Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: IsActive = true; DataFeed = DataFeedEndpoint.FileSystem; Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions]; EndOfBridge = new bool[_subscriptions]; SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); _mySQLBridgeTime = new DateTime[_subscriptions]; //Class Privates: _endOfStreams = false; _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count: _endTime = algorithm.EndDate; //Initialize arrays: for (var i = 0; i < _subscriptions; i++) { _mySQLBridgeTime[i] = algorithm.StartDate; EndOfBridge[i] = false; Bridge[i] = new ConcurrentQueue<List<BaseData>>(); SubscriptionReaderManagers[i] = new SubscriptionDataReader(Subscriptions[i], algorithm.Securities[Subscriptions[i].Symbol], DataFeedEndpoint.Database, algorithm.StartDate, algorithm.EndDate, resultHandler); } }
/// <summary> /// If this is a fillforward subscription, look at the previous time, and current time, and add new /// objects to queue until current time to fill up the gaps. /// </summary> /// <param name="manager">Subscription to process</param> /// <param name="i">Subscription position in the bridge ( which queue are we pushing data to )</param> /// <param name="increment">Timespan increment to jump the fillforward results</param> void ProcessFillForward(SubscriptionDataReader manager, int i, TimeSpan increment) { // If previous == null cannot fill forward nothing there to move forward (e.g. cases where file not found on first file). if (!Subscriptions[i].FillDataForward || manager.Previous == null) return; //Last tradebar and the current one we're about to add to queue: var previous = manager.Previous; var current = manager.Current; //Initialize the frontier: if (FillForwardFrontiers[i].Ticks == 0) FillForwardFrontiers[i] = previous.Time; //Data ended before the market closed: premature ending flag - continue filling forward until market close. if (manager.EndOfStream && manager.MarketOpen(current.Time)) { //Premature end of stream: fill manually until market closed. for (var date = FillForwardFrontiers[i] + increment; manager.MarketOpen(date); date = date + increment) { var cache = new List<BaseData>(1); var fillforward = current.Clone(true); fillforward.Time = date; FillForwardFrontiers[i] = date; cache.Add(fillforward); Bridge[i].Enqueue(cache); } return; } //Once per increment, add a new cache to the Bridge: //If the current.Time is before market close, (e.g. suspended trading at 2pm) the date is always greater than currentTime and fillforward never runs. //In this circumstance we need to keep looping till market/extended hours close even if no data. for (var date = FillForwardFrontiers[i] + increment; (date < current.Time); date = date + increment) { //If we don't want aftermarket data, rewind it backwards until the market closes. if (!Subscriptions[i].ExtendedMarketHours) { if (!manager.MarketOpen(date)) { // Move fill forward so we don't waste time in this tight loop. //Question is where to shuffle the date? // --> If BEFORE market open, shuffle forward. // --> If AFTER market close, and current.Time after market close, quit loop. date = current.Time; do { date = date - increment; } while (manager.MarketOpen(date)); continue; } } else { //If we've asked for extended hours, and the security is no longer inside extended market hours, skip: if (!manager.ExtendedMarketOpen(date)) { continue; } } var cache = new List<BaseData>(1); var fillforward = previous.Clone(true); fillforward.Time = date; FillForwardFrontiers[i] = date; cache.Add(fillforward); Bridge[i].Enqueue(cache); } }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator<BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { var cf = new CoarseFundamental(); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // load coarse data day by day var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate) let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false) let source = cf.GetSource(config, dateInDataTimeZone, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate); ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2); enumerator = enqueueable; } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/******************************************************** * CLASS CONSTRUCTOR *********************************************************/ /// <summary> /// Create a new backtesting data feed. /// </summary> /// <param name="algorithm">Instance of the algorithm</param> /// <param name="job">Algorithm work task</param> public FileSystemDataFeed(IAlgorithm algorithm, BacktestNodePacket job) { Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: DataFeed = DataFeedEndpoint.FileSystem; IsActive = true; Bridge = new ConcurrentQueue<List<BaseData>>[_subscriptions]; EndOfBridge = new bool[_subscriptions]; SubscriptionReaderManagers = new SubscriptionDataReader[_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; RealtimePrices = new List<decimal>(_subscriptions); //Class Privates: _job = job; _algorithm = algorithm; _endOfStreams = false; _bridgeMax = _bridgeMax / _subscriptions; //Set the bridge maximum count: }
private static LiveSubscription CreateSubscription(IAlgorithm algorithm, IResultHandler resultHandler, Security security, DateTime periodStart, DateTime periodEnd) { IEnumerator<BaseData> enumerator = null; if (security.IsDynamicallyLoadedData) { //Subscription managers for downloading user data: // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today var subscriptionDataReader = new SubscriptionDataReader( security.SubscriptionDataConfig, security, periodStart, Time.EndOfTime, resultHandler, Time.EachTradeableDay(algorithm.Securities.Values, periodStart, periodEnd), true, DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).Date ); // wrap the subscription data reader with a filter enumerator enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, periodEnd); } return new LiveSubscription(security, enumerator, periodStart, periodEnd, true, false); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription( IUniverse universe, DateTime startTimeUtc, DateTime endTimeUtc ) { // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder() .GetExchangeHours(config.Market, null, config.SecurityType); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>> { {config.SecurityType, new List<string>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true); _customExchange.AddEnumerator(reader); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc); return subscription; }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription( IUniverse universe, DateTime startTimeUtc, DateTime endTimeUtc ) { // grab the relevant exchange hours SubscriptionDataConfig config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder() .GetExchangeHours(config.Market, null, config.SecurityType); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false); // create the subscription var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc); // only message the user if it's one of their universe types _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return null; } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return subscription; }