/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { // copy forward Bid/Ask bars for QuoteBars if (request.Configuration.Type == typeof(QuoteBar)) { enumerator = new QuoteBarFillForwardEnumerator(enumerator); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan(), request.Configuration.DataTimeZone); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return(enumerator); }
private static LiveSubscription CreateSubscription(IAlgorithm algorithm, IResultHandler resultHandler, Security security, DateTime periodStart, DateTime periodEnd) { IEnumerator <BaseData> enumerator = null; if (security.SubscriptionDataConfig.IsCustomData) { //Subscription managers for downloading user data: // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today var subscriptionDataReader = new SubscriptionDataReader( security.SubscriptionDataConfig, periodStart, Time.EndOfTime, resultHandler, Time.EachTradeableDay(algorithm.Securities.Values, periodStart, periodEnd), true ); // wrap the subscription data reader with a filter enumerator enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, periodEnd); } return(new LiveSubscription(security, enumerator, periodStart, periodEnd, true, false)); }
private Subscription CreateSubscription(IResultHandler resultHandler, Security security, DateTime start, DateTime end, Resolution fillForwardResolution, bool userDefined) { var config = security.SubscriptionDataConfig; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { if (userDefined) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); } return(null); } var symbolResolutionDate = userDefined ? (DateTime?)null : start; // ReSharper disable once PossibleMultipleEnumeration IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, security, start, end, resultHandler, tradeableDates, false, symbolResolutionDate); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution.ToTimeSpan(), security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); var subscription = new Subscription(security, enumerator, start, end, userDefined, false); return(subscription); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return(null); } // ReSharper disable once PossibleMultipleEnumeration IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return(subscription); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable <DateTime> tradeableDates, bool useSubscriptionDataReader, bool aggregate) { IEnumerator <BaseData> enumerator; if (useSubscriptionDataReader) { enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); } else { var sourceFactory = (BaseData)Activator.CreateInstance(config.Type); enumerator = (from date in tradeableDates let source = sourceFactory.GetSource(config, date, false) let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false) let entriesForDate = factory.Read(source) from entry in entriesForDate select entry).GetEnumerator(); } if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol); } // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (config.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return(enumerator); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> protected IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward); // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours); } return(enumerator); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return(enumerator); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable <DateTime> tradeableDates) { IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (config.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return(enumerator); }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { //Subscription Count: _subscriptions = algorithm.SubscriptionManager.Subscriptions; Bridge = new BlockingCollection <TimeSlice>(); //Set Properties: _isActive = true; _endOfBridge = new bool[Subscriptions.Count]; _subscriptionManagers = new IEnumerator <BaseData> [Subscriptions.Count]; _realtimePrices = new List <decimal>(); //Set the source of the live data: _dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue")); //Class Privates: _algorithm = algorithm; if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _job = (LiveNodePacket)job; //Setup the arrays: for (var i = 0; i < Subscriptions.Count; i++) { _endOfBridge[i] = false; //This is quantconnect data source, store here for speed/ease of access var security = algorithm.Securities[_subscriptions[i].Symbol]; _isDynamicallyLoadedData.Add(security.IsDynamicallyLoadedData); // only make readers for custom data, live data will come through data queue handler if (_isDynamicallyLoadedData[i]) { //Subscription managers for downloading user data: // TODO: Update this when warmup comes in, we back up so we can get data that should have emitted at midnight today var periodStart = DateTime.Today.AddDays(-7); var subscriptionDataReader = new SubscriptionDataReader( _subscriptions[i], security, DataFeedEndpoint.LiveTrading, periodStart, DateTime.MaxValue, resultHandler, Time.EachTradeableDay(algorithm.Securities, periodStart, DateTime.MaxValue) ); // wrap the subscription data reader with a filter enumerator _subscriptionManagers[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, subscriptionDataReader, security, DateTime.MaxValue); } _realtimePrices.Add(0); } // request for data from these symbols var symbols = BuildTypeSymbolList(algorithm); if (symbols.Any()) { // don't subscribe if there's nothing there, this allows custom data to // work without an IDataQueueHandler implementation by specifying LiveDataQueue // in the configuration, that implementation throws on every method, but we actually // don't need it if we're only doing custom data _dataQueue.Subscribe(_job, symbols); } }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { _cancellationTokenSource = new CancellationTokenSource(); Subscriptions = algorithm.SubscriptionManager.Subscriptions; _subscriptions = Subscriptions.Count; //Public Properties: IsActive = true; _endOfBridge = new bool[_subscriptions]; SubscriptionReaders = new IEnumerator <BaseData> [_subscriptions]; FillForwardFrontiers = new DateTime[_subscriptions]; RealtimePrices = new List <decimal>(_subscriptions); //Class Privates: _algorithm = algorithm; // find the minimum resolution, ignoring ticks var fillForwardResolution = Subscriptions .Where(x => x.Resolution != Resolution.Tick) .Select(x => x.Resolution.ToTimeSpan()) .DefaultIfEmpty(TimeSpan.FromSeconds(1)) .Min(); // figure out how many subscriptions are at the minimum resolution var subscriptionsAtMinimumResolution = (from sub in Subscriptions where sub.Resolution == Subscriptions.Min(x => x.Resolution) select sub).Count(); Bridge = new BlockingCollection <TimeSlice>(Math.Min(1000, 50000 / subscriptionsAtMinimumResolution)); for (var i = 0; i < _subscriptions; i++) { _endOfBridge[i] = false; var config = Subscriptions[i]; var start = algorithm.StartDate; var end = algorithm.EndDate; var security = _algorithm.Securities[Subscriptions[i].Symbol]; var tradeableDates = Time.EachTradeableDay(security, start.Date, end.Date); IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, security, DataFeedEndpoint.FileSystem, start, end, resultHandler, tradeableDates); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, end, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters SubscriptionReaders[i] = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, end); FillForwardFrontiers[i] = new DateTime(); // prime the pump for iteration in Run _endOfBridge[i] = !SubscriptionReaders[i].MoveNext(); if (_endOfBridge[i]) { Log.Trace("FileSystemDataFeed.Run(): Failed to load subscription: " + Subscriptions[i].Symbol); } } }