private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source) { var factory = SubscriptionFactory.ForSource(source, _config, _tradeableDates.Current, _isLiveMode); AttachEventHandlers(factory, source); return(factory); }
private ISubscriptionFactory HandleCsvFileFormat(SubscriptionDataSource source) { // convert the date to the data time zone var dateInDataTimeZone = _tradeableDates.Current.ConvertTo(_config.ExchangeTimeZone, _config.DataTimeZone).Date; var factory = SubscriptionFactory.ForSource(source, _config, dateInDataTimeZone, _isLiveMode); // handle missing files factory.InvalidSource += (sender, args) => { switch (args.Source.TransportMedium) { case SubscriptionTransportMedium.LocalFile: // the local uri doesn't exist, write an error and return null so we we don't try to get data for today Log.Trace(string.Format("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: {0}", source)); _resultHandler.SamplePerformance(_tradeableDates.Current, 0); break; case SubscriptionTransportMedium.RemoteFile: _resultHandler.ErrorMessage(string.Format("Error downloading custom data source file, skipped: {0} Error: {1}", source, args.Exception.Message), args.Exception.StackTrace); _resultHandler.SamplePerformance(_tradeableDates.Current.Date, 0); break; case SubscriptionTransportMedium.Rest: break; default: throw new ArgumentOutOfRangeException(); } }; if (factory is TextSubscriptionFactory) { // handle empty files/instantiation errors var textSubscriptionFactory = (TextSubscriptionFactory)factory; textSubscriptionFactory.CreateStreamReaderError += (sender, args) => { Log.Error(string.Format("Failed to get StreamReader for data source({0}), symbol({1}). Skipping date({2}). Reader is null.", args.Source.Source, _mappedSymbol, args.Date.ToShortDateString())); if (_config.IsCustomData) { _resultHandler.ErrorMessage(string.Format("We could not fetch the requested data. This may not be valid data, or a failed download of custom data. Skipping source ({0}).", args.Source.Source)); } }; // handle parser errors textSubscriptionFactory.ReaderError += (sender, args) => { _resultHandler.RuntimeError(string.Format("Error invoking {0} data reader. Line: {1} Error: {2}", _config.Symbol, args.Line, args.Exception.Message), args.Exception.StackTrace); }; } return(factory); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable <DateTime> tradeableDates, bool useSubscriptionDataReader, bool aggregate) { IEnumerator <BaseData> enumerator; if (useSubscriptionDataReader) { enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); } else { var sourceFactory = (BaseData)Activator.CreateInstance(config.Type); enumerator = (from date in tradeableDates let source = sourceFactory.GetSource(config, date, false) let factory = SubscriptionFactory.ForSource(source, config, date, false) let entriesForDate = factory.Read(source) from entry in entriesForDate select entry).GetEnumerator(); } if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol); } // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (config.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return(enumerator); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator <BaseData> enumerator; var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { var cf = new CoarseFundamental(); var enqueueable = new EnqueueableEnumerator <BaseData>(true); // load coarse data day by day var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate) let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date let source = cf.GetSource(config, dateInDataTimeZone, false) let factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate); ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2); enumerator = enqueueable; } else { // normal reader for all others enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return(subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator <BaseData> enumerator; if (config.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(config.Symbol, rateLimit); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(config.Symbol, data => { aggregator.ProcessData((Tick)data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source) { return(SubscriptionFactory.ForSource(source, _config, _tradeableDates.Current, _isLiveMode)); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); // define our data enumerator IEnumerator <BaseData> enumerator; var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours); var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator(); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { var cf = new CoarseFundamental(); // load coarse data day by day enumerator = (from date in Time.EachTradeableDayInTimeZone(security.Exchange.Hours, _algorithm.StartDate, _algorithm.EndDate, config.DataTimeZone, config.ExtendedMarketHours) let source = cf.GetSource(config, date, false) let factory = SubscriptionFactory.ForSource(source, config, date, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date.AddDays(1), config.Symbol, coarseFundamentalForDate) ).GetEnumerator(); var enqueueable = new EnqueueableEnumerator <BaseData>(true); ScheduleEnumerator(enumerator, enqueueable, 5, 100000, 2); enumerator = enqueueable; } else if (config.SecurityType == SecurityType.Option && security is Option) { var configs = universe.GetSubscriptions(security); var enumerators = configs.Select(c => CreateSubscriptionEnumerator(security, c, localStartTime, localEndTime, _mapFileProvider.Get(c.Market), tradeableDates, false, true) ).ToList(); var sync = new SynchronizingEnumerator(enumerators); enumerator = new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, config.Symbol); var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } else { // normal reader for all others enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, MapFileResolver.Empty, tradeableDates, true, false); // route these custom subscriptions through the exchange for buffering var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); enumerator = enqueueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); _subscriptions.TryAdd(subscription); UpdateFillForwardResolution(); }