/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider) { var tradableDays = _tradableDaysProvider(request); var fineFundamental = new FineFundamental(); var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol); return ( from date in tradableDays let fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date) let fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataFileProvider, fineFundamentalConfiguration, date, false) let fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault() select new FineFundamental { DataType = MarketDataType.Auxiliary, Symbol = request.Configuration.Symbol, Time = date, CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(), SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(), FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(), EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(), OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(), EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(), ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios() } ).GetEnumerator(); }
/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider) { var enumerators = GetSubscriptionConfigurations(request) .Select(c => new SubscriptionRequest(request, configuration: c)) .Select(sr => _enumeratorConfigurator(request, _factory.CreateEnumerator(sr, dataFileProvider)) ); var sync = new SynchronizingEnumerator(enumerators); return new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request) { return new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, _resultHandler, _mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _includeAuxiliaryData ); }
private IEnumerable<SubscriptionDataConfig> GetSubscriptionConfigurations(SubscriptionRequest request) { // canonical also needs underlying price data var config = request.Configuration; var underlying = Symbol.Create(config.Symbol.ID.Symbol, SecurityType.Equity, config.Market); var resolution = config.Resolution == Resolution.Tick ? Resolution.Second : config.Resolution; return new[] { // rewrite the primary to be non-tick and fill forward new SubscriptionDataConfig(config, resolution: resolution, fillForward: true), // add underlying trade data new SubscriptionDataConfig(config, resolution: resolution, fillForward: true, symbol: underlying, objectType: typeof (TradeBar), tickType: TickType.Trade), }; }
/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request) { var configuration = request.Configuration; var tradableDays = _tradableDaysProvider(request); var sourceFactory = (BaseData) Activator.CreateInstance(request.Configuration.Type); return ( from date in tradableDays let source = sourceFactory.GetSource(configuration, date, false) let factory = SubscriptionDataSourceReader.ForSource(source, configuration, date, false) let coarseFundamentalForDate = factory.Read(source) select new BaseDataCollection(date.AddDays(1), configuration.Symbol, coarseFundamentalForDate) ).GetEnumerator(); }
public void ReadsFineFundamental(FineFundamentalTestParameters parameters) { var stopwatch = Stopwatch.StartNew(); var rows = new List<FineFundamental>(); var config = new SubscriptionDataConfig(typeof(FineFundamental), parameters.Symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); var security = new Security(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(CashBook.AccountCurrency, 0, 1), SymbolProperties.GetDefault(CashBook.AccountCurrency)); var request = new SubscriptionRequest(false, null, security, config, parameters.StartDate, parameters.EndDate); var fileProvider = new DefaultDataFileProvider(); var factory = new FineFundamentalSubscriptionEnumeratorFactory(); var enumerator = factory.CreateEnumerator(request, fileProvider); while (enumerator.MoveNext()) { var current = enumerator.Current as FineFundamental; rows.Add(current); } stopwatch.Stop(); Console.WriteLine("Total rows: {0}, elapsed time: {1}", rows.Count, stopwatch.Elapsed); Assert.AreEqual(parameters.RowCount, rows.Count); if (parameters.RowCount != 1) return; var row = rows[0]; Assert.AreEqual(parameters.CompanyShortName, row.CompanyReference.ShortName); Assert.AreEqual(parameters.Symbol, row.Symbol); Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.CompanyReference.PrimarySymbol); Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.SecurityReference.SecuritySymbol); Assert.AreEqual(parameters.Ebitda3M, row.FinancialStatements.IncomeStatement.EBITDA.ThreeMonths); Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA.TwelveMonths); Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA); Assert.AreEqual(parameters.CostOfRevenue3M, row.FinancialStatements.IncomeStatement.CostOfRevenue.ThreeMonths); Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue.TwelveMonths); Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue); Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth.OneYear); Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth); Assert.AreEqual(parameters.PeRatio, row.ValuationRatios.PERatio); }
/// <summary> /// Adds a new subscription to provide data for the specified security. /// </summary> /// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param> /// <returns>True if the subscription was created and added successfully, false otherwise</returns> public bool AddSubscription(SubscriptionRequest request) { if (_subscriptions.Contains(request.Configuration)) { // duplicate subscription request return false; } // create and add the subscription to our collection var subscription = request.IsUniverseSubscription ? CreateUniverseSubscription(request) : CreateSubscription(request); // for some reason we couldn't create the subscription if (subscription == null) { Log.Trace("Unable to add subscription for: " + request.Configuration); return false; } Log.Trace("LiveTradingDataFeed.AddSubscription(): Added " + request.Configuration); _subscriptions.TryAdd(subscription); // send the subscription for the new symbol through to the data queuehandler // unless it is custom data, custom data is retrieved using the same as backtest if (!subscription.Configuration.IsCustomData) { _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol}); } // keep track of security changes, we emit these to the algorithm // as notifications, used in universe selection _changes += SecurityChanges.Added(request.Security); UpdateFillForwardResolution(); return true; }
/// <summary> /// Invokes the configuration following enumerator creation /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataFileProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataFileProvider dataFileProvider) { return _configurator(_factory.CreateEnumerator(request, dataFileProvider)); }
/// <summary> /// Creates an enumerator to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request) { return _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase) .Select(x => new Tick {Time = x, Symbol = request.Configuration.Symbol}).GetEnumerator(); }
/// <summary> /// Invokes the configuration following enumerator creation /// </summary> /// <param name="request">The subscription request to be read</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request) { return _configurator(_factory.CreateEnumerator(request)); }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is UserDefinedUniverse) { // Trigger universe selection when security added/removed after Initialize var universe = (UserDefinedUniverse) request.Universe; universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) return; var symbol = items.OfType<Symbol>().FirstOrDefault(); if (symbol == null) return; var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; return new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder()); } if (request.Configuration.Type == typeof (CoarseFundamental)) { return new BaseDataCollectionSubscriptionEnumeratorFactory(); } if (request.Configuration.Type == typeof(FineFundamental)) { return new FineFundamentalSubscriptionEnumeratorFactory(); } if (request.Universe is OptionChainUniverse) { return new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)); } } var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; return new PostCreateConfigureSubscriptionEnumeratorFactory( new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true), enumerator => ConfigureEnumerator(request, false, enumerator) ); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // grab the relevant exchange hours var config = request.Configuration; // define our data enumerator var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request); var firstLoopCount = 5; var lowerThreshold = GetLowerThreshold(config.Resolution); var upperThreshold = GetUpperThreshold(config.Resolution); if (config.Type == typeof (CoarseFundamental)) { firstLoopCount = 2; lowerThreshold = 5; upperThreshold = 100000; } var enqueueable = new EnqueueableEnumerator<BaseData>(true); ScheduleEnumerator(enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount); enumerator = enqueueable; // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); return new Subscription(request.Universe, request.Security, config, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); }
/// <summary> /// Adds a new subscription to provide data for the specified security. /// </summary> /// <param name="request">Defines the subscription to be added, including start/end times the universe and security</param> /// <returns>True if the subscription was created and added successfully, false otherwise</returns> public bool AddSubscription(SubscriptionRequest request) { if (_subscriptions.Contains(request.Configuration)) { // duplicate subscription request return false; } var subscription = request.IsUniverseSubscription ? CreateUniverseSubscription(request) : CreateSubscription(request); if (subscription == null) { // subscription will be null when there's no tradeable dates for the security between the requested times, so // don't even try to load the data return false; } Log.Debug("FileSystemDataFeed.AddSubscription(): Added " + request.Configuration + " Start: " + request.StartTimeUtc + " End: " + request.EndTimeUtc); if (_subscriptions.TryAdd(subscription)) { UpdateFillForwardResolution(); } return true; }
private Subscription CreateSubscription(SubscriptionRequest request) { // ReSharper disable once PossibleMultipleEnumeration if (!request.TradableDays.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", request.Security.Symbol)); return null; } // ReSharper disable once PossibleMultipleEnumeration var enumeratorFactory = GetEnumeratorFactory(request); var enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = ConfigureEnumerator(request, false, enumerator); var enqueueable = new EnqueueableEnumerator<BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(request.Configuration.Resolution), GetUpperThreshold(request.Configuration.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); var subscription = new Subscription(request.Universe, request.Security, request.Configuration, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); return subscription; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionRequest"/> class /// </summary> public SubscriptionRequest(SubscriptionRequest template, bool? isUniverseSubscription = null, Universe universe = null, Security security = null, SubscriptionDataConfig configuration = null, DateTime? startTimeUtc = null, DateTime? endTimeUtc = null ) : this(isUniverseSubscription ?? template.IsUniverseSubscription, universe ?? template.Universe, security ?? template.Security, configuration ?? template.Configuration, startTimeUtc ?? template.StartTimeUtc, endTimeUtc ?? template.EndTimeUtc ) { }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone}); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) return; var symbol = items.OfType<Symbol>().FirstOrDefault(); if (symbol == null) return; var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return subscription; }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return enumerator; }