public void FredApiParsesDataCorrectly() { // Arrange var fileInfo = new FileInfo("./TestData/FredVixData.json"); var content = File.ReadAllText(fileInfo.FullName); var sid = SecurityIdentifier.GenerateBase(typeof(BaseData), Fred.CBOE.VIX, QuantConnect.Market.USA, false); var symbol = new Symbol(sid, Fred.CBOE.VIX); var subscriptionDataConfig = new SubscriptionDataConfig(typeof(BaseData), symbol, Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true); var fredApi = new FredApi(); // Act var data = (BaseDataCollection)fredApi.Reader(subscriptionDataConfig, content, new DateTime(2019, 01, 04), false); // Assert Assert.AreEqual(18, data.Data.Count); Assert.AreEqual(23.22, data.Data.First().Value); Assert.AreEqual(18.87, data.Data.Last().Value); Assert.AreEqual(357.66m, data.Data.Sum(d => d.Value)); }
/// <summary> /// Initializes the <see cref="SubscriptionDataReader"/> instance /// </summary> /// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set, /// since it will produce events.</remarks> public void Initialize() { if (_initialized) { return; } //Save the type of data we'll be getting from the source. try { _dataFactory = _config.GetBaseDataInstance(); } catch (ArgumentException exception) { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(_config.Symbol, exception.Message)); _endOfStream = true; return; } //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } // If Tiingo data, set the access token in data factory var tiingo = _dataFactory as TiingoPrice; if (tiingo != null) { if (!Tiingo.IsAuthCodeSet) { Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } } // If USEnergyAPI data, set the access token in data factory var energyInformation = _dataFactory as USEnergyAPI; if (energyInformation != null) { if (!USEnergyAPI.IsAuthCodeSet) { USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token")); } } // If Fred data, set the access token in data factory var fred = _dataFactory as FredApi; if (fred != null) { if (!FredApi.IsAuthCodeSet) { FredApi.SetAuthCode(Config.Get("fred-auth-token")); } } _factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>()); _mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>()); // load up the map files for equities, options, and custom data if it supports it. // Only load up factor files for equities if (_dataFactory.RequiresMapping()) { try { var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } if (!_config.IsCustomData && !_config.SecurityType.IsOption()) { var factorFile = _factorFileProvider.Get(_config.Symbol); _hasScaleFactors = factorFile != null; if (_hasScaleFactors) { _factorFile = factorFile; // if factor file has minimum date, update start period if before minimum date if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue) { if (_periodStart < _factorFile.FactorFileMinimumDate.Value) { _periodStart = _factorFile.FactorFileMinimumDate.Value; OnNumericalPrecisionLimited( new NumericalPrecisionLimitedEventArgs(_config.Symbol, $"[{_config.Symbol.Value}, {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}]")); } } } if (_periodStart < mapFile.FirstDate) { _periodStart = mapFile.FirstDate; OnStartDateLimited( new StartDateLimitedEventArgs(_config.Symbol, $"[{_config.Symbol.Value}," + $" {mapFile.FirstDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}]")); } } } catch (Exception err) { Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": "); } } _delistingDate = _config.Symbol.GetDelistingDate(_mapFile); // adding a day so we stop at EOD _delistingDate = _delistingDate.AddDays(1); UpdateDataEnumerator(true); _initialized = true; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateDataSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (!_channelProvider.ShouldStreamSubscription(request.Configuration)) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } if (!Tiingo.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } if (!USEnergyAPI.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token")); } if (!FredApi.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here FredApi.SetAuthCode(Config.Get("fred-auth-token")); } var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); UpdateSubscriptionRealTimePrice( subscription, timeZoneOffsetProvider, request.Security.Exchange.Hours, data); }); enumerator = enqueable; } else { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front switch (request.Configuration.Type.Name) { case nameof(QuoteBar): var quoteBarAggregator = new QuoteBarBuilderEnumerator( request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider, true, (sender, args) => subscription.OnNewDataAvailable()); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick?.TickType == TickType.Quote && !tick.Suspicious) { quoteBarAggregator.ProcessData(tick); UpdateSubscriptionRealTimePrice( subscription, timeZoneOffsetProvider, request.Security.Exchange.Hours, data); } }); enumerator = quoteBarAggregator; break; case nameof(TradeBar): var tradeBarAggregator = new TradeBarBuilderEnumerator( request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider, true, (sender, args) => subscription.OnNewDataAvailable()); var auxDataEnumerator = new LiveAuxiliaryDataEnumerator( request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler( request.Configuration.Symbol, data => { if (data.DataType == MarketDataType.Auxiliary) { auxDataEnumerator.Enqueue(data); subscription.OnNewDataAvailable(); } else { var tick = data as Tick; if (tick?.TickType == TickType.Trade && !tick.Suspicious) { tradeBarAggregator.ProcessData(tick); UpdateSubscriptionRealTimePrice( subscription, timeZoneOffsetProvider, request.Security.Exchange.Hours, data); } } }); enumerator = request.Configuration.SecurityType == SecurityType.Equity ? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator) : tradeBarAggregator; break; case nameof(OpenInterest): var oiAggregator = new OpenInterestEnumerator( request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider, true, (sender, args) => subscription.OnNewDataAvailable()); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious) { oiAggregator.ProcessData(tick); } }); enumerator = oiAggregator; break; case nameof(Tick): default: // tick or streaming custom data subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.AddDataHandler( request.Configuration.Symbol, data => { var tick = data as Tick; if (tick != null) { if (tick.TickType == request.Configuration.TickType) { tickEnumerator.Enqueue(data); subscription.OnNewDataAvailable(); if (tick.TickType != TickType.OpenInterest) { UpdateSubscriptionRealTimePrice( subscription, timeZoneOffsetProvider, request.Security.Exchange.Hours, data); } } } else { tickEnumerator.Enqueue(data); subscription.OnNewDataAvailable(); } }); enumerator = tickEnumerator; break; } } if (request.Configuration.FillDataForward) { var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration); enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateDataSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration)) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } if (!Tiingo.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } if (!USEnergyAPI.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token")); } if (!FredApi.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here FredApi.SetAuthCode(Config.Get("fred-auth-token")); } if (!TradingEconomicsCalendar.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token")); } var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); }); enumerator = enqueable; } else { EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable(); enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler); if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration)) { var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler); var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler); enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits); } } if (request.Configuration.FillDataForward) { var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration); enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider); } catch (Exception err) { Log.Error(err); } return(subscription); }