public void FredApiParsesDataCorrectly()
{
// Arrange
var fileInfo = new FileInfo("./TestData/FredVixData.json");
var content = File.ReadAllText(fileInfo.FullName);
var sid = SecurityIdentifier.GenerateBase(typeof(BaseData), Fred.CBOE.VIX, QuantConnect.Market.USA, false);
var symbol = new Symbol(sid, Fred.CBOE.VIX);
var subscriptionDataConfig = new SubscriptionDataConfig(typeof(BaseData), symbol, Resolution.Daily,
DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true);
var fredApi = new FredApi();
// Act
var data = (BaseDataCollection)fredApi.Reader(subscriptionDataConfig, content, new DateTime(2019, 01, 04), false);
// Assert
Assert.AreEqual(18, data.Data.Count);
Assert.AreEqual(23.22, data.Data.First().Value);
Assert.AreEqual(18.87, data.Data.Last().Value);
Assert.AreEqual(357.66m, data.Data.Sum(d => d.Value));
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(_config.Symbol, exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoPrice;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyAPI data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyAPI;
if (energyInformation != null)
{
if (!USEnergyAPI.IsAuthCodeSet)
{
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
// If Fred data, set the access token in data factory
var fred = _dataFactory as FredApi;
if (fred != null)
{
if (!FredApi.IsAuthCodeSet)
{
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_dataFactory.RequiresMapping())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && !_config.SecurityType.IsOption())
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}, {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}]"));
}
}
}
if (_periodStart < mapFile.FirstDate)
{
_periodStart = mapFile.FirstDate;
OnStartDateLimited(
new StartDateLimitedEventArgs(_config.Symbol,
$"[{_config.Symbol.Value}," +
$" {mapFile.FirstDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}]"));
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
_delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
UpdateDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
});
enumerator = enqueable;
}
else
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.Type.Name)
{
case nameof(QuoteBar):
var quoteBarAggregator = new QuoteBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.Quote && !tick.Suspicious)
{
quoteBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
});
enumerator = quoteBarAggregator;
break;
case nameof(TradeBar):
var tradeBarAggregator = new TradeBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
request.Security.Exchange.TimeZone,
_timeProvider);
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
if (data.DataType == MarketDataType.Auxiliary)
{
auxDataEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
else
{
var tick = data as Tick;
if (tick?.TickType == TickType.Trade && !tick.Suspicious)
{
tradeBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
});
enumerator = request.Configuration.SecurityType == SecurityType.Equity
? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
: tradeBarAggregator;
break;
case nameof(OpenInterest):
var oiAggregator = new OpenInterestEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
case nameof(Tick):
default:
// tick or streaming custom data subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
var tick = data as Tick;
if (tick != null)
{
if (tick.TickType == request.Configuration.TickType)
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
if (tick.TickType != TickType.OpenInterest)
{
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
}
else
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
});
enumerator = tickEnumerator;
break;
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
if (!TradingEconomicsCalendar.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration))
{
var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler);
var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler);
enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
