public void EquityTradeBar()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
new Lazy <FactorFile>(() => _factorFile));
_rawDataEnumerator.CurrentValue = new TradeBar(
new DateTime(2018, 1, 1),
_config.Symbol,
10,
10,
10,
10,
100);
Assert.IsTrue(enumerator.MoveNext());
var tradeBar = enumerator.Current as TradeBar;
var expectedValue = 10 * _config.PriceScaleFactor;
Assert.Less(expectedValue, 10);
Assert.AreEqual(expectedValue, tradeBar.Price);
Assert.AreEqual(expectedValue, tradeBar.Open);
Assert.AreEqual(expectedValue, tradeBar.Close);
Assert.AreEqual(expectedValue, tradeBar.High);
Assert.AreEqual(expectedValue, tradeBar.Low);
Assert.AreEqual(expectedValue, tradeBar.Value);
enumerator.Dispose();
}
/// <summary>
/// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
/// corporate event providers
/// </summary>
/// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradableDayNotifier">Tradable dates provider</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="includeAuxiliaryData">True to emit auxiliary data</param>
/// <param name="startTime">Start date for the data request</param>
/// <returns>The new auxiliary data enumerator</returns>
public static IEnumerator <BaseData> CreateEnumerators(
IEnumerator <BaseData> rawDataEnumerator,
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
ITradableDatesNotifier tradableDayNotifier,
MapFileResolver mapFileResolver,
bool includeAuxiliaryData,
DateTime startTime)
{
var lazyFactorFile =
new Lazy <FactorFile>(() => GetFactorFileToUse(config, factorFileProvider));
var enumerator = new AuxiliaryDataEnumerator(
config,
lazyFactorFile,
new Lazy <MapFile>(() => GetMapFileToUse(config, mapFileResolver)),
new ITradableDateEventProvider[]
{
new MappingEventProvider(),
new SplitEventProvider(),
new DividendEventProvider(),
new DelistingEventProvider()
},
tradableDayNotifier,
includeAuxiliaryData,
startTime);
var priceScaleFactorEnumerator = new PriceScaleFactorEnumerator(
rawDataEnumerator,
config,
lazyFactorFile);
return(new SynchronizingEnumerator(priceScaleFactorEnumerator, enumerator));
}
public void UpdatesFactorFileCorrectly()
{
var dateBeforeUpadate = new DateTime(2018, 3, 14);
var dateAtUpadate = new DateTime(2018, 3, 15);
var dateAfterUpadate = new DateTime(2018, 3, 16);
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
TestGlobals.FactorFileProvider);
// Before factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateBeforeUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol);
var expectedFactor = factorFile.GetPriceFactor(dateBeforeUpadate, DataNormalizationMode.Adjusted);
var tick = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor, tick.Price);
Assert.AreEqual(10 * expectedFactor, tick.Value);
// At factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateAtUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var expectedFactor2 = factorFile.GetPriceFactor(dateAtUpadate, DataNormalizationMode.Adjusted);
var tick2 = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor2, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor2, tick2.Price);
Assert.AreEqual(10 * expectedFactor2, tick2.Value);
// After factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateAfterUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var expectedFactor3 = factorFile.GetPriceFactor(dateAfterUpadate, DataNormalizationMode.Adjusted);
var tick3 = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor3, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor3, tick3.Price);
Assert.AreEqual(10 * expectedFactor3, tick3.Value);
enumerator.Dispose();
}
public void UpdatesFactorFileCorrectly()
{
var dateBeforeUpadate = new DateTime(2018, 3, 14);
var dateAtUpadate = new DateTime(2018, 3, 15);
var dateAfterUpadate = new DateTime(2018, 3, 16);
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
new Lazy <FactorFile>(() => _factorFile));
// Before factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateBeforeUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var expectedFactor = _factorFile.GetPriceScaleFactor(dateBeforeUpadate);
var tick = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor, tick.Price);
Assert.AreEqual(10 * expectedFactor, tick.Value);
// At factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateAtUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var expectedFactor2 = _factorFile.GetPriceScaleFactor(dateAtUpadate);
var tick2 = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor2, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor2, tick2.Price);
Assert.AreEqual(10 * expectedFactor2, tick2.Value);
// After factor file update date (2018, 3, 15)
_rawDataEnumerator.CurrentValue = new Tick(
dateAfterUpadate,
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var expectedFactor3 = _factorFile.GetPriceScaleFactor(dateAfterUpadate);
var tick3 = enumerator.Current as Tick;
Assert.AreEqual(expectedFactor3, _config.PriceScaleFactor);
Assert.AreEqual(10 * expectedFactor3, tick3.Price);
Assert.AreEqual(10 * expectedFactor3, tick3.Value);
enumerator.Dispose();
}
public void RawEnumeratorCurrentIsNull()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
_factorFile);
_rawDataEnumerator.CurrentValue = null;
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
/// <summary>
/// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
/// corporate event providers
/// </summary>
/// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradableDayNotifier">Tradable dates provider</param>
/// <param name="mapFileResolver">Used for resolving the correct map files</param>
/// <param name="includeAuxiliaryData">True to emit auxiliary data</param>
/// <param name="startTime">Start date for the data request</param>
/// <param name="enablePriceScaling">Applies price factor</param>
/// <returns>The new auxiliary data enumerator</returns>
public static IEnumerator <BaseData> CreateEnumerators(
IEnumerator <BaseData> rawDataEnumerator,
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
ITradableDatesNotifier tradableDayNotifier,
MapFileResolver mapFileResolver,
bool includeAuxiliaryData,
DateTime startTime,
bool enablePriceScaling = true)
{
var lazyFactorFile =
new Lazy <FactorFile>(() => SubscriptionUtils.GetFactorFileToUse(config, factorFileProvider));
var tradableEventProviders = new List <ITradableDateEventProvider>();
if (config.Symbol.SecurityType == SecurityType.Equity)
{
tradableEventProviders.Add(new SplitEventProvider());
tradableEventProviders.Add(new DividendEventProvider());
}
if (config.Symbol.SecurityType == SecurityType.Equity ||
config.Symbol.SecurityType == SecurityType.Base ||
config.Symbol.SecurityType == SecurityType.Option)
{
tradableEventProviders.Add(new MappingEventProvider());
}
tradableEventProviders.Add(new DelistingEventProvider());
var enumerator = new AuxiliaryDataEnumerator(
config,
lazyFactorFile,
new Lazy <MapFile>(() => GetMapFileToUse(config, mapFileResolver)),
tradableEventProviders.ToArray(),
tradableDayNotifier,
includeAuxiliaryData,
startTime);
// avoid price scaling for backtesting; calculate it directly in worker
// and allow subscription to extract the the data depending on config data mode
var dataEnumerator = rawDataEnumerator;
if (enablePriceScaling)
{
dataEnumerator = new PriceScaleFactorEnumerator(
rawDataEnumerator,
config,
lazyFactorFile);
}
return(new SynchronizingEnumerator(dataEnumerator, enumerator));
}
public void RawEnumeratorCurrentIsNull()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
TestGlobals.FactorFileProvider);
_rawDataEnumerator.CurrentValue = null;
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
enumerator.Dispose();
}
/// <summary>
/// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
/// corporate event providers
/// </summary>
/// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradableDayNotifier">Tradable dates provider</param>
/// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
/// <param name="startTime">Start date for the data request</param>
/// <param name="enablePriceScaling">Applies price factor</param>
/// <returns>The new auxiliary data enumerator</returns>
public static IEnumerator <BaseData> CreateEnumerators(
IEnumerator <BaseData> rawDataEnumerator,
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
ITradableDatesNotifier tradableDayNotifier,
IMapFileProvider mapFileProvider,
DateTime startTime,
bool enablePriceScaling = true)
{
var tradableEventProviders = new List <ITradableDateEventProvider>();
if (config.EmitSplitsAndDividends())
{
tradableEventProviders.Add(new SplitEventProvider());
tradableEventProviders.Add(new DividendEventProvider());
}
if (config.TickerShouldBeMapped())
{
tradableEventProviders.Add(new MappingEventProvider());
}
tradableEventProviders.Add(new DelistingEventProvider());
var enumerator = new AuxiliaryDataEnumerator(
config,
factorFileProvider,
mapFileProvider,
tradableEventProviders.ToArray(),
tradableDayNotifier,
startTime);
// avoid price scaling for backtesting; calculate it directly in worker
// and allow subscription to extract the the data depending on config data mode
var dataEnumerator = rawDataEnumerator;
if (enablePriceScaling && config.PricesShouldBeScaled())
{
dataEnumerator = new PriceScaleFactorEnumerator(
rawDataEnumerator,
config,
factorFileProvider);
}
return(new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator));
}
public void RawEnumeratorReturnsFalse()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
_factorFile);
_rawDataEnumerator.CurrentValue = new Tick(
new DateTime(2018, 1, 1),
_config.Symbol,
10,
10,
10);
_rawDataEnumerator.MoveNextReturnValue = false;
Assert.IsFalse(enumerator.MoveNext());
Assert.AreEqual(_rawDataEnumerator.CurrentValue, enumerator.Current);
}
public void FactorFileIsNull()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
null);
_rawDataEnumerator.CurrentValue = new Tick(
new DateTime(2018, 1, 1),
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var tick = enumerator.Current as Tick;
Assert.AreEqual(10, tick.Price);
Assert.AreEqual(10, tick.Value);
}
public void EquityTick()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
_factorFile);
_rawDataEnumerator.CurrentValue = new Tick(
new DateTime(2018, 1, 1),
_config.Symbol,
10,
10,
10);
Assert.IsTrue(enumerator.MoveNext());
var tick = enumerator.Current as Tick;
var expectedValue = 10 * _config.PriceScaleFactor;
Assert.Less(expectedValue, 10);
Assert.AreEqual(expectedValue, tick.Price);
Assert.AreEqual(expectedValue, tick.Value);
}
public void EquityQuoteBar()
{
var enumerator = new PriceScaleFactorEnumerator(
_rawDataEnumerator,
_config,
new Lazy <FactorFile>(() => _factorFile));
_rawDataEnumerator.CurrentValue = new QuoteBar(
new DateTime(2018, 1, 1),
_config.Symbol,
new Bar(10, 10, 10, 10),
100,
new Bar(10, 10, 10, 10),
100);
Assert.IsTrue(enumerator.MoveNext());
var quoteBar = enumerator.Current as QuoteBar;
var expectedValue = 10 * _config.PriceScaleFactor;
Assert.Less(expectedValue, 10);
Assert.AreEqual(expectedValue, quoteBar.Price);
Assert.AreEqual(expectedValue, quoteBar.Value);
Assert.AreEqual(expectedValue, quoteBar.Open);
Assert.AreEqual(expectedValue, quoteBar.Close);
Assert.AreEqual(expectedValue, quoteBar.High);
Assert.AreEqual(expectedValue, quoteBar.Low);
// bid
Assert.AreEqual(expectedValue, quoteBar.Bid.Open);
Assert.AreEqual(expectedValue, quoteBar.Bid.Close);
Assert.AreEqual(expectedValue, quoteBar.Bid.High);
Assert.AreEqual(expectedValue, quoteBar.Bid.Low);
// ask
Assert.AreEqual(expectedValue, quoteBar.Ask.Open);
Assert.AreEqual(expectedValue, quoteBar.Ask.Close);
Assert.AreEqual(expectedValue, quoteBar.Ask.High);
Assert.AreEqual(expectedValue, quoteBar.Ask.Low);
enumerator.Dispose();
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
private Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
// during warmup we might get requested to add some asset which has already expired in which case the live enumerator will be empty
if (!IsExpired(request.Configuration))
{
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
// scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
// and before fill forwarding so we don't happen to apply twice the factor
if (request.Configuration.PricesShouldBeScaled(liveMode: true))
{
enumerator = new PriceScaleFactorEnumerator(
enumerator,
request.Configuration,
_factorFileProvider,
liveMode: true);
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
// define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
}
else
{
enumerator = Enumerable.Empty <BaseData>().GetEnumerator();
}
enumerator = GetWarmupEnumerator(request, enumerator);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
