public void ReturnsExpectedTimestamps()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> {
Symbols.AAPL
});
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
var dateStart = new DateTime(2014, 3, 25);
var dateEnd = new DateTime(2014, 3, 26);
var days = (dateEnd - dateStart).Days + 1;
var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
for (var i = 0; i < days; i++)
{
Assert.IsTrue(enumerator.MoveNext());
var current = enumerator.Current as BaseDataCollection;
Assert.IsNotNull(current);
Assert.AreEqual(dateStart.AddDays(i), current.Time);
Assert.AreEqual(dateStart.AddDays(i), current.EndTime);
Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time);
Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime);
}
Assert.IsFalse(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
}
}
public void DoesNotLeakMemory()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, x => new List <Symbol> {
Symbols.AAPL
});
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(2014, 3, 25);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(true, universe, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
{
enumerator.MoveNext();
}
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
// for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
// and we have to be able to inject selection data points into the enumerator
return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider)));
}
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
if (request.Universe is OptionChainUniverse)
{
factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
{
if (!req.Configuration.SecurityType.IsOption())
{
var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
enumerator = new FilterEnumerator <BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
return(ConfigureEnumerator(req, true, enumerator));
}
var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider)));
});
}
if (request.Universe is FuturesChainUniverse)
{
factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true));
}
protected IEnumerator <BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func <SubscriptionRequest, IEnumerator <BaseData> > createUnderlyingEnumerator)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
return(factory.CreateEnumerator(request, _dataProvider));
}
}
else if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
else if (request.Configuration.Type == typeof(ZipEntryName))
{
// TODO: subscription should already come in correctly built
var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution;
// TODO: subscription should already come in as fill forward true
request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution));
var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider)
.CreateEnumerator(request, _dataProvider);
result = ConfigureEnumerator(request, true, result);
return(TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator));
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(enumerator);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
// for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
// and we have to be able to inject selection data points into the enumerator
return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider)));
}
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
if (request.Universe is OptionChainUniverse)
{
factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e),
_mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider);
}
if (request.Universe is FuturesChainUniverse)
{
factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator));
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
return(new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
