public void DoesNotLeakMemory()
{
var symbol = Symbols.AAPL;
var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market);
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider);
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(1998, 1, 1);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(false, null, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
enumerator.MoveNext();
}
date = date.AddDays(1);
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
// for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
// and we have to be able to inject selection data points into the enumerator
return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider)));
}
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
if (request.Universe is OptionChainUniverse)
{
factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
{
if (!req.Configuration.SecurityType.IsOption())
{
var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
enumerator = new FilterEnumerator <BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
return(ConfigureEnumerator(req, true, enumerator));
}
var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider)));
});
}
if (request.Universe is FuturesChainUniverse)
{
factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true));
}
protected IEnumerator <BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func <SubscriptionRequest, IEnumerator <BaseData> > createUnderlyingEnumerator)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
return(factory.CreateEnumerator(request, _dataProvider));
}
}
else if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
else if (request.Configuration.Type == typeof(ZipEntryName))
{
// TODO: subscription should already come in correctly built
var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution;
// TODO: subscription should already come in as fill forward true
request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution));
var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider)
.CreateEnumerator(request, _dataProvider);
result = ConfigureEnumerator(request, true, result);
return(TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator));
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(enumerator);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.TickType)
{
case TickType.Quote:
var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Quote)
{
quoteBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = quoteBarAggregator;
break;
case TickType.Trade:
default:
var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.Trade)
{
tradeBarAggregator.ProcessData(tick);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
}
});
enumerator = tradeBarAggregator;
break;
case TickType.OpenInterest:
var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick.TickType == TickType.OpenInterest)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
}
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Initializes a new instance of the <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
public OptionChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator)
{
_enumeratorConfigurator = enumeratorConfigurator;
_factory = new BaseDataSubscriptionEnumeratorFactory();
}
public void DoesNotEmitInvalidData()
{
var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
var endTime = new DateTime(2014, 06, 09, 20, 0, 0);
var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()));
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req) =>
{
var input = enumeratorFactory.CreateEnumerator(req, new DefaultDataProvider());
input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
return(new FillForwardEnumerator(
input,
option.Exchange,
fillForwardResolution,
false,
endTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
startTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc);
var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
var emittedCount = 0;
foreach (var data in enumerator.AsEnumerable())
{
emittedCount++;
var optionData = data as OptionChainUniverseDataCollection;
Assert.IsNotNull(optionData);
Assert.IsNotNull(optionData.Underlying);
Assert.AreNotEqual(0, optionData.Data.Count);
}
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, emittedCount);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator<BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
return new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider);
});
// rate limit the refreshing of the stack to the requested interval
// At Tick resolution, it will refresh at full speed
// At Second and Minute resolution, it will refresh every second and minute respectively
// At Hour and Daily resolutions, it will refresh every 30 minutes
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
_customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);
var enqueable = new EnqueueableEnumerator<BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick) data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator<BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null) subscription.RealtimePrice = data.Value;
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return subscription;
}