public void ReturnsExpectedTimestamps()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> {
Symbols.AAPL
});
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
var dateStart = new DateTime(2014, 3, 26);
var dateEnd = new DateTime(2014, 3, 27);
var days = (dateEnd - dateStart).Days + 1;
var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
dateStart = dateStart.AddDays(-1);
for (var i = 0; i <= days; i++)
{
Assert.IsTrue(enumerator.MoveNext());
var current = enumerator.Current as BaseDataCollection;
Assert.IsNotNull(current);
Assert.AreEqual(dateStart.AddDays(i), current.Time);
Assert.AreEqual(dateStart.AddDays(i), current.EndTime);
Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time);
Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime);
}
Assert.IsFalse(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
}
}
public void DoesNotLeakMemory()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> {
Symbols.AAPL
});
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(2014, 3, 25);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(true, universe, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
enumerator.MoveNext();
}
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (request.Configuration.IsCustomData)
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
// each time we exhaust we'll new up this enumerator stack
var refresher = new RefreshEnumerator <BaseData>(() =>
{
var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] { dateInDataTimeZone });
var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request);
var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
return(new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge));
});
// rate limit the refreshing of the stack to the requested interval
var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = enqueable;
}
else if (request.Configuration.Resolution != Resolution.Tick)
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = aggregator;
}
else
{
// tick subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
tickEnumerator.Enqueue(data);
if (subscription != null)
{
subscription.RealtimePrice = data.Value;
}
});
enumerator = tickEnumerator;
}
if (request.Configuration.FillDataForward)
{
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
