private void AddSubscriptionForUniverseSelectionMarket(string market)
{
var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(market, null, SecurityType.Equity);
var symbolName = market + "-market";
var subscriptionDataConfig = new SubscriptionDataConfig(typeof(CoarseFundamental), SecurityType.Equity, symbolName, Resolution.Daily, market, exchangeHours.TimeZone,
true, false, true);
var security = new Security(exchangeHours, subscriptionDataConfig, 1);
var cf = new CoarseFundamental();
var list = new List <BaseData>();
foreach (var date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate))
{
var factory = new BaseDataSubscriptionFactory(subscriptionDataConfig, date, false);
var source = cf.GetSource(subscriptionDataConfig, date, false);
var coarseFundamentalForDate = factory.Read(source);
list.AddRange(coarseFundamentalForDate);
}
// spoof a subscription for the market that emits at midnight of each tradeable day
var subscription = new Subscription(security,
list.GetEnumerator(),
_algorithm.StartDate.ConvertToUtc(exchangeHours.TimeZone),
_algorithm.EndDate.ConvertToUtc(exchangeHours.TimeZone),
false,
true
);
// let user know if we fail to load the universe subscription, very important for when understanding backtest results!
PrimeSubscriptionPump(subscription, true);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
/// <summary>
/// Creates a new univese and adds it to the algorithm. This is for coarse fundamntal US Equity data and
/// will be executed on day changes in the NewYork time zone (<see cref="TimeZones.NewYork"/>
/// </summary>
/// <param name="selector">Defines an initial coarse selection</param>
public void AddUniverse(Func <IEnumerable <CoarseFundamental>, IEnumerable <Symbol> > selector)
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, true, isFilteredSubscription: false);
AddUniverse(new FuncUniverse(config, UniverseSettings, SecurityInitializer, selectionData => selector(selectionData.OfType <CoarseFundamental>())));
}
/// <summary>
/// Gets the <see cref="CoarseFundamental"/> data for the specified market/date
/// </summary>
public static IEnumerable <CoarseFundamental> GetCoarseFundamentals(string market, DateTimeZone timeZone, DateTime date, bool isLiveMode)
{
var factory = new CoarseFundamental();
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), SecurityType.Equity, new Symbol(market + "-coarse"), Resolution.Daily, market, timeZone, true, false, true, false);
var reader = new BaseDataSubscriptionFactory(config, date, isLiveMode);
var source = factory.GetSource(config, date, isLiveMode);
return(reader.Read(source).OfType <CoarseFundamental>());
}
/// <summary>
/// Creates a new coarse universe that contains the bottom count of stocks
/// by daily dollar volume
/// </summary>
/// <param name="count">The number of stock to select</param>
/// <param name="universeSettings">The settings for stocks added by this universe.
/// Defaults to <see cref="QCAlgorithm.UniverseSettings"/></param>
/// <returns>A new coarse universe for the bottom count of stocks by dollar volume</returns>
public Universe Bottom(int count, UniverseSettings universeSettings = null)
{
universeSettings = universeSettings ?? _algorithm.UniverseSettings;
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, true);
return(new FuncUniverse(config, universeSettings, _algorithm.SecurityInitializer, selectionData => (
from c in selectionData.OfType <CoarseFundamental>()
orderby c.DollarVolume descending
select c.Symbol).Take(count)
));
}
public void ReturnsExpectedTimestamps()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> {
Symbols.AAPL
});
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
var dateStart = new DateTime(2014, 3, 26);
var dateEnd = new DateTime(2014, 3, 27);
var days = (dateEnd - dateStart).Days + 1;
var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
dateStart = dateStart.AddDays(-1);
for (var i = 0; i <= days; i++)
{
Assert.IsTrue(enumerator.MoveNext());
var current = enumerator.Current as BaseDataCollection;
Assert.IsNotNull(current);
Assert.AreEqual(dateStart.AddDays(i), current.Time);
Assert.AreEqual(dateStart.AddDays(i), current.EndTime);
Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time);
Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime);
}
Assert.IsFalse(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
}
}
public void DoesNotLeakMemory()
{
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
config,
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1));
var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> {
Symbols.AAPL
});
var fileProvider = new DefaultDataProvider();
var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
GC.Collect();
var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;
var date = new DateTime(2014, 3, 25);
const int iterations = 1000;
for (var i = 0; i < iterations; i++)
{
var request = new SubscriptionRequest(true, universe, security, config, date, date);
using (var enumerator = factory.CreateEnumerator(request, fileProvider))
{
enumerator.MoveNext();
}
}
GC.Collect();
var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;
Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");
Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
}
public void ToRow(string coarseLine)
{
var config = new SubscriptionDataConfig(typeof(CoarseFundamental),
Symbols.SPY,
Resolution.Second,
DateTimeZone.Utc,
DateTimeZone.Utc,
false,
false,
false);
var factory = new CoarseFundamental();
var coarseRead = (CoarseFundamental)factory.Reader(config, coarseLine, DateTime.UtcNow, false);
var row = CoarseFundamental.ToRow(coarseRead);
Assert.AreEqual(coarseLine, row);
}
/// <summary>
/// Creates a new coarse universe that contains stocks in the specified
/// dollar volume percentile
/// </summary>
/// <param name="percentile">The desired dollar volume percentile (0 to 100 inclusive)</param>
/// <param name="universeSettings">The settings for stocks added by this universe.
/// Defaults to <see cref="QCAlgorithm.UniverseSettings"/></param>
/// <returns>A new coarse universe for the bottom count of stocks by dollar volume</returns>
public Universe Percentile(double percentile, UniverseSettings universeSettings = null)
{
universeSettings = universeSettings ?? _algorithm.UniverseSettings;
var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, true);
return(new FuncUniverse(config, universeSettings, _algorithm.SecurityInitializer, selectionData =>
{
var list = selectionData as IReadOnlyList <CoarseFundamental> ?? selectionData.OfType <CoarseFundamental>().ToList();
// using quantiles since the Percentile implementation requires integers, so scale into quantile space
var lowerBound = (decimal)list.Select(x => (double)x.DollarVolume).Quantile(percentile / 100d);
return from c in list
where c.DollarVolume >= lowerBound
orderby c.DollarVolume descending
select c.Symbol;
}));
}
public void ParsesCoarseCsvLine(string line, bool hasFundamentalData, decimal price, decimal priceFactor, decimal splitFactor, decimal adjustedPrice)
{
var cf = new CoarseFundamental();
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.AAPL,
Resolution.Second,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false,
false,
TickType.Trade,
false);
cf = (CoarseFundamental)cf.Reader(config, line, DateTime.MinValue, false);
Assert.AreEqual(hasFundamentalData, cf.HasFundamentalData);
Assert.AreEqual(price, cf.Price);
Assert.AreEqual(priceFactor, cf.PriceFactor);
Assert.AreEqual(splitFactor, cf.SplitFactor);
Assert.AreEqual(adjustedPrice, cf.AdjustedPrice);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
// create a canonical security object
var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = config.Symbol
}).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
var cf = new CoarseFundamental();
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// load coarse data day by day
var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
let source = cf.GetSource(config, dateInDataTimeZone, false)
let factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);
ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="universe">The universe to add a subscription for</param>
/// <param name="startTimeUtc">The start time of the subscription in utc</param>
/// <param name="endTimeUtc">The end time of the subscription in utc</param>
public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
// define our data enumerator
IEnumerator <BaseData> enumerator;
var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);
var userDefined = universe as UserDefinedUniverse;
if (userDefined != null)
{
// spoof a tick on the requested interval to trigger the universe selection function
enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = config.Symbol
}).GetEnumerator();
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
var cf = new CoarseFundamental();
// load coarse data day by day
enumerator = (from date in Time.EachTradeableDayInTimeZone(security.Exchange.Hours, _algorithm.StartDate, _algorithm.EndDate, config.DataTimeZone, config.ExtendedMarketHours)
let source = cf.GetSource(config, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
let coarseFundamentalForDate = factory.Read(source)
select new BaseDataCollection(date.AddDays(1), config.Symbol, coarseFundamentalForDate)
).GetEnumerator();
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
ScheduleEnumerator(enumerator, enqueueable, 5, 100000, 2);
enumerator = enqueueable;
}
else if (config.SecurityType == SecurityType.Option && security is Option)
{
var configs = universe.GetSubscriptions(security);
var enumerators = configs.Select(c =>
CreateSubscriptionEnumerator(security, c, localStartTime, localEndTime, _mapFileProvider.Get(c.Market), tradeableDates, false, true)
).ToList();
var sync = new SynchronizingEnumerator(enumerators);
enumerator = new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
else
{
// normal reader for all others
enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, MapFileResolver.Empty, tradeableDates, true, false);
// route these custom subscriptions through the exchange for buffering
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
enumerator = enqueueable;
}
// create the subscription
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
_subscriptions.TryAdd(subscription);
UpdateFillForwardResolution();
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 28);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25),
new DateTime(2014, 3, 25, 23, 0, 0),
new DateTime(2014, 3, 27, 1, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false);
var coarseDataEmittedCount = 0;
var lastTime = DateTime.MinValue;
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (time != lastTime)
{
lastTime = time;
if (coarseTimes.Contains(time))
{
// emit coarse data at selected times
var coarseData = new BaseDataCollection {
Symbol = coarseUsaSymbol
};
foreach (var symbol in coarseSymbols)
{
coarseData.Data.Add(
new CoarseFundamental
{
Symbol = symbol,
Time = time,
Market = Market.USA,
Value = 100
});
}
coarseDataEmittedCount++;
return(new List <BaseData> {
coarseData
});
}
}
return(Enumerable.Empty <BaseData>());
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer);
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
coarseUniverseSelectionCount++;
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(50);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, TimeSpan.FromSeconds(1), timerInterval);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.Dispose();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount);
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount);
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.SetUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new SubscriptionSettings(Resolution.Second, 1, true, false),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
feed.UniverseSelection += (sender, args) =>
{
universeSelectionHadAllData = args.Data.Count == coarseDataPointCount;
Console.WriteLine(DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).ToString("o") + ": Fired universe selection. Count: " + args.Data.Count);
return(SecurityChanges.None);
};
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
public void FineCoarseFundamentalDataGetsPipedCorrectly()
{
var lck = new object();
BaseDataCollection list = null;
var timer = new Timer(state =>
{
lock (state)
{
list = new BaseDataCollection {
Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false)
};
list.Data.Add(new CoarseFundamental
{
Symbol = Symbols.AAPL,
Time = new DateTime(2014, 04, 24)
});
}
}, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1));
var yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
var fineWasCalled = false;
_algorithm.AddUniverse(coarse => coarse.Take(1).Select(x => x.Symbol),
fine =>
{
var symbol = fine.First().Symbol;
if (symbol == Symbols.AAPL)
{
fineWasCalled = true;
}
return(new[] { symbol });
});
var receivedFundamentalsData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts =>
{
if (ts.UniverseData.Count > 0 &&
ts.UniverseData.First().Value.Data.First() is Fundamentals)
{
receivedFundamentalsData = true;
}
}, sendUniverseData: true);
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(receivedFundamentalsData);
Assert.IsTrue(fineWasCalled);
}
public void CoarseFundamentalDataGetsPipedCorrectly()
{
var lck = new object();
BaseDataCollection list = null;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
lock (state)
{
list = new BaseDataCollection {
Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false)
};
list.Data.Add(new CoarseFundamental
{
Symbol = Symbols.SPY,
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
});
}
}, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1));
var yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
_algorithm.AddUniverse(coarse => coarse.Take(10).Select(x => x.Symbol));
var receivedCoarseData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts =>
{
if (ts.UniverseData.Count > 0 &&
ts.UniverseData.First().Value.Data.First() is CoarseFundamental)
{
receivedCoarseData = true;
}
}, sendUniverseData: true);
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(receivedCoarseData);
}
public void HandlesCoarseFundamentalData()
{
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
_algorithm.AddUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new UniverseSettings(Resolution.Second, 1, true, false, TimeSpan.Zero), SecurityInitializer.Null,
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
});
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
if (items == null || currentFrontierUtcTime == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
subscription.OnNewDataAvailable();
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: {config.Symbol.ID}");
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// Will try to pull coarse data from the data folder every 30min, file with today's date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, normalizedSymbol, true);
_customExchange.AddEnumerator(normalizedSymbol, aggregator);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(normalizedSymbol, data =>
{
var coarseData = data as BaseDataCollection;
enqueable.Enqueue(new BaseDataCollection(coarseData.Time, config.Symbol, coarseData.Data));
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = _dataQueueHandler.Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var timeProvider = new PredicateTimeProvider(_timeProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(_job, subscription.Configuration))
{
_dataQueueHandler.Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
}
return(subscription);
}
