/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var enumerator = (IEnumerator <BaseData>)_universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = request.Configuration.Symbol
})
.GetEnumerator();
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
enumerator = new InjectionEnumerator(enumerator);
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
// If it is an add we will set time 1 tick ahead to properly sync data
// with next timeslice, if it is a remove then we will set time to now
IList items;
DateTime time;
if (args.Action == NotifyCollectionChangedAction.Add)
{
items = args.NewItems;
time = _timeProvider.GetUtcNow().AddTicks(1);
}
else if (args.Action == NotifyCollectionChangedAction.Remove)
{
items = args.OldItems;
time = _timeProvider.GetUtcNow();
}
else
{
items = null;
time = DateTime.MinValue;
}
// Check that we have our items and time
if (items == null || time == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
// the data point time should always be in exchange timezone
time = time.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
var collection = new BaseDataCollection(time, symbol);
((InjectionEnumerator)enumerator).InjectDataPoint(collection);
};
}
return(enumerator);
}
public void FundamentalUniverseSelectionModelCanBeInherited()
{
var code = @"
from AlgorithmImports import *
from Selection.FundamentalUniverseSelectionModel import FundamentalUniverseSelectionModel
class MockUniverseSelectionModel(FundamentalUniverseSelectionModel):
def __init__(self):
super().__init__(False)
def SelectCoarse(self, algorithm, coarse):
return [Symbol.Create('SPY', SecurityType.Equity, Market.USA)]";
using (Py.GIL())
{
dynamic pyModel = PythonEngine
.ModuleFromString(Guid.NewGuid().ToString(), code)
.GetAttr("MockUniverseSelectionModel");
var model = new UniverseSelectionModelPythonWrapper(pyModel());
var universes = model.CreateUniverses(new QCAlgorithm()).ToList();
Assert.AreEqual(1, universes.Count);
var data = new BaseDataCollection();
data.Data.Add(new CoarseFundamental());
var universe = universes.First();
var symbols = universe.SelectSymbols(DateTime.Now, data).ToList();
Assert.AreEqual(1, symbols.Count);
var expected = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
var symbol = symbols.First();
Assert.AreEqual(expected, symbol);
}
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is UserDefinedUniverse)
{
// Trigger universe selection when security added/removed after Initialize
var universe = (UserDefinedUniverse)request.Universe;
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Configuration.Type == typeof(FineFundamental))
{
return(new FineFundamentalSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return(new PostCreateConfigureSubscriptionEnumeratorFactory(
new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true),
enumerator => ConfigureEnumerator(request, false, enumerator)
));
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
var instances = new BaseDataCollection();
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache);
break;
}
var raw = "";
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
}
}
catch (Exception err)
{
OnReaderError(raw, err);
}
foreach (var instance in instances.Data)
{
yield return(instance);
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is ITimeTriggeredUniverse)
{
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_algorithm.UtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e),
_mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider));
}
if (request.Universe is FuturesChainUniverse)
{
return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
return(_subscriptionfactory);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var enumerator = (IEnumerator <BaseData>)_universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
.Select(x => new Tick {
Time = x, Symbol = request.Configuration.Symbol
})
.GetEnumerator();
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
enumerator = new InjectionEnumerator(enumerator);
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var time = _timeProvider.GetUtcNow();
if (items == null || time == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
// the data point time should always be in exchange timezone
time = time.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
var collection = new BaseDataCollection(time, symbol);
((InjectionEnumerator)enumerator).InjectDataPoint(collection);
};
}
return(enumerator);
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
foreach (var point in base.Read(source))
{
if (point is BaseDataCollection)
{
// if underlying already is returning a collection let it through as is
yield return(point);
}
else
{
if (_collection != null && _collection.EndTime != point.EndTime)
{
// when we get a new time we flush current collection instance, if any
yield return(_collection);
_collection = null;
}
if (_collection == null)
{
_collection = (BaseDataCollection)Activator.CreateInstance(_collectionType);
_collection.Time = point.Time;
_collection.Symbol = point.Symbol;
_collection.EndTime = point.EndTime;
}
// aggregate the data points
_collection.Data.Add(point);
}
}
// underlying reader ended, flush current collection instance if any
if (_collection != null)
{
yield return(_collection);
_collection = null;
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
reader = CreateStreamReader(source);
if (reader == null)
{
yield break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances = null;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, IsLiveMode);
instances = result as BaseDataCollection;
if (instances == null && !reader.ShouldBeRateLimited)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
if (!reader.ShouldBeRateLimited)
{
continue;
}
}
if (IsLiveMode
// this shouldn't happen, rest reader is the only one to be rate limited
// and in live mode, but just in case...
|| instances == null && reader.ShouldBeRateLimited)
{
// in live trading these data points will be unrolled at the
// 'LiveCustomDataSubscriptionEnumeratorFactory' level
yield return(instances);
}
else
{
foreach (var instance in instances.Data)
{
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
}
}
finally
{
reader.DisposeSafely();
}
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator);
var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
// use all available threads, the entire system is waiting for this to complete
var options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(selectSymbolsResult, options, symbol =>
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = _securityService.CreateSecurity(symbol, config);
var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, dataProvider))
{
if (enumerator.MoveNext())
{
lock (fineCollection.Data)
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
});
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var coarseData = universeData.Data.OfType <CoarseFundamental>()
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Market = coarse.Market;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
// set the fine fundamental price property to yesterday's closing price
fine.Value = coarse.Value;
}
universeData.Data.Add(fundamentals);
}
// END -- HACK ATTACK -- END
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
var keys = _pendingSecurityAdditions.Keys;
if (keys.Any() && keys.Single() != dateTimeUtc)
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
universe.UniverseSettings.Resolution,
universe.UniverseSettings.FillForward,
universe.UniverseSettings.ExtendedMarketHours);
security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, symbol.ID.SecurityType == SecurityType.Option);
pendingAdditions.Add(symbol, security);
SetUnderlyingSecurity(universe, security);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
_algorithm.SubscriptionManager.SubscriptionDataConfigService))
{
if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw.
!security.Subscriptions.Contains(request.Configuration))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
security.AddData(request.Configuration);
}
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSubscriptionDataConfigs = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(
_algorithm.Securities,
_algorithm.SubscriptionManager,
_algorithm.BrokerageModel.DefaultMarkets,
securityChanges,
_securityService);
foreach (var subscriptionDataConfig in addedSubscriptionDataConfigs)
{
var security = _algorithm.Securities[subscriptionDataConfig.Symbol];
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, subscriptionDataConfig, dateTimeUtc, algorithmEndDateUtc));
}
}
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source, source.Headers, _isLiveMode);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache, source.Headers);
break;
}
}
catch (Exception e)
{
OnInvalidSource(source, e);
yield break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances = null;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null && !reader.ShouldBeRateLimited)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
if (!reader.ShouldBeRateLimited)
{
continue;
}
}
if (_isLiveMode
// this shouldn't happen, rest reader is the only one to be rate limited
// and in live mode, but just in case...
|| instances == null && reader.ShouldBeRateLimited)
{
yield return(instances);
}
else
{
foreach (var instance in instances.Data)
{
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
public TimeSlice Sync(IEnumerable <Subscription> subscriptions)
{
long earlyBirdTicks;
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>();
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
var universeData = new Dictionary <Universe, BaseDataCollection>();
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
SecurityChanges newChanges;
do
{
earlyBirdTicks = MaxDateTimeTicks;
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
var packet = new DataFeedPacket(subscription.Security, subscription.Configuration);
while (subscription.Current.EmitTimeUtc <= _frontier)
{
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
break;
}
}
if (packet.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData.TryGetValue(subscription.Universe, out collection))
{
collection.Data.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(_frontier, subscription.Configuration.Symbol, packetData);
}
universeData[subscription.Universe] = collection;
}
}
}
if (subscription.Current != null)
{
if (earlyBirdTicks == MaxDateTimeTicks)
{
earlyBirdTicks = subscription.Current.EmitTimeUtc.Ticks;
}
else
{
// take the earliest between the next piece of data or the current earliest bird
earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks);
}
}
}
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, _frontier, baseDataCollection);
}
changes += newChanges;
}while (newChanges != SecurityChanges.None);
var timeSlice = TimeSlice.Create(_frontier, _sliceTimeZone, _cashBook, data, changes, universeDataForTimeSliceCreate);
// next frontier time
_frontier = new DateTime(Math.Max(earlyBirdTicks, _frontier.Ticks), DateTimeKind.Utc);
return(timeSlice);
}
public void CoarseUniverseRotatesActiveSecurity()
{
var startDate = new DateTime(2014, 3, 24);
var endDate = new DateTime(2014, 3, 28);
var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
timeProvider.SetCurrentTime(startDate);
var coarseTimes = new List <DateTime>
{
new DateTime(2014, 3, 25),
new DateTime(2014, 3, 25, 23, 0, 0),
new DateTime(2014, 3, 27, 1, 0, 0)
}.ToHashSet();
var coarseSymbols = new List <Symbol> {
Symbols.SPY, Symbols.AAPL, Symbols.MSFT
};
var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false);
var coarseDataEmittedCount = 0;
var lastTime = DateTime.MinValue;
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (time != lastTime)
{
lastTime = time;
if (coarseTimes.Contains(time))
{
// emit coarse data at selected times
var coarseData = new BaseDataCollection {
Symbol = coarseUsaSymbol
};
foreach (var symbol in coarseSymbols)
{
coarseData.Data.Add(
new CoarseFundamental
{
Symbol = symbol,
Time = time,
Market = Market.USA,
Value = 100
});
}
coarseDataEmittedCount++;
return(new List <BaseData> {
coarseData
});
}
}
return(Enumerable.Empty <BaseData>());
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var algorithm = new AlgorithmStub(feed);
algorithm.SetLiveMode(true);
var mock = new Mock <ITransactionHandler>();
mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(mock.Object);
var synchronizer = new TestableLiveSynchronizer(timeProvider);
synchronizer.Initialize(algorithm, algorithm.DataManager);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer);
var symbolIndex = 0;
var coarseUniverseSelectionCount = 0;
algorithm.AddUniverse(
coarse =>
{
coarseUniverseSelectionCount++;
// rotate single symbol in universe
if (symbolIndex == coarseSymbols.Count)
{
symbolIndex = 0;
}
return(new[] { coarseSymbols[symbolIndex++] });
});
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
Exception exceptionThrown = null;
// create a timer to advance time much faster than realtime
var timerInterval = TimeSpan.FromMilliseconds(50);
var timer = Ref.Create <Timer>(null);
timer.Value = new Timer(state =>
{
try
{
// stop the timer to prevent reentrancy
timer.Value.Change(Timeout.Infinite, Timeout.Infinite);
var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
if (currentTime.Date > endDate.Date)
{
feed.Exit();
cancellationTokenSource.Cancel();
return;
}
timeProvider.Advance(TimeSpan.FromHours(1));
var activeSecuritiesCount = algorithm.ActiveSecurities.Count;
Assert.That(activeSecuritiesCount <= 1);
// restart the timer
timer.Value.Change(timerInterval, timerInterval);
}
catch (Exception exception)
{
Log.Error(exception);
exceptionThrown = exception;
feed.Exit();
cancellationTokenSource.Cancel();
}
}, null, TimeSpan.FromSeconds(1), timerInterval);
foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
{
}
timer.Value.Dispose();
if (exceptionThrown != null)
{
throw new Exception("Exception in timer: ", exceptionThrown);
}
Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount);
Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
_algorithm.Securities.Add(security);
}
var request = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc);
var enumerator = factory.CreateEnumerator(request, dataProvider);
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
public void HandlesCoarseFundamentalData()
{
var algorithm = new AlgorithmStub();
Symbol symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA);
algorithm.SetUniverse(new FuncUniverse(
new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false),
new SubscriptionSettings(Resolution.Second, 1, true, false),
coarse => coarse.Take(10).Select(x => x.Symbol)
));
var lck = new object();
BaseDataCollection list = null;
const int coarseDataPointCount = 100000;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
Console.WriteLine(currentTime + ": timer.Elapsed");
lock (state)
{
list = new BaseDataCollection {
Symbol = symbol
};
list.Data.AddRange(Enumerable.Range(0, coarseDataPointCount).Select(x => new CoarseFundamental
{
Symbol = SymbolCache.GetSymbol(x.ToString()),
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
}));
}
}, lck, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(500));
bool yieldedUniverseData = false;
var feed = RunDataFeed(algorithm, fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
Assert.IsTrue(feed.Subscriptions.Any(x => x.IsUniverseSelectionSubscription));
var universeSelectionHadAllData = false;
feed.UniverseSelection += (sender, args) =>
{
universeSelectionHadAllData = args.Data.Count == coarseDataPointCount;
Console.WriteLine(DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).ToString("o") + ": Fired universe selection. Count: " + args.Data.Count);
return(SecurityChanges.None);
};
ConsumeBridge(feed, TimeSpan.FromSeconds(5), ts =>
{
});
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(universeSelectionHadAllData);
}
public void FineCoarseFundamentalDataGetsPipedCorrectly()
{
var lck = new object();
BaseDataCollection list = null;
var timer = new Timer(state =>
{
lock (state)
{
list = new BaseDataCollection {
Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false)
};
list.Data.Add(new CoarseFundamental
{
Symbol = Symbols.AAPL,
Time = new DateTime(2014, 04, 24)
});
}
}, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1));
var yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
var fineWasCalled = false;
_algorithm.AddUniverse(coarse => coarse.Take(1).Select(x => x.Symbol),
fine =>
{
var symbol = fine.First().Symbol;
if (symbol == Symbols.AAPL)
{
fineWasCalled = true;
}
return(new[] { symbol });
});
var receivedFundamentalsData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts =>
{
if (ts.UniverseData.Count > 0 &&
ts.UniverseData.First().Value.Data.First() is Fundamentals)
{
receivedFundamentalsData = true;
}
}, sendUniverseData: true);
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(receivedFundamentalsData);
Assert.IsTrue(fineWasCalled);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var exchangeHours = _marketHoursDatabase.GetEntry(symbol.ID.Market, symbol, symbol.ID.SecurityType).ExchangeHours;
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.ID.SecurityType, CashBook.AccountCurrency);
var quoteCash = _algorithm.Portfolio.CashBook[symbolProperties.QuoteCurrency];
var security = new Equity(symbol, exchangeHours, quoteCash, symbolProperties);
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), dateTimeUtc, dateTimeUtc);
using (var enumerator = factory.CreateEnumerator(request, dataProvider))
{
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
if (IsSafeToRemove(member))
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
var keys = _pendingSecurityAdditions.Keys;
if (keys.Any() && keys.Single() != dateTimeUtc)
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
pendingAdditions.Add(symbol, security);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets, securityChanges);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, new SubscriptionDataConfig(security.Subscriptions.First()), dateTimeUtc, algorithmEndDateUtc));
}
}
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.UniverseSettings;
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List <Security>();
var removals = new List <Security>();
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
var config = member.SubscriptionDataConfig;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
_dataFeed.RemoveSubscription(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol))
{
continue;
}
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(settings.Resolution, out reason))
{
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason);
break;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(dateTimeUtc, security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
}
// return None if there's no changes, otherwise return what we've modified
return(additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None);
}
/// <summary>
/// Returns the symbols defined by the user for this universe
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(_symbols);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc))
{
if (subscription.IsUniverseSubscription)
{
removals.Remove(member);
}
else
{
_dataFeed.RemoveSubscription(subscription.Configuration);
}
}
// remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD!
SymbolCache.TryRemove(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedMember = universe.AddMember(dateTimeUtc, security);
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = addedMember;
}
}
if (addedSubscription)
{
additions.Add(security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="cancellationToken">The cancellation token to stop enumeration</param>
public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions,
CancellationToken cancellationToken)
{
var delayedSubscriptionFinished = new Queue <Subscription>();
while (!cancellationToken.IsCancellationRequested)
{
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>(1);
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
var frontierUtc = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTimeUtc(frontierUtc);
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
DataFeedPacket packet = null;
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(
subscription.Security,
subscription.Configuration,
subscription.RemovedFromUniverse
);
}
// If our subscription is a universe, and we get a delisting event emitted for it, then
// the universe itself should be unselected and removed, because the Symbol that the
// universe is based on has been delisted. Doing the disposal here allows us to
// process the delisting at this point in time before emitting out to the algorithm.
// This is very useful for universes that can be delisted, such as ETF constituent
// universes (e.g. for ETF constituent universes, since the ETF itself is used to create
// the universe Symbol (and set as its underlying), once the ETF is delisted, the
// universe should cease to exist, since there are no more constituents of that ETF).
if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting)
{
subscription.Universes.Single().Dispose();
}
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished.Enqueue(subscription);
break;
}
}
if (packet?.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData != null &&
universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData);
}
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested)
{
var universe = subscription.Universes.Single();
// check if a universe selection isn't already scheduled for this disposed universe
if (universeData == null || !universeData.ContainsKey(universe))
{
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
// we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added
universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol);
}
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
if (universeData != null && universeData.Count > 0)
{
// if we are going to perform universe selection we emit an empty
// time pulse to align algorithm time with current frontier
yield return(_timeSliceFactory.CreateTimePulse(frontierUtc));
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
}
changes += newChanges;
}while (newChanges != SecurityChanges.None ||
_universeSelection.AddPendingInternalDataFeeds(frontierUtc));
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
while (delayedSubscriptionFinished.Count > 0)
{
// these subscriptions added valid data to the packet
// we need to trigger OnSubscriptionFinished after we create the TimeSlice
// else it will drop the data
var subscription = delayedSubscriptionFinished.Dequeue();
OnSubscriptionFinished(subscription);
}
yield return(timeSlice);
}
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null
// if the universe has been disposed we don't perform selection. This us handled bellow by 'Universe.PerformSelection'
// but in this case we directly call 'SelectSymbols' because we want to perform fine selection even if coarse returns the same
// symbols, see 'Universe.PerformSelection', which detects this and returns 'Universe.Unchanged'
&& !universe.DisposeRequested)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
// Create a dictionary of CoarseFundamental keyed by Symbol that also has FineFundamental
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var allCoarse = universeData.Data.OfType <CoarseFundamental>();
var coarseData = allCoarse.Where(c => c.HasFundamentalData)
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
// Remove selected symbols that does not have fine fundamental data
var anyDoesNotHaveFundamentalData = false;
// only pre filter selected symbols if there actually is any coarse data. This way we can support custom universe filtered by fine fundamental data
// which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem,
// 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it
if (allCoarse.Any())
{
selectSymbolsResult = selectSymbolsResult
.Where(
symbol =>
{
var result = coarseData.ContainsKey(symbol);
anyDoesNotHaveFundamentalData |= !result;
return(result);
}
);
}
if (!_anyDoesNotHaveFundamentalDataWarningLogged && anyDoesNotHaveFundamentalData)
{
_algorithm.Debug("Note: Your coarse selection filter was updated to exclude symbols without fine fundamental data. Make sure your coarse filter excludes symbols where HasFundamental is false.");
_anyDoesNotHaveFundamentalDataWarningLogged = true;
}
// use all available threads, the entire system is waiting for this to complete
var options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(selectSymbolsResult, options, symbol =>
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = _securityService.CreateSecurity(symbol,
config,
addToSymbolCache: false);
var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, _dataProvider))
{
if (enumerator.MoveNext())
{
lock (fineCollection.Data)
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
});
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
AssetClassification = fine.AssetClassification,
CompanyProfile = fine.CompanyProfile,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios,
Market = fine.Symbol.ID.Market
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
// set the fine fundamental price property to yesterday's closing price
fine.Value = coarse.Value;
}
universeData.Data.Add(fundamentals);
}
// END -- HACK ATTACK -- END
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// first check for no pending removals, even if the universe selection
// didn't change we might need to remove a security because a position was closed
RemoveSecurityFromUniverse(
_pendingRemovalsManager.CheckPendingRemovals(selections, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
// check for no changes second
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Securities.Values.OrderBy(member => member.Security.Symbol.SecurityType))
{
var security = member.Security;
// if we've selected this subscription again, keep it
if (selections.Contains(security.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, security))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(security);
RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
if (universe.Securities.ContainsKey(symbol))
{
// if its already part of the universe no need to re add it
continue;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
universe.UniverseSettings.Resolution,
universe.UniverseSettings.FillForward,
universe.UniverseSettings.ExtendedMarketHours,
dataNormalizationMode: universe.UniverseSettings.DataNormalizationMode);
security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, (symbol.ID.SecurityType == SecurityType.Option || symbol.ID.SecurityType == SecurityType.FutureOption));
pendingAdditions.Add(symbol, security);
}
var addedSubscription = false;
var dataFeedAdded = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
_algorithm.SubscriptionManager.SubscriptionDataConfigService))
{
if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw.
!security.Subscriptions.Contains(request.Configuration))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
security.AddData(request.Configuration);
}
var toRemove = _currencySubscriptionDataConfigManager.GetSubscriptionDataConfigToRemove(request.Configuration.Symbol);
if (toRemove != null)
{
Log.Trace($"UniverseSelection.ApplyUniverseSelection(): Removing internal currency data feed {toRemove}");
_dataManager.RemoveSubscription(toRemove);
}
// 'dataFeedAdded' will help us notify the user for security changes only once per non internal subscription
// for example two universes adding the sample configuration, we don't want two notifications
dataFeedAdded = _dataManager.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
_internalSubscriptionManager.AddedSubscriptionRequest(request);
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember && dataFeedAdded)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
EnsureCurrencyDataFeeds(securityChanges);
}
if (securityChanges != SecurityChanges.None && Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
public void CoarseFundamentalDataGetsPipedCorrectly()
{
var lck = new object();
BaseDataCollection list = null;
var timer = new Timer(state =>
{
var currentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
lock (state)
{
list = new BaseDataCollection {
Symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false)
};
list.Data.Add(new CoarseFundamental
{
Symbol = Symbols.SPY,
Time = currentTime - Time.OneDay, // hard-coded coarse period of one day
});
}
}, lck, TimeSpan.FromSeconds(0.5), TimeSpan.FromMilliseconds(-1));
var yieldedUniverseData = false;
var feed = RunDataFeed(getNextTicksFunction: fdqh =>
{
lock (lck)
{
if (list != null)
{
try
{
var tmp = list;
return(new List <BaseData> {
tmp
});
}
finally
{
list = null;
yieldedUniverseData = true;
}
}
}
return(Enumerable.Empty <BaseData>());
});
_algorithm.AddUniverse(coarse => coarse.Take(10).Select(x => x.Symbol));
var receivedCoarseData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(2), ts =>
{
if (ts.UniverseData.Count > 0 &&
ts.UniverseData.First().Value.Data.First() is CoarseFundamental)
{
receivedCoarseData = true;
}
}, sendUniverseData: true);
timer.Dispose();
Assert.IsTrue(yieldedUniverseData);
Assert.IsTrue(receivedCoarseData);
}
