/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
foreach (var symbol in selectSymbolsResult)
{
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { dateTimeUtc });
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
_algorithm.Securities.Add(security);
}
var request = new SubscriptionRequest(true, universe, security, config, dateTimeUtc, dateTimeUtc);
var enumerator = factory.CreateEnumerator(request, dataProvider);
if (enumerator.MoveNext())
{
fineCollection.Data.Add(enumerator.Current);
}
}
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// remove previously deselected members which were kept in the universe because of holdings or open orders
foreach (var member in _pendingRemovals.ToList())
{
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
_pendingRemovals.Remove(member);
}
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
RemoveSecurityFromUniverse(universe, member, removals, dateTimeUtc, algorithmEndDateUtc);
}
else
{
_pendingRemovals.Add(member);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.UniverseSettings;
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List <Security>();
var removals = new List <Security>();
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
var config = member.SubscriptionDataConfig;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
_dataFeed.RemoveSubscription(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol))
{
continue;
}
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(settings.Resolution, out reason))
{
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason);
break;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, universe.SecurityInitializer,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(dateTimeUtc, security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
}
// return None if there's no changes, otherwise return what we've modified
return(additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
// perform initial filtering and limit the result
var selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == member.Symbol);
if (!member.HoldStock && !openOrders.Any())
{
// safe to remove the member from the universe
universe.RemoveMember(dateTimeUtc, member);
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
member.Cache.Reset();
foreach (var subscription in universe.GetSubscriptionRequests(member, dateTimeUtc, algorithmEndDateUtc))
{
if (subscription.IsUniverseSubscription)
{
removals.Remove(member);
}
else
{
_dataFeed.RemoveSubscription(subscription.Configuration);
}
}
// remove symbol mappings for symbols removed from universes // TODO : THIS IS BAD!
SymbolCache.TryRemove(member.Symbol);
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = universe.CreateSecurity(symbol, _algorithm, _marketHoursDatabase, _symbolPropertiesDatabase);
}
var addedMember = universe.AddMember(dateTimeUtc, security);
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc))
{
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(request.Configuration.Resolution, out reason))
{
// should we be counting universe subscriptions against user subscriptions limits?
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscription: " + request.Configuration.Symbol.ToString() + ": " + reason);
continue;
}
// add the new subscriptions to the data feed
_dataFeed.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = addedMember;
}
}
if (addedSubscription)
{
additions.Add(security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, _marketHoursDatabase, _symbolPropertiesDatabase, _algorithm.BrokerageModel.DefaultMarkets);
foreach (var security in addedSecurities)
{
// assume currency feeds are always one subscription per, these are typically quote subscriptions
_dataFeed.AddSubscription(new SubscriptionRequest(false, universe, security, security.Subscriptions.First(), dateTimeUtc, algorithmEndDateUtc));
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}