/// <summary>
/// Creates a new universe that restricts the universe selection data to symbols that passed the
/// first universe's selection critera
///
/// NOTE: The <paramref name="second"/> universe instance provided to this method should not be manually
/// added to the algorithm. The <paramref name="first"/> should still be manually (assuming no other changes).
/// </summary>
/// <param name="second">The universe to be filtere</param>
/// <param name="first">The universe providing the set of symbols used for filtered</param>
/// <returns>A new universe that can be added to the algorithm that represents invoking the second
/// using the selections from the first as a filter.</returns>
public static Universe PrefilterUsing(this Universe second, Universe first)
{
return(new SelectSymbolsUniverseDecorator(second, (utcTime, data) =>
{
var clone = (BaseDataCollection)data.Clone();
clone.Data = clone.Data.Where(d => first.ContainsMember(d.Symbol)).ToList();
return second.SelectSymbols(utcTime, clone);
}));
}
/// <summary>
/// Performs universe selection using the data specified
/// </summary>
/// <param name="utcTime">The current utc time</param>
/// <param name="data">The symbols to remain in the universe</param>
/// <returns>The data that passes the filter</returns>
public override IEnumerable <Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
{
return(Universe.SelectSymbols(utcTime, data));
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.UniverseSettings;
// perform initial filtering and limit the result
var selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData.Data);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return SecurityChanges.None;
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List<Security>();
var removals = new List<Security>();
// determine which data subscriptions need to be removed for this market
foreach (var subscription in _dataFeed.Subscriptions)
{
// universes can only remove members of their own
if (!universe.ContainsMember(subscription.Security)) continue;
// never remove universe selection subscriptions
if (subscription.IsUniverseSelectionSubscription) continue;
var config = subscription.Configuration;
// never remove internal feeds
if (config.IsInternalFeed) continue;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol)) continue;
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, subscription.Security)) continue;
// let the algorithm know this security has been removed from the universe
removals.Add(subscription.Security);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!subscription.Security.HoldStock && !openOrders.Any())
{
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
subscription.Security.Cache.Reset();
if (_dataFeed.RemoveSubscription(subscription))
{
universe.RemoveMember(dateTimeUtc, subscription.Security);
}
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol)) continue;
// ask the limiter if we can add another subscription at that resolution
string reason;
if (!_limiter.CanAddSubscription(settings.Resolution, out reason))
{
_algorithm.Error(reason);
Log.Trace("UniverseSelection.ApplyUniverseSelection(): Skipping adding subscriptions: " + reason);
break;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
// added securities should use the brokerage model provided models
_algorithm.UpdateModel(_algorithm.BrokerageModel, security);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(dateTimeUtc, security);
}
}
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
var addedSecurities = _algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(_algorithm.Securities, _algorithm.SubscriptionManager, MarketHoursDatabase.FromDataFolder());
foreach (var security in addedSecurities)
{
_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
}
// return None if there's no changes, otherwise return what we've modified
return additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var settings = universe.SubscriptionSettings;
var limit = 1000; //daily/hourly limit
var resolution = settings.Resolution;
switch (resolution)
{
case Resolution.Tick:
limit = _algorithm.Securities.TickLimit;
break;
case Resolution.Second:
limit = _algorithm.Securities.SecondLimit;
break;
case Resolution.Minute:
limit = _algorithm.Securities.MinuteLimit;
break;
}
// subtract current subscriptions that can't be removed
limit -= _algorithm.Securities.Count(x => x.Value.Resolution == resolution && x.Value.HoldStock);
if (limit < 1)
{
// if we don't have room for more securities then we can't really do anything here.
_algorithm.Error("Unable to add more securities from universe selection due to holding stock.");
return SecurityChanges.None;
}
// perform initial filtering and limit the result
var selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData.Data);
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return SecurityChanges.None;
}
var selections = selectSymbolsResult.Take(limit).ToHashSet();
// create a hash set of our existing subscriptions by sid
var existingSubscriptions = _dataFeed.Subscriptions.ToHashSet(x => x.Security.Symbol);
var additions = new List<Security>();
var removals = new List<Security>();
// determine which data subscriptions need to be removed for this market
foreach (var subscription in _dataFeed.Subscriptions)
{
// universes can only remove members of their own
if (!universe.ContainsMember(subscription.Security)) continue;
// never remove universe selection subscriptions
if (subscription.IsUniverseSelectionSubscription) continue;
var config = subscription.Configuration;
// never remove internal feeds
if (config.IsInternalFeed) continue;
// if we've selected this subscription again, keep it
if (selections.Contains(config.Symbol)) continue;
// let the algorithm know this security has been removed from the universe
removals.Add(subscription.Security);
// but don't physically remove it from the algorithm if we hold stock or have open orders against it
var openOrders = _algorithm.Transactions.GetOrders(x => x.Status.IsOpen() && x.Symbol == config.Symbol);
if (!subscription.Security.HoldStock && !openOrders.Any())
{
// we need to mark this security as untradeable while it has no data subscription
// it is expected that this function is called while in sync with the algo thread,
// so we can make direct edits to the security here
subscription.Security.Cache.Reset();
if (_dataFeed.RemoveSubscription(subscription))
{
universe.RemoveMember(subscription.Security);
}
}
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// we already have a subscription for this symbol so don't re-add it
if (existingSubscriptions.Contains(symbol)) continue;
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!_algorithm.Securities.TryGetValue(symbol, out security))
{
security = SecurityManager.CreateSecurity(_algorithm.Portfolio, _algorithm.SubscriptionManager, _marketHoursDatabase,
symbol,
settings.Resolution,
settings.FillForward,
settings.Leverage,
settings.ExtendedMarketHours,
false,
false,
false);
}
additions.Add(security);
// add the new subscriptions to the data feed
if (_dataFeed.AddSubscription(universe, security, dateTimeUtc, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)))
{
universe.AddMember(security);
}
}
// return None if there's no changes, otherwise return what we've modified
return additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
}
