/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null
// if the universe has been disposed we don't perform selection. This us handled bellow by 'Universe.PerformSelection'
// but in this case we directly call 'SelectSymbols' because we want to perform fine selection even if coarse returns the same
// symbols, see 'Universe.PerformSelection', which detects this and returns 'Universe.Unchanged'
&& !universe.DisposeRequested)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
// Create a dictionary of CoarseFundamental keyed by Symbol that also has FineFundamental
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var allCoarse = universeData.Data.OfType <CoarseFundamental>();
var coarseData = allCoarse.Where(c => c.HasFundamentalData)
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
// Remove selected symbols that does not have fine fundamental data
var anyDoesNotHaveFundamentalData = false;
// only pre filter selected symbols if there actually is any coarse data. This way we can support custom universe filtered by fine fundamental data
// which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem,
// 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it
if (allCoarse.Any())
{
selectSymbolsResult = selectSymbolsResult
.Where(
symbol =>
{
var result = coarseData.ContainsKey(symbol);
anyDoesNotHaveFundamentalData |= !result;
return(result);
}
);
}
if (!_anyDoesNotHaveFundamentalDataWarningLogged && anyDoesNotHaveFundamentalData)
{
_algorithm.Debug("Note: Your coarse selection filter was updated to exclude symbols without fine fundamental data. Make sure your coarse filter excludes symbols where HasFundamental is false.");
_anyDoesNotHaveFundamentalDataWarningLogged = true;
}
// use all available threads, the entire system is waiting for this to complete
var options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(selectSymbolsResult, options, symbol =>
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = _securityService.CreateSecurity(symbol,
config,
addToSymbolCache: false);
var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, _dataProvider))
{
if (enumerator.MoveNext())
{
lock (fineCollection.Data)
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
});
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
AssetClassification = fine.AssetClassification,
CompanyProfile = fine.CompanyProfile,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios,
Market = fine.Symbol.ID.Market
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
// set the fine fundamental price property to yesterday's closing price
fine.Value = coarse.Value;
}
universeData.Data.Add(fundamentals);
}
// END -- HACK ATTACK -- END
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
// first check for no pending removals, even if the universe selection
// didn't change we might need to remove a security because a position was closed
RemoveSecurityFromUniverse(
_pendingRemovalsManager.CheckPendingRemovals(selections, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
// check for no changes second
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Securities.Values.OrderBy(member => member.Security.Symbol.SecurityType))
{
var security = member.Security;
// if we've selected this subscription again, keep it
if (selections.Contains(security.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, security))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(security);
RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(security, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
if (universe.Securities.ContainsKey(symbol))
{
// if its already part of the universe no need to re add it
continue;
}
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
universe.UniverseSettings.Resolution,
universe.UniverseSettings.FillForward,
universe.UniverseSettings.ExtendedMarketHours,
dataNormalizationMode: universe.UniverseSettings.DataNormalizationMode);
security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, (symbol.ID.SecurityType == SecurityType.Option || symbol.ID.SecurityType == SecurityType.FutureOption));
pendingAdditions.Add(symbol, security);
}
var addedSubscription = false;
var dataFeedAdded = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
_algorithm.SubscriptionManager.SubscriptionDataConfigService))
{
if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw.
!security.Subscriptions.Contains(request.Configuration))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
security.AddData(request.Configuration);
}
var toRemove = _currencySubscriptionDataConfigManager.GetSubscriptionDataConfigToRemove(request.Configuration.Symbol);
if (toRemove != null)
{
Log.Trace($"UniverseSelection.ApplyUniverseSelection(): Removing internal currency data feed {toRemove}");
_dataManager.RemoveSubscription(toRemove);
}
// 'dataFeedAdded' will help us notify the user for security changes only once per non internal subscription
// for example two universes adding the sample configuration, we don't want two notifications
dataFeedAdded = _dataManager.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
_internalSubscriptionManager.AddedSubscriptionRequest(request);
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember && dataFeedAdded)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
EnsureCurrencyDataFeeds(securityChanges);
}
if (securityChanges != SecurityChanges.None && Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}
/// <summary>
/// Applies universe selection the the data feed and algorithm
/// </summary>
/// <param name="universe">The universe to perform selection on</param>
/// <param name="dateTimeUtc">The current date time in utc</param>
/// <param name="universeData">The data provided to perform selection with</param>
public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTimeUtc, BaseDataCollection universeData)
{
var algorithmEndDateUtc = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
if (dateTimeUtc > algorithmEndDateUtc)
{
return(SecurityChanges.None);
}
IEnumerable <Symbol> selectSymbolsResult;
// check if this universe must be filtered with fine fundamental data
var fineFiltered = universe as FineFundamentalFilteredUniverse;
if (fineFiltered != null)
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.SelectSymbols(dateTimeUtc, universeData);
if (!ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
// prepare a BaseDataCollection of FineFundamental instances
var fineCollection = new BaseDataCollection();
var dataProvider = new DefaultDataProvider();
// use all available threads, the entire system is waiting for this to complete
var options = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
Parallel.ForEach(selectSymbolsResult, options, symbol =>
{
var config = FineFundamentalUniverse.CreateConfiguration(symbol);
var security = _securityService.CreateSecurity(symbol,
config,
addToSymbolCache: false);
var localStartTime = dateTimeUtc.ConvertFromUtc(config.ExchangeTimeZone).AddDays(-1);
var factory = new FineFundamentalSubscriptionEnumeratorFactory(_algorithm.LiveMode, x => new[] { localStartTime });
var request = new SubscriptionRequest(true, universe, security, new SubscriptionDataConfig(config), localStartTime, localStartTime);
using (var enumerator = factory.CreateEnumerator(request, dataProvider))
{
if (enumerator.MoveNext())
{
lock (fineCollection.Data)
{
fineCollection.Data.Add(enumerator.Current);
}
}
}
});
// WARNING -- HACK ATTACK -- WARNING
// Fine universes are considered special due to their chaining behavior.
// As such, we need a means of piping the fine data read in here back to the data feed
// so that it can be properly emitted via a TimeSlice.Create call. There isn't a mechanism
// in place for this function to return such data. The following lines are tightly coupled
// to the universeData dictionaries in SubscriptionSynchronizer and LiveTradingDataFeed and
// rely on reference semantics to work.
// Coarse raw data has SID collision on: CRHCY R735QTJ8XC9X
var coarseData = universeData.Data.OfType <CoarseFundamental>()
.DistinctBy(c => c.Symbol)
.ToDictionary(c => c.Symbol);
universeData.Data = new List <BaseData>();
foreach (var fine in fineCollection.Data.OfType <FineFundamental>())
{
var fundamentals = new Fundamentals
{
Symbol = fine.Symbol,
Time = fine.Time,
EndTime = fine.EndTime,
DataType = fine.DataType,
CompanyReference = fine.CompanyReference,
EarningReports = fine.EarningReports,
EarningRatios = fine.EarningRatios,
FinancialStatements = fine.FinancialStatements,
OperationRatios = fine.OperationRatios,
SecurityReference = fine.SecurityReference,
ValuationRatios = fine.ValuationRatios
};
CoarseFundamental coarse;
if (coarseData.TryGetValue(fine.Symbol, out coarse))
{
// the only time the coarse data won't exist is if the selection function
// doesn't use the data provided, and instead returns a constant list of
// symbols -- coupled with a potential hole in the data
fundamentals.Value = coarse.Value;
fundamentals.Market = coarse.Market;
fundamentals.Volume = coarse.Volume;
fundamentals.DollarVolume = coarse.DollarVolume;
fundamentals.HasFundamentalData = coarse.HasFundamentalData;
// set the fine fundamental price property to yesterday's closing price
fine.Value = coarse.Value;
}
universeData.Data.Add(fundamentals);
}
// END -- HACK ATTACK -- END
// perform the fine fundamental universe selection
selectSymbolsResult = fineFiltered.FineFundamentalUniverse.PerformSelection(dateTimeUtc, fineCollection);
}
}
else
{
// perform initial filtering and limit the result
selectSymbolsResult = universe.PerformSelection(dateTimeUtc, universeData);
}
// check for no changes first
if (ReferenceEquals(selectSymbolsResult, Universe.Unchanged))
{
return(SecurityChanges.None);
}
// materialize the enumerable into a set for processing
var selections = selectSymbolsResult.ToHashSet();
var additions = new List <Security>();
var removals = new List <Security>();
RemoveSecurityFromUniverse(
_pendingRemovalsManager.CheckPendingRemovals(selections, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
// determine which data subscriptions need to be removed from this universe
foreach (var member in universe.Members.Values)
{
// if we've selected this subscription again, keep it
if (selections.Contains(member.Symbol))
{
continue;
}
// don't remove if the universe wants to keep him in
if (!universe.CanRemoveMember(dateTimeUtc, member))
{
continue;
}
// remove the member - this marks this member as not being
// selected by the universe, but it may remain in the universe
// until open orders are closed and the security is liquidated
removals.Add(member);
RemoveSecurityFromUniverse(_pendingRemovalsManager.TryRemoveMember(member, universe),
removals,
dateTimeUtc,
algorithmEndDateUtc);
}
var keys = _pendingSecurityAdditions.Keys;
if (keys.Any() && keys.Single() != dateTimeUtc)
{
// if the frontier moved forward then we've added these securities to the algorithm
_pendingSecurityAdditions.Clear();
}
Dictionary <Symbol, Security> pendingAdditions;
if (!_pendingSecurityAdditions.TryGetValue(dateTimeUtc, out pendingAdditions))
{
// keep track of created securities so we don't create the same security twice, leads to bad things :)
pendingAdditions = new Dictionary <Symbol, Security>();
_pendingSecurityAdditions[dateTimeUtc] = pendingAdditions;
}
// find new selections and add them to the algorithm
foreach (var symbol in selections)
{
// create the new security, the algorithm thread will add this at the appropriate time
Security security;
if (!pendingAdditions.TryGetValue(symbol, out security) && !_algorithm.Securities.TryGetValue(symbol, out security))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
var configs = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
universe.UniverseSettings.Resolution,
universe.UniverseSettings.FillForward,
universe.UniverseSettings.ExtendedMarketHours);
security = _securityService.CreateSecurity(symbol, configs, universe.UniverseSettings.Leverage, symbol.ID.SecurityType == SecurityType.Option);
pendingAdditions.Add(symbol, security);
SetUnderlyingSecurity(universe, security);
}
var addedSubscription = false;
foreach (var request in universe.GetSubscriptionRequests(security, dateTimeUtc, algorithmEndDateUtc,
_algorithm.SubscriptionManager.SubscriptionDataConfigService))
{
if (security.Symbol == request.Configuration.Symbol && // Just in case check its the same symbol, else AddData will throw.
!security.Subscriptions.Contains(request.Configuration))
{
// For now this is required for retro compatibility with usages of security.Subscriptions
security.AddData(request.Configuration);
}
_dataManager.AddSubscription(request);
// only update our security changes if we actually added data
if (!request.IsUniverseSubscription)
{
addedSubscription = true;
}
}
if (addedSubscription)
{
var addedMember = universe.AddMember(dateTimeUtc, security);
if (addedMember)
{
additions.Add(security);
}
}
}
// return None if there's no changes, otherwise return what we've modified
var securityChanges = additions.Count + removals.Count != 0
? new SecurityChanges(additions, removals)
: SecurityChanges.None;
// Add currency data feeds that weren't explicitly added in Initialize
if (additions.Count > 0)
{
EnsureCurrencyDataFeeds(securityChanges);
foreach (var subscriptionDataConfig in _addedCurrencySubscriptionDataConfigs)
{
var security = _algorithm.Securities[subscriptionDataConfig.Symbol];
_dataManager.AddSubscription(new SubscriptionRequest(false, universe, security, subscriptionDataConfig, dateTimeUtc, algorithmEndDateUtc));
}
_addedCurrencySubscriptionDataConfigs.Clear();
}
if (securityChanges != SecurityChanges.None)
{
Log.Debug("UniverseSelection.ApplyUniverseSelection(): " + dateTimeUtc + ": " + securityChanges);
}
return(securityChanges);
}