/// <summary>
/// Adds the security to the user defined universe for the specified
/// </summary>
private void AddToUserDefinedUniverse(Security security)
{
// add this security to the user defined universe
UserDefinedUniverse universe;
var key = new SecurityTypeMarket(security.Type, security.SubscriptionDataConfig.Market);
if (_userDefinedUniverses.TryGetValue(key, out universe))
{
universe.Add(security.Symbol);
}
else
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var securityConfig = security.SubscriptionDataConfig;
var universeSymbol = UserDefinedUniverse.CreateSymbol(securityConfig.SecurityType, securityConfig.Market);
var uconfig = new SubscriptionDataConfig(securityConfig, symbol: universeSymbol, isInternalFeed: true, fillForward: false);
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(security.Resolution, security.Leverage, security.IsFillDataForward, security.IsExtendedMarketHours, TimeSpan.Zero),
QuantConnect.Time.OneDay,
new List <Symbol> {
security.Symbol
}
);
_userDefinedUniverses[key] = universe;
UniverseManager.Add(universe.Configuration.Symbol, universe);
}
}
public void AddsSubscription_NewUserUniverse()
{
FuncDataQueueHandler dataQueueHandler;
var feed = RunDataFeed(out dataQueueHandler, equities: new List <string> {
"SPY"
});
var forexFxcmUserUniverse = UserDefinedUniverse.CreateSymbol(SecurityType.Forex, Market.FXCM);
var emittedData = false;
var newDataCount = 0;
var securityChanges = 0;
ConsumeBridge(feed, TimeSpan.FromSeconds(5), true, ts =>
{
securityChanges += ts.SecurityChanges.Count;
if (!emittedData)
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
if (ts.Data.Count > 0)
{
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
}
Assert.AreEqual(1, dataQueueHandler.Subscriptions.Count);
_algorithm.AddSecurities(forex: new List <string> {
"EURUSD"
});
emittedData = true;
}
else
{
if (dataQueueHandler.Subscriptions.Count == 2) // there could be some slices with no data
{
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.SPY));
if (ts.Data.Count > 0)
{
Assert.IsTrue(ts.Slice.Keys.Contains(Symbols.SPY));
}
Assert.IsTrue(dataQueueHandler.Subscriptions.Contains(Symbols.EURUSD) ||
dataQueueHandler.Subscriptions.Contains(forexFxcmUserUniverse));
// Might delay a couple of Slices to send over the data, so we will count them
// and assert a minimum amount
if (ts.Slice.Keys.Contains(Symbols.EURUSD))
{
newDataCount++;
}
}
else
{
Assert.Fail($"Subscriptions.Count: {dataQueueHandler.Subscriptions.Count}");
}
}
});
Console.WriteLine("newDataCount: " + newDataCount);
Assert.AreEqual(2, securityChanges);
Assert.GreaterOrEqual(newDataCount, 1000);
Assert.IsTrue(emittedData);
}
/// <summary>
/// Adds the security to the user defined universe for the specified
/// </summary>
private void AddToUserDefinedUniverse(Security security)
{
// if we are adding a non-internal security which already has an internal feed, we remove it first
Security existingSecurity;
if (Securities.TryGetValue(security.Symbol, out existingSecurity))
{
if (!security.IsInternalFeed() && existingSecurity.IsInternalFeed())
{
var securityUniverse = UniverseManager.Select(x => x.Value).OfType <UserDefinedUniverse>().FirstOrDefault(x => x.Members.ContainsKey(security.Symbol));
securityUniverse?.Remove(security.Symbol);
Securities.Remove(security.Symbol);
}
}
Securities.Add(security);
// add this security to the user defined universe
Universe universe;
var subscription = security.Subscriptions.First();
var universeSymbol = UserDefinedUniverse.CreateSymbol(subscription.SecurityType, subscription.Market);
if (!UniverseManager.TryGetValue(universeSymbol, out universe))
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var uconfig = new SubscriptionDataConfig(subscription, symbol: universeSymbol, isInternalFeed: true, fillForward: false);
if (security.Type == SecurityType.Base)
{
// set entry in market hours database for the universe subscription to match the custom data
var symbolString = MarketHoursDatabase.GetDatabaseSymbolKey(uconfig.Symbol);
MarketHoursDatabase.SetEntry(uconfig.Market, symbolString, uconfig.SecurityType, security.Exchange.Hours, uconfig.DataTimeZone);
}
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(security.Resolution, security.Leverage, security.IsFillDataForward, security.IsExtendedMarketHours, TimeSpan.Zero),
SecurityInitializer,
QuantConnect.Time.MaxTimeSpan,
new List <Symbol> {
security.Symbol
}
);
UniverseManager.Add(universeSymbol, universe);
}
var userDefinedUniverse = universe as UserDefinedUniverse;
if (userDefinedUniverse != null)
{
userDefinedUniverse.Add(security.Symbol);
}
else
{
// should never happen, someone would need to add a non-user defined universe with this symbol
throw new Exception("Expected universe with symbol '" + universeSymbol.Value + "' to be of type UserDefinedUniverse.");
}
}
/// <summary>
/// Used to send data updates to algorithm framework models
/// </summary>
/// <param name="slice">The current data slice</param>
public void OnFrameworkData(Slice slice)
{
if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc())
{
var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
// remove deselected universes by symbol
foreach (var ukvp in UniverseManager)
{
var universeSymbol = ukvp.Key;
var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market);
if (universeSymbol.Equals(qcUserDefined))
{
// prevent removal of qc algorithm created user defined universes
continue;
}
Universe universe;
if (!universes.TryGetValue(universeSymbol, out universe))
{
if (ukvp.Value.DisposeRequested)
{
UniverseManager.Remove(universeSymbol);
}
// mark this universe as disposed to remove all child subscriptions
ukvp.Value.Dispose();
}
}
// add newly selected universes
foreach (var ukvp in universes)
{
// note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
UniverseManager.Add(ukvp);
}
}
// we only want to run universe selection if there's no data available in the slice
if (!slice.HasData)
{
return;
}
// insight timestamping handled via InsightsGenerated event handler
var insightsEnumerable = Alpha.Update(this, slice);
// for performance only call 'ToArray' if not empty enumerable (which is static)
var insights = insightsEnumerable == Enumerable.Empty <Insight>()
? new Insight[] { } : insightsEnumerable.ToArray();
// only fire insights generated event if we actually have insights
if (insights.Length != 0)
{
OnInsightsGenerated(insights.Select(InitializeInsightFields));
}
ProcessInsights(insights);
}
public UserDefinedUniverseAddition(
UserDefinedUniverse universe,
List <SubscriptionDataConfig> subscriptionDataConfigs,
Security security)
{
Universe = universe;
SubscriptionDataConfigs = subscriptionDataConfigs;
Security = security;
}
public override void Initialize()
{
SetStartDate(2015, 12, 24);
SetEndDate(2015, 12, 24);
var goog = AddEquity(UnderlyingTicker);
// expect GOOG equity
_expectedData.Add(goog.Symbol);
_expectedSecurities.Add(goog.Symbol);
// expect user defined universe holding GOOG equity
_expectedUniverses.Add(UserDefinedUniverse.CreateSymbol(SecurityType.Equity, Market.USA));
}
/// <summary>
/// Adds the security to the user defined universe for the specified
/// </summary>
private void AddToUserDefinedUniverse(Security security)
{
Securities.Add(security);
// add this security to the user defined universe
Universe universe;
var securityConfig = security.SubscriptionDataConfig;
var universeSymbol = UserDefinedUniverse.CreateSymbol(securityConfig.SecurityType, securityConfig.Market);
if (!UniverseManager.TryGetValue(universeSymbol, out universe))
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var uconfig = new SubscriptionDataConfig(securityConfig, symbol: universeSymbol, isInternalFeed: true, fillForward: false);
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(security.Resolution, security.Leverage, security.IsFillDataForward, security.IsExtendedMarketHours, TimeSpan.Zero),
SecurityInitializer,
QuantConnect.Time.OneDay,
new List <Symbol> {
security.Symbol
}
);
UniverseManager.Add(universeSymbol, universe);
}
var userDefinedUniverse = universe as UserDefinedUniverse;
if (userDefinedUniverse != null)
{
userDefinedUniverse.Add(security.Symbol);
}
else
{
// should never happen, someone would need to add a non-user defined universe with this symbol
throw new Exception("Expected universe with symbol '" + universeSymbol.Value + "' to be of type UserDefinedUniverse.");
}
}
/// <summary>
/// Adds the security to the user defined universe
/// </summary>
/// <param name="security">The security to add</param>
/// <param name="configurations">The <see cref="SubscriptionDataConfig"/> instances we want to add</param>
private Security AddToUserDefinedUniverse(
Security security,
List <SubscriptionDataConfig> configurations)
{
var subscription = configurations.First();
// if we are adding a non-internal security which already has an internal feed, we remove it first
if (Securities.TryGetValue(security.Symbol, out var existingSecurity))
{
if (!subscription.IsInternalFeed && existingSecurity.IsInternalFeed())
{
var securityUniverse = UniverseManager.Select(x => x.Value).OfType <UserDefinedUniverse>().FirstOrDefault(x => x.Members.ContainsKey(security.Symbol));
securityUniverse?.Remove(security.Symbol);
Securities.Remove(security.Symbol);
Securities.Add(security);
}
else
{
var isTradable = security.IsTradable;
// We will reuse existing so we return it to the user.
// We will use the IsTradable flag of the new security, since existing could of been set to false when removed
security = existingSecurity;
security.IsTradable = isTradable;
}
}
else
{
Securities.Add(security);
}
// add this security to the user defined universe
Universe universe;
var universeSymbol = UserDefinedUniverse.CreateSymbol(security.Type, security.Symbol.ID.Market);
lock (_pendingUniverseAdditionsLock)
{
if (!UniverseManager.TryGetValue(universeSymbol, out universe))
{
universe = _pendingUniverseAdditions.FirstOrDefault(x => x.Configuration.Symbol == universeSymbol);
if (universe == null)
{
// create a new universe, these subscription settings don't currently get used
// since universe selection proper is never invoked on this type of universe
var uconfig = new SubscriptionDataConfig(subscription, symbol: universeSymbol, isInternalFeed: true, fillForward: false,
exchangeTimeZone: DateTimeZone.Utc,
dataTimeZone: DateTimeZone.Utc);
// this is the universe symbol, has no real entry in the mhdb, will default to market and security type
// set entry in market hours database for the universe subscription to match the config
var symbolString = MarketHoursDatabase.GetDatabaseSymbolKey(uconfig.Symbol);
MarketHoursDatabase.SetEntry(uconfig.Market, symbolString, uconfig.SecurityType,
SecurityExchangeHours.AlwaysOpen(uconfig.ExchangeTimeZone), uconfig.DataTimeZone);
universe = new UserDefinedUniverse(uconfig,
new UniverseSettings(
subscription.Resolution,
security.Leverage,
subscription.FillDataForward,
subscription.ExtendedMarketHours,
TimeSpan.Zero),
QuantConnect.Time.MaxTimeSpan,
new List <Symbol>());
AddUniverse(universe);
}
}
}
var userDefinedUniverse = universe as UserDefinedUniverse;
if (userDefinedUniverse != null)
{
lock (_pendingUniverseAdditionsLock)
{
_pendingUserDefinedUniverseSecurityAdditions.Add(
new UserDefinedUniverseAddition(userDefinedUniverse, configurations, security));
}
}
else
{
// should never happen, someone would need to add a non-user defined universe with this symbol
throw new Exception($"Expected universe with symbol '{universeSymbol.Value}' to be of type {nameof(UserDefinedUniverse)} but was {universe.GetType().Name}.");
}
return(security);
}
/// <summary>
/// Initializes a new instance of the <see cref="UserDefinedUniverseSubscriptionEnumeratorFactory"/> class
/// </summary>
/// <param name="universe">The user defined universe</param>
/// <param name="marketHoursDatabase">The market hours database</param>
public UserDefinedUniverseSubscriptionEnumeratorFactory(UserDefinedUniverse universe, MarketHoursDatabase marketHoursDatabase)
{
_universe = universe;
_marketHoursDatabase = marketHoursDatabase;
}
/// <summary>
/// Used to send data updates to algorithm framework models
/// </summary>
/// <param name="slice">The current data slice</param>
public sealed override void OnFrameworkData(Slice slice)
{
if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc())
{
var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
// remove deselected universes by symbol
foreach (var ukvp in UniverseManager)
{
var universeSymbol = ukvp.Key;
var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market);
if (universeSymbol.Equals(qcUserDefined))
{
// prevent removal of qc algorithm created user defined universes
continue;
}
Universe universe;
if (!universes.TryGetValue(universeSymbol, out universe))
{
if (ukvp.Value.DisposeRequested)
{
UniverseManager.Remove(universeSymbol);
}
// mark this universe as disposed to remove all child subscriptions
ukvp.Value.Dispose();
}
}
// add newly selected universes
foreach (var ukvp in universes)
{
// note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
UniverseManager.Add(ukvp);
}
}
// we only want to run universe selection if there's no data available in the slice
if (!slice.HasData)
{
return;
}
// insight timestamping handled via InsightsGenerated event handler
var insights = Alpha.Update(this, slice).ToArray();
// only fire insights generated event if we actually have insights
if (insights.Length != 0)
{
// debug printing of generated insights
if (DebugMode)
{
Log($"{Time}: ALPHA: {string.Join(" | ", insights.Select(i => i.ToString()).OrderBy(i => i))}");
}
OnInsightsGenerated(insights);
}
// construct portfolio targets from insights
var targets = PortfolioConstruction.CreateTargets(this, insights).ToArray();
// set security targets w/ those generated via portfolio construction module
foreach (var target in targets)
{
var security = Securities[target.Symbol];
security.Holdings.Target = target;
}
if (DebugMode)
{
// debug printing of generated targets
if (targets.Length > 0)
{
Log($"{Time}: PORTFOLIO: {string.Join(" | ", targets.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
var riskTargetOverrides = RiskManagement.ManageRisk(this, targets).ToArray();
// override security targets w/ those generated via risk management module
foreach (var target in riskTargetOverrides)
{
var security = Securities[target.Symbol];
security.Holdings.Target = target;
}
if (DebugMode)
{
// debug printing of generated risk target overrides
if (riskTargetOverrides.Length > 0)
{
Log($"{Time}: RISK: {string.Join(" | ", riskTargetOverrides.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
// execute on the targets, overriding targets for symbols w/ risk targets
var riskAdjustedTargets = riskTargetOverrides.Concat(targets).DistinctBy(pt => pt.Symbol).ToArray();
if (DebugMode)
{
// only log adjusted targets if we've performed an adjustment
if (riskTargetOverrides.Length > 0)
{
Log($"{Time}: RISK ADJUSTED TARGETS: {string.Join(" | ", riskAdjustedTargets.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
Execution.Execute(this, riskAdjustedTargets);
}
