/// <summary>
/// Event fired each time the we add/remove securities from the data feed
/// </summary>
/// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
/// <param name="changes">The security additions and removals from the algorithm</param>
public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
{
base.OnSecuritiesChanged(algorithm, changes);
// Get removed symbol and invalidate them in the insight collection
_removedSymbols = changes.RemovedSecurities.Select(x => x.Symbol).ToList();
InsightCollection.Clear(_removedSymbols.ToArray());
}
/// <summary>
/// Determines the target percent for each insight
/// </summary>
/// <param name="activeInsights">The active insights to generate a target for</param>
/// <returns>A target percent for each insight</returns>
protected override Dictionary <Insight, double> DetermineTargetPercent(ICollection <Insight> activeInsights)
{
var percentPerSymbol = new Dictionary <Symbol, double>();
foreach (var insight in InsightCollection.GetActiveInsights(_currentUtcTime)
.OrderBy(insight => insight.GeneratedTimeUtc))
{
double targetPercent;
if (percentPerSymbol.TryGetValue(insight.Symbol, out targetPercent))
{
if (insight.Direction == InsightDirection.Flat)
{
// We received a Flat
// if adding or subtracting will push past 0, then make it 0
if (Math.Abs(targetPercent) < _percent)
{
targetPercent = 0;
}
else
{
// otherwise, we flatten by percent
targetPercent += (targetPercent > 0 ? -_percent : _percent);
}
}
}
targetPercent += _percent * (int)insight.Direction;
percentPerSymbol[insight.Symbol] = targetPercent;
}
return(activeInsights
.ToDictionary(insight => insight, insight => percentPerSymbol[insight.Symbol]));
}
/// <summary>
/// Event fired each time the we add/remove securities from the data feed
/// </summary>
/// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
/// <param name="changes">The security additions and removals from the algorithm</param>
public virtual void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
{
_securityChanges = changes != SecurityChanges.None;
// Get removed symbol and invalidate them in the insight collection
_removedSymbols = changes.RemovedSecurities.Select(x => x.Symbol).ToList();
InsightCollection.Clear(_removedSymbols.ToArray());
}
/// <summary>
/// Event fired each time the we add/remove securities from the data feed
/// </summary>
/// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
/// <param name="changes">The security additions and removals from the algorithm</param>
public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
{
base.OnSecuritiesChanged(algorithm, changes);
// Get removed symbol and invalidate them in the insight collection
_removedSymbols = changes.RemovedSecurities.Select(x => x.Symbol).ToList();
InsightCollection.Clear(_removedSymbols.ToArray());
foreach (var symbol in _removedSymbols)
{
if (_symbolDataDict.ContainsKey(symbol))
{
_symbolDataDict[symbol].Reset();
_symbolDataDict.Remove(symbol);
}
}
// initialize data for added securities
var addedSymbols = changes.AddedSecurities.Select(x => x.Symbol).ToList();
algorithm.History(addedSymbols, _lookback * _period, _resolution)
.PushThrough(bar =>
{
ReturnsSymbolData symbolData;
if (!_symbolDataDict.TryGetValue(bar.Symbol, out symbolData))
{
symbolData = new ReturnsSymbolData(bar.Symbol, _lookback, _period);
_symbolDataDict.Add(bar.Symbol, symbolData);
}
symbolData.Update(bar.EndTime, bar.Value);
});
}
/// <summary>
/// Gets the target insights to calculate a portfolio target percent for
/// </summary>
/// <returns>An enumerable of the target insights</returns>
protected virtual List <Insight> GetTargetInsights()
{
// Get insight that haven't expired of each symbol that is still in the universe
var activeInsights = InsightCollection.GetActiveInsights(Algorithm.UtcTime);
// Get the last generated active insight for each symbol
return((from insight in activeInsights
group insight by insight.Symbol into g
select g.OrderBy(x => x.GeneratedTimeUtc).Last()).ToList());
}
public static void InsightCollectionShouldBeAbleToBeConvertedToListWithoutStackOverflow()
{
var aapl = Symbol.Create("AAPL", SecurityType.Equity, "usa");
var insightCollection = new InsightCollection();
insightCollection.Add(new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Price, InsightDirection.Up)
{
CloseTimeUtc = new DateTime(2019, 1, 1),
});
insightCollection.Add(new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Volatility, InsightDirection.Up)
{
CloseTimeUtc = new DateTime(2019, 1, 2),
});
Assert.DoesNotThrow(() => insightCollection.OrderBy(x => x.CloseTimeUtc).ToList());
}
/// <summary>
/// Determines if the portfolio should be rebalanced base on the provided rebalancing func,
/// if any security change have been taken place or if an insight has expired or a new insight arrived
/// If the rebalancing function has not been provided will return true.
/// </summary>
/// <param name="insights">The insights to create portfolio targets from</param>
/// <param name="algorithmUtc">The current algorithm UTC time</param>
/// <returns>True if should rebalance</returns>
protected virtual bool IsRebalanceDue(Insight[] insights, DateTime algorithmUtc)
{
// if there is no rebalance func set, just return true but refresh state
// just in case the rebalance func is going to be set.
if (_rebalancingFunc == null)
{
RefreshRebalance(algorithmUtc);
return(true);
}
// we always get the next expiry time
// we don't know if a new insight was added or removed
var nextInsightExpiryTime = InsightCollection.GetNextExpiryTime();
if (_rebalancingTime == null)
{
_rebalancingTime = _rebalancingFunc(algorithmUtc);
if (_rebalancingTime != null && _rebalancingTime <= algorithmUtc)
{
// if the rebalancing time stopped being null and is current time
// we will ask for the next rebalance time in the next loop.
// we don't want to call the '_rebalancingFunc' twice in the same loop,
// since its internal state machine will probably be in the same state.
_rebalancingTime = null;
_securityChanges = false;
return(true);
}
}
if (_rebalancingTime != null && _rebalancingTime <= algorithmUtc ||
RebalanceOnSecurityChanges && _securityChanges ||
RebalanceOnInsightChanges &&
(insights.Length != 0 ||
nextInsightExpiryTime != null && nextInsightExpiryTime < algorithmUtc))
{
RefreshRebalance(algorithmUtc);
return(true);
}
return(false);
}
/// <summary>
/// Create portfolio targets from the specified insights
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="insights">The insights to create portfolio targets from</param>
/// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
public virtual IEnumerable <IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
{
Algorithm = algorithm;
// always add new insights
if (insights.Length > 0)
{
// Validate we should create a target for this insight
InsightCollection.AddRange(insights
.Where(insight => PythonWrapper?.ShouldCreateTargetForInsight(insight)
?? ShouldCreateTargetForInsight(insight)));
}
if (!(PythonWrapper?.IsRebalanceDue(insights, algorithm.UtcTime)
?? IsRebalanceDue(insights, algorithm.UtcTime)))
{
return(Enumerable.Empty <IPortfolioTarget>());
}
var targets = new List <IPortfolioTarget>();
// Create flatten target for each security that was removed from the universe unless RebalanceOnSecurityChanges is False
if (_removedSymbols != null)
{
public void SetNextExpiration(DateTime nextExpiration)
{
InsightCollection.Add(
new Insight(Symbols.SPY, time => nextExpiration, InsightType.Price, InsightDirection.Down));
}
/// <summary>
/// Create portfolio targets from the specified insights
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="insights">The insights to create portfolio targets from</param>
/// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
public override IEnumerable <IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
{
// always add new insights
if (insights.Length > 0)
{
// Validate we should create a target for this insight
InsightCollection.AddRange(insights.Where(ShouldCreateTargetForInsight));
}
if (!IsRebalanceDue(insights, algorithm.UtcTime))
{
return(Enumerable.Empty <IPortfolioTarget>());
}
var targets = new List <IPortfolioTarget>();
// Create flatten target for each security that was removed from the universe
if (_removedSymbols != null)
{
var universeDeselectionTargets = _removedSymbols.Select(symbol => new PortfolioTarget(symbol, 0));
targets.AddRange(universeDeselectionTargets);
_removedSymbols = null;
}
// Get insight that haven't expired of each symbol that is still in the universe
var activeInsights = InsightCollection.GetActiveInsights(algorithm.UtcTime);
// Get the last generated active insight for each symbol
var lastActiveInsights = (from insight in activeInsights
group insight by insight.Symbol into g
select g.OrderBy(x => x.GeneratedTimeUtc).Last()).ToList();
var errorSymbols = new HashSet <Symbol>();
// Determine target percent for the given insights
var percents = DetermineTargetPercent(lastActiveInsights);
foreach (var insight in lastActiveInsights)
{
var target = PortfolioTarget.Percent(algorithm, insight.Symbol, percents[insight]);
if (target != null)
{
targets.Add(target);
}
else
{
errorSymbols.Add(insight.Symbol);
}
}
// Get expired insights and create flatten targets for each symbol
var expiredInsights = InsightCollection.RemoveExpiredInsights(algorithm.UtcTime);
var expiredTargets = from insight in expiredInsights
group insight.Symbol by insight.Symbol into g
where !InsightCollection.HasActiveInsights(g.Key, algorithm.UtcTime) && !errorSymbols.Contains(g.Key)
select new PortfolioTarget(g.Key, 0);
targets.AddRange(expiredTargets);
return(targets);
}
/// <summary>
/// Initialize a new instance of <see cref="PortfolioConstructionModel"/>
/// </summary>
/// <param name="rebalancingFunc">For a given algorithm UTC DateTime returns the next expected rebalance time
/// or null if unknown, in which case the function will be called again in the next loop. Returning current time
/// will trigger rebalance. If null will be ignored</param>
public PortfolioConstructionModel(Func <DateTime, DateTime?> rebalancingFunc)
{
_rebalancingFunc = rebalancingFunc;
InsightCollection = new InsightCollection();
}
/// <summary>
/// Create portfolio targets from the specified insights
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="insights">The insights to create portfolio targets from</param>
/// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
public virtual IEnumerable <IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
{
Algorithm = algorithm;
// always add new insights
if (insights.Length > 0)
{
// Validate we should create a target for this insight
InsightCollection.AddRange(insights
.Where(insight => PythonWrapper?.ShouldCreateTargetForInsight(insight)
?? ShouldCreateTargetForInsight(insight)));
}
if (!(PythonWrapper?.IsRebalanceDue(insights, algorithm.UtcTime)
?? IsRebalanceDue(insights, algorithm.UtcTime)))
{
return(Enumerable.Empty <IPortfolioTarget>());
}
var targets = new List <IPortfolioTarget>();
// Create flatten target for each security that was removed from the universe
if (_removedSymbols != null)
{
var universeDeselectionTargets = _removedSymbols.Select(symbol => new PortfolioTarget(symbol, 0));
targets.AddRange(universeDeselectionTargets);
_removedSymbols = null;
}
var lastActiveInsights = PythonWrapper?.GetTargetInsights()
?? GetTargetInsights();
var errorSymbols = new HashSet <Symbol>();
// Determine target percent for the given insights
var percents = PythonWrapper?.DetermineTargetPercent(lastActiveInsights)
?? DetermineTargetPercent(lastActiveInsights);
foreach (var insight in lastActiveInsights)
{
double percent;
if (!percents.TryGetValue(insight, out percent))
{
continue;
}
var target = PortfolioTarget.Percent(algorithm, insight.Symbol, percent);
if (target != null)
{
targets.Add(target);
}
else
{
errorSymbols.Add(insight.Symbol);
}
}
// Get expired insights and create flatten targets for each symbol
var expiredInsights = InsightCollection.RemoveExpiredInsights(algorithm.UtcTime);
var expiredTargets = from insight in expiredInsights
group insight.Symbol by insight.Symbol into g
where !InsightCollection.HasActiveInsights(g.Key, algorithm.UtcTime) && !errorSymbols.Contains(g.Key)
select new PortfolioTarget(g.Key, 0);
targets.AddRange(expiredTargets);
return(targets);
}
/// <summary>
/// Steps the manager forward in time, accepting new state information and potentialy newly generated insights
/// </summary>
/// <param name="frontierTimeUtc">The frontier time of the insight analysis</param>
/// <param name="securityValuesCollection">Snap shot of the securities at the frontier time</param>
/// <param name="generatedInsights">Any insight generated by the algorithm at the frontier time</param>
public void Step(DateTime frontierTimeUtc, ReadOnlySecurityValuesCollection securityValuesCollection, InsightCollection generatedInsights)
{
if (generatedInsights != null && generatedInsights.Insights.Count > 0)
{
foreach (var insight in generatedInsights.Insights)
{
// save initial security values and deterine analysis period
var initialValues = securityValuesCollection[insight.Symbol];
var analysisPeriod = insight.Period + TimeSpan.FromTicks((long)(_extraAnalysisPeriodRatio * insight.Period.Ticks));
// set this as an open analysis context
var context = new InsightAnalysisContext(insight, initialValues, analysisPeriod);
_openInsightContexts.Add(context);
// let everyone know we've received an insight
_extensions.ForEach(e => e.OnInsightGenerated(context));
}
}
UpdateScores(securityValuesCollection);
foreach (var extension in _extensions)
{
extension.Step(frontierTimeUtc);
}
}
/// <summary>
/// Gets the target insights to calculate a portfolio target percent for
/// </summary>
/// <returns>An enumerable of the target insights</returns>
protected override List <Insight> GetTargetInsights()
{
return(InsightCollection.GetActiveInsights(Algorithm.UtcTime)
.OrderBy(insight => insight.GeneratedTimeUtc)
.ToList());
}
/// <summary>
/// Create portfolio targets from the specified insights
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="insights">The insights to create portfolio targets from</param>
/// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
public override IEnumerable <IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
{
// always add new insights
if (insights.Length > 0)
{
insights = FilterInvalidInsightMagnitude(algorithm, insights);
InsightCollection.AddRange(insights);
}
if (!IsRebalanceDue(insights, algorithm.UtcTime))
{
return(Enumerable.Empty <IPortfolioTarget>());
}
var targets = new List <IPortfolioTarget>();
// Create flatten target for each security that was removed from the universe
if (_removedSymbols != null)
{
var universeDeselectionTargets = _removedSymbols.Select(symbol => new PortfolioTarget(symbol, 0));
targets.AddRange(universeDeselectionTargets);
_removedSymbols = null;
}
// Get insight that haven't expired of each symbol that is still in the universe
var activeInsights = InsightCollection.GetActiveInsights(algorithm.UtcTime);
// Get the last generated active insight for each symbol
var lastActiveInsights = (from insight in activeInsights
group insight by new { insight.Symbol, insight.SourceModel } into g
select g.OrderBy(x => x.GeneratedTimeUtc).Last())
.OrderBy(x => x.Symbol).ToArray();
double[,] P;
double[] Q;
if (TryGetViews(lastActiveInsights, out P, out Q))
{
// Updates the ReturnsSymbolData with insights
foreach (var insight in lastActiveInsights)
{
ReturnsSymbolData symbolData;
if (_symbolDataDict.TryGetValue(insight.Symbol, out symbolData))
{
if (insight.Magnitude == null)
{
algorithm.SetRunTimeError(new ArgumentNullException("BlackLittermanOptimizationPortfolioConstructionModel does not accept \'null\' as Insight.Magnitude. Please make sure your Alpha Model is generating Insights with the Magnitude property set."));
}
symbolData.Add(algorithm.Time, insight.Magnitude.Value.SafeDecimalCast());
}
}
// Get symbols' returns
var symbols = lastActiveInsights.Select(x => x.Symbol).Distinct().ToList();
var returns = _symbolDataDict.FormReturnsMatrix(symbols);
// Calculate posterior estimate of the mean and uncertainty in the mean
double[,] Σ;
var Π = GetEquilibriumReturns(returns, out Σ);
ApplyBlackLittermanMasterFormula(ref Π, ref Σ, P, Q);
// Create portfolio targets from the specified insights
var W = _optimizer.Optimize(returns, Π, Σ);
var sidx = 0;
foreach (var symbol in symbols)
{
var weight = W[sidx].SafeDecimalCast();
var target = PortfolioTarget.Percent(algorithm, symbol, weight);
if (target != null)
{
targets.Add(target);
}
sidx++;
}
}
// Get expired insights and create flatten targets for each symbol
var expiredInsights = InsightCollection.RemoveExpiredInsights(algorithm.UtcTime);
var expiredTargets = from insight in expiredInsights
group insight.Symbol by insight.Symbol into g
where !InsightCollection.HasActiveInsights(g.Key, algorithm.UtcTime)
select new PortfolioTarget(g.Key, 0);
targets.AddRange(expiredTargets);
return(targets);
}
public InsightQueueItem(DateTime frontierTimeUtc, ReadOnlySecurityValuesCollection securityValues, InsightCollection generatedInsights = null)
{
FrontierTimeUtc = frontierTimeUtc;
SecurityValues = securityValues;
GeneratedInsights = generatedInsights ?? new InsightCollection(frontierTimeUtc, Enumerable.Empty <Insight>());
}
public static void InsightCollectionCopyTo()
{
var aapl = Symbol.Create("AAPL", SecurityType.Equity, "usa");
var insightCollection = new InsightCollection();
insightCollection.Add(new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Price, InsightDirection.Up)
{
GeneratedTimeUtc = new DateTime(2019, 1, 1),
CloseTimeUtc = new DateTime(2019, 1, 1),
});
insightCollection.Add(new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Volatility, InsightDirection.Up)
{
GeneratedTimeUtc = new DateTime(2019, 1, 2),
CloseTimeUtc = new DateTime(2019, 1, 2),
});
var insights = new Insight[7] {
new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Price, InsightDirection.Up)
{
GeneratedTimeUtc = new DateTime(2019, 1, 5),
CloseTimeUtc = new DateTime(2019, 1, 5),
},
new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Price, InsightDirection.Flat)
{
GeneratedTimeUtc = new DateTime(2019, 1, 6),
CloseTimeUtc = new DateTime(2019, 1, 6),
},
new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Price, InsightDirection.Down)
{
GeneratedTimeUtc = new DateTime(2019, 1, 7),
CloseTimeUtc = new DateTime(2019, 1, 7),
},
new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Volatility, InsightDirection.Up)
{
GeneratedTimeUtc = new DateTime(2019, 1, 8),
CloseTimeUtc = new DateTime(2019, 1, 8),
},
new Insight(aapl, new TimeSpan(1, 0, 0, 0), InsightType.Volatility, InsightDirection.Down)
{
GeneratedTimeUtc = new DateTime(2019, 1, 9),
CloseTimeUtc = new DateTime(2019, 1, 9),
},
null,
null
};
insightCollection.CopyTo(insights, 5);
var orderedInsights = insights.OrderBy(x => x.CloseTimeUtc).ToList();
var orderedInsightsDe = JsonConvert.DeserializeObject <InsightCollection>(_expectedResultCopyTo);
var equals = orderedInsights.Zip(orderedInsightsDe, (i, de) =>
{
return(i.CloseTimeUtc == de.CloseTimeUtc &&
i.GeneratedTimeUtc == de.GeneratedTimeUtc &&
i.Symbol.Value == de.Symbol.Value &&
i.Type == de.Type &&
i.ReferenceValue == de.ReferenceValue &&
i.Direction == de.Direction &&
i.Period == de.Period);
});
Assert.IsTrue(equals.All((x) => x));
}