/// <summary>
/// Will first check and add all the required conversion rate securities
/// and later will seed an initial value to them.
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="universeSelection">The universe selection instance</param>
public static void SetupCurrencyConversions(
IAlgorithm algorithm,
UniverseSelection universeSelection)
{
// this is needed to have non-zero currency conversion rates during warmup
// will also set the Cash.ConversionRateSecurity
universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
// now set conversion rates
var cashToUpdate = algorithm.Portfolio.CashBook.Values
.Where(x => x.ConversionRateSecurity != null && x.ConversionRate == 0)
.ToList();
var historyRequestFactory = new HistoryRequestFactory(algorithm);
var historyRequests = new List <HistoryRequest>();
foreach (var cash in cashToUpdate)
{
var configs = algorithm
.SubscriptionManager
.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(cash.ConversionRateSecurity.Symbol,
includeInternalConfigs: true);
// we need to order and select a specific configuration type
// so the conversion rate is deterministic
var configToUse = configs.OrderBy(x => x.TickType).First();
var hours = cash.ConversionRateSecurity.Exchange.Hours;
var resolution = configs.GetHighestResolution();
var startTime = historyRequestFactory.GetStartTimeAlgoTz(
cash.ConversionRateSecurity.Symbol,
10,
resolution,
hours,
configToUse.DataTimeZone);
var endTime = algorithm.Time;
historyRequests.Add(historyRequestFactory.CreateHistoryRequest(
configToUse,
startTime,
endTime,
cash.ConversionRateSecurity.Exchange.Hours,
resolution));
}
var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone);
slices.PushThrough(data =>
{
foreach (var cash in cashToUpdate
.Where(x => x.ConversionRateSecurity.Symbol == data.Symbol))
{
cash.Update(data);
}
});
Log.Trace("BaseSetupHandler.SetupCurrencyConversions():" +
$"{Environment.NewLine}{algorithm.Portfolio.CashBook}");
}
/// <summary>
/// Will first check and add all the required conversion rate securities
/// and later will seed an initial value to them.
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="universeSelection">The universe selection instance</param>
public static void SetupCurrencyConversions(
IAlgorithm algorithm,
UniverseSelection universeSelection)
{
// this is needed to have non-zero currency conversion rates during warmup
// will also set the Cash.ConversionRateSecurity
universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
// now set conversion rates
var cashToUpdate = algorithm.Portfolio.CashBook.Values
.Where(x => x.ConversionRateSecurity != null && x.ConversionRate == 0)
.ToList();
var historyRequestFactory = new HistoryRequestFactory(algorithm);
var historyRequests = new List <HistoryRequest>();
foreach (var cash in cashToUpdate)
{
// if we already added a history request for this security, skip
if (historyRequests.Any(x => x.Symbol == cash.ConversionRateSecurity.Symbol))
{
continue;
}
var configs = algorithm
.SubscriptionManager
.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(cash.ConversionRateSecurity.Symbol);
var resolution = configs.GetHighestResolution();
var startTime = historyRequestFactory.GetStartTimeAlgoTz(
cash.ConversionRateSecurity.Symbol,
1,
resolution,
cash.ConversionRateSecurity.Exchange.Hours);
var endTime = algorithm.Time.RoundDown(resolution.ToTimeSpan());
// we need to order and select a specific configuration type
// so the conversion rate is deterministic
var configToUse = configs.OrderBy(x => x.TickType).First();
historyRequests.Add(historyRequestFactory.CreateHistoryRequest(
configToUse,
startTime,
endTime,
cash.ConversionRateSecurity.Exchange.Hours,
resolution));
}
var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone);
slices.PushThrough(data =>
{
foreach (var cash in cashToUpdate
.Where(x => x.ConversionRateSecurity.Symbol == data.Symbol))
{
cash.Update(data);
}
});
}
/// <summary>
/// Used to send data updates to algorithm framework models
/// </summary>
/// <param name="slice">The current data slice</param>
public void OnFrameworkData(Slice slice)
{
if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc())
{
var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
// remove deselected universes by symbol
foreach (var ukvp in UniverseManager)
{
var universeSymbol = ukvp.Key;
var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market);
if (universeSymbol.Equals(qcUserDefined))
{
// prevent removal of qc algorithm created user defined universes
continue;
}
Universe universe;
if (!universes.TryGetValue(universeSymbol, out universe))
{
if (ukvp.Value.DisposeRequested)
{
UniverseManager.Remove(universeSymbol);
}
// mark this universe as disposed to remove all child subscriptions
ukvp.Value.Dispose();
}
}
// add newly selected universes
foreach (var ukvp in universes)
{
// note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
UniverseManager.Add(ukvp);
}
}
// we only want to run universe selection if there's no data available in the slice
if (!slice.HasData)
{
return;
}
// insight timestamping handled via InsightsGenerated event handler
var insightsEnumerable = Alpha.Update(this, slice);
// for performance only call 'ToArray' if not empty enumerable (which is static)
var insights = insightsEnumerable == Enumerable.Empty <Insight>()
? new Insight[] { } : insightsEnumerable.ToArray();
// only fire insights generated event if we actually have insights
if (insights.Length != 0)
{
OnInsightsGenerated(insights.Select(InitializeInsightFields));
}
ProcessInsights(insights);
}
/// <summary>
/// Called by setup handlers after Initialize and allows the algorithm a chance to organize
/// the data gather in the Initialize method
/// </summary>
public override void PostInitialize()
{
CheckModels();
foreach (var universe in UniverseSelection.CreateUniverses(this))
{
AddUniverse(universe);
}
base.PostInitialize();
}
/// <summary>
/// Called by setup handlers after Initialize and allows the algorithm a chance to organize
/// the data gather in the Initialize method
/// </summary>
public void FrameworkPostInitialize()
{
foreach (var universe in UniverseSelection.CreateUniverses(this))
{
AddUniverse(universe);
}
if (DebugMode)
{
InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}");
}
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var feed = new AlgorithmManagerTests.MockDataFeed();
var universeSelection = new UniverseSelection(feed, algorithm);
var synchronizer = new SubscriptionSynchronizer(universeSelection, algorithm.TimeZone, algorithm.Portfolio.CashBook, startTimeUtc);
var totalDataPoints = 0;
var subscriptions = new List <Subscription>();
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.Add(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// force JIT
synchronizer.Sync(subscriptions);
// log what we're doing
Console.WriteLine($"Running {subscriptions.Count} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime;
var dateTimeMaxValue = DateTime.MaxValue;
var stopwatch = Stopwatch.StartNew();
do
{
var timeSlice = synchronizer.Sync(subscriptions);
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != dateTimeMaxValue);
stopwatch.Stop();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
algorithm.Initialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var feed = new AlgorithmManagerTests.MockDataFeed();
var universeSelection = new UniverseSelection(feed, algorithm);
var synchronizer = new SubscriptionSynchronizer(universeSelection, algorithm.TimeZone, algorithm.Portfolio.CashBook, startTimeUtc);
var subscriptions = new List <Subscription>();
foreach (var kvp in algorithm.Securities)
{
subscriptions.Add(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc));
}
var count = 0;
double kps = 0;
var currentTime = default(DateTime);
var stopwatch = new Stopwatch();
var timer = new Timer(_ =>
{
kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
});
timer.Change(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
stopwatch.Start();
do
{
var timeSlice = synchronizer.Sync(subscriptions);
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime < endTimeUtc);
stopwatch.Stop();
timer.Dispose();
kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
}
/// <summary>
/// Called by setup handlers after Initialize and allows the algorithm a chance to organize
/// the data gather in the Initialize method
/// </summary>
public void FrameworkPostInitialize()
{
foreach (var universe in UniverseSelection.CreateUniverses(this))
{
// on purpose we don't call 'AddUniverse' here so that these universes don't get registered as user added
// this is so that later during 'UniverseSelection.CreateUniverses' we wont remove them from UniverseManager
_pendingUniverseAdditions.Add(universe);
}
if (DebugMode)
{
InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}");
}
}
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="universeSelection">The universe selection instance</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="algorithmNodePacket">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configured transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
public SetupHandlerParameters(UniverseSelection universeSelection,
IAlgorithm algorithm,
IBrokerage brokerage,
AlgorithmNodePacket algorithmNodePacket,
IResultHandler resultHandler,
ITransactionHandler transactionHandler,
IRealTimeHandler realTimeHandler
)
{
UniverseSelection = universeSelection;
Algorithm = algorithm;
Brokerage = brokerage;
AlgorithmNodePacket = algorithmNodePacket;
ResultHandler = resultHandler;
TransactionHandler = transactionHandler;
RealTimeHandler = realTimeHandler;
}
private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchronizer, IDataAggregator dataAggregator)
{
_dataFeed = new TestableLiveTradingDataFeed(dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? new AggregationManager()));
_algorithm = new AlgorithmStub(createDataManager: false);
_synchronizer = synchronizer ?? new LiveSynchronizer();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
_algorithm,
registeredTypesProvider,
new SecurityCacheProvider(_algorithm.Portfolio));
var universeSelection = new UniverseSelection(
_algorithm,
securityService,
new DataPermissionManager(),
TestGlobals.DataProvider,
Resolution.Second);
_dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork),
MarketHoursDatabase.FromDataFolder(),
true,
new RegisteredSecurityDataTypesProvider(),
new DataPermissionManager());
_resultHandler = new TestResultHandler();
_synchronizer.Initialize(_algorithm, _dataManager);
_dataFeed.Initialize(_algorithm,
new LiveNodePacket(),
_resultHandler,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
TestGlobals.DataProvider,
_dataManager,
_synchronizer,
new DataChannelProvider());
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.Securities.SetSecurityService(securityService);
_algorithm.SetFinishedWarmingUp();
var backtestingTransactionHandler = new BacktestingTransactionHandler();
backtestingTransactionHandler.Initialize(_algorithm, new PaperBrokerage(_algorithm, new LiveNodePacket()), _resultHandler);
_algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler);
_algorithm.PostInitialize();
}
/// <summary>
/// Adds a new universe selection model
/// </summary>
/// <param name="universeSelection">Model defining universes for the algorithm to add</param>
public void AddUniverseSelection(IUniverseSelectionModel universeSelection)
{
if (UniverseSelection.GetType() != typeof(NullUniverseSelectionModel))
{
var compositeUniverseSelection = UniverseSelection as CompositeUniverseSelectionModel;
if (compositeUniverseSelection != null)
{
compositeUniverseSelection.AddUniverseSelection(universeSelection);
}
else
{
UniverseSelection = new CompositeUniverseSelectionModel(UniverseSelection, universeSelection);
}
}
else
{
UniverseSelection = universeSelection;
}
}
public void Setup()
{
_dataFeed = new TestableLiveTradingDataFeed(new FakeDataQueue());
_algorithm = new AlgorithmStub(createDataManager: false);
_synchronizer = new LiveSynchronizer();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
_algorithm,
registeredTypesProvider,
new SecurityCacheProvider(_algorithm.Portfolio));
var universeSelection = new UniverseSelection(
_algorithm,
securityService,
new DataPermissionManager(),
new DefaultDataProvider(),
Resolution.Second);
_dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork),
MarketHoursDatabase.FromDataFolder(),
true,
new RegisteredSecurityDataTypesProvider(),
new DataPermissionManager());
_synchronizer.Initialize(_algorithm, _dataManager);
_dataFeed.Initialize(_algorithm,
new LiveNodePacket(),
new TestResultHandler(),
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
new DefaultDataProvider(),
_dataManager,
_synchronizer,
new DataChannelProvider());
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.Securities.SetSecurityService(securityService);
_algorithm.SetFinishedWarmingUp();
_algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
}
/// <summary>
/// Called by setup handlers after Initialize and allows the algorithm a chance to organize
/// the data gather in the Initialize method
/// </summary>
public void FrameworkPostInitialize()
{
//Prevents execution in the case of cash brokerage with IExecutionModel and IPortfolioConstructionModel
if (PortfolioConstruction.GetType() != typeof(NullPortfolioConstructionModel) &&
Execution.GetType() != typeof(NullExecutionModel) &&
BrokerageModel.AccountType == AccountType.Cash)
{
throw new InvalidOperationException($"Non null {nameof(IExecutionModel)} and {nameof(IPortfolioConstructionModel)} are currently unsuitable for Cash Modeled brokerages (e.g. GDAX) and may result in unexpected trades."
+ " To prevent possible user error we've restricted them to Margin trading. You can select margin account types with"
+ $" SetBrokerage( ... AccountType.Margin). Or please set them to {nameof(NullExecutionModel)}, {nameof(NullPortfolioConstructionModel)}");
}
foreach (var universe in UniverseSelection.CreateUniverses(this))
{
// on purpose we don't call 'AddUniverse' here so that these universes don't get registered as user added
// this is so that later during 'UniverseSelection.CreateUniverses' we wont remove them from UniverseManager
_pendingUniverseAdditions.Add(universe);
}
if (DebugMode)
{
InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}");
}
}
/// <summary>
/// Creates a new instance
/// </summary>
/// <param name="universeSelection">The universe selection instance</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="algorithmNodePacket">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configured transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <param name="objectStore">The configured object store</param>
/// <param name="dataCacheProvider">The configured data cache provider</param>
/// <param name="mapFileProvider">The map file provider</param>
public SetupHandlerParameters(UniverseSelection universeSelection,
IAlgorithm algorithm,
IBrokerage brokerage,
AlgorithmNodePacket algorithmNodePacket,
IResultHandler resultHandler,
ITransactionHandler transactionHandler,
IRealTimeHandler realTimeHandler,
IObjectStore objectStore,
IDataCacheProvider dataCacheProvider,
IMapFileProvider mapFileProvider
)
{
UniverseSelection = universeSelection;
Algorithm = algorithm;
Brokerage = brokerage;
AlgorithmNodePacket = algorithmNodePacket;
ResultHandler = resultHandler;
TransactionHandler = transactionHandler;
RealTimeHandler = realTimeHandler;
ObjectStore = objectStore;
DataCacheProvider = dataCacheProvider;
MapFileProvider = mapFileProvider;
}
/// <summary>
/// Called by setup handlers after Initialize and allows the algorithm a chance to organize
/// the data gather in the Initialize method
/// </summary>
public override void PostInitialize()
{
CheckModels();
foreach (var universe in UniverseSelection.CreateUniverses(this))
{
AddUniverse(universe);
}
if (DebugMode)
{
InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}");
}
// emit warning message about using the framework with cash modelling
if (BrokerageModel.AccountType == AccountType.Cash)
{
Error("These models are currently unsuitable for Cash Modeled brokerages (e.g. GDAX) and may result in unexpected trades."
+ " To prevent possible user error we've restricted them to Margin trading. You can select margin account types with"
+ " SetBrokerage( ... AccountType.Margin)");
}
base.PostInitialize();
}
/// <summary>
/// Used to send data updates to algorithm framework models
/// </summary>
/// <param name="slice">The current data slice</param>
public sealed override void OnFrameworkData(Slice slice)
{
if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc())
{
var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
// remove deselected universes by symbol
foreach (var ukvp in UniverseManager)
{
var universeSymbol = ukvp.Key;
var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market);
if (universeSymbol.Equals(qcUserDefined))
{
// prevent removal of qc algorithm created user defined universes
continue;
}
Universe universe;
if (!universes.TryGetValue(universeSymbol, out universe))
{
if (ukvp.Value.DisposeRequested)
{
UniverseManager.Remove(universeSymbol);
}
// mark this universe as disposed to remove all child subscriptions
ukvp.Value.Dispose();
}
}
// add newly selected universes
foreach (var ukvp in universes)
{
// note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
UniverseManager.Add(ukvp);
}
}
// we only want to run universe selection if there's no data available in the slice
if (!slice.HasData)
{
return;
}
// insight timestamping handled via InsightsGenerated event handler
var insights = Alpha.Update(this, slice).ToArray();
// only fire insights generated event if we actually have insights
if (insights.Length != 0)
{
// debug printing of generated insights
if (DebugMode)
{
Log($"{Time}: ALPHA: {string.Join(" | ", insights.Select(i => i.ToString()).OrderBy(i => i))}");
}
OnInsightsGenerated(insights);
}
// construct portfolio targets from insights
var targets = PortfolioConstruction.CreateTargets(this, insights).ToArray();
// set security targets w/ those generated via portfolio construction module
foreach (var target in targets)
{
var security = Securities[target.Symbol];
security.Holdings.Target = target;
}
if (DebugMode)
{
// debug printing of generated targets
if (targets.Length > 0)
{
Log($"{Time}: PORTFOLIO: {string.Join(" | ", targets.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
var riskTargetOverrides = RiskManagement.ManageRisk(this, targets).ToArray();
// override security targets w/ those generated via risk management module
foreach (var target in riskTargetOverrides)
{
var security = Securities[target.Symbol];
security.Holdings.Target = target;
}
if (DebugMode)
{
// debug printing of generated risk target overrides
if (riskTargetOverrides.Length > 0)
{
Log($"{Time}: RISK: {string.Join(" | ", riskTargetOverrides.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
// execute on the targets, overriding targets for symbols w/ risk targets
var riskAdjustedTargets = riskTargetOverrides.Concat(targets).DistinctBy(pt => pt.Symbol).ToArray();
if (DebugMode)
{
// only log adjusted targets if we've performed an adjustment
if (riskTargetOverrides.Length > 0)
{
Log($"{Time}: RISK ADJUSTED TARGETS: {string.Join(" | ", riskAdjustedTargets.Select(t => t.ToString()).OrderBy(t => t))}");
}
}
Execution.Execute(this, riskAdjustedTargets);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = new AlgorithmManager(_liveMode);
//Start monitoring the backtest active status:
var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job);
var statusPingThread = new Thread(statusPing.Run);
statusPingThread.Start();
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetChannel(job.Channel);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language);
// set the history provider before setting up the algorithm
_algorithmHandlers.HistoryProvider.Initialize(job, progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
});
algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Wire up the universe selection event handler before kicking off the data feed
var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode);
_algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args);
//Load the associated handlers for data, transaction and realtime events:
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results);
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message);
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (_algorithmHandlers.DataFeed != null)
{
_algorithmHandlers.DataFeed.Exit();
}
if (_algorithmHandlers.Results != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
_algorithmHandlers.Results.LogMessage(message);
_algorithmHandlers.Results.RuntimeError(message, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var banner = new Dictionary <string, string>();
var statisticsResults = new StatisticsResults();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"),
dataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
statusPing.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive ||
(_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) ||
(_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) ||
(_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) &&
ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
if (brokerage != null)
{
brokerage.Disconnect();
}
if (_algorithmHandlers.Setup != null)
{
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <param name="token">Cancellation token</param>
/// <remarks>Modify with caution</remarks>
public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, CancellationToken token)
{
//Initialize:
_dataPointCount = 0;
var portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
var delistingTickets = new List <OrderTicket>();
//Initialize Properties:
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = algorithm.StartDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
// Algorithm 2.0 data accessors
var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers);
// dividend and split events
var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers);
var hasOnDataDelistings = AddMethodInvoker <Delistings>(algorithm, methodInvokers);
var hasOnDataSymbolChangedEvents = AddMethodInvoker <SymbolChangedEvents>(algorithm, methodInvokers);
// Algorithm 3.0 data accessors
var hasOnDataSlice = algorithm.GetType().GetMethods()
.Where(x => x.Name == "OnData" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == typeof(Slice))
.FirstOrDefault(x => x.DeclaringType == algorithm.GetType()) != null;
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions.Select(x => x.Configuration))
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick" && !config.IsInternalFeed)
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null && !hasOnDataSlice)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
if (genericMethod != null)
{
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
}
// wire up universe selection. it is assumed that the data feed will perform the
// required thread synchronization.
var universeSelection = new UniverseSelection(feed, algorithm, _liveMode);
feed.Fundamental += (sender, args) =>
{
var market = args.Configuration.Market;
var localTime = args.DateTimeUtc.ConvertFromUtc(args.Configuration.TimeZone);
universeSelection.ApplyUniverseSelection(localTime, market, args.Data.OfType <CoarseFundamental>());
};
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Trace("AlgorithmManager.Run(): Begin DataStream - Start: " + algorithm.StartDate + " Stop: " + algorithm.EndDate);
foreach (var timeSlice in Stream(job, algorithm, feed, results, token))
{
// reset our timer on each loop
_currentTimeStepTime = DateTime.UtcNow;
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
Log.Error(string.Format("AlgorithmManager.Run(): Algorthm state changed to {0} at {1}", _algorithmState, timeSlice.Time));
break;
}
//Execute with TimeLimit Monitor:
if (token.IsCancellationRequested)
{
Log.Error("AlgorithmManager.Run(): CancellationRequestion at " + timeSlice.Time);
return;
}
var time = timeSlice.Time;
_dataPointCount += timeSlice.DataPointCount;
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
SampleBenchmark(algorithm, results, _previousTime.Date);
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (portfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
}
portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
else
{
// live mode continously sample the benchmark
SampleBenchmark(algorithm, results, time);
}
//Update algorithm state after capturing performance from previous day
//Set the algorithm and real time handler's time
algorithm.SetDateTime(time);
if (timeSlice.Slice.SymbolChangedEvents.Count != 0)
{
if (hasOnDataSymbolChangedEvents)
{
methodInvokers[typeof(SymbolChangedEvents)](algorithm, timeSlice.Slice.SymbolChangedEvents);
}
foreach (var symbol in timeSlice.Slice.SymbolChangedEvents.Keys)
{
// cancel all orders for the old symbol
foreach (var ticket in transactions.GetOrderTickets(x => x.Status.IsOpen() && x.Symbol == symbol))
{
ticket.Cancel("Open order cancelled on symbol changed event");
}
}
}
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
foreach (var security in timeSlice.SecurityChanges.AddedSecurities)
{
if (!algorithm.Securities.ContainsKey(security.Symbol))
{
// add the new security
algorithm.Securities.Add(security);
}
}
}
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
foreach (var kvp in timeSlice.CashBookUpdateData)
{
kvp.Key.Update(kvp.Value);
}
//Update the securities properties: first before calling user code to avoid issues with data
foreach (var kvp in timeSlice.SecuritiesUpdateData)
{
kvp.Key.SetMarketPrice(kvp.Value);
// Send market price updates to the TradeBuilder
if (kvp.Value != null)
{
algorithm.TradeBuilder.SetMarketPrice(kvp.Key.Symbol, kvp.Value.Price);
}
}
// fire real time events after we've updated based on the new data
realtime.SetTime(timeSlice.Time);
// process fill models on the updated data before entering algorithm, applies to all non-market orders
transactions.ProcessSynchronousEvents();
if (delistingTickets.Count != 0)
{
for (int i = 0; i < delistingTickets.Count; i++)
{
var ticket = delistingTickets[i];
if (ticket.Status == OrderStatus.Filled)
{
algorithm.Securities.Remove(ticket.Symbol);
delistingTickets.RemoveAt(i--);
Log.Trace("AlgorithmManager.Run(): Delisted Security removed: " + ticket.Symbol.Permtick);
}
}
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
Log.Trace("AlgorithmManager.Run(): Algorithm quit requested.");
break;
}
if (algorithm.RunTimeError != null)
{
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Trace(string.Format("AlgorithmManager.Run(): Algorithm encountered a runtime error at {0}. Error: {1}", timeSlice.Time, algorithm.RunTimeError));
break;
}
// perform margin calls, in live mode we can also use realtime to emit these
if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.Now))
{
// determine if there are possible margin call orders to be executed
bool issueMarginCallWarning;
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
if (marginCallOrders.Count != 0)
{
var executingMarginCall = false;
try
{
// tell the algorithm we're about to issue the margin call
algorithm.OnMarginCall(marginCallOrders);
executingMarginCall = true;
// execute the margin call orders
var executedTickets = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var ticket in executedTickets)
{
algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, ticket.Symbol, ticket.Quantity, ticket.AverageFillPrice));
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
var locator = executingMarginCall ? "Portfolio.MarginCallModel.ExecuteMarginCall" : "OnMarginCall";
Log.Error(string.Format("AlgorithmManager.Run(): RuntimeError: {0}: ", locator) + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
// we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
else if (issueMarginCallWarning)
{
try
{
algorithm.OnMarginCallWarning();
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
nextMarginCallTime = time + marginCallFrequency;
}
// before we call any events, let the algorithm know about universe changes
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
try
{
algorithm.OnSecuritiesChanged(timeSlice.SecurityChanges);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnSecuritiesChanged event: " + err.Message);
return;
}
}
// apply dividends
foreach (var dividend in timeSlice.Slice.Dividends.Values)
{
Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol, true);
algorithm.Portfolio.ApplyDividend(dividend);
}
// apply splits
foreach (var split in timeSlice.Slice.Splits.Values)
{
try
{
Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol, true);
algorithm.Portfolio.ApplySplit(split);
// apply the split to open orders as well in raw mode, all other modes are split adjusted
if (_liveMode || algorithm.Securities[split.Symbol].SubscriptionDataConfig.DataNormalizationMode == DataNormalizationMode.Raw)
{
// in live mode we always want to have our order match the order at the brokerage, so apply the split to the orders
var openOrders = transactions.GetOrderTickets(ticket => ticket.Status.IsOpen() && ticket.Symbol == split.Symbol);
algorithm.BrokerageModel.ApplySplit(openOrders.ToList(), split);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Split event: " + err.Message);
return;
}
}
//Update registered consolidators for this symbol index
try
{
foreach (var kvp in timeSlice.ConsolidatorUpdateData)
{
var consolidators = kvp.Key.Consolidators;
foreach (var dataPoint in kvp.Value)
{
foreach (var consolidator in consolidators)
{
consolidator.Update(dataPoint);
}
}
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
// fire custom event handlers
foreach (var kvp in timeSlice.CustomData)
{
MethodInvoker methodInvoker;
if (!methodInvokers.TryGetValue(kvp.Key.SubscriptionDataConfig.Type, out methodInvoker))
{
continue;
}
try
{
foreach (var dataPoint in kvp.Value)
{
methodInvoker(algorithm, dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
try
{
// fire off the dividend and split events before pricing events
if (hasOnDataDividends && timeSlice.Slice.Dividends.Count != 0)
{
methodInvokers[typeof(Dividends)](algorithm, timeSlice.Slice.Dividends);
}
if (hasOnDataSplits && timeSlice.Slice.Splits.Count != 0)
{
methodInvokers[typeof(Splits)](algorithm, timeSlice.Slice.Splits);
}
if (hasOnDataDelistings && timeSlice.Slice.Delistings.Count != 0)
{
methodInvokers[typeof(Delistings)](algorithm, timeSlice.Slice.Delistings);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Dividends/Splits/Delistings: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// run the delisting logic after firing delisting events
HandleDelistedSymbols(algorithm, timeSlice.Slice.Delistings, delistingTickets);
//After we've fired all other events in this second, fire the pricing events:
try
{
if (hasOnDataTradeBars && timeSlice.Slice.Bars.Count > 0)
{
methodInvokers[typeof(TradeBars)](algorithm, timeSlice.Slice.Bars);
}
if (hasOnDataTicks && timeSlice.Slice.Ticks.Count > 0)
{
methodInvokers[typeof(Ticks)](algorithm, timeSlice.Slice.Ticks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
try
{
if (timeSlice.Slice.Count != 0)
{
// EVENT HANDLER v3.0 -- all data in a single event
algorithm.OnData(timeSlice.Slice);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Slice: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach feed.Bridge.GetConsumingEnumerable
// stop timing the loops
_currentTimeStepTime = DateTime.MinValue;
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Error("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated && _liveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
SampleBenchmark(algorithm, results, _previousTime);
results.SamplePerformance(_previousTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
} // End of Run();
/// <summary>
/// Will first check and add all the required conversion rate securities
/// and later will seed an initial value to them.
/// </summary>
/// <param name="algorithm">The algorithm instance</param>
/// <param name="universeSelection">The universe selection instance</param>
public static void SetupCurrencyConversions(
IAlgorithm algorithm,
UniverseSelection universeSelection)
{
// this is needed to have non-zero currency conversion rates during warmup
// will also set the Cash.ConversionRateSecurity
universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
// now set conversion rates
var cashToUpdate = algorithm.Portfolio.CashBook.Values
.Where(x => x.CurrencyConversion != null && x.ConversionRate == 0)
.ToList();
var securitiesToUpdate = cashToUpdate
.SelectMany(x => x.CurrencyConversion.ConversionRateSecurities)
.Distinct()
.ToList();
var historyRequestFactory = new HistoryRequestFactory(algorithm);
var historyRequests = new List <HistoryRequest>();
foreach (var security in securitiesToUpdate)
{
var configs = algorithm
.SubscriptionManager
.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(security.Symbol,
includeInternalConfigs: true);
// we need to order and select a specific configuration type
// so the conversion rate is deterministic
var configToUse = configs.OrderBy(x => x.TickType).First();
var hours = security.Exchange.Hours;
var resolution = configs.GetHighestResolution();
var startTime = historyRequestFactory.GetStartTimeAlgoTz(
security.Symbol,
10,
resolution,
hours,
configToUse.DataTimeZone);
var endTime = algorithm.Time;
historyRequests.Add(historyRequestFactory.CreateHistoryRequest(
configToUse,
startTime,
endTime,
security.Exchange.Hours,
resolution));
}
// Attempt to get history for these requests and update cash
var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone);
slices.PushThrough(data =>
{
foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol))
{
security.SetMarketPrice(data);
}
});
foreach (var cash in cashToUpdate)
{
cash.Update();
}
// Any remaining unassigned cash will attempt to fall back to a daily resolution history request to resolve
var unassignedCash = cashToUpdate.Where(x => x.ConversionRate == 0).ToList();
if (unassignedCash.Any())
{
Log.Error(
$"Failed to assign conversion rates for the following cash: {string.Join(",", unassignedCash.Select(x => x.Symbol))}." +
$" Attempting to request daily resolution history to resolve conversion rate");
var unassignedCashSymbols = unassignedCash
.SelectMany(x => x.SecuritySymbols)
.ToHashSet();
var replacementHistoryRequests = new List <HistoryRequest>();
foreach (var request in historyRequests.Where(x =>
unassignedCashSymbols.Contains(x.Symbol) && x.Resolution < Resolution.Daily))
{
var newRequest = new HistoryRequest(request.EndTimeUtc.AddDays(-10), request.EndTimeUtc,
request.DataType,
request.Symbol, Resolution.Daily, request.ExchangeHours, request.DataTimeZone,
request.FillForwardResolution,
request.IncludeExtendedMarketHours, request.IsCustomData, request.DataNormalizationMode,
request.TickType);
replacementHistoryRequests.Add(newRequest);
}
slices = algorithm.HistoryProvider.GetHistory(replacementHistoryRequests, algorithm.TimeZone);
slices.PushThrough(data =>
{
foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol))
{
security.SetMarketPrice(data);
}
});
foreach (var cash in unassignedCash)
{
cash.Update();
}
}
Log.Trace("BaseSetupHandler.SetupCurrencyConversions():" +
$"{Environment.NewLine}{algorithm.Portfolio.CashBook}");
}
/// <summary>
/// Used to send data updates to algorithm framework models
/// </summary>
/// <param name="slice">The current data slice</param>
public void OnFrameworkData(Slice slice)
{
if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc())
{
var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol);
// remove deselected universes by symbol
foreach (var ukvp in UniverseManager)
{
var universeSymbol = ukvp.Key;
if (_userAddedUniverses.Contains(universeSymbol))
{
// prevent removal of qc algorithm created user defined universes
continue;
}
if (ukvp.Value.DisposeRequested)
{
// have to remove in the next loop after the universe is marked as disposed, when 'Dispose()' is called it will trigger universe selection
// and deselect all symbols, sending the removed security changes, which are picked up by the AlgorithmManager and tags securities
// as non tradable as long as they are not active in any universe (uses UniverseManager.ActiveSecurities)
// but they will remain tradable if a position is still being hold since they won't be remove from the UniverseManager
// but this last part will not happen if we remove the universe from the UniverseManager right away, since it won't be part of 'UniverseManager'.
// And we have to remove the universe even if it's present at 'universes' because that one is another New universe that should get added!
// 'UniverseManager' will skip duplicate entries getting added.
UniverseManager.Remove(universeSymbol);
}
Universe universe;
if (!universes.TryGetValue(universeSymbol, out universe))
{
// mark this universe as disposed to remove all child subscriptions
ukvp.Value.Dispose();
}
}
// add newly selected universes
foreach (var ukvp in universes)
{
// note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here
UniverseManager.Add(ukvp);
}
}
// we only want to run universe selection if there's no data available in the slice
if (!slice.HasData)
{
return;
}
// insight timestamping handled via InsightsGenerated event handler
var insightsEnumerable = Alpha.Update(this, slice);
// for performance only call 'ToArray' if not empty enumerable (which is static)
var insights = insightsEnumerable == Enumerable.Empty <Insight>()
? new Insight[] { } : insightsEnumerable.ToArray();
// only fire insights generated event if we actually have insights
if (insights.Length != 0)
{
insights = InitializeInsights(insights);
OnInsightsGenerated(insights);
}
ProcessInsights(insights);
}
