/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
// Log our numerical precision limited warnings if any
if (!_numericalPrecisionLimitedWarnings.IsNullOrEmpty())
{
var message = "Due to numerical precision issues in the factor file, data for the following" +
$" symbols was adjust to a later starting date: {string.Join(", ", _numericalPrecisionLimitedWarnings.Values.Take(_numericalPrecisionLimitedWarningsMaxCount))}";
// If we reached our max warnings count suggest that more may have been left out
if (_numericalPrecisionLimitedWarnings.Count >= _numericalPrecisionLimitedWarningsMaxCount)
{
message += "...";
}
_resultHandler.DebugMessage(message);
}
// Log our start date adjustments because of map files
if (!_startDateLimitedWarnings.IsNullOrEmpty())
{
var message = "The starting dates for the following symbols have been adjusted to match their" +
$" map files first date: {string.Join(", ", _startDateLimitedWarnings.Values.Take(_startDateLimitedWarningsMaxCount))}";
// If we reached our max warnings count suggest that more may have been left out
if (_startDateLimitedWarnings.Count >= _startDateLimitedWarningsMaxCount)
{
message += "...";
}
_resultHandler.DebugMessage(message);
}
_zipDataCacheProvider?.DisposeSafely();
}
public void TickResolutionOpenInterestHistoryRequestIsNotFilteredWhenRequestedExplicitly()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var zipCacheProvider = new ZipDataCacheProvider(_dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
_dataProvider,
zipCacheProvider,
_mapFileProvider,
_factorFileProvider,
null,
false,
new DataPermissionManager()));
var start = new DateTime(2014, 6, 05);
_algorithm.SetStartDate(start);
_algorithm.SetDateTime(start.AddDays(2));
_algorithm.UniverseSettings.FillForward = false;
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var openInterests = _algorithm.History <OpenInterest>(optionSymbol, start, start.AddDays(2), Resolution.Minute).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(openInterests);
Assert.AreEqual(2, openInterests.Count);
Assert.AreEqual(new DateTime(2014, 06, 05, 6, 32, 0), openInterests[0].Time);
Assert.AreEqual(optionSymbol, openInterests[0].Symbol);
Assert.AreEqual(new DateTime(2014, 06, 06, 6, 32, 0), openInterests[1].Time);
Assert.AreEqual(optionSymbol, openInterests[1].Symbol);
}
public void TickResolutionHistoryRequest()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var zipCacheProvider = new ZipDataCacheProvider(_dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
_dataProvider,
zipCacheProvider,
_mapFileProvider,
_factorFileProvider,
null,
false,
new DataPermissionManager()));
_algorithm.SetStartDate(2013, 10, 08);
var start = new DateTime(2013, 10, 07);
// Trades and quotes
var result = _algorithm.History(new [] { Symbols.SPY }, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList();
// Just Trades
var result2 = _algorithm.History <Tick>(Symbols.SPY, start.AddHours(9.8), start.AddHours(10), Resolution.Tick).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(result);
Assert.IsNotEmpty(result2);
Assert.IsTrue(result2.All(tick => tick.TickType == TickType.Trade));
// (Trades and quotes).Count > Trades * 2
Assert.Greater(result.Count, result2.Count * 2);
}
public void TickResolutionOpenInterestHistoryRequestIsFilteredByDefault_SingleSymbol()
{
_algorithm = new QCAlgorithm();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
_algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var dataProvider = new DefaultDataProvider();
var zipCacheProvider = new ZipDataCacheProvider(dataProvider);
_algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
dataProvider,
zipCacheProvider,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
var start = new DateTime(2014, 6, 05);
_algorithm.SetStartDate(start);
_algorithm.SetDateTime(start.AddDays(2));
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var result = _algorithm.History(new[] { optionSymbol }, start, start.AddDays(2), Resolution.Minute, fillForward: false).ToList();
zipCacheProvider.DisposeSafely();
Assert.IsNotEmpty(result);
Assert.IsTrue(result.Any(slice => slice.ContainsKey(optionSymbol)));
var openInterests = result.Select(slice => slice.Get(typeof(OpenInterest)) as DataDictionary <OpenInterest>).Where(dataDictionary => dataDictionary.Count > 0).ToList();
Assert.AreEqual(0, openInterests.Count);
}
public void GetLastKnownPriceOfIlliquidAsset_RealData()
{
var algorithm = new QCAlgorithm();
algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm));
algorithm.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider());
algorithm.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
new DefaultDataProvider(),
cacheProvider,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
algorithm.SetDateTime(new DateTime(2014, 6, 6, 15, 0, 0));
//20140606_twx_minute_quote_american_call_230000_20150117.csv
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var option = algorithm.AddOptionContract(optionSymbol);
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.IsNotNull(lastKnownPrice);
// Data gap of more than 15 minutes
Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15);
cacheProvider.DisposeSafely();
}
public void ZipDataCacheProviderEphemeralDataIsRespected(bool isDataEphemeral)
{
var cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), isDataEphemeral: isDataEphemeral);
var remoteReader = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(1, TestDownloadProvider.DownloadCount);
var remoteReader2 = new RemoteFileSubscriptionStreamReader(
cacheProvider,
@"https://www.quantconnect.com/api/v2/proxy/quandl/api/v3/datasets/BCHARTS/BITSTAMPUSD.csv?order=asc&api_key=WyAazVXnq7ATy_fefTqm",
Globals.Cache,
null);
Assert.IsFalse(remoteReader.EndOfStream);
Assert.AreEqual(isDataEphemeral ? 2 : 1, TestDownloadProvider.DownloadCount);
remoteReader.Dispose();
remoteReader2.Dispose();
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceOption()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09));
var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15)));
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void OptionsAreMappedCorrectly()
{
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCache,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
var symbol = Symbol.CreateOption(
"FOXA",
Market.USA,
OptionStyle.American,
OptionRight.Call,
32,
new DateTime(2013, 07, 20));
var result = historyProvider.GetHistory(
new[]
{
new HistoryRequest(new DateTime(2013, 06, 28),
new DateTime(2013, 07, 03),
typeof(QuoteBar),
symbol,
Resolution.Minute,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
TickType.Quote)
},
TimeZones.NewYork).ToList();
Assert.IsNotEmpty(result);
// assert we fetch the data for the previous and new symbol
var firstBar = result.First().Values.Single();
var lastBar = result.Last().Values.Single();
Assert.IsTrue(firstBar.Symbol.Value.Contains("NWSA"));
Assert.AreEqual(28, firstBar.Time.Date.Day);
Assert.IsTrue(lastBar.Symbol.Value.Contains("FOXA"));
Assert.AreEqual(2, lastBar.Time.Date.Day);
zipCache.DisposeSafely();
}
public void GetLastKnownPriceOfCustomData()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2018, 4, 4));
var alpha = algorithm.AddData <AlphaStreamsPortfolioState>("9fc8ef73792331b11dbd5429a");
var lastKnownPrice = algorithm.GetLastKnownPrice(alpha);
Assert.IsNotNull(lastKnownPrice);
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceFuture()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var future = algorithm.AddSecurity(Symbols.CreateFutureSymbol(Futures.Indices.SP500EMini, new DateTime(2013, 12, 20)));
var lastKnownPrice = algorithm.GetLastKnownPrice(future);
Assert.AreEqual(typeof(QuoteBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void GetLastKnownPriceEquity()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var equity = algorithm.AddEquity("SPY");
var lastKnownPrice = algorithm.GetLastKnownPrice(equity);
Assert.AreEqual(typeof(TradeBar), lastKnownPrice.GetType());
cacheProvider.DisposeSafely();
}
public void GetLastKnownPricesOption()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 06, 09));
var option = algorithm.AddOptionContract(Symbols.CreateOptionSymbol("AAPL", OptionRight.Call, 250m, new DateTime(2016, 01, 15)));
var lastKnownPrices = algorithm.GetLastKnownPrices(option).ToList();;
Assert.AreEqual(2, lastKnownPrices.Count);
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar)));
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar)));
cacheProvider.DisposeSafely();
}
public void GetLastKnownPricesEquity()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2013, 10, 8));
var equity = algorithm.AddEquity("SPY");
var lastKnownPrices = algorithm.GetLastKnownPrices(equity.Symbol).ToList();
Assert.AreEqual(2, lastKnownPrices.Count);
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(TradeBar)));
Assert.AreEqual(1, lastKnownPrices.Count(data => data.GetType() == typeof(QuoteBar)));
cacheProvider.DisposeSafely();
}
public void WarmUpPythonIndicatorProperly()
{
var algo = new AlgorithmStub
{
HistoryProvider = new SubscriptionDataReaderHistoryProvider()
};
var zipCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
algo.HistoryProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCacheProvider,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
algo.SetStartDate(2013, 10, 08);
algo.AddEquity("SPY", Resolution.Minute);
// Different types of indicators
var indicatorDataPoint = new SimpleMovingAverage("SPY", 10);
var indicatorDataBar = new AverageTrueRange("SPY", 10);
var indicatorTradeBar = new VolumeWeightedAveragePriceIndicator("SPY", 10);
using (Py.GIL())
{
var sma = indicatorDataPoint.ToPython();
var atr = indicatorTradeBar.ToPython();
var vwapi = indicatorDataBar.ToPython();
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", sma, Resolution.Minute));
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", atr, Resolution.Minute));
Assert.DoesNotThrow(() => algo.WarmUpIndicator("SPY", vwapi, Resolution.Minute));
var smaIsReady = ((dynamic)sma).IsReady;
var atrIsReady = ((dynamic)atr).IsReady;
var vwapiIsReady = ((dynamic)vwapi).IsReady;
Assert.IsTrue(smaIsReady.IsTrue());
Assert.IsTrue(atrIsReady.IsTrue());
Assert.IsTrue(vwapiIsReady.IsTrue());
}
zipCacheProvider.DisposeSafely();
}
public void GetLastKnownPriceOfIlliquidAsset_RealData()
{
var cacheProvider = new ZipDataCacheProvider(_dataProvider);
var algorithm = GetAlgorithm(cacheProvider, new DateTime(2014, 6, 6, 11, 0, 0));
//20140606_twx_minute_quote_american_call_230000_20150117.csv
var optionSymbol = Symbol.CreateOption("TWX", Market.USA, OptionStyle.American, OptionRight.Call, 23, new DateTime(2015, 1, 17));
var option = algorithm.AddOptionContract(optionSymbol);
var lastKnownPrice = algorithm.GetLastKnownPrice(option);
Assert.IsNotNull(lastKnownPrice);
// Data gap of more than 15 minutes
Assert.Greater((algorithm.Time - lastKnownPrice.EndTime).TotalMinutes, 15);
cacheProvider.DisposeSafely();
}
public void CurrencyConversionRateResolved()
{
// Unit test to prove that in the event that default resolution (minute) history request returns
// no data for our currency conversion that BaseSetupHandler will use a daily history request
// to determine the the conversion rate if possible.
// Setup history provider and algorithm
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
TestGlobals.DataProvider,
zipCache,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
null,
false,
new DataPermissionManager()));
var algorithm = new BrokerageSetupHandlerTests.TestAlgorithm {
UniverseSettings = { Resolution = Resolution.Minute }
};
algorithm.SetHistoryProvider(historyProvider);
// Pick a date range where we do NOT have BTCUSD minute data
algorithm.SetStartDate(2015, 1, 24);
algorithm.SetCash("USD", 0);
algorithm.SetCash("BTC", 10);
// Have BaseSetupHandler resolve the currency conversion
BaseSetupHandler.SetupCurrencyConversions(algorithm, algorithm.DataManager.UniverseSelection);
// Assert that our portfolio has some value and that value is bitcoin
Assert.IsTrue(algorithm.Portfolio.Cash > 0);
Assert.IsTrue(algorithm.Portfolio.CashBook["BTC"].ValueInAccountCurrency > 0);
zipCache.DisposeSafely();
}
public void EquitiesAreMappedCorrectly()
{
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
historyProvider.Initialize(new HistoryProviderInitializeParameters(
null,
null,
new DefaultDataProvider(),
zipCache,
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
null,
false,
new DataPermissionManager()));
var symbol = Symbol.Create("WM", SecurityType.Equity, Market.USA);
var result = historyProvider.GetHistory(
new[]
{
new HistoryRequest(new DateTime(2008, 01, 01),
new DateTime(2008, 01, 05),
typeof(TradeBar),
symbol,
Resolution.Daily,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
TickType.Trade)
},
TimeZones.NewYork).ToList();
var firstBar = result.First().Values.Single();
Assert.AreEqual("WMI", firstBar.Symbol.Value);
Assert.IsNotEmpty(result);
zipCache.DisposeSafely();
}
public void Performance(Type streamReaderType, Type readLineReaderType, TickType tickType)
{
var streamReaderMilliSeconds = 0L;
var streamReaderCount = 0;
var getLineReaderMilliSeconds = 0L;
var getLineReaderCount = 0;
var stopWatch = new Stopwatch();
{
var config = new SubscriptionDataConfig(
streamReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = streamReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
streamReaderCount = reader.Read(source).Count();
streamReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
streamReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
streamReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
{
var config = new SubscriptionDataConfig(
readLineReaderType,
Symbols.SPY,
Resolution.Minute,
TimeZones.NewYork,
TimeZones.NewYork,
false,
true,
false,
tickType: tickType
);
var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
var date = new DateTime(2013, 10, 07);
var reader = new TextSubscriptionDataSourceReader(
zipCache,
config,
date,
false);
var source = readLineReaderType.GetBaseDataInstance().GetSource(config, date, false);
// warmup
getLineReaderCount = reader.Read(source).Count();
getLineReaderCount = 0;
// start test
stopWatch.Start();
for (int i = 0; i < 200; i++)
{
getLineReaderCount += reader.Read(source).Count();
}
stopWatch.Stop();
getLineReaderMilliSeconds = stopWatch.ElapsedMilliseconds;
zipCache.DisposeSafely();
}
Console.WriteLine($"StreamReader: {streamReaderMilliSeconds}ms. Count {streamReaderCount}");
Console.WriteLine($"GetLine Reader: {getLineReaderMilliSeconds}ms. Count {getLineReaderCount}");
// its 50% faster but lets leave some room to avoid noise
Assert.IsTrue((streamReaderMilliSeconds * 1.85d) < getLineReaderMilliSeconds);
Assert.AreEqual(getLineReaderCount, streamReaderCount);
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager">The algorithm manager instance</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
/// <param name="workerThread">The worker thread instance</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread)
{
var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
//-> Initialize messaging system
SystemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
IBrokerage brokerage = null;
DataManager dataManager = null;
IDataCacheProvider historyDataCacheProvider = null;
var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
try
{
// we get the mhdb before creating the algorithm instance,
// since the algorithm constructor will use it
var marketHoursDatabase = marketHoursDatabaseTask.Result;
AlgorithmHandlers.Setup.WorkerThread = workerThread;
// Save algorithm to cache, load algorithm instance:
algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
SystemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
// initialize the object store
AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls);
// initialize the data permission manager
AlgorithmHandlers.DataPermissionsManager.Initialize(job);
// notify the user of any errors w/ object store persistence
AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}");
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
registeredTypesProvider,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(AlgorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService,
AlgorithmHandlers.DataPermissionsManager),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
_liveMode,
registeredTypesProvider,
AlgorithmHandlers.DataPermissionsManager);
AlgorithmHandlers.Results.SetDataManager(dataManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(algorithm, dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
AlgorithmHandlers.DataFeed.Initialize(
algorithm,
job,
AlgorithmHandlers.Results,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
AlgorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer,
AlgorithmHandlers.DataPermissionsManager.DataChannelProvider);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
SystemHandlers.Api,
AlgorithmHandlers.DataProvider,
historyDataCacheProvider,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
Invariant($"Processing history {progress}%..."));
}
},
// disable parallel history requests for live trading
parallelHistoryRequestsEnabled: !_liveMode,
dataPermissionManager: AlgorithmHandlers.DataPermissionsManager
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore));
// set this again now that we've actually added securities
AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
// alpha handler needs start/end dates to determine sample step sizes
AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = string.Join(",", algorithm.ErrorMessages);
errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value);
message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
AlgorithmHandlers.Results.RuntimeError(errorMessage);
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName);
Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
SystemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.UtcNow;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results);
AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
// Result manager scanning message queue: (started earlier)
AlgorithmHandlers.Results.DebugMessage(
$"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}");
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation, workerThread: workerThread);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F"));
throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
algorithm.RunTimeError = err;
algorithm.SetStatus(AlgorithmStatus.RuntimeError);
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
SystemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName);
}
if (!_liveMode)
{
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.UtcNow - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
var kps = dataPoints / (double)1000 / totalSeconds;
AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
dataManager?.RemoveAllSubscriptions();
workerThread?.Dispose();
}
// Close data feed, alphas. Could be running even if algorithm initialization failed
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Alphas.Exit();
//Close result handler:
AlgorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var millisecondInterval = 10;
var millisecondTotalWait = 0;
while ((AlgorithmHandlers.Results.IsActive ||
(AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) ||
(AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) ||
(AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) ||
(AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) &&
millisecondTotalWait < 30 * 1000)
{
Thread.Sleep(millisecondInterval);
if (millisecondTotalWait % (millisecondInterval * 10) == 0)
{
Log.Trace("Waiting for threads to exit...");
}
millisecondTotalWait += millisecondInterval;
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (AlgorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
AlgorithmHandlers.Setup.Dispose();
}
historyDataCacheProvider.DisposeSafely();
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
AlgorithmHandlers.Results.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
_zipDataCacheProvider?.DisposeSafely();
}
public virtual void TearDown()
{
_cacheProvider.DisposeSafely();
_dataProvider.DisposeSafely();
}