public void Py_AccessPropertyThatDoesNotExists_ReturnsEmptyList_WhenTypeIsIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
registeredTypes.RegisterType(typeof(TradeBar));
dynamic securityData = new DynamicSecurityData(registeredTypes);
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
from QuantConnect.Data.Market import *
def Test(dynamicData):
data = dynamicData.GetAll(TradeBar)
if data is None:
raise Exception('Unexpected None value')
if len(data) != 0:
raise Exception('Unexpected length')").GetAttr("Test");
Assert.DoesNotThrow(() => test(securityData));
}
}
public override void Initialize()
{
SetStartDate(2014, 5, 7); //Set Start Date
SetEndDate(2014, 5, 15); //Set End Date
SetCash(100000); //Set Strategy Cash
// Find more symbols here: http://quantconnect.com/data
//AddForex("EURUSD", Resolution.Minute);
//AddForex("EURUSD", Resolution.Minute);
//var dailyHistory = History(5, Resolution.Daily);
//var hourHistory = History(5, Resolution.Hour);
//var minuteHistory = History(5, Resolution.Minute);
//var secondHistory = History(5, Resolution.Second);
algorithm = new QCAlgorithm();
//algorithm.Initialize();
// new forex - should be quotebar
algorithm.SubscriptionManager = SubscriptionManager;
//algorithm.Securities.SetSecurityService(Securities.SetSecurityService);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
//var mapFilePrimaryExchangeProvider = new MapFilePrimaryExchangeProvider(AlgorithmHandlers.MapFileProvider);
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
Type typSecurityManager = typeof(SecurityManager);
FieldInfo typeAccessSecurityManager = typSecurityManager.GetField("_securityService", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
var sm = (SecurityService)typeAccessSecurityManager.GetValue(Securities);
// Type typSecurityService = typeof(SecurityService);
// FieldInfo typeAccessSecurityService = typSecurityService.GetField("_primaryExchangeProvider", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
// var ss = typeAccessSecurityService.GetValue(sm);
//algorithm.Securities.SetSecurityService(sm);
algorithm.Securities = Securities;
algorithm.Portfolio.Securities = Securities;
/*
* var securityService = new SecurityService(algorithm.Portfolio.CashBook,
* marketHoursDatabase,
* symbolPropertiesDatabase,
* algorithm,
* registeredTypesProvider,
* new SecurityCacheProvider(algorithm.Portfolio),
* mapFilePrimaryExchangeProvider);*/
AddForex("EURUSD", Resolution.Minute);
var forexQuote = algorithm.AddForex("EURUSD", Resolution.Minute);
// Assert.IsTrue(forexQuote.Subscriptions.Count() == 1);
// Assert.IsTrue(GetMatchingSubscription(forexQuote, typeof(TradeBar)) != null);
algorithm.Transactions.SetOrderProcessor(new BacktestingTransactionHandler());
algorithm.Portfolio.Transactions = algorithm.Transactions;
algorithm.PostInitialize();
}
public void AccessingPropertyThatDoesNotExists_ThrowsKeyNotFoundException_WhenNotIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
dynamic securityData = new DynamicSecurityData(registeredTypes, _cache);
Assert.Throws <KeyNotFoundException>(() =>
{
var _ = securityData.NotFoundProperty;
});
}
public void AccessPropertyThatDoesNotExists_ReturnsEmptyList_WhenTypeIsIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
registeredTypes.RegisterType(typeof(TradeBar));
dynamic securityData = new DynamicSecurityData(registeredTypes);
var tradeBars = securityData.TradeBar;
Assert.IsInstanceOf <List <TradeBar> >(tradeBars);
Assert.IsEmpty(tradeBars);
}
public void Py_GetTypeThatDoesNotExists_ThrowsKeyNotFoundException_WhenNotIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
dynamic securityData = new DynamicSecurityData(registeredTypes, _cache);
using (Py.GIL())
{
dynamic test = PyModule.FromString("testModule",
@"
from AlgorithmImports import *
def Test(dynamicData):
data = dynamicData.Get(TradeBar)").GetAttr("Test");
Assert.Throws <KeyNotFoundException>(() => test(securityData));
}
}
private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchronizer, IDataAggregator dataAggregator)
{
_dataFeed = new TestableLiveTradingDataFeed(dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? new AggregationManager()));
_algorithm = new AlgorithmStub(createDataManager: false);
_synchronizer = synchronizer ?? new LiveSynchronizer();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
_algorithm,
registeredTypesProvider,
new SecurityCacheProvider(_algorithm.Portfolio));
var universeSelection = new UniverseSelection(
_algorithm,
securityService,
new DataPermissionManager(),
TestGlobals.DataProvider,
Resolution.Second);
_dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork),
MarketHoursDatabase.FromDataFolder(),
true,
new RegisteredSecurityDataTypesProvider(),
new DataPermissionManager());
_resultHandler = new TestResultHandler();
_synchronizer.Initialize(_algorithm, _dataManager);
_dataFeed.Initialize(_algorithm,
new LiveNodePacket(),
_resultHandler,
TestGlobals.MapFileProvider,
TestGlobals.FactorFileProvider,
TestGlobals.DataProvider,
_dataManager,
_synchronizer,
new DataChannelProvider());
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.Securities.SetSecurityService(securityService);
_algorithm.SetFinishedWarmingUp();
var backtestingTransactionHandler = new BacktestingTransactionHandler();
backtestingTransactionHandler.Initialize(_algorithm, new PaperBrokerage(_algorithm, new LiveNodePacket()), _resultHandler);
_algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler);
_algorithm.PostInitialize();
}
public void Py_GetTypeThatDoesNotExists_ThrowsKeyNotFoundException_WhenNotIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
dynamic securityData = new DynamicSecurityData(registeredTypes);
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from clr import AddReference
AddReference(""QuantConnect.Common"")
AddReference(""System"")
from System import *
from QuantConnect import *
def Test(dynamicData):
data = dynamicData.Get(TradeBar)").GetAttr("Test");
Assert.Throws <PythonException>(() => test(securityData));
}
}
public void Setup()
{
_dataFeed = new TestableLiveTradingDataFeed(new FakeDataQueue());
_algorithm = new AlgorithmStub(createDataManager: false);
_synchronizer = new LiveSynchronizer();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
MarketHoursDatabase.FromDataFolder(),
SymbolPropertiesDatabase.FromDataFolder(),
_algorithm,
registeredTypesProvider,
new SecurityCacheProvider(_algorithm.Portfolio));
var universeSelection = new UniverseSelection(
_algorithm,
securityService,
new DataPermissionManager(),
new DefaultDataProvider(),
Resolution.Second);
_dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork),
MarketHoursDatabase.FromDataFolder(),
true,
new RegisteredSecurityDataTypesProvider(),
new DataPermissionManager());
_synchronizer.Initialize(_algorithm, _dataManager);
_dataFeed.Initialize(_algorithm,
new LiveNodePacket(),
new TestResultHandler(),
new LocalDiskMapFileProvider(),
new LocalDiskFactorFileProvider(),
new DefaultDataProvider(),
_dataManager,
_synchronizer,
new DataChannelProvider());
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.Securities.SetSecurityService(securityService);
_algorithm.SetFinishedWarmingUp();
_algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
}
public void Py_AccessPropertyThatDoesNotExists_ReturnsEmptyList_WhenTypeIsIncludedInRegisteredTypes()
{
var registeredTypes = new RegisteredSecurityDataTypesProvider();
registeredTypes.RegisterType(typeof(TradeBar));
dynamic securityData = new DynamicSecurityData(registeredTypes, _cache);
using (Py.GIL())
{
dynamic test = PyModule.FromString("testModule",
@"
from AlgorithmImports import *
def Test(dynamicData):
data = dynamicData.GetAll(TradeBar)
if data is None:
raise Exception('Unexpected None value')
if len(data) != 0:
raise Exception('Unexpected length')").GetAttr("Test");
Assert.DoesNotThrow(() => test(securityData));
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook, MarketHoursDatabase, symbolPropertiesDataBase, this, registeredTypes);
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="manager">The algorithm manager instance</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
/// <param name="workerThread">The worker thread instance</param>
public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread)
{
var marketHoursDatabaseTask = Task.Run(() => StaticInitializations());
var algorithm = default(IAlgorithm);
var algorithmManager = manager;
try
{
//Reset thread holders.
var initializeComplete = false;
//-> Initialize messaging system
SystemHandlers.Notify.SetAuthentication(job);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions);
IBrokerage brokerage = null;
DataManager dataManager = null;
var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
try
{
// we get the mhdb before creating the algorithm instance,
// since the algorithm constructor will use it
var marketHoursDatabase = marketHoursDatabaseTask.Result;
AlgorithmHandlers.Setup.WorkerThread = workerThread;
// Save algorithm to cache, load algorithm instance:
algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath);
// Set algorithm in ILeanManager
SystemHandlers.LeanManager.SetAlgorithm(algorithm);
// initialize the alphas handler with the algorithm instance
AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api);
// initialize the object store
AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls);
// notify the user of any errors w/ object store persistence
AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}");
// Initialize the brokerage
IBrokerageFactory factory;
brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory);
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
registeredTypesProvider,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
dataManager = new DataManager(AlgorithmHandlers.DataFeed,
new UniverseSelection(
algorithm,
securityService),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
_liveMode,
registeredTypesProvider);
AlgorithmHandlers.Results.SetDataManager(dataManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
synchronizer.Initialize(algorithm, dataManager);
// Initialize the data feed before we initialize so he can intercept added securities/universes via events
AlgorithmHandlers.DataFeed.Initialize(
algorithm,
job,
AlgorithmHandlers.Results,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
AlgorithmHandlers.DataProvider,
dataManager,
(IDataFeedTimeProvider)synchronizer);
// set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider)
algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions);
algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions);
// set the history provider before setting up the algorithm
var historyProvider = GetHistoryProvider(job.HistoryProvider);
if (historyProvider is BrokerageHistoryProvider)
{
(historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage);
}
var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode);
historyProvider.Initialize(
new HistoryProviderInitializeParameters(
job,
SystemHandlers.Api,
AlgorithmHandlers.DataProvider,
historyDataCacheProvider,
AlgorithmHandlers.MapFileProvider,
AlgorithmHandlers.FactorFileProvider,
progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History,
Invariant($"Processing history {progress}%..."));
}
},
// disable parallel history requests for live trading
parallelHistoryRequestsEnabled: !_liveMode
)
);
historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); };
historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); };
historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); };
historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); };
algorithm.HistoryProvider = historyProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore));
// set this again now that we've actually added securities
AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
// alpha handler needs start/end dates to determine sample step sizes
AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = string.Join(",", algorithm.ErrorMessages);
errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e =>
{
var message = e.Message;
if (e.InnerException != null)
{
var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value);
message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err);
}
return(message);
}));
Log.Error("Engine.Run(): " + errorMessage);
AlgorithmHandlers.Results.RuntimeError(errorMessage);
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
Log.Error(err);
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err;
AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString());
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName);
Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName);
if (algorithm?.HistoryProvider != null)
{
Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName);
}
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + brokerage?.GetType().FullName);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
// notify the LEAN manager that the algorithm is initialized and starting
SystemHandlers.LeanManager.OnAlgorithmStart();
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Load the associated handlers for transaction and realtime events:
AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results);
AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit);
// wire up the brokerage message handler
brokerage.Message += (sender, message) =>
{
algorithm.BrokerageMessageHandler.Handle(message);
// fire brokerage message events
algorithm.OnBrokerageMessage(message);
switch (message.Type)
{
case BrokerageMessageType.Disconnect:
algorithm.OnBrokerageDisconnect();
break;
case BrokerageMessageType.Reconnect:
algorithm.OnBrokerageReconnect();
break;
}
};
//Send status to user the algorithm is now executing.
AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running);
// Result manager scanning message queue: (started earlier)
AlgorithmHandlers.Results.DebugMessage(
$"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}");
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.Controls.RamAllocation, workerThread: workerThread);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F"));
throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
HandleAlgorithmError(job, algorithm.RunTimeError);
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
algorithm.RunTimeError = err;
algorithm.SetStatus(AlgorithmStatus.RuntimeError);
HandleAlgorithmError(job, err);
}
// notify the LEAN manager that the algorithm has finished
SystemHandlers.LeanManager.OnAlgorithmEnd();
try
{
var csvTransactionsFileName = Config.Get("transaction-log");
if (!string.IsNullOrEmpty(csvTransactionsFileName))
{
SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName);
}
if (!_liveMode)
{
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount;
var kps = dataPoints / (double)1000 / totalSeconds;
AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points.");
}
}
catch (Exception err)
{
Log.Error(err, "Error sending analysis results");
}
//Before we return, send terminate commands to close up the threads
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
dataManager?.RemoveAllSubscriptions();
workerThread?.Dispose();
}
// Close data feed, alphas. Could be running even if algorithm initialization failed
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Alphas.Exit();
//Close result handler:
AlgorithmHandlers.Results.Exit();
//Wait for the threads to complete:
var millisecondInterval = 10;
var millisecondTotalWait = 0;
while ((AlgorithmHandlers.Results.IsActive ||
(AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) ||
(AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) ||
(AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) ||
(AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) &&
millisecondTotalWait < 30 * 1000)
{
Thread.Sleep(millisecondInterval);
if (millisecondTotalWait % (millisecondInterval * 10) == 0)
{
Log.Trace("Waiting for threads to exit...");
}
millisecondTotalWait += millisecondInterval;
}
if (brokerage != null)
{
Log.Trace("Engine.Run(): Disconnecting from brokerage...");
brokerage.Disconnect();
brokerage.Dispose();
}
if (AlgorithmHandlers.Setup != null)
{
Log.Trace("Engine.Run(): Disposing of setup handler...");
AlgorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error(err, "Error running algorithm");
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
{
SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
}
AlgorithmHandlers.Results.Exit();
AlgorithmHandlers.DataFeed.Exit();
AlgorithmHandlers.Transactions.Exit();
AlgorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// Issue #4892 : Set start time relative to NY time
// when the data is available from the previous day
var newYorkTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var hourThreshold = Config.GetInt("qb-data-hour", 9);
// If it is after our hour threshold; then we can use today
if (newYorkTime.Hour >= hourThreshold)
{
SetStartDate(newYorkTime);
}
else
{
SetStartDate(newYorkTime - TimeSpan.FromDays(1));
}
// Sets PandasConverter
SetPandasConverter();
// Reset our composer; needed for re-creation of QuantBook
Composer.Instance.Reset();
var composer = Composer.Instance;
// Create our handlers with our composer instance
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
_dataProvider = algorithmHandlers.DataProvider;
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService, algorithmHandlers.DataPermissionsManager, algorithmHandlers.DataProvider),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes,
algorithmHandlers.DataPermissionsManager));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true,
algorithmHandlers.DataPermissionsManager
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataProvider)));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataProvider)));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public void SetUp()
{
_cache = new SecurityCache();
_dataTypesProvider = new RegisteredSecurityDataTypesProvider();
_dataTypesProvider.RegisterType(typeof(TradeBar));
}
