public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(CustomPythonData, "IBM", Resolution.Daily)
qcAlgorithm.Initialize();
var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY");
Assert.IsNotNull(niftySubscription);
var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type });
Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false));
var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "IBM");
Assert.IsNotNull(customDataSubscription);
Assert.IsTrue(customDataSubscription.IsCustomData);
Assert.AreEqual("custom_data.CustomPythonData", customDataSubscription.Type.ToString());
var customDataFactory = (BaseData)ObjectActivator.GetActivator(customDataSubscription.Type).Invoke(new object[] { customDataSubscription.Type });
Assert.DoesNotThrow(() => customDataFactory.GetSource(customDataSubscription, DateTime.UtcNow, false));
}
public void PythonCustomDataTypes_AreAddedToSubscriptions_Successfully()
{
var pythonPath = new System.IO.DirectoryInfo("RegressionAlgorithms");
Environment.SetEnvironmentVariable("PYTHONPATH", pythonPath.FullName);
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
qcAlgorithm.Initialize();
var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY");
Assert.IsNotNull(niftySubscription);
var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type });
Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false));
var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "SCF/CME_CL1_ON");
Assert.IsNotNull(quandlSubscription);
var quandlFactory = (BaseData)ObjectActivator.GetActivator(quandlSubscription.Type).Invoke(new object[] { quandlSubscription.Type });
Assert.DoesNotThrow(() => quandlFactory.GetSource(quandlSubscription, DateTime.UtcNow, false));
}
public void AddsOnEndOfDayEventsIfImplemented(Language language)
{
IAlgorithm algorithm;
if (language == Language.CSharp)
{
algorithm = new TestAlgorithm();
(algorithm as QCAlgorithm).AddEquity("SPY");
}
else
{
algorithm = new AlgorithmPythonWrapper("OnEndOfDayRegressionAlgorithm");
algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(new MockDataFeed(), algorithm));
algorithm.AddSecurity(SecurityType.Equity,
"SPY",
Resolution.Daily,
Market.USA,
false,
1,
false);
}
var realTimeHandler = new TestBacktestingRealTimeHandler();
realTimeHandler.Setup(algorithm,
new AlgorithmNodePacket(PacketType.AlgorithmNode)
{
Language = language
},
new TestResultHandler(),
null,
new TestTimeLimitManager());
Assert.AreEqual(2, realTimeHandler.GetScheduledEventsCount);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
//Help python find the module
Environment.SetEnvironmentVariable("PYTHONPATH", pythonFile.DirectoryName);
try
{
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
PythonEngine.BeginAllowThreads();
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
try
{
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
//Help python find the module
Environment.SetEnvironmentVariable("PYTHONPATH", pythonFile.DirectoryName);
// Initialize Python Engine
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
PythonEngine.BeginAllowThreads();
}
// Import Python module
using (Py.GIL())
{
Log.Trace($"Loader.TryCreatePythonAlgorithm(): Python version {PythonEngine.Version}: Importing python module {moduleName}");
var module = Py.Import(moduleName);
if (module == null)
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. Check for errors in the python scripts.";
return(false);
}
Log.Trace("Loader.TryCreatePythonAlgorithm(): Creating IAlgorithm instance.");
algorithmInstance = new AlgorithmPythonWrapper(module);
}
}
catch (Exception e)
{
// perform exception interpretation for error in module import
var interpreter = StackExceptionInterpreter.CreateFromAssemblies(AppDomain.CurrentDomain.GetAssemblies());
e = interpreter.Interpret(e, interpreter);
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {interpreter.GetExceptionMessageHeader(e)}";
}
//Successful load.
return(algorithmInstance != null);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = "Loader.TryCreatePythonAlgorithm(): Unable to find py file: " + assemblyPath;
return(false);
}
try
{
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
//Help python find the module
Environment.SetEnvironmentVariable("PYTHONPATH", pythonFile.DirectoryName);
// Initialize Python Engine
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
PythonEngine.BeginAllowThreads();
}
// Import Python module
using (Py.GIL())
{
Log.Trace("Loader.TryCreatePythonAlgorithm(): Importing python module " + moduleName);
var module = Py.Import(moduleName);
if (module == null)
{
errorMessage = "Loader.TryCreatePythonAlgorithm(): Unable to import python module " + assemblyPath + ". Check for errors in the python scripts.";
return(false);
}
Log.Trace("Loader.TryCreatePythonAlgorithm(): Creating IAlgorithm instance.");
algorithmInstance = new AlgorithmPythonWrapper(module);
ObjectActivator.SetPythonModule(module);
}
}
catch (Exception e)
{
Log.Error(e);
errorMessage = "Loader.TryCreatePythonAlgorithm(): Unable to import python module " + assemblyPath + ". " + e.Message;
}
//Successful load.
return(algorithmInstance != null);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
// Set the python path for loading python algorithms.
var pythonPath = new List <string>
{
pythonFile.Directory.FullName,
new DirectoryInfo(Environment.CurrentDirectory).FullName,
};
// Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH
// environment variable won't be used in the module import process
var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH");
if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable))
{
pythonPath.Add(pythonPathEnvironmentVariable);
}
Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath));
try
{
PythonInitializer.Initialize();
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
public void PythonCustomDataTypes_AreAddedToConsolidator_Successfully()
{
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
qcAlgorithm.Initialize();
var niftyConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("NIFTY", niftyConsolidator));
var quandlConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("SCF/CME_CL1_ON", quandlConsolidator));
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
try
{
PythonInitializer.Initialize();
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
// we need stdout for debugging
if (!_debugging && Config.GetBool("mute-python-library-logging", true))
{
using (Py.GIL())
{
PythonEngine.Exec(
@"
import logging, os, sys
sys.stdout = open(os.devnull, 'w')
logging.captureWarnings(True)"
);
}
}
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
// Set the python path for loading python algorithms.
var pythonPath = new[]
{
pythonFile.Directory.FullName,
new DirectoryInfo(Environment.CurrentDirectory).FullName,
Environment.GetEnvironmentVariable("PYTHONPATH")
};
Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath));
try
{
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
if (!_isBeginAllowThreadsCalled)
{
PythonEngine.BeginAllowThreads();
_isBeginAllowThreadsCalled = true;
}
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
public void AddsOnEndOfDayEventsIfImplemented(Language language)
{
Security security;
IAlgorithm algorithm;
if (language == Language.CSharp)
{
algorithm = new TestAlgorithmB();
security = (algorithm as QCAlgorithm).AddEquity("SPY");
}
else
{
algorithm = new AlgorithmPythonWrapper("OnEndOfDayRegressionAlgorithm");
algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(new MockDataFeed(), algorithm));
security = algorithm.AddSecurity(SecurityType.Equity,
"SPY",
Resolution.Daily,
Market.USA,
false,
1,
false);
}
var realTimeHandler = new TestBacktestingRealTimeHandler();
realTimeHandler.Setup(algorithm,
new AlgorithmNodePacket(PacketType.AlgorithmNode)
{
Language = language
},
_resultHandler,
null,
new TestTimeLimitManager());
// Because neither implement EOD() deprecated it should be zero
Assert.AreEqual(0, realTimeHandler.GetScheduledEventsCount);
realTimeHandler.OnSecuritiesChanged(
new SecurityChanges(new[] { security }, Enumerable.Empty <Security>()));
Assert.AreEqual(1, realTimeHandler.GetScheduledEventsCount);
realTimeHandler.Exit();
}
public void DoesNotAddOnEndOfDayEventsIfNotImplemented(Language language)
{
Security security;
IAlgorithm algorithm;
if (language == Language.CSharp)
{
algorithm = new AlgorithmStub();
security = (algorithm as QCAlgorithm).AddEquity("SPY");
}
else
{
algorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm));
security = algorithm.AddSecurity(SecurityType.Equity,
"SPY",
Resolution.Daily,
Market.USA,
false,
1,
false);
}
var realTimeHandler = new TestBacktestingRealTimeHandler();
realTimeHandler.Setup(algorithm,
new AlgorithmNodePacket(PacketType.AlgorithmNode)
{
Language = language
},
_resultHandler,
null,
new TestTimeLimitManager());
Assert.AreEqual(0, realTimeHandler.GetScheduledEventsCount);
realTimeHandler.OnSecuritiesChanged(
new SecurityChanges(new[] { security }, Enumerable.Empty <Security>()));
Assert.AreEqual(0, realTimeHandler.GetScheduledEventsCount);
realTimeHandler.Exit();
}
public void PythonCustomDataTypes_AreAddedToConsolidator_Successfully()
{
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
qcAlgorithm.SubscriptionManager.SetDataManager(new DataManagerStub(qcAlgorithm));
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(CustomPythonData, "IBM", Resolution.Daily)
qcAlgorithm.Initialize();
var niftyConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("NIFTY", niftyConsolidator));
var customDataConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("IBM", customDataConsolidator));
}
public void PythonCustomDataTypes_AreAddedToConsolidator_Successfully()
{
var pythonPath = new System.IO.DirectoryInfo("RegressionAlgorithms");
Environment.SetEnvironmentVariable("PYTHONPATH", pythonPath.FullName);
var qcAlgorithm = new AlgorithmPythonWrapper("Test_CustomDataAlgorithm");
// Initialize contains the statements:
// self.AddData(Nifty, "NIFTY")
// self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
qcAlgorithm.Initialize();
var niftyConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("NIFTY", niftyConsolidator));
var quandlConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("SCF/CME_CL1_ON", quandlConsolidator));
}
