public static void AdjustCurrentDirectory()
{
// nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder
var dir = TestContext.CurrentContext.TestDirectory;
Environment.CurrentDirectory = dir;
Directory.SetCurrentDirectory(dir);
Config.Reset();
Globals.Reset();
PythonInitializer.Initialize();
PythonInitializer.AddPythonPaths(
new[]
{
"./Alphas",
"./Execution",
"./Portfolio",
"./Risk",
"./Selection",
"./RegressionAlgorithms",
"./Research/RegressionScripts",
"../../../Algorithm",
"../../../Algorithm/Selection",
"../../../Algorithm.Framework",
"../../../Algorithm.Framework/Selection",
"../../../Algorithm.Python"
});
}
static void Main(string[] args)
{
// Adds the current working directory to the PYTHONPATH env var.
PythonInitializer.SetPythonPathEnvironmentVariable();
// Parse report arguments and merge with config to use in report creator:
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(ReportArgumentParser.ParseArguments(args));
}
var name = Config.Get("strategy-name");
var description = Config.Get("strategy-description");
var version = Config.Get("strategy-version");
var backtestDataFile = Config.Get("backtest-data-source-file");
var liveDataFile = Config.Get("live-data-source-file");
var destination = Config.Get("report-destination");
//Set the Order Parser For JSON:
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
Converters = { new OrderJsonConverter() }
};
// Parse content from source files into result objects
Log.Trace($"QuantConnect.Report.Main(): Parsing source files...{backtestDataFile}, {liveDataFile}");
var backtest = JsonConvert.DeserializeObject <BacktestResult>(File.ReadAllText(backtestDataFile));
LiveResult live = null;
if (liveDataFile != string.Empty)
{
live = JsonConvert.DeserializeObject <LiveResult>(File.ReadAllText(liveDataFile), new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
});
}
//Create a new report
Log.Trace("QuantConnect.Report.Main(): Instantiating report...");
var report = new Report(name, description, version, backtest, live);
// Generate the html content
Log.Trace("QuantConnect.Report.Main(): Starting content compile...");
var html = report.Compile();
//Write it to target destination.
if (destination != string.Empty)
{
Log.Trace($"QuantConnect.Report.Main(): Writing content to file {destination}");
File.WriteAllText(destination, html);
}
else
{
Console.Write(html);
}
Log.Trace("QuantConnect.Report.Main(): Completed.");
Console.ReadKey();
}
public void Initialize()
{
PythonInitializer.Initialize();
_algorithm = new QCAlgorithmFramework();
_algorithm.PortfolioConstruction = new NullPortfolioConstructionModel();
_algorithm.HistoryProvider = new SineHistoryProvider(_algorithm.Securities);
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
InitializeAlgorithm(_algorithm);
}
/// <summary>
/// Charting base class report element
/// </summary>
protected ChartReportElement()
{
PythonInitializer.Initialize();
using (Py.GIL())
{
dynamic module = PythonEngine.ImportModule("ReportCharts");
var classObj = module.ReportCharts;
Charting = classObj.Invoke();
}
}
public void Setup()
{
PythonInitializer.Initialize();
using (Py.GIL())
{
var module = Py.Import("Test_MethodOverload");
_algorithm = module.GetAttr("Test_MethodOverload").Invoke();
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub());
_algorithm.Initialize();
}
}
/// <summary>
/// Will start a new debugging session
/// </summary>
public static void Initialize(Language language)
{
if (language == Language.Python)
{
DebuggingMethod debuggingType;
Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType);
Log.Trace("DebuggerHelper.Initialize(): initializing python...");
PythonInitializer.Initialize();
Log.Trace("DebuggerHelper.Initialize(): python initialization done");
using (Py.GIL())
{
Log.Trace("DebuggerHelper.Initialize(): starting...");
switch (debuggingType)
{
case DebuggingMethod.LocalCmdline:
PythonEngine.RunSimpleString("import pdb; pdb.set_trace()");
break;
case DebuggingMethod.VisualStudio:
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
PythonEngine.RunSimpleString(@"import sys; import time;
while not sys.gettrace():
time.sleep(0.25)");
break;
}
Log.Trace("DebuggerHelper.Initialize(): started");
}
}
else if (language == Language.CSharp)
{
if (Debugger.IsAttached)
{
Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break.");
Debugger.Break();
}
else
{
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
while (!Debugger.IsAttached)
{
Thread.Sleep(250);
}
Log.Trace("DebuggerHelper.Initialize(): debugger attached");
}
}
else
{
throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}");
}
}
public void Setup()
{
PythonInitializer.Initialize();
using (Py.GIL())
{
var module = Py.Import("Test_MethodOverload");
_algorithm = module.GetAttr("Test_MethodOverload").Invoke();
// this is required else will get a 'RuntimeBinderException' because fails to match constructor method
dynamic algo = _algorithm.AsManagedObject((Type)_algorithm.GetPythonType().AsManagedObject(typeof(Type)));
_algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algo));
_algorithm.Initialize();
}
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
// Set the python path for loading python algorithms.
var pythonPath = new List <string>
{
pythonFile.Directory.FullName,
new DirectoryInfo(Environment.CurrentDirectory).FullName,
};
// Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH
// environment variable won't be used in the module import process
var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH");
if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable))
{
pythonPath.Add(pythonPathEnvironmentVariable);
}
Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath));
try
{
PythonInitializer.Initialize();
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
/// <summary>
/// Get the algorithm location for client side backtests.
/// </summary>
/// <returns></returns>
private string GetAlgorithmLocation()
{
if (Language == Language.Python)
{
if (!File.Exists(AlgorithmLocation))
{
throw new FileNotFoundException($"JobQueue.TryCreatePythonAlgorithm(): Unable to find py file: {AlgorithmLocation}");
}
// Add this directory to our Python Path so it may be imported properly
var pythonFile = new FileInfo(AlgorithmLocation);
PythonInitializer.AddPythonPaths(new string[] { pythonFile.Directory.FullName });
}
return(AlgorithmLocation);
}
/// <summary>
/// Create a new instance of a python algorithm
/// </summary>
/// <param name="assemblyPath"></param>
/// <param name="algorithmInstance"></param>
/// <param name="errorMessage"></param>
/// <returns></returns>
private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage)
{
algorithmInstance = null;
errorMessage = string.Empty;
//File does not exist.
if (!File.Exists(assemblyPath))
{
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}";
return(false);
}
var pythonFile = new FileInfo(assemblyPath);
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
try
{
PythonInitializer.Initialize();
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
// we need stdout for debugging
if (!_debugging && Config.GetBool("mute-python-library-logging", true))
{
using (Py.GIL())
{
PythonEngine.Exec(
@"
import logging, os, sys
sys.stdout = open(os.devnull, 'w')
logging.captureWarnings(True)"
);
}
}
}
catch (Exception e)
{
Log.Error(e);
errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}";
return(false);
}
//Successful load.
return(true);
}
public void ImportsCanBeExecutedFromDifferentThreads()
{
PythonInitializer.Initialize();
Task.Factory.StartNew(() =>
{
using (Py.GIL())
{
var module = Py.Import("Test_MethodOverload");
module.GetAttr("Test_MethodOverload").Invoke();
}
}).Wait();
PythonInitializer.Initialize();
Task.Factory.StartNew(() =>
{
using (Py.GIL())
{
var module = Py.Import("Test_AlgorithmPythonWrapper");
module.GetAttr("Test_AlgorithmPythonWrapper").Invoke();
}
}).Wait();
}
/// <summary>
/// Will start a new debugging session
/// </summary>
public static void Initialize(Language language)
{
if (language == Language.Python)
{
DebuggingMethod debuggingType;
Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType);
Log.Trace("DebuggerHelper.Initialize(): initializing python...");
PythonInitializer.Initialize();
Log.Trace("DebuggerHelper.Initialize(): python initialization done");
using (Py.GIL())
{
Log.Trace("DebuggerHelper.Initialize(): starting...");
switch (debuggingType)
{
case DebuggingMethod.LocalCmdline:
PythonEngine.RunSimpleString("import pdb; pdb.set_trace()");
break;
case DebuggingMethod.VisualStudio:
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
PythonEngine.RunSimpleString(@"import sys; import time;
while not sys.gettrace():
time.sleep(0.25)");
break;
case DebuggingMethod.PTVSD:
Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678...");
PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()");
break;
case DebuggingMethod.DebugPy:
Log.Trace("DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678...");
PythonEngine.RunSimpleString("import debugpy; debugpy.listen(('0.0.0.0', 5678)); debugpy.wait_for_client()");
break;
case DebuggingMethod.PyCharm:
Log.Trace("DebuggerHelper.Initialize(): Attempting to connect to Pycharm PyDev debugger server...");
PythonEngine.RunSimpleString(@"import pydevd_pycharm; import time;
count = 1
while count <= 10:
try:
pydevd_pycharm.settrace('localhost', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False)
print('SUCCESS: Connected to local program')
break
except ConnectionRefusedError:
pass
try:
pydevd_pycharm.settrace('host.docker.internal', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False)
print('SUCCESS: Connected to docker container')
break
except ConnectionRefusedError:
pass
print('\n')
print('Failed: Ensure your PyCharm Debugger is actively waiting for a connection at port 6000!')
print('Try ' + count.__str__() + ' out of 10')
print('\n')
count += 1
time.sleep(3)");
break;
}
Log.Trace("DebuggerHelper.Initialize(): started");
}
}
else if (language == Language.CSharp)
{
if (Debugger.IsAttached)
{
Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break.");
Debugger.Break();
}
else
{
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
while (!Debugger.IsAttached)
{
Thread.Sleep(250);
}
Log.Trace("DebuggerHelper.Initialize(): debugger attached");
}
}
else
{
throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}");
}
}
private static Dictionary <string, TestCaseData[]> GetParameterlessFunctions()
{
// Initialize the Python engine and begin allow thread
PythonInitializer.Initialize();
var functionsByType = new Dictionary <string, TestCaseData[]>();
using (Py.GIL())
{
var module = PyModule.FromString("Test",
@"import pandas
from inspect import getmembers, isfunction, signature
skipped = [ 'boxplot', 'hist', 'plot', # <- Graphics
'agg', 'aggregate', 'align', 'bool','combine', 'corrwith', 'dot', 'drop',
'equals', 'ewm', 'fillna', 'filter', 'groupby', 'join', 'mask', 'melt',
'pivot', 'pivot_table', 'reindex_like', 'rename', 'reset_index', 'select_dtypes',
'slice_shift', 'swaplevel', 'to_clipboard', 'to_excel', 'to_feather', 'to_gbq',
'to_hdf', 'to_list', 'tolist', 'to_parquet', 'to_period', 'to_pickle', 'to_sql',
'to_stata', 'to_timestamp', 'to_xarray', 'tshift', 'update', 'value_counts', 'where']
newPandas = int(pandas.__version__.split('.')[0]) >= 1
def getSimpleExceptionTestFunctions(cls):
functions = [ 'describe', 'mode']
if not newPandas:
functions.append('get_dtype_counts')
functions.append('get_ftype_counts')
for name, member in getmembers(cls):
if isfunction(member) and name.startswith('to') and name not in skipped:
functions.append(name)
return functions
DataFrame = getSimpleExceptionTestFunctions(pandas.DataFrame)
Series = getSimpleExceptionTestFunctions(pandas.Series)
skipped += set(DataFrame + Series)
def getParameterlessFunctions(cls):
functions = list()
for name, member in getmembers(cls):
if isfunction(member) and not name.startswith('_') and name not in skipped:
parameters = signature(member).parameters
count = 0
for parameter in parameters.values():
if parameter.default is parameter.empty:
count += 1
else:
break
if count < 2:
functions.append(name)
return functions
def getOtherParameterFunctions(cls):
functions = list()
for name, member in getmembers(pandas.DataFrame):
if isfunction(member) and not name.startswith('_') and name not in skipped:
parameters = signature(member).parameters
for parameter in parameters.values():
if parameter.name == 'other':
functions.append(name)
break
return functions
DataFrameParameterless = getParameterlessFunctions(pandas.DataFrame)
SeriesParameterless = getParameterlessFunctions(pandas.Series)
DataFrameOtherParameter = getOtherParameterFunctions(pandas.DataFrame)
SeriesOtherParameter = getOtherParameterFunctions(pandas.Series)
");
Func <string, string, TestCaseData[]> converter = (s, p) =>
{
var list = (List <string>)module.GetAttr(s).AsManagedObject(typeof(List <string>));
return(list.SelectMany(x => new[]
{
new TestCaseData($".{x}{p}", "'SPY'", true),
new TestCaseData($".{x}{p}", "symbol", false),
new TestCaseData($".{x}{p}", "str(symbol.ID)", false)
}
).ToArray());
};
functionsByType.Add("DataFrame", converter("DataFrame", "()"));
functionsByType.Add("Series", converter("Series", "()"));
functionsByType.Add("DataFrameParameterless", converter("DataFrameParameterless", "()"));
functionsByType.Add("SeriesParameterless", converter("SeriesParameterless", "()"));
functionsByType.Add("DataFrameOtherParameter", converter("DataFrameOtherParameter", "(other)"));
functionsByType.Add("SeriesOtherParameter", converter("SeriesOtherParameter", "(other)"));
}
return(functionsByType);
}
/// <summary>
/// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution.
/// </summary>
/// <returns></returns>
public AlgorithmNodePacket NextJob(out string location)
{
location = GetAlgorithmLocation();
Log.Trace($"JobQueue.NextJob(): Selected {location}");
// check for parameters in the config
var parameters = new Dictionary <string, string>();
var parametersConfigString = Config.Get("parameters");
if (parametersConfigString != string.Empty)
{
parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString);
}
var controls = new Controls()
{
MinuteLimit = Config.GetInt("symbol-minute-limit", 10000),
SecondLimit = Config.GetInt("symbol-second-limit", 10000),
TickLimit = Config.GetInt("symbol-tick-limit", 10000),
RamAllocation = int.MaxValue,
MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000)
};
if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python)
{
// Set the python path for loading python algorithms ("algorithm-location" config parameter)
var pythonFile = new FileInfo(location);
// PythonInitializer automatically adds the current working directory for us
PythonInitializer.SetPythonPathEnvironmentVariable(new string[] { pythonFile.Directory.FullName });
}
var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName);
//If this isn't a backtesting mode/request, attempt a live job.
if (_liveMode)
{
var liveJob = new LiveNodePacket
{
Type = PacketType.LiveNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler),
DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
DeployId = algorithmId,
Parameters = parameters,
Language = Language,
Controls = controls
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
liveJob.BrokerageData = brokerageFactory.BrokerageData;
}
catch (Exception err)
{
Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}");
}
return(liveJob);
}
//Default run a backtesting job.
var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local")
{
Type = PacketType.BacktestNode,
Algorithm = File.ReadAllBytes(AlgorithmLocation),
HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider),
Channel = AccessToken,
UserToken = AccessToken,
UserId = UserId,
ProjectId = ProjectId,
Version = Globals.Version,
BacktestId = algorithmId,
Language = Language,
Parameters = parameters,
Controls = controls
};
return(backtestJob);
}
