public static void AdjustCurrentDirectory() { // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder var dir = TestContext.CurrentContext.TestDirectory; Environment.CurrentDirectory = dir; Directory.SetCurrentDirectory(dir); Config.Reset(); Globals.Reset(); PythonInitializer.Initialize(); PythonInitializer.AddPythonPaths( new[] { "./Alphas", "./Execution", "./Portfolio", "./Risk", "./Selection", "./RegressionAlgorithms", "./Research/RegressionScripts", "../../../Algorithm", "../../../Algorithm/Selection", "../../../Algorithm.Framework", "../../../Algorithm.Framework/Selection", "../../../Algorithm.Python" }); }
static void Main(string[] args) { // Adds the current working directory to the PYTHONPATH env var. PythonInitializer.SetPythonPathEnvironmentVariable(); // Parse report arguments and merge with config to use in report creator: if (args.Length > 0) { Config.MergeCommandLineArgumentsWithConfiguration(ReportArgumentParser.ParseArguments(args)); } var name = Config.Get("strategy-name"); var description = Config.Get("strategy-description"); var version = Config.Get("strategy-version"); var backtestDataFile = Config.Get("backtest-data-source-file"); var liveDataFile = Config.Get("live-data-source-file"); var destination = Config.Get("report-destination"); //Set the Order Parser For JSON: JsonConvert.DefaultSettings = () => new JsonSerializerSettings { Converters = { new OrderJsonConverter() } }; // Parse content from source files into result objects Log.Trace($"QuantConnect.Report.Main(): Parsing source files...{backtestDataFile}, {liveDataFile}"); var backtest = JsonConvert.DeserializeObject <BacktestResult>(File.ReadAllText(backtestDataFile)); LiveResult live = null; if (liveDataFile != string.Empty) { live = JsonConvert.DeserializeObject <LiveResult>(File.ReadAllText(liveDataFile), new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); } //Create a new report Log.Trace("QuantConnect.Report.Main(): Instantiating report..."); var report = new Report(name, description, version, backtest, live); // Generate the html content Log.Trace("QuantConnect.Report.Main(): Starting content compile..."); var html = report.Compile(); //Write it to target destination. if (destination != string.Empty) { Log.Trace($"QuantConnect.Report.Main(): Writing content to file {destination}"); File.WriteAllText(destination, html); } else { Console.Write(html); } Log.Trace("QuantConnect.Report.Main(): Completed."); Console.ReadKey(); }
public void Initialize() { PythonInitializer.Initialize(); _algorithm = new QCAlgorithmFramework(); _algorithm.PortfolioConstruction = new NullPortfolioConstructionModel(); _algorithm.HistoryProvider = new SineHistoryProvider(_algorithm.Securities); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm)); InitializeAlgorithm(_algorithm); }
/// <summary> /// Charting base class report element /// </summary> protected ChartReportElement() { PythonInitializer.Initialize(); using (Py.GIL()) { dynamic module = PythonEngine.ImportModule("ReportCharts"); var classObj = module.ReportCharts; Charting = classObj.Invoke(); } }
public void Setup() { PythonInitializer.Initialize(); using (Py.GIL()) { var module = Py.Import("Test_MethodOverload"); _algorithm = module.GetAttr("Test_MethodOverload").Invoke(); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub()); _algorithm.Initialize(); } }
/// <summary> /// Will start a new debugging session /// </summary> public static void Initialize(Language language) { if (language == Language.Python) { DebuggingMethod debuggingType; Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType); Log.Trace("DebuggerHelper.Initialize(): initializing python..."); PythonInitializer.Initialize(); Log.Trace("DebuggerHelper.Initialize(): python initialization done"); using (Py.GIL()) { Log.Trace("DebuggerHelper.Initialize(): starting..."); switch (debuggingType) { case DebuggingMethod.LocalCmdline: PythonEngine.RunSimpleString("import pdb; pdb.set_trace()"); break; case DebuggingMethod.VisualStudio: Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach..."); PythonEngine.RunSimpleString(@"import sys; import time; while not sys.gettrace(): time.sleep(0.25)"); break; } Log.Trace("DebuggerHelper.Initialize(): started"); } } else if (language == Language.CSharp) { if (Debugger.IsAttached) { Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break."); Debugger.Break(); } else { Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach..."); while (!Debugger.IsAttached) { Thread.Sleep(250); } Log.Trace("DebuggerHelper.Initialize(): debugger attached"); } } else { throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}"); } }
public void Setup() { PythonInitializer.Initialize(); using (Py.GIL()) { var module = Py.Import("Test_MethodOverload"); _algorithm = module.GetAttr("Test_MethodOverload").Invoke(); // this is required else will get a 'RuntimeBinderException' because fails to match constructor method dynamic algo = _algorithm.AsManagedObject((Type)_algorithm.GetPythonType().AsManagedObject(typeof(Type))); _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algo)); _algorithm.Initialize(); } }
/// <summary> /// Create a new instance of a python algorithm /// </summary> /// <param name="assemblyPath"></param> /// <param name="algorithmInstance"></param> /// <param name="errorMessage"></param> /// <returns></returns> private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage) { algorithmInstance = null; errorMessage = string.Empty; //File does not exist. if (!File.Exists(assemblyPath)) { errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}"; return(false); } var pythonFile = new FileInfo(assemblyPath); var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", ""); // Set the python path for loading python algorithms. var pythonPath = new List <string> { pythonFile.Directory.FullName, new DirectoryInfo(Environment.CurrentDirectory).FullName, }; // Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH // environment variable won't be used in the module import process var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH"); if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable)) { pythonPath.Add(pythonPathEnvironmentVariable); } Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath)); try { PythonInitializer.Initialize(); algorithmInstance = new AlgorithmPythonWrapper(moduleName); } catch (Exception e) { Log.Error(e); errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}"; return(false); } //Successful load. return(true); }
/// <summary> /// Get the algorithm location for client side backtests. /// </summary> /// <returns></returns> private string GetAlgorithmLocation() { if (Language == Language.Python) { if (!File.Exists(AlgorithmLocation)) { throw new FileNotFoundException($"JobQueue.TryCreatePythonAlgorithm(): Unable to find py file: {AlgorithmLocation}"); } // Add this directory to our Python Path so it may be imported properly var pythonFile = new FileInfo(AlgorithmLocation); PythonInitializer.AddPythonPaths(new string[] { pythonFile.Directory.FullName }); } return(AlgorithmLocation); }
/// <summary> /// Create a new instance of a python algorithm /// </summary> /// <param name="assemblyPath"></param> /// <param name="algorithmInstance"></param> /// <param name="errorMessage"></param> /// <returns></returns> private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage) { algorithmInstance = null; errorMessage = string.Empty; //File does not exist. if (!File.Exists(assemblyPath)) { errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to find py file: {assemblyPath}"; return(false); } var pythonFile = new FileInfo(assemblyPath); var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", ""); try { PythonInitializer.Initialize(); algorithmInstance = new AlgorithmPythonWrapper(moduleName); // we need stdout for debugging if (!_debugging && Config.GetBool("mute-python-library-logging", true)) { using (Py.GIL()) { PythonEngine.Exec( @" import logging, os, sys sys.stdout = open(os.devnull, 'w') logging.captureWarnings(True)" ); } } } catch (Exception e) { Log.Error(e); errorMessage = $"Loader.TryCreatePythonAlgorithm(): Unable to import python module {assemblyPath}. {e.Message}"; return(false); } //Successful load. return(true); }
public void ImportsCanBeExecutedFromDifferentThreads() { PythonInitializer.Initialize(); Task.Factory.StartNew(() => { using (Py.GIL()) { var module = Py.Import("Test_MethodOverload"); module.GetAttr("Test_MethodOverload").Invoke(); } }).Wait(); PythonInitializer.Initialize(); Task.Factory.StartNew(() => { using (Py.GIL()) { var module = Py.Import("Test_AlgorithmPythonWrapper"); module.GetAttr("Test_AlgorithmPythonWrapper").Invoke(); } }).Wait(); }
/// <summary> /// Will start a new debugging session /// </summary> public static void Initialize(Language language) { if (language == Language.Python) { DebuggingMethod debuggingType; Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType); Log.Trace("DebuggerHelper.Initialize(): initializing python..."); PythonInitializer.Initialize(); Log.Trace("DebuggerHelper.Initialize(): python initialization done"); using (Py.GIL()) { Log.Trace("DebuggerHelper.Initialize(): starting..."); switch (debuggingType) { case DebuggingMethod.LocalCmdline: PythonEngine.RunSimpleString("import pdb; pdb.set_trace()"); break; case DebuggingMethod.VisualStudio: Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach..."); PythonEngine.RunSimpleString(@"import sys; import time; while not sys.gettrace(): time.sleep(0.25)"); break; case DebuggingMethod.PTVSD: Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678..."); PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()"); break; case DebuggingMethod.DebugPy: Log.Trace("DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678..."); PythonEngine.RunSimpleString("import debugpy; debugpy.listen(('0.0.0.0', 5678)); debugpy.wait_for_client()"); break; case DebuggingMethod.PyCharm: Log.Trace("DebuggerHelper.Initialize(): Attempting to connect to Pycharm PyDev debugger server..."); PythonEngine.RunSimpleString(@"import pydevd_pycharm; import time; count = 1 while count <= 10: try: pydevd_pycharm.settrace('localhost', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False) print('SUCCESS: Connected to local program') break except ConnectionRefusedError: pass try: pydevd_pycharm.settrace('host.docker.internal', port=6000, stdoutToServer=True, stderrToServer=True, suspend=False) print('SUCCESS: Connected to docker container') break except ConnectionRefusedError: pass print('\n') print('Failed: Ensure your PyCharm Debugger is actively waiting for a connection at port 6000!') print('Try ' + count.__str__() + ' out of 10') print('\n') count += 1 time.sleep(3)"); break; } Log.Trace("DebuggerHelper.Initialize(): started"); } } else if (language == Language.CSharp) { if (Debugger.IsAttached) { Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break."); Debugger.Break(); } else { Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach..."); while (!Debugger.IsAttached) { Thread.Sleep(250); } Log.Trace("DebuggerHelper.Initialize(): debugger attached"); } } else { throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}"); } }
private static Dictionary <string, TestCaseData[]> GetParameterlessFunctions() { // Initialize the Python engine and begin allow thread PythonInitializer.Initialize(); var functionsByType = new Dictionary <string, TestCaseData[]>(); using (Py.GIL()) { var module = PyModule.FromString("Test", @"import pandas from inspect import getmembers, isfunction, signature skipped = [ 'boxplot', 'hist', 'plot', # <- Graphics 'agg', 'aggregate', 'align', 'bool','combine', 'corrwith', 'dot', 'drop', 'equals', 'ewm', 'fillna', 'filter', 'groupby', 'join', 'mask', 'melt', 'pivot', 'pivot_table', 'reindex_like', 'rename', 'reset_index', 'select_dtypes', 'slice_shift', 'swaplevel', 'to_clipboard', 'to_excel', 'to_feather', 'to_gbq', 'to_hdf', 'to_list', 'tolist', 'to_parquet', 'to_period', 'to_pickle', 'to_sql', 'to_stata', 'to_timestamp', 'to_xarray', 'tshift', 'update', 'value_counts', 'where'] newPandas = int(pandas.__version__.split('.')[0]) >= 1 def getSimpleExceptionTestFunctions(cls): functions = [ 'describe', 'mode'] if not newPandas: functions.append('get_dtype_counts') functions.append('get_ftype_counts') for name, member in getmembers(cls): if isfunction(member) and name.startswith('to') and name not in skipped: functions.append(name) return functions DataFrame = getSimpleExceptionTestFunctions(pandas.DataFrame) Series = getSimpleExceptionTestFunctions(pandas.Series) skipped += set(DataFrame + Series) def getParameterlessFunctions(cls): functions = list() for name, member in getmembers(cls): if isfunction(member) and not name.startswith('_') and name not in skipped: parameters = signature(member).parameters count = 0 for parameter in parameters.values(): if parameter.default is parameter.empty: count += 1 else: break if count < 2: functions.append(name) return functions def getOtherParameterFunctions(cls): functions = list() for name, member in getmembers(pandas.DataFrame): if isfunction(member) and not name.startswith('_') and name not in skipped: parameters = signature(member).parameters for parameter in parameters.values(): if parameter.name == 'other': functions.append(name) break return functions DataFrameParameterless = getParameterlessFunctions(pandas.DataFrame) SeriesParameterless = getParameterlessFunctions(pandas.Series) DataFrameOtherParameter = getOtherParameterFunctions(pandas.DataFrame) SeriesOtherParameter = getOtherParameterFunctions(pandas.Series) "); Func <string, string, TestCaseData[]> converter = (s, p) => { var list = (List <string>)module.GetAttr(s).AsManagedObject(typeof(List <string>)); return(list.SelectMany(x => new[] { new TestCaseData($".{x}{p}", "'SPY'", true), new TestCaseData($".{x}{p}", "symbol", false), new TestCaseData($".{x}{p}", "str(symbol.ID)", false) } ).ToArray()); }; functionsByType.Add("DataFrame", converter("DataFrame", "()")); functionsByType.Add("Series", converter("Series", "()")); functionsByType.Add("DataFrameParameterless", converter("DataFrameParameterless", "()")); functionsByType.Add("SeriesParameterless", converter("SeriesParameterless", "()")); functionsByType.Add("DataFrameOtherParameter", converter("DataFrameOtherParameter", "(other)")); functionsByType.Add("SeriesOtherParameter", converter("SeriesOtherParameter", "(other)")); } return(functionsByType); }
/// <summary> /// Desktop/Local Get Next Task - Get task from the Algorithm folder of VS Solution. /// </summary> /// <returns></returns> public AlgorithmNodePacket NextJob(out string location) { location = GetAlgorithmLocation(); Log.Trace($"JobQueue.NextJob(): Selected {location}"); // check for parameters in the config var parameters = new Dictionary <string, string>(); var parametersConfigString = Config.Get("parameters"); if (parametersConfigString != string.Empty) { parameters = JsonConvert.DeserializeObject <Dictionary <string, string> >(parametersConfigString); } var controls = new Controls() { MinuteLimit = Config.GetInt("symbol-minute-limit", 10000), SecondLimit = Config.GetInt("symbol-second-limit", 10000), TickLimit = Config.GetInt("symbol-tick-limit", 10000), RamAllocation = int.MaxValue, MaximumDataPointsPerChartSeries = Config.GetInt("maximum-data-points-per-chart-series", 4000) }; if ((Language)Enum.Parse(typeof(Language), Config.Get("algorithm-language")) == Language.Python) { // Set the python path for loading python algorithms ("algorithm-location" config parameter) var pythonFile = new FileInfo(location); // PythonInitializer automatically adds the current working directory for us PythonInitializer.SetPythonPathEnvironmentVariable(new string[] { pythonFile.Directory.FullName }); } var algorithmId = Config.Get("algorithm-id", AlgorithmTypeName); //If this isn't a backtesting mode/request, attempt a live job. if (_liveMode) { var liveJob = new LiveNodePacket { Type = PacketType.LiveNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), Brokerage = Config.Get("live-mode-brokerage", PaperBrokerageTypeName), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), DataQueueHandler = Config.Get("data-queue-handler", DefaultDataQueueHandler), DataChannelProvider = Config.Get("data-channel-provider", DefaultDataChannelProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, DeployId = algorithmId, Parameters = parameters, Language = Language, Controls = controls }; try { // import the brokerage data for the configured brokerage var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); liveJob.BrokerageData = brokerageFactory.BrokerageData; } catch (Exception err) { Log.Error(err, $"Error resolving BrokerageData for live job for brokerage {liveJob.Brokerage}"); } return(liveJob); } //Default run a backtesting job. var backtestJob = new BacktestNodePacket(0, 0, "", new byte[] {}, "local") { Type = PacketType.BacktestNode, Algorithm = File.ReadAllBytes(AlgorithmLocation), HistoryProvider = Config.Get("history-provider", DefaultHistoryProvider), Channel = AccessToken, UserToken = AccessToken, UserId = UserId, ProjectId = ProjectId, Version = Globals.Version, BacktestId = algorithmId, Language = Language, Parameters = parameters, Controls = controls }; return(backtestJob); }