static void Main(string[] rawArgs) {
List<string> args = new List<string>(rawArgs);
args.Insert(0, Application.ExecutablePath);
PythonEngine engine = new PythonEngine();
engine.AddToPath(Path.GetDirectoryName(Application.ExecutablePath));
engine.Sys.argv = List.Make(args);
EngineModule engineModule = engine.CreateModule("__main__", new Dictionary<string, object>(), true);
engine.DefaultModule = engineModule;
string path = Path.Combine(
Path.GetDirectoryName(Application.ExecutablePath), "main.py");
engine.ExecuteFile(path);
}
protected virtual void ExecuteScript(PythonEngine engine, Assembly assembly, string script)
{
Stream stream = GetStream(assembly, script) ?? GetStream(assembly, _fixtureType, script);
if (stream == null)
{
throw new ApplicationException(
string.Format(
"Could not find script: {0} in the assembly resources, ensure the name is spelt correctly and that the C# fixture class and resource are in the same namespace.",
script));
}
using (stream)
using (var reader = new StreamReader(stream))
{
try
{
engine.Execute(reader.ReadToEnd());
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Failed to execute script: {0}", script), ex);
}
}
}
public void HookingToEvents()
{
PythonEngine engine = new PythonEngine();
ManualResetEvent waitHandle = new ManualResetEvent(false);
WebClient client = new WebClient();
List<string> results = new List<string>();
engine.Globals["client"] = client;
engine.Globals["results"] = results;
engine.Execute(
@"def storeResult(sender, e):
results.Add(e.Result)
e.UserState.Set()
# assign out storeResult function as an event handler for the client class
client.DownloadStringCompleted += storeResult
");
client.DownloadStringAsync(
new Uri(
"http://api.feedburner.com/awareness/1.0/GetFeedData?uri=http://feeds.feedburner.com/BitterCoder"),
waitHandle);
Assert.IsTrue(waitHandle.WaitOne(10000, false), "timeout occured after 10 seconds");
Assert.AreEqual(1, results.Count);
Assert.IsTrue(results[0].StartsWith("<?xml"));
}
protected virtual void ExecuteScripts(PythonEngine engine)
{
foreach (string script in _scripts)
{
ExecuteScript(engine, GetType().Assembly, script);
}
}
protected virtual void ExecuteScript(PythonEngine engine, Assembly assembly, string script)
{
Stream stream = assembly.GetManifestResourceStream(script);
if (stream == null)
{
stream = assembly.GetManifestResourceStream(GetType().Namespace + "." + script);
}
if (stream == null)
{
throw new ApplicationException(
string.Format("Could not find script: {0} in the assembly resources", script));
}
using (stream)
using (StreamReader reader = new StreamReader(stream))
{
try
{
engine.Execute(reader.ReadToEnd());
}
catch (Exception ex)
{
throw new ApplicationException(string.Format("Failed to execute script: {0}", script), ex);
}
}
}
static void Main(string[] args)
{
using (PythonEngine pe = new PythonEngine())
{
pe.ExecuteFile("test.py");
pe.ExecuteFile("test2.py");
object result = pe.Evaluate("get_courses()");
foreach (object item in result as IEnumerable<object>)
{
Console.WriteLine(item);
}
Hashtable list = pe.EvaluateAs<Hashtable>("get_courses2()");
foreach (string title in list.Values)
{
Console.WriteLine(title);
}
}
TestPythonUsingGenerics();
Console.WriteLine("�I���E�E�E");
}
// This is called when the python extension is first loaded.
public static void LoadExtension()
{
WriteOutput("IronPython-Mdbg Extension loaded");
g_python = new IronPython.Hosting.PythonEngine();
g_pythonInteractive = new PythonCommandLine(g_python);
string ver = g_python.GetType().Assembly.GetName().Version.ToString();
WriteOutput(MDbgOutputConstants.Ignore, "Binding against Iron Python version: " + ver);
// Get original load command, for use in python load command.
g_origLoadCmd = Shell.Commands.Lookup("load");
// Add Python extension commands to Shell.Commands.
MDbgAttributeDefinedCommand.AddCommandsFromType(Shell.Commands, typeof(PythonExt));
// Add the current directory to the python engine search path.
g_python.AddToPath(Environment.CurrentDirectory);
// Tell Python about some key objects in Mdbg. Python can then reflect over these objects.
// These variables live at some special "main" scope. Python Modules imported via python "import" command
// can't access them. Use PythonEngine.ExecuteFile() to import files such that they can access these vars.
g_python.Globals.Add("CommandBase", IronPython.Runtime.Types.ReflectedType.FromType(typeof(CommandBase)));
// Hook input + output. This is redirecting pythons 'sys.stdout, sys.stdin, sys.stderr'
// This connects python's sys.stdout --> Stream --> Mdbg console.
MDbgStream s = new MDbgStream();
// New codepaths for IronPython 1.0
g_python.SetStandardInput(s);
g_python.SetStandardOutput(s);
g_python.SetStandardError(s);
}
static IronPythonExtension()
{
_engine = new PythonEngine();
_engine.Import("clr");
AddReference("System");
ImportNamespace("System");
}
public void StatementsToStronglyTypedDelegate()
{
PythonEngine engine = new PythonEngine();
List<string> parameters = new List<string>(new string[] {"age"});
Converter<int, string> converter =
engine.CreateMethod<Converter<int, string>>("return str(age+1)", parameters);
Assert.AreEqual("11", converter(10));
}
public void Setup()
{
PythonEngine engine = new PythonEngine();
PrepareEngine(engine);
ExecuteScripts(engine);
_engine = engine;
}
protected virtual void PrepareEngine(PythonEngine engine)
{
engine.Import("clr");
engine.Import("System");
engine.LoadAssembly(typeof (TestAttribute).Assembly);
engine.Execute("from NUnit.Framework import *");
ExecuteScript(engine, typeof (AbstractPythonSuite).Assembly, "IronPythonLib.TestingSupport.py");
}
public static void Main(string[] args) {
PythonEngine pe = new PythonEngine();
if (args.Length > 0) {
try {
Console.WriteLine(pe.Evaluate(args[0]));
} catch {
Console.WriteLine("Error");
}
} else Console.WriteLine("eval <expression>");
}
void InitializePythonEngine() {
engine = new PythonEngine();
engine.Import("site");
engine.Globals["dateentries"] = dateEntries;
try {
engine.Execute("print y");
} catch (Exception exc) {
MessageBox.Show(exc.ToString());
}
}
public PythonCommandLine(PythonEngine pEngine)
{
engine = pEngine;
if (typeof(IMDbgIO3).IsAssignableFrom(CommandBase.Shell.IO.GetType()))
{
console = new MDbgSuperConsole(engine, false);
}
else
{
console = new MDbgBasicConsole();
}
}
void InitializePythonEngine() {
engine = new PythonEngine();
engine.Import("site");
engine.Globals["dateentries"] = dateEntries;
try {
engine.Execute("print 'Hello world!'");
} catch (IronPython.Runtime.Exceptions.PythonNameErrorException exc) {
MessageBox.Show("IronPython engine complained: " + exc.Message);
} catch (Exception exc) {
MessageBox.Show("Unexpected exception " + exc.ToString() + "\n Shutting down!");
System.Environment.Exit(-1);
}
}
public Form1()
{
EngineOptions options = new EngineOptions();
eng = new PythonEngine(options);
eng.Import("site");
System.IO.FileStream fs = new System.IO.FileStream("scripting-log.txt", System.IO.FileMode.Create);
eng.SetStandardOutput(fs);
eng.SetStandardError(fs);
mod1 = eng.CreateModule("mr", false);
InitializeComponent();
}
public PythonFixtureBuilder(PythonEngine engine)
{
if (engine == null) throw new ArgumentNullException("engine");
_engine = engine;
_findTestMethods =
_engine.CreateMethod<Func<List, OldClass, string>>(@"return getTestCaseNames(testCaseClass, prefix)",
Params("testCaseClass", "prefix"));
_isNUnitFixture =
_engine.CreateMethod<Func<bool, OldClass>>(
string.Format("return (obj != {0}) and (issubclass(obj, {0}))", FixtureBaseClass), Params("obj"));
_createInstance = _engine.CreateMethod<Func<OldInstance, OldClass>>("return toCreate()", Params("toCreate"));
}
public void SetPath(PythonEngine engine, String rootPath)
{
engine.AddToPath(rootPath);
// This path entry allows the framework modules to be loaded from
// rootPath/bin/Lib
engine.Sys.prefix = Path.Combine(rootPath, "bin");
string libpath = Path.Combine(Path.Combine(rootPath, "bin"), "Lib");
engine.AddToPath(libpath);
// This path entry allows the application specific modules to be
// loaded from rootPath/wsgiapp. This allows seperation of the
// the framework and application.
engine.AddToPath(Path.Combine(rootPath, "wsgiapp"));
}
private static void TestPythonUsingGenerics()
{
using (PythonEngine pe = new PythonEngine())
{
pe.ExecuteFile("test3.py");
IStore store = pe.EvaluateAs<IStore>("StoreImpl()");
List<Course> result = store.GetCourses();
foreach(Course item in result)
{
Console.WriteLine("id:{0}, title:{1}", item.Id, item.Title);
}
}
}
public void InitializePythonEngine()
{
try
{
Thread.CurrentThread.CurrentCulture = clsGlobal.MyCulture;
this.PythonEngineOutput = new ListBoxStream(ref this.pythonOutput);
this.pythonOutput.SelectedIndex = this.pythonOutput.Items.Add("Initializing Python Engine...");
EngineOptions engineOptions = new EngineOptions();
engineOptions.ClrDebuggingEnabled = true;
this.engine = new PythonEngine(engineOptions);
this.engine.AddToPath(ConfigurationManager.AppSettings["InstalledDirectory"] + @"\PythonStdLib");
this.engine.AddToPath(ConfigurationManager.AppSettings["InstalledDirectory"] + @"\scripts");
this.engine.AddToPath(ConfigurationManager.AppSettings["InstalledDirectory"] + @"\scripts\core");
this.engine.AddToPath(Environment.CurrentDirectory);
this.engine.AddToPath(Path.GetDirectoryName(Application.ExecutablePath));
this.engine.Import("Site");
this.engine.Import("System");
this.engine.Import("System.Diagnostics");
this.engine.Import("sys");
this.engine.Import("nt");
this.engine.Import("os");
this.engine.Import("shutil");
this.engine.Import("ConfigParser");
this.engine.Import("time");
this.engine.Import("array");
this.engine.Import("pdb");
this.engine.Execute("from System import Array");
this.engine.Import("clr");
this.engine.Execute("clr.AddReference('System.Messaging')");
this.engine.Execute("from System.Messaging import MessageQueue");
this.engine.Execute("MessageQueue.EnableConnectionCache = False");
this.engine.Import("clr");
this.engine.Execute("clr.AddReferenceByPartialName('CommonClassLibrary')");
this.engine.Execute("from CommonClassLibrary import *");
this.engine.SetStandardOutput(this.PythonEngineOutput);
this.engine.SetStandardError(this.PythonEngineOutput);
this.pythonOutput.SelectedIndex = this.pythonOutput.Items.Add("Python Engine Running.");
this.engine.Execute("import clr");
this.engine.Execute("clr.AddReferenceByPartialName('SiRFLive')");
this.engine.Execute("from SiRFLive import *");
this.engine.Execute("print dir()");
this.engine.Execute("nt.chdir(\"..\\scripts\")");
}
catch (Exception exception)
{
this.pythonOutput.SelectedIndex = this.pythonOutput.Items.Add("Python Engine Failed to Start Properly: " + exception.Message);
}
}
public void ExtendingClass()
{
PythonEngine engine = new PythonEngine();
engine.Import("clr");
engine.LoadAssembly(typeof (IStringTransformer).Assembly);
engine.ExecuteToConsole("from IronPythonLib import *");
engine.Execute(
@"class MyTransformer(IStringTransformer):
def Transform(self, input):
return input + "" is now transformed""");
IStringTransformer transformer = engine.EvaluateAs<IStringTransformer>("MyTransformer()");
Assert.AreEqual("input is now transformed", transformer.Transform("input"));
}
void InitializePythonEngine() {
engine = new PythonEngine();
engine.Import("site");
System.IO.FileStream fs = new System.IO.FileStream("scripting-log.txt",
System.IO.FileMode.Create);
engine.SetStandardOutput(fs);
engine.SetStandardError(fs);
engine.Globals["dateentries"] = dateEntries;
try {
engine.Execute("print 'Hello world!'");
} catch (IronPython.Runtime.Exceptions.PythonNameErrorException exc) {
MessageBox.Show("IronPython engine complained: " + exc.Message);
} catch (Exception exc) {
MessageBox.Show("Unexpected exception " + exc.ToString() + "\n Shutting down!");
System.Environment.Exit(-1);
}
}
static WSGIHandler()
{
String reloadVal = ConfigurationManager.AppSettings["WSGIReloadIronPython"];
if (reloadVal == null || reloadVal.ToLower() != "true")
{
engine = CreateIronPythonEngine();
}
else
{
engine = null;
}
moduleName = ConfigurationManager.AppSettings["WSGIApplicationModule"];
if (moduleName == null || moduleName == "")
{
moduleName = DEFAULT_MODULE;
}
}
protected bool Eval(string source, object propertyValue)
{
string typeName = this.GetType().Name;
if (_engineCache.ContainsKey(typeName))
_engine = _engineCache[this.GetType().Name] ;
else
_engine = _engineCache[typeName] = new PythonEngine();
_engine.Execute("from System import *");
_engine.Globals["property_value"] = propertyValue;
_engine.Globals["result"] = true;
_engine.Execute(source);
bool result = (bool)_engine.Globals["result"];
_engine.Shutdown();
return result;
}
/// <summary>
/// Creates a new PythonInterpreter
/// </summary>
public PythonInterpreter(Game1 game, SpriteFont font) : base((Game)game)
{
this.PythonEngine = new PythonEngine();
this.PythonOutput = new MemoryStream();
this.PythonEngine.SetStandardOutput(PythonOutput);
this.ASCIIEncoder = new ASCIIEncoding();
ClrModule clr = this.PythonEngine.Import("clr") as ClrModule;
clr.AddReference("Microsoft.Xna.Framework");
clr.AddReference("Microsoft.Xna.Framework.Game");
this.PythonEngine.Execute("from Microsoft.Xna.Framework import *");
this.PythonEngine.Execute("from Microsoft.Xna.Framework.Graphics import *");
this.PythonEngine.Execute("from Microsoft.Xna.Framework.Content import *");
multi = "";
Console = new XnaConsoleComponent(game, font);
game.Components.Add(Console);
Console.Prompt(Prompt, Execute);
AddGlobal("Console", Console);
}
public void Dispose()
{
PythonEngine.EndAllowThreads(_threadState);
PythonEngine.Shutdown();
}
void InitializePythonEngine()
{
if (Script.mainEngine == null || Script.savedGlobals == null)
{
Script.mainEngine = new PythonEngine();
Script.savedGlobals = new List<string>();
}
else
{
foreach (string g in Script.savedGlobals)
Script.mainEngine.Globals.Remove(g);
Script.savedGlobals.Clear();
}
engine = Script.mainEngine;
string script = Encoding.UTF8.GetString(Properties.Resources.Script);
engine.Execute(script);
}
protected override void Execute(CodeActivityContext context)
{
var code = Code.Get(context);
var language = Language.Get(context);
var variables = new Dictionary <string, Type>();
var variablevalues = new Dictionary <string, object>();
var vars = context.DataContext.GetProperties();
foreach (dynamic v in vars)
{
Type rtype = v.PropertyType as Type;
var value = v.GetValue(context.DataContext);
if (rtype == null && value != null)
{
rtype = value.GetType();
}
if (rtype == null)
{
continue;
}
variables.Add(v.DisplayName, rtype);
variablevalues.Add(v.DisplayName, value);
}
string sourcecode = code;
if (namespaces == null)
{
throw new Exception("InvokeCode is missing namespaces, please open workflow in designer and save changes");
}
if (language == "VB")
{
sourcecode = GetVBHeaderText(variables, "Expression", namespaces) + code + GetVBFooterText();
}
if (language == "C#")
{
sourcecode = GetCSharpHeaderText(variables, "Expression", namespaces) + code + GetCSharpFooterText();
}
if (language == "PowerShell")
{
if (runspace == null)
{
runspace = RunspaceFactory.CreateRunspace();
runspace.Open();
}
using (var pipeline = runspace.CreatePipeline())
{
Command cmd = new Command(sourcecode, true);
foreach (var parameter in variablevalues)
{
// cmd.Parameters.Add(parameter.Key, parameter.Value);
runspace.SessionStateProxy.SetVariable(parameter.Key, parameter.Value);
}
pipeline.Commands.Add(cmd);
var res = pipeline.Invoke();
foreach (var o in res)
{
if (o != null)
{
Log.Output(o.ToString());
}
}
foreach (dynamic v in vars)
{
var value = runspace.SessionStateProxy.GetVariable(v.DisplayName);
var myVar = context.DataContext.GetProperties().Find(v.DisplayName, true);
if (myVar != null && value != null && value != "")
{
//var myValue = myVar.GetValue(context.DataContext);
myVar.SetValue(context.DataContext, value);
}
}
PipelineOutput.Set(context, res);
}
return;
}
if (language == "AutoHotkey")
{
AppDomain Temporary = null;
try
{
AppDomainSetup domaininfo = new AppDomainSetup();
domaininfo.ApplicationBase = System.Environment.CurrentDirectory;
System.Security.Policy.Evidence adevidence = AppDomain.CurrentDomain.Evidence;
Temporary = AppDomain.CreateDomain("Temporary", adevidence, domaininfo);
Temporary.AssemblyResolve += AHKProxy.CurrentDomain_AssemblyResolve;
//var ahk = (AutoHotkey.Interop.AutoHotkeyEngine)Temporary.CreateInstanceAndUnwrap("sharpAHK, Version=1.0.0.5, Culture=neutral, PublicKeyToken=null", "AutoHotkey.Interop.AutoHotkeyEngine");
Type type = typeof(AHKProxy);
var ahk = (AHKProxy)Temporary.CreateInstanceAndUnwrap(type.Assembly.FullName, type.FullName);
foreach (var parameter in variablevalues)
{
if (parameter.Value == null)
{
continue;
}
ahk.SetVar(parameter.Key, parameter.Value.ToString());
}
ahk.ExecRaw(code);
foreach (dynamic v in vars)
{
var value = ahk.GetVar(v.DisplayName);
PropertyDescriptor myVar = context.DataContext.GetProperties().Find(v.DisplayName, true);
if (myVar != null && value != null && value != "")
{
if (myVar.PropertyType == typeof(string))
{
myVar.SetValue(context.DataContext, value);
}
else if (myVar.PropertyType == typeof(int))
{
myVar.SetValue(context.DataContext, int.Parse(value.ToString()));
}
else if (myVar.PropertyType == typeof(bool))
{
myVar.SetValue(context.DataContext, bool.Parse(value.ToString()));
}
else
{
Log.Information("Ignorering variable " + v.DisplayName + " of type " + myVar.PropertyType.FullName);
}
}
}
}
catch (Exception ex)
{
Log.Error(ex.ToString());
throw;
}
finally
{
if (Temporary != null)
{
AppDomain.Unload(Temporary);
}
}
return;
}
if (language == "Python")
{
try
{
GenericTools.RunUI(() =>
{
IntPtr lck = IntPtr.Zero;
try
{
lck = PythonEngine.AcquireLock();
using (var scope = Py.CreateScope())
{
foreach (var parameter in variablevalues)
{
PyObject pyobj = parameter.Value.ToPython();
scope.Set(parameter.Key, pyobj);
}
try
{
PythonEngine.RunSimpleString(@"
import sys
from System import Console
class output(object):
def write(self, msg):
Console.Out.Write(msg)
def writelines(self, msgs):
for msg in msgs:
Console.Out.Write(msg)
def flush(self):
pass
def close(self):
pass
sys.stdout = sys.stderr = output()
");
}
catch (Exception ex)
{
Log.Debug(ex.ToString());
}
scope.Exec(code);
foreach (var parameter in variablevalues)
{
PyObject pyobj = scope.Get(parameter.Key);
if (pyobj == null)
{
continue;
}
PropertyDescriptor myVar = context.DataContext.GetProperties().Find(parameter.Key, true);
if (myVar.PropertyType == typeof(string))
{
myVar.SetValue(context.DataContext, pyobj.ToString());
}
else if (myVar.PropertyType == typeof(int))
{
myVar.SetValue(context.DataContext, int.Parse(pyobj.ToString()));
}
else if (myVar.PropertyType == typeof(bool))
{
myVar.SetValue(context.DataContext, bool.Parse(pyobj.ToString()));
}
else
{
Log.Information("Ignorering variable " + parameter.Key + " of type " + myVar.PropertyType.FullName);
}
}
}
//lck = PythonEngine.AcquireLock();
//PythonEngine.Exec(code);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
PythonEngine.ReleaseLock(lck);
}
});
//using (Python.Runtime.Py.GIL())
//{
// IntPtr lck = Python.Runtime.PythonEngine.AcquireLock();
// Python.Runtime.PythonEngine.Exec(code);
// Python.Runtime.PythonEngine.ReleaseLock(lck);
// //// create a Python scope
// //using (var scope = Python.Runtime.Py.CreateScope())
// //{
// // //// convert the Person object to a PyObject
// // //PyObject pyPerson = person.ToPython();
// // // create a Python variable "person"
// // // scope.Set("person", pyPerson);
// // // the person object may now be used in Python
// // // string code = "fullName = person.FirstName + ' ' + person.LastName";
// // scope.Exec(code);
// //}
//}
}
catch (Exception)
{
throw;
}
finally
{
try
{
// Python.Runtime.PythonEngine.Shutdown();
}
catch (Exception ex)
{
Log.Error(ex.ToString());
}
}
return;
}
var assemblyLocations = GetAssemblyLocations();
CompileAndRun(language, sourcecode, assemblyLocations.ToArray(), variablevalues, context);
}
public DnaFunctions(string scriptPath)
{
this.ScriptPath = scriptPath;
engine = new PythonEngine(this.ScriptPath);
}
public ClientCore()
{
s_clientCore = this;
//Ansi.SendAnsiInit();
m_synchronizedInvoke = new SynchronizedInvoke();
// Services
m_baseServicesDispatcher = new BaseServicesDispatcher();
// Init console
m_textConsole = new TextConsole();
ChiConsole.SetChiConsole(m_textConsole);
// Initialize ironpython
IronPython.Compiler.Options.GenerateModulesAsSnippets = true;
m_pythonEngine = new PythonEngine();
ChiPythonStream s = new ChiPythonStream();
m_pythonEngine.SetStandardOutput(s);
m_pythonEngine.SetStandardError(s);
m_pythonEngine.SetStandardInput(s);
//m_pythonEngine.AddToPath(Environment.CurrentDirectory);
//m_pythonEngine.AddToPath(Application.StartupPath + "/lib");
#if DEBUG
m_pythonEngine.AddToPath(@"../../../scripts/lib");
#endif
m_pythonEngine.LoadAssembly(typeof(TriggerManager).Assembly); // load BatClientBase
m_pythonEngine.LoadAssembly(typeof(System.Drawing.Bitmap).Assembly); // load System.Drawing
m_pythonEngine.LoadAssembly(typeof(System.Windows.Forms.Keys).Assembly); // load System.Windows.Forms
// Network
m_telnet = new Telnet();
m_telnet.connectEvent += new Telnet.ConnectDelegate(_ConnectEvent);
m_telnet.disconnectEvent += new Telnet.DisconnectDelegate(_DisconnectEvent);
m_telnet.receiveEvent += new Telnet.ReceiveDelegate(_ReceiveEvent);
m_telnet.promptEvent += new Telnet.PromptDelegate(_PromptEvent);
m_telnet.telnetEvent += new Telnet.TelnetDelegate(_TelnetEvent);
m_netPipe = UnixPipes.CreatePipes();
m_commandManager = new CommandManager(m_baseServicesDispatcher);
AddBuiltinCommands();
m_triggerManager = new TriggerManager(m_baseServicesDispatcher);
m_keyManager = new KeyManager(m_baseServicesDispatcher);
m_hiliteManager = new HiliteManager(m_triggerManager);
PythonInterface.Initialize(m_baseServicesDispatcher, m_triggerManager, m_commandManager, this,
m_textConsole, m_pythonEngine, m_keyManager, m_hiliteManager);
// run init script
Version currentVersion = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
Version baseVersion = System.Reflection.Assembly.GetAssembly(typeof(Telnet)).GetName().Version;
ChiConsole.WriteLine("Chiroptera version {0} (base {1})", currentVersion.ToString(2), baseVersion.ToString(2));
ChiConsole.WriteLine("Using {0}", PythonEngine.VersionString);
try
{
#if DEBUG
PythonInterface.RunScript(Path.GetFullPath("../../../scripts/std/init_std.bc"));
#else
PythonInterface.RunScript(Path.Combine(Environment.CurrentDirectory, "std/init_std.bc"));
#endif
}
catch (Exception e)
{
ChiConsole.WriteError("Error running init_std.bc", e);
}
/*
* m_pythonEngine.Import("site");
*
* try
* {
* m_pythonEngine.ExecuteFile("init.py");
* }
* catch (Exception e)
* {
* ChiConsole.WriteError("Eval failed", e);
* }
*/
}
public void PostInitialise()
{
m_log.Info("[PYTHON] Initialising IronPython engine.");
m_python = new PythonEngine();
m_python.AddToPath(System.Environment.CurrentDirectory + System.IO.Path.DirectorySeparatorChar + "Python");
}
/// <summary>
/// Initializes an instance of <see cref="PandasData"/>
/// </summary>
public PandasData(object data)
{
if (_pandas == null)
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
// this python Remapper class will work as a proxy and adjust the
// input to its methods using the provided 'mapper' callable object
_remapperFactory = PythonEngine.ModuleFromString("remapper",
@"import wrapt
import pandas
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
originalConcat = pandas.concat
def PandasConcatWrapper(objs, axis=0, join='outer', join_axes=None, ignore_index=False, keys=None, levels=None, names=None, verify_integrity=False, sort=None, copy=True):
return Remapper(originalConcat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy))
pandas.concat = PandasConcatWrapper
class Remapper(wrapt.ObjectProxy):
def __init__(self, wrapped):
super(Remapper, self).__init__(wrapped)
def _self_mapper(self, key):
''' Our remapping method.
Originally implemented in C# but some cases were not working correctly and using Py did the trick.
This is to improve user experience, they can use Symbol as key and we convert it to string for pandas
'''
if isinstance(key, Symbol):
return str(key.ID)
# this is the most normal use case
if isinstance(key, str):
tupleResult = SymbolCache.TryGetSymbol(key, None)
if tupleResult[0]:
return str(tupleResult[1].ID)
# If the key is a tuple, convert the items
if isinstance(key, tuple) and 2 >= len(key) >= 1:
item0 = self._self_mapper(key[0])
return (item0,) if len(key) == 1 else \
(item0, self._self_mapper(key[1]))
return key
def _wrap_if_pandas_object(self, result):
''' For these cases we wrap the result too. Can't apply the wrap around all
results because it causes issues in pandas for some of our use cases
specifically pandas timestamp type'''
if isinstance(result, (pandas.Series, pandas.DataFrame, pandas.Index, pandas.MultiIndex)):
return Remapper(result)
return result
def __contains__(self, key):
key = self._self_mapper(key)
return self.__wrapped__.__contains__(key)
def __getitem__(self, name):
name = self._self_mapper(name)
result = self.__wrapped__.__getitem__(name)
return self._wrap_if_pandas_object(result)
def __setitem__(self, name, value):
name = self._self_mapper(name)
return self.__wrapped__.__setitem__(name, value)
def __delitem__(self, name):
name = self._self_mapper(name)
return self.__wrapped__.__delitem__(name)
def __str__(self):
return self.__wrapped__.__str__()
def __repr__(self):
return self.__wrapped__.__repr__()
def abs(self):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.abs.html'''
result = self.__wrapped__.abs()
return Remapper(result)
def add(self, other, axis='columns', level=None, fill_value=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.add.html'''
result = self.__wrapped__.add(other, axis=axis, level=level, fill_value=fill_value)
return Remapper(result)
def add_prefix(self, prefix):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.add_prefix.html'''
result = self.__wrapped__.add_prefix(prefix)
return Remapper(result)
def add_suffix(self, suffix):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.add_suffix.html'''
result = self.__wrapped__.add_suffix(suffix)
return Remapper(result)
def agg(self, func, axis=0, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.agg.html'''
result = self.__wrapped__.agg(func, axis=axis, *args, **kwargs)
return self._wrap_if_pandas_object(result)
def aggregate(self, func, axis=0, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.aggregate.html'''
result = self.__wrapped__.aggregate(func, axis=axis, *args, **kwargs)
return self._wrap_if_pandas_object(result)
def align(self, other, join='outer', axis=None, level=None, copy=True, fill_value=None, method=None, limit=None, fill_axis=0, broadcast_axis=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.align.html'''
df1, df2 = self.__wrapped__.align(other, join=join, axis=axis, level=level, copy=copy, fill_value=fill_value, method=method, limit=limit, fill_axis=fill_axis, broadcast_axis=broadcast_axis)
return self._wrap_if_pandas_object(df1), self._wrap_if_pandas_object(df2)
def all(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.all.html'''
result = self.__wrapped__.all(axis=axis, bool_only=bool_only, skipna=skipna, level=level, **kwargs)
return Remapper(result)
def any(self, axis=0, bool_only=None, skipna=True, level=None, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.any.html'''
result = self.__wrapped__.any(axis=axis, bool_only=bool_only, skipna=skipna, level=level, **kwargs)
return Remapper(result)
def append(self, other, ignore_index=False, verify_integrity=False, sort=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.append.html'''
result = self.__wrapped__.append(other, ignore_index=ignore_index, verify_integrity=verify_integrity, sort=sort)
return Remapper(result)
def apply(self, func, axis=0, broadcast=None, raw=False, reduce=None, result_type=None, args=(), **kwds):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.apply.html'''
result = self.__wrapped__.apply(func, axis=axis, broadcast=broadcast, raw=raw, reduce=reduce, result_type= result_type, args=args, **kwds)
return Remapper(result)
def applymap(self, func):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.applymap.html'''
result = self.__wrapped__.applymap(func)
return Remapper(result)
def asfreq(self, freq, method=None, how=None, normalize=False, fill_value=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.asfreq.html'''
result = self.__wrapped__.asfreq(freq, method=method, how=how, normalize=normalize, fill_value=fill_value)
return self._wrap_if_pandas_object(result)
def asof(self, where, subset=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.asof.html'''
result = self.__wrapped__.asof(where, subset=subset)
return self._wrap_if_pandas_object(result)
def assign(self, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.assign.html'''
result = self.__wrapped__.assign(**kwargs)
return Remapper(result)
def astype(self, dtype, copy=True, errors='raise', **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.astype.html'''
result = self.__wrapped__.astype(dtype, copy=copy, errors=errors, **kwargs)
return self._wrap_if_pandas_object(result)
@property
def at(self):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.at.html'''
return Remapper(self.__wrapped__.at)
def at_time(self, time, asof=False, axis=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.at_time.html'''
result = self.__wrapped__.at_time(time, asof=asof, axis=axis)
return Remapper(result)
@property
def axes(self):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.axes.html'''
return [self._wrap_if_pandas_object(result) for result in self.__wrapped__.axes]
def between_time(self, start_time, end_time, include_start=True, include_end=True, axis=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.between_time.html'''
result = self.__wrapped__.between_time(start_time, end_time, include_start=include_start, include_end=include_end, axis=axis)
return Remapper(result)
def bfill(self, axis=None, inplace=False, limit=None, downcast=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.bfill.html'''
result = self.__wrapped__.bfill(axis=axis, inplace=inplace, limit=limit, downcast=downcast)
return Remapper(result)
def clip(self, lower=None, upper=None, axis=None, inplace=False, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.clip.html'''
result = self.__wrapped__.clip(lower=lower, upper=upper, axis=axis, inplace=inplace, *args, **kwargs)
return Remapper(result)
def clip_lower(self, threshold, axis=None, inplace=False):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.clip_lower.html'''
result = self.__wrapped__.clip_lower(threshold, axis=axis, inplace=inplace)
return Remapper(result)
def clip_upper(self, threshold, axis=None, inplace=False):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.clip_upper.html'''
result = self.__wrapped__.clip_upper(threshold, axis=axis, inplace=inplace)
return Remapper(result)
@property
def columns(self):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.columns.html'''
return Remapper(self.__wrapped__.columns)
def combine(self, other, func, fill_value=None, overwrite=True):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.combine.html'''
result = self.__wrapped__.combine(other, func, fill_value=fill_value, overwrite=overwrite)
return Remapper(result)
def combine_first(self, other):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.combine_first.html'''
result = self.__wrapped__.combine_first(other)
return Remapper(result)
def compound(self, axis=None, skipna=None, level=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.compound.html'''
result = self.__wrapped__.compound(axis=axis, skipna=skipna, level=level)
return Remapper(result)
def copy(self, deep=True):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.copy.html'''
result = self.__wrapped__.copy(deep=deep)
return Remapper(result)
def corr(self, method='pearson', min_periods=1):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.corr.html'''
result = self.__wrapped__.corr(method=method, min_periods=min_periods)
return Remapper(result)
def corrwith(self, other, axis=0, drop=False, method='pearson'):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.corrwith.html'''
result = self.__wrapped__.corrwith(other, axis=axis, drop=drop, method=method)
return Remapper(result)
def count(self, axis=0, level=None, numeric_only=False):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.count.html'''
result = self.__wrapped__.count(axis=axis, level=level, numeric_only=numeric_only)
return Remapper(result)
def cov(self, min_periods=None):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.cov.html'''
result = self.__wrapped__.cov(min_periods=min_periods)
return Remapper(result)
def cummax(self, axis=None, skipna=True, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.cummax.html'''
result = self.__wrapped__.cummax(axis=axis, skipna=skipna, *args, **kwargs)
return Remapper(result)
def cummin(self, axis=None, skipna=True, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.cummin.html'''
result = self.__wrapped__.cummin(axis=axis, skipna=skipna, *args, **kwargs)
return Remapper(result)
def cumprod(self, axis=None, skipna=True, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.cumprod.html'''
result = self.__wrapped__.cumprod(axis=axis, skipna=skipna, *args, **kwargs)
return Remapper(result)
def cumsum(self, axis=None, skipna=True, *args, **kwargs):
'''https://pandas.pydata.org/pandas-docs/version/0.25.3/reference/api/pandas.DataFrame.cumsum.html'''
result = self.__wrapped__.cumsum(axis=axis, skipna=skipna, *args, **kwargs)
return Remapper(result)
def get(self, key, default=None):
key = self._self_mapper(key)
return self.__wrapped__.get(key=key, default=default)
def join(self, other, on=None, how='left', lsuffix='', rsuffix='', sort=False):
result = self.__wrapped__.join(other=other, on=on, how=how, lsuffix=lsuffix, rsuffix=rsuffix, sort=sort)
return Remapper(result)
def merge(self, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None):
result = self.__wrapped__.merge(right=right, how=how, on=on, left_on=left_on, right_on=right_on, left_index=left_index, right_index=right_index, sort=sort, suffixes=suffixes, copy=copy, indicator=indicator, validate=validate)
return Remapper(result)
# we wrap the result of 'unstack'
def unstack(self, level=-1, fill_value=None):
result = self.__wrapped__.unstack(level=level, fill_value=fill_value)
return Remapper(result)
# we wrap the result and input of 'xs'
def xs(self, key, axis=0, level=None, drop_level=True):
key = self._self_mapper(key)
result = self.__wrapped__.xs(key=key, axis=axis, level=level, drop_level=drop_level)
return Remapper(result)
# we wrap 'loc' to cover the: df.loc['SPY'] case
@property
def loc(self):
return Remapper(self.__wrapped__.loc)
@property
def ix(self):
return Remapper(self.__wrapped__.ix)
@property
def iloc(self):
return Remapper(self.__wrapped__.iloc)
@property
def index(self):
return Remapper(self.__wrapped__.index)
@property
def levels(self):
return Remapper(self.__wrapped__.levels)
# we wrap the following properties so that when 'unstack', 'loc' are called we wrap them
@property
def open(self):
return Remapper(self.__wrapped__.open)
@property
def high(self):
return Remapper(self.__wrapped__.high)
@property
def close(self):
return Remapper(self.__wrapped__.close)
@property
def low(self):
return Remapper(self.__wrapped__.low)
@property
def lastprice(self):
return Remapper(self.__wrapped__.lastprice)
@property
def volume(self):
return Remapper(self.__wrapped__.volume)
@property
def askopen(self):
return Remapper(self.__wrapped__.askopen)
@property
def askhigh(self):
return Remapper(self.__wrapped__.askhigh)
@property
def asklow(self):
return Remapper(self.__wrapped__.asklow)
@property
def askclose(self):
return Remapper(self.__wrapped__.askclose)
@property
def askprice(self):
return Remapper(self.__wrapped__.askprice)
@property
def asksize(self):
return Remapper(self.__wrapped__.asksize)
@property
def quantity(self):
return Remapper(self.__wrapped__.quantity)
@property
def suspicious(self):
return Remapper(self.__wrapped__.suspicious)
@property
def bidopen(self):
return Remapper(self.__wrapped__.bidopen)
@property
def bidhigh(self):
return Remapper(self.__wrapped__.bidhigh)
@property
def bidlow(self):
return Remapper(self.__wrapped__.bidlow)
@property
def bidclose(self):
return Remapper(self.__wrapped__.bidclose)
@property
def bidprice(self):
return Remapper(self.__wrapped__.bidprice)
@property
def bidsize(self):
return Remapper(self.__wrapped__.bidsize)
@property
def exchange(self):
return Remapper(self.__wrapped__.exchange)
@property
def openinterest(self):
return Remapper(self.__wrapped__.openinterest)
").GetAttr("Remapper");
}
}
var enumerable = data as IEnumerable;
if (enumerable != null)
{
foreach (var item in enumerable)
{
data = item;
}
}
var type = data.GetType();
IsCustomData = type.Namespace != typeof(Bar).Namespace;
_members = new List <MemberInfo>();
_symbol = ((IBaseData)data).Symbol;
if (_symbol.SecurityType == SecurityType.Future)
{
Levels = 3;
}
if (_symbol.SecurityType == SecurityType.Option)
{
Levels = 5;
}
var columns = new HashSet <string>
{
"open", "high", "low", "close", "lastprice", "volume",
"askopen", "askhigh", "asklow", "askclose", "askprice", "asksize", "quantity", "suspicious",
"bidopen", "bidhigh", "bidlow", "bidclose", "bidprice", "bidsize", "exchange", "openinterest"
};
if (IsCustomData)
{
var keys = (data as DynamicData)?.GetStorageDictionary().ToHashSet(x => x.Key);
// C# types that are not DynamicData type
if (keys == null)
{
if (_membersByType.TryGetValue(type, out _members))
{
keys = _members.ToHashSet(x => x.Name.ToLowerInvariant());
}
else
{
var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property).ToList();
var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
foreach (var duplicateKey in duplicateKeys)
{
throw new ArgumentException($"PandasData.ctor(): More than one \'{duplicateKey}\' member was found in \'{type.FullName}\' class.");
}
// If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys
keys = members.ToHashSet(x => x.Name.ToLowerInvariant());
keys.ExceptWith(_baseDataProperties);
keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type));
keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type));
keys.ExceptWith(GetPropertiesNames(typeof(Tick), type));
keys.Add("value");
_members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList();
_membersByType.TryAdd(type, _members);
}
}
columns.Add("value");
columns.UnionWith(keys);
}
_series = columns.ToDictionary(k => k, v => Tuple.Create(new List <DateTime>(), new List <object>()));
}
/// <summary>
/// Initializes an instance of <see cref="PandasData"/>
/// </summary>
public PandasData(object data)
{
if (_pandas == null)
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
// this python Remapper class will work as a proxy and adjust the
// input to its methods using the provided 'mapper' callable object
_remapperFactory = PythonEngine.ModuleFromString("remapper",
@"import wrapt
import pandas
from clr import AddReference
AddReference(""QuantConnect.Common"")
from QuantConnect import *
class Remapper(wrapt.ObjectProxy):
def __init__(self, wrapped):
super(Remapper, self).__init__(wrapped)
# Our remapping method. Originally implemented in C# but some cases were not working
# correctly and using Py did the trick
def _self_mapper(self, key):
# this is the most normal use case
if isinstance(key, str):
tupleResult = SymbolCache.TryGetSymbol(key, None)
if tupleResult[0]:
return str(tupleResult[1].ID)
# this case is required to cover 'df.at[]'
elif isinstance(key, tuple) and 2 >= len(key) >= 1:
keyElement = key[0]
if isinstance(keyElement, tuple) and 2 >= len(keyElement) >= 1:
keyElement = keyElement[0]
if isinstance(keyElement, str):
tupleResult = SymbolCache.TryGetSymbol(keyElement, None)
if tupleResult[0]:
result = str(tupleResult[1].ID)
# tuples can not be modified in Py so we generate new ones
if len(key[0]) == 1:
firstTuple = (result,)
else:
firstTuple = (result, key[0][1])
if len(key[1]) == 1:
return (firstTuple,)
else:
return (firstTuple, key[1])
return key
def __contains__(self, key):
key = self._self_mapper(key)
return self.__wrapped__.__contains__(key)
def __getitem__(self, name):
name = self._self_mapper(name)
result = self.__wrapped__.__getitem__(name)
if isinstance(result, (pandas.Series, pandas.Index)):
# For these cases we wrap the result too. Can't apply the wrap around all
# results because it causes issues in pandas for some of our use cases
# specifically pandas timestamp type
return Remapper(result)
return result
def __setitem__(self, name, value):
name = self._self_mapper(name)
return self.__wrapped__.__setitem__(name, value)
def __delitem__(self, name):
name = self._self_mapper(name)
return self.__wrapped__.__delitem__(name)
# we wrap the result and input of 'xs'
def xs(self, key, axis=0, level=None, drop_level=True):
key = self._self_mapper(key)
result = self.__wrapped__.xs(key=key, axis=axis, level=level, drop_level=drop_level)
return Remapper(result)
def get(self, key, default=None):
key = self._self_mapper(key)
return self.__wrapped__.get(key=key, default=default)
# we wrap the result of 'unstack'
def unstack(self, level=-1, fill_value=None):
result = self.__wrapped__.unstack(level=level, fill_value=fill_value)
return Remapper(result)
# we wrap 'loc' to cover the: df.loc['SPY'] case
@property
def loc(self):
return Remapper(self.__wrapped__.loc)
@property
def at(self):
return Remapper(self.__wrapped__.at)
@property
def index(self):
return Remapper(self.__wrapped__.index)
@property
def levels(self):
return Remapper(self.__wrapped__.levels)
# we wrap the following properties so that when 'unstack', 'loc' are called we wrap them
@property
def open(self):
return Remapper(self.__wrapped__.open)
@property
def high(self):
return Remapper(self.__wrapped__.high)
@property
def close(self):
return Remapper(self.__wrapped__.close)
@property
def low(self):
return Remapper(self.__wrapped__.low)
@property
def lastprice(self):
return Remapper(self.__wrapped__.lastprice)
@property
def volume(self):
return Remapper(self.__wrapped__.volume)
@property
def askopen(self):
return Remapper(self.__wrapped__.askopen)
@property
def askhigh(self):
return Remapper(self.__wrapped__.askhigh)
@property
def asklow(self):
return Remapper(self.__wrapped__.asklow)
@property
def askclose(self):
return Remapper(self.__wrapped__.askclose)
@property
def askprice(self):
return Remapper(self.__wrapped__.askprice)
@property
def asksize(self):
return Remapper(self.__wrapped__.asksize)
@property
def quantity(self):
return Remapper(self.__wrapped__.quantity)
@property
def suspicious(self):
return Remapper(self.__wrapped__.suspicious)
@property
def bidopen(self):
return Remapper(self.__wrapped__.bidopen)
@property
def bidhigh(self):
return Remapper(self.__wrapped__.bidhigh)
@property
def bidlow(self):
return Remapper(self.__wrapped__.bidlow)
@property
def bidclose(self):
return Remapper(self.__wrapped__.bidclose)
@property
def bidprice(self):
return Remapper(self.__wrapped__.bidprice)
@property
def bidsize(self):
return Remapper(self.__wrapped__.bidsize)
@property
def exchange(self):
return Remapper(self.__wrapped__.exchange)
@property
def openinterest(self):
return Remapper(self.__wrapped__.openinterest)
").GetAttr("Remapper");
}
}
var enumerable = data as IEnumerable;
if (enumerable != null)
{
foreach (var item in enumerable)
{
data = item;
}
}
var type = data.GetType() as Type;
_isCustomData = type.Namespace != "QuantConnect.Data.Market";
_members = Enumerable.Empty <MemberInfo>();
_symbol = (data as IBaseData)?.Symbol;
_levels = 2;
if (_symbol.SecurityType == SecurityType.Future)
{
_levels = 3;
}
if (_symbol.SecurityType == SecurityType.Option)
{
_levels = 5;
}
var columns = new List <string>
{
"open", "high", "low", "close", "lastprice", "volume",
"askopen", "askhigh", "asklow", "askclose", "askprice", "asksize", "quantity", "suspicious",
"bidopen", "bidhigh", "bidlow", "bidclose", "bidprice", "bidsize", "exchange", "openinterest"
};
if (_isCustomData)
{
var keys = (data as DynamicData)?.GetStorageDictionary().Select(x => x.Key);
// C# types that are not DynamicData type
if (keys == null)
{
var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property);
var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
foreach (var duplicateKey in duplicateKeys)
{
throw new ArgumentException($"PandasData.ctor(): More than one \'{duplicateKey}\' member was found in \'{type.FullName}\' class.");
}
keys = members.Select(x => x.Name.ToLowerInvariant()).Except(_baseDataProperties).Concat(new[] { "value" });
_members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant()));
}
columns.Add("value");
columns.AddRange(keys);
}
_series = columns.Distinct().ToDictionary(k => k, v => Tuple.Create(new List <DateTime>(), new List <object>()));
}
public void Setup()
{
PythonEngine.Initialize();
module = PythonEngine.ModuleFromString("module", testModule).GetAttr("PythonModule").Invoke();
}
public void InvokeNull()
{
var list = PythonEngine.Eval("list");
Assert.Throws <ArgumentNullException>(() => list.Invoke(new PyObject[] { null }));
}
public void Dispose()
{
PythonEngine.Shutdown();
}
public void SetUp()
{
PythonEngine.Initialize();
}
public void PythonSlice_performance()
{
using (Py.GIL())
{
dynamic test = PythonEngine.ModuleFromString("testModule",
@"
from AlgorithmImports import *
AddReference(""QuantConnect.Tests"")
from QuantConnect.Tests.Common.Data import *
def Test(slice, symbol):
msg = '__contains__'
if 'SPY' in slice:
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = 'SPY' in slice
span1 = (datetime.now()-now).total_seconds()
if slice.ContainsKey('SPY'):
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = slice.ContainsKey('SPY')
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
msg += '\n\n__len__'
if len(slice) > 0:
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = len(slice)
span1 = (datetime.now()-now).total_seconds()
if slice.Count > 0:
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = slice.Count
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
msg += '\n\nkeys()'
if len(slice.keys()) > 0:
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = slice.keys()
span1 = (datetime.now()-now).total_seconds()
if len(slice.Keys) > 0:
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = slice.Keys
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
msg += '\n\nvalues()'
if len(slice.values()) > 0:
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = slice.values()
span1 = (datetime.now()-now).total_seconds()
if len(slice.Values) > 0:
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = slice.Values
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
msg += '\n\nget()'
if slice.get(symbol):
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = slice.get(symbol)
span1 = (datetime.now()-now).total_seconds()
dummy = None
if slice.TryGetValue(symbol, dummy):
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = slice.TryGetValue(symbol, dummy)
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
msg += '\n\nitems()'
if slice.items():
msg += ' Py'
now = datetime.now()
for i in range(0,1000000):
result = list(slice.items())
span1 = (datetime.now()-now).total_seconds()
msg += ' C#\n'
now = datetime.now()
for i in range(0,1000000):
result = [x for x in slice]
span2 = (datetime.now()-now).total_seconds()
msg += f'Py: {span1}\nC#: {span2}\nRatio: {span1/span2}'
return msg").GetAttr("Test");
var message = string.Empty;
Assert.DoesNotThrow(() => message = test(GetPythonSlice(), Symbols.SPY));
Assert.Ignore(message);
}
}
public void Dispose()
{
m_GIL.Dispose();
PythonEngine.Shutdown();
}
public void RebalanceFunctionNull(Language language)
{
TestPortfolioConstructionModel constructionModel;
if (language == Language.Python)
{
constructionModel = new TestPortfolioConstructionModel();
using (Py.GIL())
{
var func = PythonEngine.ModuleFromString(
"RebalanceFunc",
@"
from datetime import timedelta
def RebalanceFunc(time):
if time.day == 17:
return time + timedelta(hours=1)
if time.day == 18:
return time
return None"
).GetAttr("RebalanceFunc");
constructionModel.SetRebalancingFunc(func);
}
}
else
{
constructionModel = new TestPortfolioConstructionModel(
time =>
{
if (time.Day == 18)
{
return(time);
}
if (time.Day == 17)
{
return(time.AddHours(1));
}
return(null);
});
}
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 1), new Insight[0]));
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(
new DateTime(2020, 1, 1, 23, 0, 0), new Insight[0]));
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 2), new Insight[0]));
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(
new DateTime(2020, 1, 2, 0, 0, 1), new Insight[0]));
// day number '17' should trigger rebalance in the next hour
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 17), new Insight[0]));
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(
new DateTime(2020, 1, 17, 0, 59, 59), new Insight[0]));
Assert.IsTrue(constructionModel.IsRebalanceDueWrapper(
new DateTime(2020, 1, 17, 1, 0, 0), new Insight[0]));
constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 20), new Insight[0]);
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 21), new Insight[0]));
// day number '18' should trigger rebalance immediately
Assert.IsTrue(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 18), new Insight[0]));
Assert.IsFalse(constructionModel.IsRebalanceDueWrapper(new DateTime(2020, 1, 21), new Insight[0]));
}
public void Dispose()
{
PythonEngine.ReleaseLock(_gs);
PythonEngine.Shutdown();
}
private PuffinPython()
{
if (srcDir == null || srcDir.Length == 0)
srcDir = @"c:\vlab\python\source";
if (runDir == null || runDir.Length == 0)
runDir = @"c:\vlab\python\runtime";
engine = new PythonEngine();
sbout = new MemoryStream();
sberr = new MemoryStream();
loaded = new StringBuilder();
EngineModule agent = engine.CreateModule("agentModule", true);
puffinAgent = new PuffinAgent();
agent.Globals.Add("agent", puffinAgent);
engine.SetStandardOutput(sbout);
engine.SetStandardError(sberr);
engine.AddToPath(runDir);
WritePy("INTRINSICS", Code());
WriteDB();
}
public frmMain()
{
InitializeComponent();
//DirectoryInfo di = new DirectoryInfo();
outstream = new MyStream(Shell);
Shell.CommandEntered += new UILibrary.EventCommandEntered(Shell_CommandEntered);
engine = new PythonEngine();
engine.AddToPath(Environment.CurrentDirectory);
string exepath = Environment.CurrentDirectory;
engine.Import("clr");
engine.SetStandardOutput(outstream);
engine.SetStandardError(new MyStream(Shell));
engine.Execute("import clr;clr.AddReferenceToFile(\"Interop.SAPbobsCOM.dll\")");
engine.Execute("import SAPbobsCOM;import System;import System.IO;import System.Text;import System.Collections");
IronPython.Runtime.List objecttypes = engine.EvaluateAs<IronPython.Runtime.List>("dir(SAPbobsCOM.BoObjectTypes)");
foreach (string otype in objecttypes)
{
if (otype.StartsWith("o"))
{
string handle;
int endcut = 2;
if (otype.ToUpper().EndsWith("US"))
{
endcut = 1;
}
else if (otype.ToUpper().EndsWith("HES"))
{
endcut = 3;
}
else if (otype.ToUpper().EndsWith("S"))
{
endcut = 2;
}
else
{
endcut = 1;
}
handle = otype.Substring(1, otype.Length - endcut).ToUpper();
if (handle.EndsWith("IE"))
{
handle = handle.Substring(0, handle.Length - 2) + "Y";
}
engine.Globals[handle] = engine.EvaluateAs<BoObjectTypes>("SAPbobsCOM.BoObjectTypes." + otype);
B1ObjectList.Add(handle, engine.Globals[handle]);
}
}
System.Collections.Generic.List<string> enums = new System.Collections.Generic.List<string>();
Assembly[] domainAssemblies = AppDomain.CurrentDomain.GetAssemblies();
foreach (Assembly ass in domainAssemblies)
{
if (ass.FullName.Contains("SAPbobsCOM"))
{
Type[] types = ass.GetTypes();
foreach (Type type in types)
{
if (type.IsEnum)
{
string[] enumnames = System.Enum.GetNames(type);
Array enumvalues = System.Enum.GetValues(type);
for (int enumindex = 0; enumindex < enumnames.Length; enumindex++)
{
engine.Globals[enumnames[enumindex]] = enumvalues.GetValue(enumindex);
if (!enumnames[enumindex].StartsWith("o"))
{
B1EnumList.Add(enumnames[enumindex], enumvalues.GetValue(enumindex));
B1EnumValues.Add(enumnames[enumindex], enumvalues.GetValue(enumindex).GetType().Name);
Type enumclass = enumvalues.GetValue(enumindex).GetType();
if (!B1EnumClassList.Contains(enumclass.Name))
{
B1EnumClassList.Add(enumclass.Name);
}
}
engine.Globals[enumnames[enumindex].ToLower()] = enumvalues.GetValue(enumindex);
engine.Globals[enumnames[enumindex].ToUpper()] = enumvalues.GetValue(enumindex);
enums.Add(type.Name + ":" + enumvalues.GetValue(enumindex));
}
}
}
}
}
engine.Globals["enumvalues"] = enums;
foreach (string key in B1ObjectList.Keys)
{
this.cmbRefObjects.Items.Add(key);
}
foreach (string key in B1EnumList.Keys)
{
this.cmbRefEnum.Items.Add(key);
}
foreach (string key in B1EnumClassList)
{
this.cmbEnumInfo.Items.Add(key);
}
SyntaxDocument doc = new SyntaxDocument();
doc.SyntaxFile = "Python.syn";
this.codeEditor.Document = doc;
}
/// <summary>
/// <see cref = "AlgorithmPythonWrapper"/> constructor.
/// Creates and wraps the algorithm written in python.
/// </summary>
/// <param name="moduleName">Name of the module that can be found in the PYTHONPATH</param>
public AlgorithmPythonWrapper(string moduleName)
{
try
{
using (Py.GIL())
{
Logging.Log.Trace($"AlgorithmPythonWrapper(): Python version {PythonEngine.Version}: Importing python module {moduleName}");
var module = Py.Import(moduleName);
var pyList = module.Dir();
foreach (var name in pyList)
{
Type type;
var attr = module.GetAttr(name.ToString());
var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
if (repr.StartsWith(moduleName) && // Must be defined in the module
attr.TryConvert(out type) && // Must be a Type
typeof(QCAlgorithm).IsAssignableFrom(type)) // Must inherit from QCAlgorithm
{
Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");
_algorithm = attr.Invoke();
// Set pandas
_algorithm.SetPandasConverter();
// IAlgorithm reference for LEAN internal C# calls (without going from C# to Python and back)
_baseAlgorithm = _algorithm.AsManagedObject(type);
// determines whether OnData method was defined or inherits from QCAlgorithm
// If it is not, OnData from the base class will not be called
_onData = (_algorithm as PyObject).GetAttr("OnData");
var pythonType = _onData.GetPythonType();
_isOnDataDefined = pythonType.Repr().Equals("<class \'method\'>");
_onOrderEvent = (_algorithm as PyObject).GetAttr("OnOrderEvent");
}
attr.Dispose();
}
module.Dispose();
pyList.Dispose();
// If _algorithm could not be set, throw exception
if (_algorithm == null)
{
throw new Exception("Please ensure that one class inherits from QCAlgorithm.");
}
if (Config.GetBool("mute-python-library-logging", true))
{
PythonEngine.Exec("import os, sys; sys.stdout = open(os.devnull, 'w')");
}
}
}
catch (Exception e)
{
// perform exception interpretation for error in module import
var interpreter = StackExceptionInterpreter.CreateFromAssemblies(AppDomain.CurrentDomain.GetAssemblies());
e = interpreter.Interpret(e, interpreter);
throw new Exception($"AlgorithmPythonWrapper(): {interpreter.GetExceptionMessageHeader(e)}");
}
}
public void Dispose()
{
ExtraBaseTypeProvider.ExtraBase.Dispose();
PythonEngine.Shutdown();
}
internal GILState()
{
state = PythonEngine.AcquireLock();
}
public void SymmetricalOperatorOverloads()
{
string name = string.Format("{0}.{1}",
typeof(OperableObject).DeclaringType.Name,
typeof(OperableObject).Name);
string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace;
PythonEngine.Exec($@"
from {module} import *
cls = {name}
a = cls(-2)
b = cls(10)
c = ~a
assert c.Num == ~a.Num
c = +a
assert c.Num == +a.Num
a = cls(2)
c = -a
assert c.Num == -a.Num
c = a + b
assert c.Num == a.Num + b.Num
c = a - b
assert c.Num == a.Num - b.Num
c = a * b
assert c.Num == a.Num * b.Num
c = a / b
assert c.Num == a.Num // b.Num
c = a % b
assert c.Num == a.Num % b.Num
c = a & b
assert c.Num == a.Num & b.Num
c = a | b
assert c.Num == a.Num | b.Num
c = a ^ b
assert c.Num == a.Num ^ b.Num
c = a == b
assert c == (a.Num == b.Num)
c = a != b
assert c == (a.Num != b.Num)
c = a <= b
assert c == (a.Num <= b.Num)
c = a >= b
assert c == (a.Num >= b.Num)
c = a < b
assert c == (a.Num < b.Num)
c = a > b
assert c == (a.Num > b.Num)
");
}
public static PyObject Import(string name)
{
return(PythonEngine.ImportModule(name));
}
public void TearDown()
{
PythonEngine.ReleaseLock(gs);
PythonEngine.Shutdown();
}
public void SetUp()
{
PythonEngine.Initialize();
gs = PythonEngine.AcquireLock();
}
public void LoadModules(PythonEngine engine, String appModule)
{
engine.Import("site");
engine.Import("wsgi");
engine.Import(appModule);
}
public static IntPtr allowThreads()
{
return(PythonEngine.BeginAllowThreads());
}
public static void endAllowPython(IntPtr thread_state)
{
PythonEngine.EndAllowThreads(thread_state);
}
public void SetUp()
{
PythonEngine.Initialize();
_threadState = PythonEngine.BeginAllowThreads();
}
public int Run()
{
dynamic datasets = Py.Import("sklearn.datasets");
dynamic slice = PythonEngine.Eval("slice");
var iris = datasets.load_iris();
dynamic firstTwoFeaturesIndex = new PyTuple(new PyObject[] {
slice(null),
slice(null, 2)
});
var input = iris.data.__getitem__(firstTwoFeaturesIndex);
IEnumerable target = iris.target;
var expectedOutput = target.Cast <dynamic>()
.Select(l => (int)l == 0 ? 1 : -1)
.ToArray();
int trainCount = expectedOutput.Length * 4 / 5;
var trainIn = numpy.np.array(((IEnumerable)input).Cast <dynamic>().Take(trainCount));
var trainOut = numpy.np.array(expectedOutput.Take(trainCount));
var testIn = numpy.np.array(((IEnumerable)input).Cast <dynamic>().Skip(trainCount));
var testOut = numpy.np.array(expectedOutput.Skip(trainCount));
var inPlace = tf.placeholder(shape: new int?[] { null, input.shape[1] }.Cast <object>(), dtype: tf.float32);
var outPlace = tf.placeholder(shape: new int?[] { null, 1 }.Cast <object>(), dtype: tf.float32);
var w = new Variable(tf.random_normal(shape: new TensorShape((int)input.shape[1], 1)));
var b = new Variable(tf.random_normal(shape: new TensorShape(1, 1)));
var totalLoss = Loss(w, b, inPlace, outPlace);
var accuracy = Inference(w, b, inPlace, outPlace);
var trainOp = new GradientDescentOptimizer(this.flags.InitialLearningRate).minimize(totalLoss);
var expectedTrainOut = trainOut.reshape((trainOut.Length, 1));
var expectedTestOut = testOut.reshape((testOut.Length, 1));
new Session().UseSelf(sess =>
{
var init = tensorflow.tf.global_variables_initializer();
sess.run(init);
for (int step = 0; step < this.flags.StepCount; step++)
{
(numpy.ndarray @in, numpy.ndarray @out) = NextBatch(trainIn, trainOut, sampleCount: this.flags.BatchSize);
var feed = new PythonDict <object, object> {
[inPlace] = @in,
[outPlace] = @out,
};
sess.run(trainOp, feed_dict: feed);
var loss = sess.run(totalLoss, feed_dict: feed);
var trainAcc = sess.run(accuracy, new PythonDict <object, object>
{
[inPlace] = trainIn,
[outPlace] = expectedTrainOut,
});
var testAcc = sess.run(accuracy, new PythonDict <object, object>
{
[inPlace] = testIn,
[outPlace] = expectedTestOut,
});
if ((step + 1) % 100 == 0)
{
Console.WriteLine($"Step{step}: test acc {testAcc}, train acc {trainAcc}");
}
}
//if (this.flags.IsEvaluation)
//{
//}
});
return(0);
}
public void Dispose()
{
keras?.Dispose();
PythonEngine.Shutdown();
}
public void SetUp()
{
_oldThreshold = Finalizer.Instance.Threshold;
PythonEngine.Initialize();
Exceptions.Clear();
}
public void TearDown()
{
Finalizer.Instance.Threshold = _oldThreshold;
PythonEngine.Shutdown();
}
public void TearDown()
{
PythonEngine.Shutdown();
}
public IronPythonConverter()
{
_engine = new PythonEngine();
_engine.Import("clr");
}
