public void PythonMacdAlphaModelWarmsUpProperly()
{
using (Py.GIL())
{
SetUpHistoryProvider();
Algorithm.SetStartDate(2013, 10, 08);
Algorithm.SetUniverseSelection(new ManualUniverseSelectionModel());
// Create and set alpha model
dynamic model = Py.Import("MacdAlphaModel").GetAttr("MacdAlphaModel");
var instance = model();
Algorithm.SetAlpha(instance);
var changes = new SecurityChanges(AddedSecurities, RemovedSecurities);
Algorithm.OnFrameworkSecuritiesChanged(changes);
// Get the dictionary of macd indicators
var symbolData = instance.symbolData;
// Check the dictionary is not empty
Assert.NotZero(symbolData.Length());
// Check all MACD indicators from the alpha are ready and have at least
// one datapoint
foreach (var item in symbolData)
{
var macd = symbolData[item].MACD;
Assert.IsTrue(macd.IsReady.IsTrue());
Assert.NotZero(((PyObject)macd.Samples).GetAndDispose <int>());
}
ZipCacheProvider.DisposeSafely();
}
}
public void Run()
{
using (PyScope scope = Py.CreateScope())
{
dynamic np = Py.Import("numpy");
Console.WriteLine(np.cos(np.pi * 2));
dynamic sin = np.sin;
Console.WriteLine(sin(5));
Double c = np.cos(5) + sin(5);
Console.WriteLine(c);
dynamic a = np.array(new List <float> {
1, 2, 3
});
Console.WriteLine(a.dtype);
dynamic b = np.array(new List <float> {
6, 5, 4
}, dtype: np.int32);
Console.WriteLine(b.dtype);
Console.WriteLine(a * b);
}
}
public static HttpResponseLite GetFromPythonNET(string url)
{
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
}
var sw = Stopwatch.StartNew();
var result = new HttpResponseLite();
try
{
// https://github.com/pythonnet/pythonnet/wiki/Threading
var mThreadState = PythonEngine.BeginAllowThreads();
using (Py.GIL())
{
dynamic cfscrape = Py.Import("cfscrape");
dynamic scraper = cfscrape.create_scraper();
PyObject response = scraper.get(url);
result.StatusCode = (HttpStatusCode)response.GetAttr("status_code").As <int>();
result.Html = response.GetAttr("text").As <string>();
result.ElapsedMs = sw.ElapsedMilliseconds;
}
PythonEngine.EndAllowThreads(mThreadState);
}
catch (Exception e)
{
result.ElapsedMs = sw.ElapsedMilliseconds;
result.Exception = e;
}
return(result);
}
private void SetPortfolioConstruction(Language language)
{
_algorithm.SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel());
if (language == Language.Python)
{
try
{
using (Py.GIL())
{
var name = nameof(EqualWeightingPortfolioConstructionModel);
var instance = Py.Import(name).GetAttr(name).Invoke();
var model = new PortfolioConstructionModelPythonWrapper(instance);
_algorithm.SetPortfolioConstruction(model);
}
}
catch (Exception e)
{
Assert.Ignore(e.Message);
}
}
var changes = SecurityChanges.Added(_algorithm.Securities.Values.ToArray());
_algorithm.PortfolioConstruction.OnSecuritiesChanged(_algorithm, changes);
}
private static PyObject InstallAndImport(bool force = false)
{
PythonEngine.Initialize();
var mod = Py.Import("torch");
return(mod);
}
static void Main(string[] args)
{
using (Py.GIL())
{
dynamic np = Py.Import("numpy");
Console.WriteLine(np.cos(np.pi * 2));
dynamic sin = np.sin;
Console.WriteLine(sin(5));
double c = np.cos(5) + sin(5);
Console.WriteLine(c);
dynamic a = np.array(new List <float> {
1, 2, 3
});
Console.WriteLine(a.dtype);
dynamic b = np.array(new List <float> {
6, 5, 4
}, dtype: np.int32);
Console.WriteLine(b.dtype);
Console.WriteLine(a * b);
Console.ReadKey();
}
}
private void SetPortfolioConstruction(Language language, QCAlgorithm algorithm, PortfolioBias bias = PortfolioBias.LongShort)
{
algorithm.SetPortfolioConstruction(new AccumulativeInsightPortfolioConstructionModel((Func <DateTime, DateTime>)null, bias));
if (language == Language.Python)
{
using (Py.GIL())
{
var name = nameof(AccumulativeInsightPortfolioConstructionModel);
var instance = Py.Import(name).GetAttr(name).Invoke(((object)null).ToPython(), ((int)bias).ToPython());
var model = new PortfolioConstructionModelPythonWrapper(instance);
algorithm.SetPortfolioConstruction(model);
}
}
foreach (var kvp in _algorithm.Portfolio)
{
kvp.Value.SetHoldings(kvp.Value.Price, 0);
}
_algorithm.Portfolio.SetCash(_startingCash);
SetUtcTime(new DateTime(2018, 7, 31));
var changes = SecurityChangesTests.AddedNonInternal(_algorithm.Securities.Values.ToArray());
algorithm.PortfolioConstruction.OnSecuritiesChanged(_algorithm, changes);
}
private void Copy(DocBin docBin)
{
_docs = docBin._docs;
// I'd rather copy Python object no matter the serialization mode
// If set to DotNet, the variable will be initialized to null
// disregarding its current value which might be a default object
_pyDocBin = docBin._pyDocBin;
if (Serialization.Selected == Serialization.Mode.SpacyAndDotNet)
{
using (Py.GIL())
{
dynamic spacy = Py.Import("spacy");
dynamic pyVocab = spacy.vocab.Vocab.__call__();
dynamic pyDocs = _pyDocBin.get_docs(pyVocab);
dynamic builtins = Py.Import("builtins");
dynamic listDocs = builtins.list(pyDocs);
var pyCount = new PyInt(builtins.len(listDocs));
var count = pyCount.ToInt32();
for (var i = 0; i < count; i++)
{
dynamic pyDoc = listDocs[i];
_docs[i].PyDoc = pyDoc;
_docs[i].Vocab.PyVocab = pyDoc.vocab;
}
}
}
}
private void Button1_Click(object sender, EventArgs e)
{
MessageBox.Show
using (Py.GIL())
{
dynamic sys = Py.Import("sys");
dynamic sysAppend = sys.path.append("D:\\WORK");
dynamic sysPath = sys.path;
dynamic sysVersion = sys.version;
Console.WriteLine(sysPath);
Console.WriteLine(sysVersion);
StringBuilder sb = new StringBuilder();
foreach (dynamic Path in sysPath)
{
sb.AppendLine(Convert.ToString(Path));
}
richTextBox1.Text = sb.ToString();
dynamic test = Py.Import("example");
dynamic f = test.Calculator(int.Parse(textBox1.Text), int.Parse(textBox2.Text));
dynamic sumRst = (f.add());
textBox3.Text = Convert.ToString(sumRst);
}
}
private void SetPortfolioConstruction(Language language, QCAlgorithm algorithm, dynamic paramenter = null)
{
paramenter = paramenter ?? Resolution.Daily;
algorithm.SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel(paramenter));
if (language == Language.Python)
{
using (Py.GIL())
{
var name = nameof(EqualWeightingPortfolioConstructionModel);
var instance = Py.Import(name).GetAttr(name).Invoke(((object)paramenter).ToPython());
var model = new PortfolioConstructionModelPythonWrapper(instance);
algorithm.SetPortfolioConstruction(model);
}
}
foreach (var kvp in _algorithm.Portfolio)
{
kvp.Value.SetHoldings(kvp.Value.Price, 0);
}
_algorithm.Portfolio.SetCash(_startingCash);
SetUtcTime(new DateTime(2018, 7, 31));
var changes = SecurityChanges.Added(_algorithm.Securities.Values.ToArray());
algorithm.PortfolioConstruction.OnSecuritiesChanged(_algorithm, changes);
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public static Task <double[, ]> VectorizeDocumentsTFIDF(IEnumerable <string> documents)
{
return(Task.Run(() =>
{
try
{
using (Py.GIL())
{
dynamic sklear_feature_extraction_text = Py.Import("sklearn.feature_extraction.text");
dynamic tfidfVectorizer = sklear_feature_extraction_text.TfidfVectorizer();
dynamic countMatrixObject = tfidfVectorizer.fit_transform(documents);
PyList countMatrix = PyList.AsList(countMatrixObject.toarray());
var matrix = (double[][])countMatrix.AsManagedObject(typeof(double[][]));
return ConvertMatrix(matrix);
}
}
catch (PythonException e)
{
if (e.Message.Contains("NoneType"))
{
return new double[0, 0];
}
else
{
throw e;
}
}
}));
}
public void ReturnsExpectedPortfolioTarget(
Language language,
decimal maxDrawdownPercent,
bool invested,
decimal unrealizedProfit,
decimal absoluteHoldingsCost,
bool shouldLiquidate)
{
var security = new Mock <Equity>(
Symbols.AAPL,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
new Cash(Currencies.USD, 0, 1),
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance
);
security.Setup(m => m.Invested).Returns(invested);
var holding = new Mock <EquityHolding>(security.Object,
new IdentityCurrencyConverter(Currencies.USD));
holding.Setup(m => m.UnrealizedProfit).Returns(unrealizedProfit);
holding.Setup(m => m.AbsoluteHoldingsCost).Returns(absoluteHoldingsCost);
security.Object.Holdings = holding.Object;
var algorithm = new QCAlgorithm();
algorithm.SetPandasConverter();
algorithm.Securities.Add(Symbols.AAPL, security.Object);
if (language == Language.Python)
{
using (Py.GIL())
{
const string name = nameof(MaximumDrawdownPercentPerSecurity);
var instance = Py.Import(name).GetAttr(name).Invoke(maxDrawdownPercent.ToPython());
var model = new RiskManagementModelPythonWrapper(instance);
algorithm.SetRiskManagement(model);
}
}
else
{
var model = new MaximumDrawdownPercentPerSecurity(maxDrawdownPercent);
algorithm.SetRiskManagement(model);
}
var targets = algorithm.RiskManagement.ManageRisk(algorithm, null).ToList();
if (shouldLiquidate)
{
Assert.AreEqual(1, targets.Count);
Assert.AreEqual(Symbols.AAPL, targets[0].Symbol);
Assert.AreEqual(0, targets[0].Quantity);
}
else
{
Assert.AreEqual(0, targets.Count);
}
}
public ResponseWrapper <string> CreateCity(string cityName, int wkid)
{
var errors = new List <Error>();
cityName = NormalizeCityName(cityName);
ClearConnectionFiles();
using var state = Py.GIL();
using dynamic arcpy = Py.Import("arcpy");
using var spatialReference = arcpy.SpatialReference(wkid);
using var management = arcpy.management;
var database = new DataBase("GeodataCadaster", @"C:\CadasterSystem\sde\cadaster-ad.sde");
try
{
errors.AddRange(CreateCityUser(cityName, arcpy, database));
}
catch (Exception e)
{
HandleError(e, errors);
}
return(new ResponseWrapper <string>(errors));
}
private void TestRunning()
{
Bitmap bitmap = new Bitmap(@"./GMFN/bird.bmp");
Rectangle rect = new Rectangle(0, 0, bitmap.Width, bitmap.Height);
BitmapData bmpData = bitmap.LockBits(rect, ImageLockMode.ReadWrite, bitmap.PixelFormat);
IntPtr ptr = bmpData.Scan0;
int size = Math.Abs(bmpData.Stride) * bitmap.Height;
byte[] array = new byte[size];
//byte[] array_x4 = new byte[size * 16];
Marshal.Copy(ptr, array, 0, size);
bitmap.UnlockBits(bmpData);
using (Py.GIL())
{
dynamic test = Py.Import("GMFN.gmfn_test");
DL = test.SR(@"./GMFN/test_GMFN_example.json");
Tensor _tensor = torch.randn(new Shape(size));
_tensor = array;
_tensor = _tensor.reshape(new Shape(new int[] { bmpData.Height, bmpData.Width, 3 }));
_tensor = _tensor.transpose(0, 2).transpose(1, 2);
_tensor = _tensor.unsqueeze(0);
DL.TensorToList(_tensor);
Stopwatch sw = new Stopwatch();
sw.Start();
Tensor batch = new Tensor(DL.images_bmp(array, bmpData.Width, bmpData.Height, bmpData.Stride));
sw.Stop();
Console.WriteLine("data ms: " + sw.ElapsedMilliseconds.ToString() + "ms");
}
}
private void SetPortfolioConstruction(Language language, QCAlgorithm algorithm)
{
algorithm.SetPortfolioConstruction(new InsightWeightingPortfolioConstructionModel());
if (language == Language.Python)
{
using (Py.GIL())
{
var name = nameof(InsightWeightingPortfolioConstructionModel);
var instance = Py.Import(name).GetAttr(name).Invoke();
var model = new PortfolioConstructionModelPythonWrapper(instance);
algorithm.SetPortfolioConstruction(model);
}
}
foreach (var kvp in _algorithm.Portfolio)
{
kvp.Value.SetHoldings(kvp.Value.Price, 0);
}
_algorithm.Portfolio.SetCash(_startingCash);
SetUtcTime(new DateTime(2018, 7, 31));
var changes = SecurityChanges.Added(_algorithm.Securities.Values.ToArray());
algorithm.PortfolioConstruction.OnSecuritiesChanged(_algorithm, changes);
}
public TimeSeriesResultModel GetForcasting(Dictionary <string, decimal> data)
{
try
{
dynamic predictResult;
var values = new Dictionary <string, List <string> >
{
{ "Date", data.Keys.Select(x => x.ToString()).ToList() },
{ "riders", data.Values.Select(x => x.ToString()).ToList() }
};
dynamic clrPredict;
dynamic clrForecast;
var v = PythonEngine.Version;
using (Py.GIL())
{
var converter = (new PythonHelper()).GetConverter();
dynamic syspy = Py.Import("sys");
syspy.path.append(_pythonFilePaths);
dynamic ps = Py.Import("TimeSeries");
predictResult = ps.main(values);
clrPredict = converter.ToClr(predictResult[0]);
clrForecast = converter.ToClr(predictResult[1]);
}
return(new TimeSeriesResultModel(clrPredict, clrForecast));
}
catch (Exception e)
{
//Todo Handle Errors
throw e;
}
}
private static PyObject InstallAndImport(string module)
{
if (!PythonEngine.IsInitialized)
{
PythonEngine.Initialize();
}
sys = Py.Import("sys");
if (DisablePySysConsoleLog && !alreadyDisabled)
{
string codeToRedirectOutput =
"import sys\n" +
"from io import StringIO\n" +
"sys.stdout = mystdout = StringIO()\n" +
"sys.stdout.flush()\n" +
"sys.stderr = mystderr = StringIO()\n" +
"sys.stderr.flush()\n";
PythonEngine.RunSimpleString(codeToRedirectOutput);
alreadyDisabled = true;
}
var mod = Py.Import(module);
return(mod);
}
private static PyObject InstallAndImport(bool force = false)
{
Compute.Install(force).Wait();
Compute.PipInstall("numpy");
PythonEngine.Initialize();
return(Py.Import("numpy"));
}
public override InitializationResult Init()
{
var cts = new CancellationTokenSource();
_pythonTaskScheduler = new SingleThreadTaskScheduler(cts.Token);
_pythonTaskScheduler.Schedule(() =>
{
try
{
_state = Py.GIL();
_sys = Py.Import("sys");
_np = Py.Import("numpy");
}
catch (Exception ex)
{
Logger.LogError(ex, "");
}
});
_pythonTaskScheduler.Start();
_code = File.ReadAllText(Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location),
"pythagorean.py"));
return(string.IsNullOrEmpty(_code) ? InitializationResult.Error : InitializationResult.Ok);
}
public void FullTrain(LudwigArgs args)
{
using (Py.GIL())
{
PyObject train = Py.Import("ludwig.train");
//train(
// model_definition, <---- docs forgot to document this positional argument
// data_df = None,
// data_train_df = None,
// data_validation_df = None,
// data_test_df = None,
// data_csv = None,
// data_train_csv = None,
// data_validation_csv = None,
// data_test_csv = None,
// data_hdf5 = None,
// data_train_hdf5 = None,
// data_validation_hdf5 = None,
// data_test_hdf5 = None,
// data_dict = None,
// train_set_metadata_json = None,
// experiment_name = 'api_experiment',
// model_name = 'run',
// model_load_path = None,
// model_resume_path = None,
// skip_save_model = False,
// skip_save_progress = False,
// skip_save_log = False,
// skip_save_processed_input = False,
// output_directory = 'results',
// gpus = None,
// gpu_fraction = 1.0,
// use_horovod = False,
// random_seed = 42,
// logging_level = 40,
// debug = False
//)
//var model_definition = Py.CreateScope("asdf").Eval(args.ModelDefinition);
// var arg = new PyTuple(new PyObject[] { args.ModelDefinition });
var arg = Util.ToTuple(new object[]
{
args.ModelDefinition
});
var kwargs = new PyDict();
if (args.DataCsv != null)
{
kwargs["data_csv"] = Util.ToPython(args.DataCsv);
}
if (args.ModelDefinitionFile != null)
{
kwargs["model_definition_file"] = Util.ToPython(args.ModelDefinitionFile);
}
// if (order != null) kwargs["order"] = ToPython(order);
dynamic py = train.InvokeMethod("full_train", arg, kwargs);
}
}
/// <summary>
/// <see cref = "AlgorithmPythonWrapper"/> constructor.
/// Creates and wraps the algorithm written in python.
/// </summary>
/// <param name="moduleName">Name of the module that can be found in the PYTHONPATH</param>
public AlgorithmPythonWrapper(string moduleName)
{
try
{
using (Py.GIL())
{
Logging.Log.Trace($"AlgorithmPythonWrapper(): Python version {PythonEngine.Version}: Importing python module {moduleName}");
var module = Py.Import(moduleName);
var pyList = module.Dir();
foreach (var name in pyList)
{
Type type;
var attr = module.GetAttr(name.ToString());
var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
if (repr.StartsWith(moduleName) && // Must be defined in the module
attr.TryConvert(out type) && // Must be a Type
typeof(QCAlgorithm).IsAssignableFrom(type)) // Must inherit from QCAlgorithm
{
Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");
_algorithm = attr.Invoke();
// Set pandas
_algorithm.SetPandasConverter();
// IAlgorithm reference for LEAN internal C# calls (without going from C# to Python and back)
_baseAlgorithm = _algorithm.AsManagedObject(type);
// determines whether OnData method was defined or inherits from QCAlgorithm
// If it is not, OnData from the base class will not be called
var pyAlgorithm = _algorithm as PyObject;
_onData = pyAlgorithm.GetPythonMethod("OnData");
_onMarginCall = pyAlgorithm.GetPythonMethod("OnMarginCall");
_onOrderEvent = pyAlgorithm.GetAttr("OnOrderEvent");
}
attr.Dispose();
}
module.Dispose();
pyList.Dispose();
// If _algorithm could not be set, throw exception
if (_algorithm == null)
{
throw new Exception("Please ensure that one class inherits from QCAlgorithm.");
}
}
}
catch (Exception e)
{
// perform exception interpretation for error in module import
var interpreter = StackExceptionInterpreter.CreateFromAssemblies(AppDomain.CurrentDomain.GetAssemblies());
e = interpreter.Interpret(e, interpreter);
throw new Exception($"AlgorithmPythonWrapper(): {interpreter.GetExceptionMessageHeader(e)}");
}
}
public string Call_simpleDefaultArg_WithNull(string moduleName)
{
using (Py.GIL())
{
dynamic module = Py.Import(moduleName);
return(module.simpleDefaultArg(null));
}
}
public DocBin()
{
using (Py.GIL())
{
dynamic spacy = Py.Import("spacy");
_pyDocBin = spacy.tokens.DocBin.__call__();
}
}
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
var state = Py.GIL();
var sys = Py.Import("sys");
var np = Py.Import("numpy");
}
private void button1_Click_1(object sender, EventArgs e)
{
using (Py.GIL())
{
dynamic test = Py.Import("example2");
dynamic plot = test.plot();
}
}
public void Setup()
{
using (Py.GIL())
{
_module = Py.Import("PandasIndexingTests");
_pandasIndexingTests = _module.PandasIndexingTests();
}
}
public Vocab()
{
using (Py.GIL())
{
dynamic spacy = Py.Import("spacy");
PyVocab = spacy.vocab.Vocab.__call__();
}
}
public void Connect()
{
using (Py.GIL())
{
_msgpackrpc = Py.Import("msgpackrpc");
_client = _msgpackrpc.Client(_msgpackrpc.Address("localhost", Port), timeout: 99999);
}
}
/// <summary>
/// <see cref = "AlgorithmPythonWrapper"/> constructor.
/// Creates and wraps the algorithm written in python.
/// </summary>
/// <param name="moduleName">Name of the module that can be found in the PYTHONPATH</param>
public AlgorithmPythonWrapper(string moduleName)
{
try
{
using (Py.GIL())
{
Logging.Log.Trace($"AlgorithmPythonWrapper(): Python version {PythonEngine.Version}: Importing python module {moduleName}");
var module = Py.Import(moduleName);
foreach (var name in module.Dir())
{
Type type;
var attr = module.GetAttr(name.ToString());
var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
if (repr.StartsWith(moduleName) && // Must be defined in the module
attr.TryConvert(out type) && // Must be a Type
typeof(QCAlgorithm).IsAssignableFrom(type)) // Must inherit from QCAlgorithm
{
Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");
_algorithm = attr.Invoke();
// Set pandas
_algorithm.SetPandasConverter();
// IAlgorithm reference for LEAN internal C# calls (without going from C# to Python and back)
_baseAlgorithm = _algorithm.AsManagedObject(type);
// write events such that when the base handles an event it
// will also invoke event handlers defined on this instance
_baseAlgorithm.InsightsGenerated += InsightsGenerated;
// determines whether OnData method was defined or inherits from QCAlgorithm
// If it is not, OnData from the base class will not be called
var pythonType = (_algorithm as PyObject).GetAttr("OnData").GetPythonType();
_isOnDataDefined = pythonType.Repr().Equals("<class \'method\'>");
}
}
// If _algorithm could not be set, throw exception
if (_algorithm == null)
{
throw new Exception("Please ensure that one class inherits from QCAlgorithm or QCAlgorithmFramework.");
}
}
}
catch (Exception e)
{
// perform exception interpretation for error in module import
var interpreter = StackExceptionInterpreter.CreateFromAssemblies(AppDomain.CurrentDomain.GetAssemblies());
e = interpreter.Interpret(e, interpreter);
throw new Exception($"AlgorithmPythonWrapper(): {interpreter.GetExceptionMessageHeader(e)}");
}
}