/// <summary> /// Creates a pandas DataFrame from an enumerable of slice containing the requested historical data /// </summary> /// <param name="symbols">The symbols to retrieve historical data for</param> /// <param name="history">an enumerable of slice containing the requested historical data</param> /// <returns>A python dictionary with pandas DataFrame containing the requested historical data</returns> private PyObject CreatePandasDataFrame(List <Symbol> symbols, IEnumerable <Slice> history) { // If pandas is null (cound not be imported), return null if (_pandas == null) { return(null); } using (Py.GIL()) { var pyDict = new PyDict(); foreach (var symbol in symbols) { var index = Securities[symbol].Type == SecurityType.Equity ? history.Get <TradeBar>(symbol).Select(x => x.Time) : history.Get <QuoteBar>(symbol).Select(x => x.Time); var dataframe = new PyDict(); dataframe.SetItem("open", _pandas.Series(history.Get(symbol, Field.Open).ToList(), index)); dataframe.SetItem("high", _pandas.Series(history.Get(symbol, Field.High).ToList(), index)); dataframe.SetItem("low", _pandas.Series(history.Get(symbol, Field.Low).ToList(), index)); dataframe.SetItem("close", _pandas.Series(history.Get(symbol, Field.Close).ToList(), index)); dataframe.SetItem("volume", _pandas.Series(history.Get(symbol, Field.Volume).ToList(), index)); pyDict.SetItem(symbol.Value, _pandas.DataFrame(dataframe, columns: new[] { "open", "high", "low", "close", "volume" }.ToList())); } return(pyDict); } }
/// <summary> /// Generate the top N drawdown plot using the python libraries. /// </summary> public override string Render() { var backtestPoints = ResultsUtil.EquityPoints(_backtest); var livePoints = ResultsUtil.EquityPoints(_live); var liveSeries = new Series <DateTime, double>(livePoints.Keys, livePoints.Values); var strategySeries = DrawdownCollection.NormalizeResults(_backtest, _live); var seriesUnderwaterPlot = DrawdownCollection.GetUnderwater(strategySeries).DropMissing(); var liveUnderwaterPlot = backtestPoints.Count == 0 ? seriesUnderwaterPlot : seriesUnderwaterPlot.After(backtestPoints.Last().Key); var drawdownCollection = DrawdownCollection.FromResult(_backtest, _live, periods: 5); var base64 = ""; using (Py.GIL()) { var backtestList = new PyList(); if (liveUnderwaterPlot.IsEmpty) { backtestList.Append(seriesUnderwaterPlot.Keys.ToList().ToPython()); backtestList.Append(seriesUnderwaterPlot.Values.ToList().ToPython()); } else { backtestList.Append(seriesUnderwaterPlot.Before(liveUnderwaterPlot.FirstKey()).Keys.ToList().ToPython()); backtestList.Append(seriesUnderwaterPlot.Before(liveUnderwaterPlot.FirstKey()).Values.ToList().ToPython()); } var liveList = new PyList(); liveList.Append(liveUnderwaterPlot.Keys.ToList().ToPython()); liveList.Append(liveUnderwaterPlot.Values.ToList().ToPython()); var worstList = new PyList(); var previousDrawdownPeriods = new List <KeyValuePair <DateTime, DateTime> >(); foreach (var group in drawdownCollection.Drawdowns) { // Skip drawdown periods that are overlapping if (previousDrawdownPeriods.Where(kvp => (group.Start >= kvp.Key && group.Start <= kvp.Value) || (group.End >= kvp.Key && group.End <= kvp.Value)).Any()) { continue; } var worst = new PyDict(); worst.SetItem("Begin", group.Start.ToPython()); worst.SetItem("End", group.End.ToPython()); worst.SetItem("Total", group.PeakToTrough.ToPython()); worstList.Append(worst); previousDrawdownPeriods.Add(new KeyValuePair <DateTime, DateTime>(group.Start, group.End)); } base64 = Charting.GetDrawdown(backtestList, liveList, worstList); } return(base64); }
public override bool Init() { if (!base.Init()) { return(false); } globals = new PyDict(); commentsDict = new PyDict(); globals.SetItem("nltk", Py.Import("nltk")); globals.SetItem("vader", Py.Import("vaderSentiment.vaderSentiment")); globals.SetItem("comments", commentsDict); return(true); }
public void TestEval() { dynamic sys = Py.Import("sys"); sys.attr1 = 100; var locals = new PyDict(); locals.SetItem("sys", sys); locals.SetItem("a", new PyInt(10)); object b = PythonEngine.Eval("sys.attr1 + a + 1", null, locals.Handle) .AsManagedObject(typeof(int)); Assert.AreEqual(111, b); }
public void TestExec() { dynamic sys = Py.Import("sys"); sys.attr1 = 100; var locals = new PyDict(); locals.SetItem("sys", sys); locals.SetItem("a", new PyInt(10)); PythonEngine.Exec("c = sys.attr1 + a + 1", null, locals); object c = locals.GetItem("c").AsManagedObject(typeof(int)); Assert.AreEqual(111, c); }
public static void Main(string[] args) { Console.WriteLine("Hello World!"); // TODO: Implement Functionality Here using (Py.GIL()) { PyDict locals = new PyDict(); locals.SetItem("a", new PyInt(10)); if (false) { int b = (int)PythonEngine.Eval("a * 2 + 1", null, locals.Handle) .AsManagedObject(typeof(int)); Console.WriteLine(b); } else { PythonEngine.Exec("print(a * 2 + 1)", null, locals.Handle); } } Console.Write("Press any key to continue . . . "); Console.ReadKey(true); }
private static PyDict CreateGlobals() { var globals = new PyDict(); globals.SetItem("random", PythonEngine.ImportModule("random")); return(globals); }
/// <summary> /// Creates a pandas.DataFrame from an enumerable of slice /// </summary> /// <param name="requests">The history requests to execute</param> /// <param name="slices">An enumerable of slice containing the requested historical data</param> /// <param name="selector">Selects a value from the BaseData to filter the request output, if null retuns all OHLCV</param> /// <returns>A pandas.DataFrame containing the requested historical data</returns> private PyObject CreateDataFrame(IEnumerable <HistoryRequest> requests, IEnumerable <Slice> slices, Func <IBaseData, decimal> selector) { if (selector == null) { return(CreateMultiIndex(requests, slices)); } using (Py.GIL()) { var history = slices.ToList(); var data = new PyDict(); foreach (var request in requests) { var symbol = request.Symbol; var list = history.Get(symbol, selector).Select(x => (double)x).ToList(); if (list.Count == 0) { continue; } var index = request.DataType.Equals(typeof(QuoteBar)) ? history.Get <QuoteBar>(symbol).Select(x => x.Time) : history.Get <TradeBar>(symbol).Select(x => x.Time); data.SetItem(symbol, _pandas.Series(list, index)); } return(_pandas.DataFrame(data)); } }
/// <summary> /// Python implementation of GetFundamental, get fundamental data for input symbols or tickers /// </summary> /// <param name="input">The symbols or tickers to retrieve fundamental data for</param> /// <param name="selector">Selects a value from the Fundamental data to filter the request output</param> /// <param name="start">The start date of selected data</param> /// <param name="end">The end date of selected data</param> /// <returns>pandas DataFrame</returns> public PyObject GetFundamental(PyObject input, string selector, DateTime?start = null, DateTime?end = null) { //Null selector is not allowed for Python DataFrame if (string.IsNullOrWhiteSpace(selector)) { throw new ArgumentException("Invalid selector. Cannot be None, empty or consist only of white-space characters"); } //Covert to symbols var symbols = PythonUtil.ConvertToSymbols(input); //Fetch the data var fundamentalData = GetAllFundamental(symbols, selector, start, end); using (Py.GIL()) { var data = new PyDict(); foreach (var day in fundamentalData.OrderBy(x => x.Key)) { var orderedValues = day.Value.OrderBy(x => x.Key.ID.ToString()).ToList(); var columns = orderedValues.Select(x => x.Key.ID.ToString()); var values = orderedValues.Select(x => x.Value); var row = _pandas.Series(values, columns); data.SetItem(day.Key.ToPython(), row); } return(_pandas.DataFrame.from_dict(data, orient: "index")); } }
/// <summary> /// Get a list of backtests for a specific project id /// </summary> /// <param name="projectId">Project id to search</param> /// <returns>pandas.DataFrame for list of backtests</returns> public PyObject ListBacktests(int projectId) { var listBacktests = _api.ListBacktests(projectId); if (listBacktests.Success) { try { using (Py.GIL()) { var pyDict = new PyDict(); var backtests = listBacktests.Backtests .Select(x => ReadBacktest(projectId, x.BacktestId)) .Where(x => x.Result != null).ToList(); var index = backtests.Select(x => x.BacktestId); var columns = new List <string>() { "Name" }; pyDict.SetItem("Name", _pandas.Series(backtests.Select(x => x.Name).ToList(), index)); if (backtests.Count() > 0) { var CsDict = backtests .SelectMany(x => x.Result.Statistics) .GroupBy(k => k.Key) .ToDictionary(k => k.Key, v => v.Select(i => i.Value).ToList()); foreach (var kvp in CsDict) { pyDict.SetItem(kvp.Key, _pandas.Series(kvp.Value, index)); } columns.AddRange(CsDict.Keys); } return(_pandas.DataFrame(pyDict, columns: columns)); } } catch (Exception exception) { return(exception.Message.ToPython()); } } return(string.Join(Environment.NewLine, listBacktests.Errors).ToPython()); }
private void UpdateLocals(string name, dynamic item) { using (Py.GIL()) { PyDict update = new PyDict(); update.SetItem(name, item); Local.Update(update); } }
/// <summary> /// Get fundamental data from given symbols /// </summary> /// <param name="pyObject">The symbols to retrieve fundamental data for</param> /// <param name="selector">Selects a value from the Fundamental data to filter the request output</param> /// <param name="start">The start date of selected data</param> /// <param name="end">The end date of selected data</param> /// <returns></returns> public PyObject GetFundamental(PyObject tickers, string selector, DateTime?start = null, DateTime?end = null) { if (string.IsNullOrWhiteSpace(selector)) { return("Invalid selector. Cannot be None, empty or consist only of white-space characters".ToPython()); } using (Py.GIL()) { // If tickers are not a PyList, we create one if (!PyList.IsListType(tickers)) { var tmp = new PyList(); tmp.Append(tickers); tickers = tmp; } var list = new List <Tuple <Symbol, DateTime, object> >(); foreach (var ticker in tickers) { var symbol = QuantConnect.Symbol.Create(ticker.ToString(), SecurityType.Equity, Market.USA); var dir = new DirectoryInfo(Path.Combine(Globals.DataFolder, "equity", symbol.ID.Market, "fundamental", "fine", symbol.Value.ToLowerInvariant())); if (!dir.Exists) { continue; } var config = new SubscriptionDataConfig(typeof(FineFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false); foreach (var fileName in dir.EnumerateFiles()) { var date = DateTime.ParseExact(fileName.Name.Substring(0, 8), DateFormat.EightCharacter, CultureInfo.InvariantCulture); if (date < start || date > end) { continue; } var factory = new TextSubscriptionDataSourceReader(_dataCacheProvider, config, date, false); var source = new SubscriptionDataSource(fileName.FullName, SubscriptionTransportMedium.LocalFile); var value = factory.Read(source).Select(x => GetPropertyValue(x, selector)).First(); list.Add(Tuple.Create(symbol, date, value)); } } var data = new PyDict(); foreach (var item in list.GroupBy(x => x.Item1)) { var index = item.Select(x => x.Item2); data.SetItem(item.Key, _pandas.Series(item.Select(x => x.Item3).ToList(), index)); } return(_pandas.DataFrame(data)); } }
/// <summary> /// Read back a list of all projects on the account for a user. /// </summary> /// <returns>pandas.DataFrame for list of projects</returns> public PyObject ListProjects() { var listProject = _api.ListProjects(); if (listProject.Success) { using (Py.GIL()) { var pyDict = new PyDict(); var index = listProject.Projects.Select(x => x.ProjectId); pyDict.SetItem("Name", _pandas.Series(listProject.Projects.Select(x => x.Name).ToList(), index)); pyDict.SetItem("Created", _pandas.Series(listProject.Projects.Select(x => x.Created).ToList(), index)); pyDict.SetItem("Modified", _pandas.Series(listProject.Projects.Select(x => x.Modified).ToList(), index)); return(_pandas.DataFrame(pyDict, columns: new[] { "Name", "Created", "Modified" }.ToList())); } } return(null); }
/// <summary> /// Get an equity curve for an alpha. /// </summary> /// <param name="alphaId">Alpha Id for strategy we're downloading.</param> /// <param name="dateFormat">Preferred date format</param> /// <param name="format">Preferred format of returned equity curve</param> /// <returns>Equity curve list of points</returns> public PyObject GetAlphaEquityCurve(string alphaId, string dateFormat = "date", string format = "json") { var equityCurve = GetAlphaEquityCurveCSharp(alphaId, dateFormat, format); using (Py.GIL()) { dynamic pandas = Py.Import("pandas"); var index = equityCurve.Select(x => x.Time).ToList(); var equity = equityCurve.Select(x => x.Equity); var sample = equityCurve.Select(x => x.Sample); var pyDict = new PyDict(); pyDict.SetItem("equity", pandas.Series(equity, index)); pyDict.SetItem("sample", pandas.Series(sample, index)); return(pandas.DataFrame(pyDict)); } }
public static PyDict FromClr(Dictionary <string, object> obj) { PyDict dict = new PyDict(); foreach (var item in obj) { dict.SetItem(GymObject.ToPython(item.Key), GymObject.ToPython(item.Value)); } return(dict); }
/// <summary> /// Gets symbol information from a list of tickers /// </summary> /// <param name="pyObject">The tickers to retrieve information for</param> /// <returns>A pandas.DataFrame containing the symbol information</returns> public PyObject PrintSymbols(PyObject pyObject) { var symbols = GetSymbolsFromPyObject(pyObject); if (symbols == null) { return("Invalid ticker(s). Please use get_symbol to add symbols.".ToPython()); } using (Py.GIL()) { var data = new PyDict(); var index = symbols.Select(x => x.ID); data.SetItem("Value", _pandas.Series(symbols.Select(x => x.Value).ToList(), index)); data.SetItem("Asset Type", _pandas.Series(symbols.Select(x => x.SecurityType.ToString()).ToList(), index)); data.SetItem("Market", _pandas.Series(symbols.Select(x => x.ID.Market.ToString()).ToList(), index)); data.SetItem("Start Date", _pandas.Series(symbols.Select(x => x.SecurityType == SecurityType.Equity ? x.ID.Date.ToShortDateString() : null).ToList(), index)); return(_pandas.DataFrame(data, columns: "Value,Asset Type,Market,Start Date".Split(',').ToList())); } }
public static PyDict ToPyDict <K, V>(this Dictionary <K, V> dict) { var pyDict = new PyDict(); foreach (K key in dict.Keys) { pyDict.SetItem(key.ToPython(), dict[key].ToPython()); } return(pyDict); }
public void WriteInt64Property() { using (Py.GIL()) { var locals = new PyDict(); locals.SetItem("a", new NetObject().ToPython()); Exec($@" s = 0 for i in range(50000): a.{nameof(NetObject.LongProperty)} += i ", locals: locals); } }
/// <summary> /// Creates a series from a list of <see cref="IndicatorDataPoint"/> and adds it to the /// <see cref="PyDict"/> as the value of the given <paramref name="key"/> /// </summary> /// <param name="key">Key to insert in the <see cref="PyDict"/></param> /// <param name="points">List of <see cref="IndicatorDataPoint"/> that will make up the resulting series</param> /// <param name="pyDict"><see cref="PyDict"/> where the resulting key-value pair will be inserted into</param> private void AddSeriesToPyDict(string key, List <IndicatorDataPoint> points, PyDict pyDict) { var index = new List <DateTime>(); var values = new List <double>(); foreach (var point in points) { index.Add(point.EndTime); values.Add((double)point.Value); } pyDict.SetItem(key.ToLowerInvariant(), _pandas.Series(values, index)); }
/// <summary> /// Creates a new dictionary from the given sequence of elements with a value provided by the user. /// </summary> /// <param name="sequence">Sequence of elements which is to be used as keys for the new dictionary</param> /// <param name="value">Value which is set to each each element of the dictionary</param> /// <returns>Returns a new dictionary with the given sequence of elements as the keys of the dictionary. /// Each element of the newly created dictionary is set to the provided value.</returns> public PyDict fromkeys(Symbol[] sequence, T value) { using (Py.GIL()) { var dict = new PyDict(); foreach (var key in sequence) { var pyValue = get(key, value); dict.SetItem(key.ToPython(), pyValue.ToPython()); } return(dict); } }
public void ReadInt64Property() { using (Py.GIL()) { var locals = new PyDict(); locals.SetItem("a", new NetObject().ToPython()); PythonEngine.Exec($@" s = 0 for i in range(50000): s += a.{nameof(NetObject.LongProperty)} ", locals: locals.Handle); } }
/// <summary> /// Get the pandas.DataFrame of the current <see cref="PandasData"/> state /// </summary> /// <param name="pandas">pandas module</param> /// <param name="levels">Number of levels of the multi index</param> /// <returns>pandas.DataFrame object</returns> public PyObject ToPandasDataFrame(dynamic pandas, int levels = 2) { var seriesDict = GetPandasSeries(pandas, levels); using (Py.GIL()) { var pyDict = new PyDict(); foreach (var series in seriesDict) { pyDict.SetItem(series.Key, series.Value); } return(pandas.DataFrame(pyDict)); } }
/// <summary> /// Generate the monthly returns plot using the python libraries. /// </summary> public override string Render() { var backtestPoints = ResultsUtil.EquityPoints(_backtest); var livePoints = ResultsUtil.EquityPoints(_live); var backtestSeries = new Series <DateTime, double>(backtestPoints.Keys, backtestPoints.Values); var liveSeries = new Series <DateTime, double>(livePoints.Keys, livePoints.Values); // Equivalent to python pandas line: `backtestSeries.resample('M').apply(lambda x: x.pct_change().sum())` var backtestMonthlyReturns = backtestSeries.ResampleEquivalence(date => new DateTime(date.Year, date.Month, 1).AddMonths(1).AddDays(-1)) .Select(kvp => kvp.Value.PercentChange().Sum()); var liveMonthlyReturns = liveSeries.ResampleEquivalence(date => new DateTime(date.Year, date.Month, 1).AddMonths(1).AddDays(-1)) .Select(kvp => kvp.Value.PercentChange().Sum()); var base64 = ""; using (Py.GIL()) { var backtestResults = new PyDict(); foreach (var kvp in backtestMonthlyReturns.GroupBy(kvp => kvp.Key.Year).GetObservations()) { var key = kvp.Key.ToStringInvariant(); var values = (kvp.Value * 100).Values.ToList(); while (values.Count != 12) { values.Add(double.NaN); } backtestResults.SetItem(key.ToPython(), values.ToPython()); } var liveResults = new PyDict(); foreach (var kvp in liveMonthlyReturns.GroupBy(kvp => kvp.Key.Year).GetObservations()) { var key = kvp.Key.ToStringInvariant(); var values = (kvp.Value * 100).Values.ToList(); while (values.Count != 12) { values.Add(double.NaN); } liveResults.SetItem(key.ToPython(), values.ToPython()); } base64 = Charting.GetMonthlyReturns(backtestResults, liveResults); } return(base64); }
public static Task <double[, ]> FindSimilarities(string stringifiedMatrix) { return(Task.Run(() => { using (Py.GIL()) { dynamic pairwise = Py.Import("sklearn.metrics.pairwise"); dynamic cosine_similarity = pairwise.cosine_similarity; dynamic np = Py.Import("numpy"); dynamic matrix = np.array(np.mat(stringifiedMatrix)); PyDict locals = new PyDict(); locals.SetItem("X", matrix); locals.SetItem("cosine_similarity", cosine_similarity); dynamic cosineSimilarityObj = PythonEngine.Eval("cosine_similarity(X, X)", null, locals.Handle); PyList cosineSimilarity = PyList.AsList(cosineSimilarityObj); return ConvertSimilarityPythonListToArray(cosineSimilarity); } })); }
public static PyObject ToPyObj(this YoutubeDLOptions opts) { var d = GetPythonSettings(opts); PyDict dict = new PyDict(); foreach (var kv in d) { if (!kv.Value.GetType().IsPrimitive) { continue; } dict.SetItem(kv.Key.ToPython(), kv.Value.ToPython()); } return(dict); }
/// <summary> /// Converts a dictionary with a list of <see cref="IndicatorDataPoint"/> in a pandas.DataFrame /// </summary> /// <param name="data">Dictionary with a list of <see cref="IndicatorDataPoint"/></param> /// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns> public PyObject GetIndicatorDataFrame(IDictionary <string, List <IndicatorDataPoint> > data) { using (Py.GIL()) { var pyDict = new PyDict(); foreach (var kvp in data) { var index = kvp.Value.Select(x => x.EndTime).ToList(); var values = kvp.Value.Select(x => (double)x.Value).ToList(); pyDict.SetItem(kvp.Key.ToLower(), _pandas.Series(values, index)); } return(_pandas.DataFrame(pyDict, columns: data.Keys.Select(x => x.ToLower()).OrderBy(x => x))); } }
/// <summary> /// Read out the P/L of a specific backtest /// </summary> /// <param name="projectId">Project id for the backtest we'd like to read</param> /// <param name="backtestId">Backtest id for the backtest we'd like to read</param> /// <returns>andas.DataFrame for list of P/L</returns> public PyObject GetProfitLoss(int projectId, string backtestId) { var backtest = ReadBacktest(projectId, backtestId); if (backtest.Success) { using (Py.GIL()) { var pyDict = new PyDict(); pyDict.SetItem("ProfitLoss", _pandas.Series( backtest.Result.ProfitLoss.Values.Select(x => (double)x).ToList(), backtest.Result.ProfitLoss.Keys.ToList())); return(_pandas.DataFrame(pyDict)); } } return(string.Join(Environment.NewLine, backtest.Errors).ToPython()); }
public void Download_With_Python_Parameter_Successfully() { var algo = new QCAlgorithm(); var byteKey = Encoding.ASCII.GetBytes($"UserName:Password"); var value = $"Basic ({Convert.ToBase64String(byteKey)})"; var headers = new PyDict(); using (Py.GIL()) { headers.SetItem("Authorization".ToPython(), value.ToPython()); } var content = string.Empty; Assert.DoesNotThrow(() => content = algo.Download("https://www.quantconnect.com/", headers)); Assert.IsNotEmpty(content); }
private static PyObject Encode(Dictionary <string, Array> data) { using (var scope = Py.CreateScope()) { dynamic pandas = scope.Import("pandas"); var pyDict = new PyDict(); foreach (var pair in data) { var series = pandas.Series(Codecs.Encode(pair.Value)); if (pair.Value is string[]) { series = series.astype("string"); } pyDict.SetItem(pair.Key, series); } return(pandas.DataFrame(pyDict)); } }
/// <summary> /// Converts a dictionary with a list of <see cref="IndicatorDataPoint"/> in a pandas.DataFrame /// </summary> /// <param name="data">Dictionary with a list of <see cref="IndicatorDataPoint"/></param> /// <returns><see cref="PyObject"/> containing a pandas.DataFrame</returns> public PyObject GetIndicatorDataFrame(IDictionary <string, List <IndicatorDataPoint> > data) { using (Py.GIL()) { var pyDict = new PyDict(); foreach (var kvp in data) { var index = new List <DateTime>(); var values = new List <double>(); foreach (var item in kvp.Value) { index.Add(item.EndTime); values.Add((double)item.Value); } pyDict.SetItem(kvp.Key.ToLowerInvariant(), _pandas.Series(values, index)); } return(_pandas.DataFrame(pyDict, columns: data.Keys.Select(x => x.ToLowerInvariant()).OrderBy(x => x))); } }