public static DataFrame<double> read_csv(this Pandas pd, string filepath_or_buffer)
{
using (StreamReader reader = new StreamReader(filepath_or_buffer))
{
string line = String.Empty;
while (!String.IsNullOrEmpty(line = reader.ReadLine()))
{
var tokens = line.Split(',');
/*x1.AddRange(tokens.Select(x => double.Parse(x)).Take(tokens.Length - 1));
var _y = int.Parse(tokens[tokens.Length - 1]);
if (!labels.Contains(_y))
{
labels.Add(_y);
}
y1.Add(labels.FindIndex(l => l == _y));
length1d++;
length2d = tokens.Length - 1;*/
}
}
return null;
}
public void Slice_Row_Test()
{
var dict = new Dictionary <string, NDArray>
{
{ "one", np.arange(100) },
{ "two", np.arange(101, 201) }
};
var pd = new Pandas();
IDataFrame df1 = new DataFrame <string>(dict);
var dfSl1 = df1[(Slice)"0:12:5"];
var one = dfSl1["one"];
Assert.Equal("one", (one as SeriesBase).Name);
Assert.Equal(10, (one as SeriesBase)[2]);
var dfSl2 = df1[(Slice)"0:10"];
var two = dfSl2["two"];
Assert.Equal("two", (two as SeriesBase).Name);
Assert.Equal(110, (two as SeriesBase)[9]);
var dfSl3 = df1[(Slice)"6:10:3"];
var two2 = dfSl3["two"];
Assert.Equal("two", (two2 as SeriesBase).Name);
Assert.Equal(110, (two2 as SeriesBase)[1]);
var dfSl4 = df1[(Slice)":"];
var one2 = dfSl4["one"];
Assert.Equal("one", (one2 as SeriesBase).Name);
Assert.Equal(0, (one2 as SeriesBase)[0]);
}
public static DataFrame <TInd> DataFrame <TInd, TValue>(this Pandas pd, NDArray[] data, IList <TInd> index = null, IList <string> columns = null)
{
var df = new DataFrame <TInd>();
if (columns == null)
{
columns = Enumerable.Range(0, (data.Length - 1)).Select(x => x.ToString()).ToArray();
}
Type indexType = typeof(TInd);
if (index == null)
{
dynamic indexDyn = index;
switch (indexType.Name)
{
case ("Double"): indexDyn = Enumerable.Range(0, data[0].size).Select(x => (double)x).ToList(); break;
case ("Int32"): indexDyn = Enumerable.Range(0, data[0].size).ToList(); break;
}
index = (List <TInd>)indexDyn;
}
else
{
}
/*for(int idx = 0; idx < columns.Count;idx++)
* df[columns[idx]] = data[idx];*/
//df.Index = new Index<TInd>();
df.Index.values = new NDArray(typeof(int));
df.Index.values.reshape(new Shape(index.Count));
return(df);
}
public void SetColumn_Test()
{
var dict = new Dictionary <string, NDArray>
{
{ "one", np.arange(1000) },
{ "two", np.arange(1001, 2001) }
};
var pd = new Pandas();
var df1 = pd.DataFrame <string>(dict);
df1["three"] = new Series(np.arange(2001, 3001));
Assert.Equal(3000, df1.Size);
df1.SingleColumn("four", 1);
Assert.Equal(1, df1["four"][500]);
Assert.Equal(4000, df1.Size);
df1.SingleColumn(1, 1);
Assert.Equal(1, df1["two"][500]);
df1.Column("five", np.arange(3001, 4001));
Assert.Equal(3001, df1["five"][0]);
df1.SingleColumn(4, 1);
Assert.Equal(1, df1["five"][0]);
Assert.Equal(3000, df1["three"][999]);
df1["three"] = new Series(np.arange(1000));
Assert.Equal(999, df1["three"][999]);
}
public void DataFrame()
{
var np = new NumPy <int>();
var pd = new Pandas();
var array = np.random.randint(low: 0, high: 10, size: new Shape(5, 5));
var df = pd.DataFrame(array, columns: new string[] { "a", "b", "c", "d", "e" });
}
public static Series Series(this Pandas pd, NDArray nd, Index index = null)
{
var res = new Series(nd);
res.index = index;
return(res);
}
public static void ShowPandas()
{
var p = new Pandas();
foreach (var poo in p.PandaZoo)
{
Debug.WriteLine($"Name: {poo.Name}, Age: {poo.Age}, Location: {poo.Location}");
}
}
public static DataFrame <T> DataFrame <T>(this Pandas pd, NDArray <T> data, IList <int> index = null, IList <string> columns = null)
{
var df = new DataFrame <T>();
df.Columns.Array(columns);
df.Data = data.Data;
df.Shape = data.Shape;
return(df);
}
public void Read_iloc_Test()
{
var dict = new Dictionary <string, NDArray>
{
{ "one", np.arange(1000) },
{ "two", np.arange(1001, 2001) }
};
var pd = new Pandas();
IDataFrame df1 = new DataFrame <string>(dict);
Assert.Equal(2, df1.iloc[2].Name);
Assert.Equal(2, df1.iloc[2]["one"]);
}
public void read_csv()
{
var pd = new Pandas();
var trainData = System.IO.Path.GetFullPath("../../../../../data/train.csv");
var a = pd.read_csv(trainData);
/*var column1 = a["Lag1"];
* var column2 = a["Lag2"];
*
* Assert.IsTrue(column1.Size == 998);
* Assert.IsTrue(column2.Size == 998);*/
}
public void DataFrame()
{
var pd = new Pandas();
// var array = np.random.randint(low: 0, high: 10, shape: new Shape(5, 5));
// var df = pd.DataFrame<int, >((NDArray<object>)array, columns: new string[] { "a", "b", "c", "d", "e" });
/*var column1 = df["a"];
*
* for (int idx = 0; idx < 5; idx++)
* Assert.IsTrue(column1[idx] == array[idx,0]);
*
* var column2 = df["b"];
*
* for (int idx = 0; idx < 5; idx++)
* Assert.IsTrue(column2[idx] == array[idx,1]);*/
}
private void CreateSliceModel()
{
int row = 10;
int col = 10;
var nd = np.random.randn(row, col);
NDArray array = nd;
//array.reshape(5, 4);
var pd = new Pandas();
List <string> indexs = new List <string>();
List <string> columns = new List <string>();
CreateRowAndCol(row, col, ref indexs, ref columns);
IDataFrame df1 = pd.DataFrame(array, indexs, columns, typeof(object));
_dataFrame = df1;
}
public static DataFrame <TInd> DataFrame <TInd>(this Pandas pd, NDArray data, IList <TInd> index = null, IList <string> columns = null)
{
var vectors = new NDArray[data.shape[1]];
for (int idx = 0; idx < data.shape[1]; idx++)
{
vectors[idx] = new NDArray(typeof(int));
for (int jdx = 0; jdx < data.shape[0]; jdx++)
{
vectors[idx][jdx] = data[jdx, idx];
}
}
// return pd.DataFrame<TInd>(vectors,index,columns);
return(null);
}
public void Create_WithNDArray_Test()
{
NDArray array = np.arange(10);
array.reshape(5, 2);
var pd = new Pandas();
IDataFrame df1 = new DataFrame <string>(array, null, null, typeof(object));
var one = df1[0];
Assert.Equal(0, (one as SeriesBase).Name);
Assert.Equal(4, (one as SeriesBase)[2]);
var oneAndTwo = df1[0, 1];
var s = oneAndTwo.iloc[4];
Assert.Equal(8, s[0]);
Assert.Equal(9, s[1]);
}
public static DataFrame <int> read_csv(this Pandas pd, string filepath_or_buffer, string sep = ",", string delimiter = ",", int?header = 0,
List <IColumnType> dtype = null)
{
var data = new List <double[]>();
string[] headers = null;
using (StreamReader reader = new StreamReader(filepath_or_buffer))
{
string line = String.Empty;
int row = 0;
while (!String.IsNullOrEmpty(line = reader.ReadLine()))
{
var tokens = line.Split(',');
// for header
if (row == header)
{
headers = tokens;
row++;
continue;
}
row++;
data.Add(tokens.Select(x => double.Parse(x)).Take(tokens.Length).ToArray());
}
}
if (dtype == null)
{
dtype = new List <IColumnType>();
}
for (int i = 0; i < headers.Length; i++)
{
dtype.Add(ColumnHelper.Infer(headers[i], data[i]));
}
var nd = np.array(data.ToArray());
//var df = pd.DataFrame<int,double>(nd);
return(null);
}
public void WriteCsvQuoted_ToFile_Test()
{
var filepath = "write_quoted_test.csv";
var array = np.arange(100).reshape(20, 5);
var columnNames = new string[] { "first", "second", "third",
"fourth", "fifth" };
var pd = new Pandas();
IDataFrame df1 = pd.DataFrame(array, null, columnNames, typeof(object));
df1.to_csv(filepath, quoting: 1);
using (var fr = File.OpenText(filepath))
{
Assert.Equal(string.Join(',', columnNames), fr.ReadLine());
for (var i = 0; i < array.shape[0]; i++)
{
Assert.Equal('"' + string.Join("\",\"", array[i].Data <int>()) + '"', fr.ReadLine());
}
}
}
public static Series Series(this Pandas pd, object obj)
{
Series res = null;
Type type = obj.GetType();
if (type.Name.Contains("AnonymousType"))
{
var properties = type.GetProperties();
var inferredDtype = properties[0].GetValue(obj).GetType();
var nd = new NDArray(inferredDtype, properties.Length);
res = new Series(nd);
res.values.Storage.SetData(properties.Select(x => x.GetValue(obj)).ToArray());
res.index = new Index(properties.Select(x => x.Name).ToArray());
}
return(res);
}
public void Create_Group_Test()
{
var dict = new Dictionary <string, NDArray>
{
{ "col1", np.arange(5) },
{ "col2", np.arange(5) },
{ "col3", np.arange(5) },
{ "col4", np.arange(5) },
{ "col5", np.arange(5) },
};
var pd = new Pandas();
var df1 = pd.DataFrame <string>(dict, new string[] {
"row1", "row2", "row3", "row4", "row5"
});
var gro = df1.groupby("col2");
var group = gro.Groups;
var a = gro.Groups[1];
Assert.Equal("row2", a.Values[0]);
}
public void Create_WithDict_Test()
{
var dict = new Dictionary <string, NDArray>
{
{ "one", np.arange(10000) },
{ "two", np.arange(10001, 20001) }
};
var pd = new Pandas();
IDataFrame df1 = new DataFrame <string>(dict);
var one = df1["one"];
Assert.Equal("one", (one as SeriesBase).Name);
Assert.Equal(2, (one as SeriesBase)[2]);
var oneAndTwo = df1["one", "two"];
var s = oneAndTwo.iloc[9999];
Assert.Equal(9999, s[0]);
Assert.Equal(9999, s["one"]);
Assert.Equal(20000, s[1]);
Assert.Equal(20000, s["two"]);
}
public void WriteCsvFormated_ToFile_Test()
{
var filepath = "write_quoted_test.csv";
var array = np.arange(0, 50, 0.5).reshape(20, 5);
var columnNames = new string[] { "first", "second", "third",
"fourth", "fifth" };
var floatFormat = "E03";
var pd = new Pandas();
IDataFrame df1 = pd.DataFrame(array, null, columnNames, typeof(object));
df1.to_csv(filepath, float_format: floatFormat);
using (var fr = File.OpenText(filepath))
{
Assert.Equal(string.Join(',', columnNames), fr.ReadLine());
for (var i = 0; i < array.shape[0]; i++)
{
var formattedData = array[i].Data <double>().Select(
x => x.ToString(floatFormat));
Assert.Equal(string.Join(",", formattedData), fr.ReadLine());
}
}
}
private static IDataFrame CreateDataFrame(IDataFrame df, List <int> indexs)
{
var value = df.Values.Array;
int colSize = df.Columns.Size;
int rowSize = df.Index.Size;
var cols = df.Columns.Values.Array;
var rows = df.Index.Values.Array;
var rowArray = Array.CreateInstance(df.Index.DType, rows.Length);
int index = 0;
foreach (var ind in indexs)
{
rowArray.SetValue(rows.GetValue(ind), index);
index++;
}
object[] objs = new object[df.Size];
int ins = 0;
for (int c = 0; c < colSize; c++)
{
for (int r = 0; r < rowSize; r++)
{
objs[ins] = value.GetValue(indexs[r] + c * rowSize);
ins++;
}
}
NDArray nDArray = new NDArray(df.DType, new Shape(rowSize, colSize));
nDArray.SetData(objs);
var pd = new Pandas();
IDataFrame dataFrame = pd.DataFrame(nDArray, null, null, df.DType);
dataFrame.Index.Values.SetData(rowArray);
dataFrame.Columns.Values.SetData(cols);
return(dataFrame);
}
public void TestPandas()
{
var pd = new Pandas();
}
public TestBase()
{
np = new NumPy();
pd = new Pandas();
}
public TestBase()
{
pd = new Pandas();
}
public SeriesTest()
{
pd = new Pandas();
}
public static Index Index(this Pandas pd, params string[] items)
{
return(new Index(items));
}
public MongoReturnDataList <StockMongoData> Stock_FQ(string code, MongoReturnDataList <StockMongoData> org_bfq_data = null, MongoReturnDataList <XDXRData> org_xdxr_data = null, PriceAdj fqType = PriceAdj.Fore)
{
/*
* 1、一般的股票交易软件中，都有计算复权的功能。股票行情有除权与复权，在计算股票涨跌幅时采用复权价格，这是经常要用到的。系统计算分为以下步骤：
* ①每次除权行为日，在登记日计算除权价；
* 除权价=（除权前一日收盘价+配股价X配股比率－每股派息）/（1+配股比率+送股比率）
* ②用登记日的收盘价除以除权价得到单次除权因子；
* 除权因子=收盘价/除权价
* ③将公司上市以来的除权因子连乘积，得到对应每一交易日的除权因子；
* ④向后复权价=股票收盘价（实际交易价）*每一交易日的除权因子；
* ⑤复权涨幅（不特别说明，涨幅均指复权涨幅）=复权价/前一日复权价-1。
* 2、所谓复权就是对股价和成交量进行权息修复，按照股票的实际涨跌绘制股价走势图,并把成交量调整为相同的股本口径。
*/
if (org_bfq_data == null)
{
org_bfq_data = new SecurityReader(base.DbTypeName, base.TableName, new string[] { code }).GetAllCodeDateSerialDataList <StockMongoData>()?[code];
}
if (org_xdxr_data == null)
{
org_xdxr_data = new XDXRReader().getXDXRList(DbTypeName, code);
}
MongoReturnDataList <XDXRData> xdxr_data = org_xdxr_data.Copy(false);
MongoReturnDataList <StockMongoData> bfq_data = org_bfq_data.Copy(false);
MongoReturnDataList <StockMongoData> ret = bfq_data;
if (bfq_data.Count == 0)
{
return(ret);
}
if (xdxr_data.Count == 0)
{
return(ret);
}
MongoReturnDataList <XDXRData> info = xdxr_data.Query <int>("category", 1);
MongoReturnDataList <StockMongoData> retList = null;
List <StockMongoData> list = null;
#region category = 1
if (info.Count > 0)
{
list = Pandas.Concat <StockMongoData, XDXRData, string>(
ret,
info,
a => a.date,
a => a.date,
(s, a, b) => {
XDXRData ExObj = s.ExtentData as XDXRData;
if (ExObj == null)
{
ExObj = new XDXRData();
}
if (b)
{
ExObj.date = s.date;
ExObj.category = a.category;
}
s.ExtentData = ExObj;
});
var testlist = list;
//retList = new MongoReturnDataList<StockMongoData>(list);
list = Pandas.FillNa <StockMongoData, string, XDXRData>(list,
a => a.ExtentData as XDXRData,
a => (a.ExtentData as XDXRData).date,
(a, b, c) =>
{
XDXRData data = a.ExtentData as XDXRData;
if (data == null)
{
data = new XDXRData();
}
//data.date = a.date;
data.category = b.category ?? 0;
a.ExtentData = data;
}
, FillType.FFill);
//retList = new MongoReturnDataList<StockMongoData>(list);
//(a, b) =>
//{
// return (a.ExtentData as XDXRData).category;
//}, (a, b) => { a.ExtentData = b; }, FillType.FFill);
list = Pandas.Concat <StockMongoData, XDXRData, string>(new MongoReturnDataList <StockMongoData>(list), info,
a => a.date,
a => a.date,
(s, a, b) => {
XDXRData ExObj = s.ExtentData as XDXRData;
if (ExObj == null)
{
ExObj = new XDXRData();
}
//ExObj.date = s.date;
if (b)
{
ExObj.date = a.date;
ExObj.fenhong = a.fenhong;
ExObj.peigu = a.peigu;
ExObj.peigujia = a.peigujia;
ExObj.songzhuangu = a.songzhuangu;
}
s.ExtentData = ExObj;
});
//ret.FillNa({ return a => a.ExtentData; }, "category", FillType.FFill);
//retList = new MongoReturnDataList<StockMongoData>(list);
}
#endregion
#region 其他分红
else
{
list = Pandas.Concat <StockMongoData, XDXRData, string>(ret, info,
a => a.date,
a => a.date,
(s, a, b) => {
XDXRData ExObj = s.ExtentData as XDXRData;
if (ExObj == null)
{
ExObj = new XDXRData();
}
//ExObj.date = s.date;
if (b)
{
ExObj.date = s.date;
ExObj.category = a.category;
ExObj.fenhong = a.fenhong;
ExObj.peigu = a.peigu;
ExObj.peigujia = a.peigujia;
ExObj.songzhuangu = a.songzhuangu;
}
s.ExtentData = ExObj;
});
}
#endregion
list = Pandas.FillNa <StockMongoData, string, XDXRData>(list,
a => a.ExtentData as XDXRData,
a => (a.ExtentData as XDXRData).date,
(a, b, c) =>
{
XDXRData ExObj = a.ExtentData as XDXRData;
if (ExObj == null)
{
ExObj = new XDXRData();
}
ExObj.date = a.date;
ExObj.fenhong = 0;
ExObj.peigu = 0;
ExObj.peigujia = 0;
ExObj.songzhuangu = 0;
}, FillType.None);
ret = new MongoReturnDataList <StockMongoData>(list);
int xscnt = GlobalClass.TypeDataPoints[base.DbTypeName].RuntimeInfo.SecurityInfoList[code].decimal_point;
long Lbase = (long)Math.Pow(10, xscnt);
List <long?> CloseList = ret.ToList(a => (long?)(a.close * Lbase));
List <long?> closes = Pandas.ShiftX <long?>(CloseList, 1);
List <double?> xdxrcloses = new List <double?>();
for (int i = 0; i < ret.Count; i++)
{
XDXRData xdata = ret[i].ExtentData as XDXRData;
double midval = ((closes[i] ?? 0) * 10 - xdata.fenhong.Value * Lbase + xdata.peigu.Value * xdata.peigujia.Value * Lbase) / (10 + xdata.peigu.Value + xdata.songzhuangu.Value);
//long xdprice = Math.Round(midval);
xdxrcloses.Add(midval);
}
//precloses = Pandas.ShiftX<double>(precloses, -1);
List <double?> ProAdjList = new List <double?>();
List <double?> ppclose = Pandas.ShiftX <double?>(xdxrcloses, -1);//
for (int i = 0; i < ppclose.Count; i++)
{
if (ppclose[i] == null)
{
ProAdjList.Add(1);
continue;
}
//double? midval = (double?)(1*((double)ppclose[i].Value)/ ((double)CloseList[i]) / 1);
double?midval = (double?)(1 * ((double)ppclose[i].Value) / ((double)CloseList[i]) / 1);
ProAdjList.Add((double?)midval);
}
List <double?> adj = new List <double?>();
if (fqType == PriceAdj.Fore)
{
adj = ProAdjList;
adj = Pandas.Recovert <double?>(adj);
adj = Pandas.Cumprod(adj);
adj = Pandas.Recovert <double?>(adj);
double?curr = adj[adj.Count - 1];
if (curr.Value != 1)
{
//adj.ForEach(a => a = a / curr);
for (int i = 0; i < adj.Count; i++)
{
adj[i] /= curr;
}
}
//adj = Pandas.Recovert<double?>(adj);
}
else
{
adj = Pandas.Cumprod(ProAdjList);
Pandas.ShiftX <double?>(adj, -1);
}
for (int i = 0; i < ret.Count; i++)
{
ret[i].close *= adj[i].Value;
ret[i].open *= adj[i].Value;
ret[i].high *= adj[i].Value;
ret[i].low *= adj[i].Value;
ret[i].vol /= adj[i].Value;
}
return(ret);
}
