public void Test_Upload_DataTable()
{
DataTable dt = new DataTable();
DataColumn dc = new DataColumn("dt_string", Type.GetType("System.String"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_short", Type.GetType("System.Int16"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_int", Type.GetType("System.Int32"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_long", Type.GetType("System.Int64"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_double", Type.GetType("System.Int32"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_datetime", Type.GetType("System.DateTime"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_bool", Type.GetType("System.Boolean"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_byte", Type.GetType("System.Byte"));
dt.Columns.Add(dc);
dc = new DataColumn("dt_string", Type.GetType("System.String"));
dt.Columns.Add(dc);
DataRow dr = dt.NewRow();
dr["dt_short"] = 1;
dr["dt_int"] = 2147483646;
dr["dt_long"] = 2147483649;
dr["dt_double"] = 3.14159893984;
dr["dt_datetime"] = new DateTime(2018, 03, 30, 14, 59, 02, 111);
dr["dt_bool"] = false;
dr["dt_byte"] = (byte)97;
dr["dt_string"] = "test_string";
dt.Rows.Add(dr);
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
BasicTable bt = new BasicTable(dt);
Dictionary <string, IEntity> obj = new Dictionary <string, IEntity>();
obj.Add("up_datatable", (IEntity)bt);
db.upload(obj);
BasicIntVector v = (BasicIntVector)db.run("up_datatable.dt_int");
Assert.AreEqual(2147483646, v.get(0));
}
public void Test_WriteLocalTableToDfs()
{
//=======================prepare data to writing into dfs database =======================
List <string> colNames = new List <string>()
{
"sym", "dt", "prc", "cnt"
};
BasicStringVector symVec = new BasicStringVector(new List <string>()
{
"MS", "GOOG", "FB"
});
BasicDateTimeVector dtVec = new BasicDateTimeVector(new List <int?>()
{
Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now)
});
BasicDoubleVector prcVec = new BasicDoubleVector(new double[] { 101.5, 132.75, 37.96 });
BasicIntVector cntVec = new BasicIntVector(new int[] { 50, 78, 88 });
List <IVector> cols = new List <IVector>()
{
(IVector)symVec, (IVector)dtVec, (IVector)prcVec, (IVector)cntVec
};
BasicTable table1 = new BasicTable(colNames, cols);
//======================================================================================
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
db.login("admin", "123456", false);//login
//prepare dfs database and table
db.run("if(existsDatabase('dfs://testDatabase')){dropDatabase('dfs://testDatabase')}");
db.run("db = database('dfs://testDatabase',VALUE,'MS' 'GOOG' 'FB')");
db.run("tb= table('MS' as sym,datetime(now()) as dt,1.01 as prc,1 as cnt)");
db.run("db.createPartitionedTable(tb,'tb1','sym')");
db.run("def saveQuotes(t){ loadTable('dfs://testDatabase','tb1').append!(t)}");
List <IEntity> args = new List <IEntity>(1);
args.Add(table1);
db.run("saveQuotes", args);
}
public void Test_GetList()
{
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
BasicIntVector iv = (BasicIntVector)db.run("1 2 3 4");
int[] r = (int[])iv.getList();
Assert.AreEqual(4, r.Length);
BasicDoubleVector dv = (BasicDoubleVector)db.run("1.1 2.01 3.25 4.251441 3.32222");
double[] dr = (double[])dv.getList();
Assert.AreEqual(5, dr.Length);
Assert.AreEqual(1.1, dr[0]);
BasicLongVector lv = (BasicLongVector)db.run("102012522 12345678900 12221114455");
long[] lr = (long[])lv.getList();
Assert.AreEqual(3, lr.Length);
Assert.AreEqual(12345678900, lr[1]);
}
public autoFitTableAppender(string dbUrl, string tableName, bool asynTask, DBConnection conn)
{
this.dbUrl = dbUrl;
this.tableName = tableName;
this.asynTask = asynTask;
this.conn = conn;
conn.setasynTask(false);
BasicDictionary tableInfo = (BasicDictionary)conn.run("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
BasicTable colDefs;
colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs")));
BasicStringVector names = (BasicStringVector)colDefs.getColumn("name");
BasicIntVector types = (BasicIntVector)colDefs.getColumn("typeInt");
int rows = names.rows();
this.schema = new Dictionary <string, DATA_TYPE>();
for (int i = 0; i < rows; ++i)
{
schema.Add(names.getString(i), (DATA_TYPE)types.getInt(i));
}
conn.setasynTask(asynTask);
}
private BasicTable createBasicTable()
{
List <string> colNames = new List <string>();
colNames.Add("cbool");
colNames.Add("cchar");
colNames.Add("cshort");
colNames.Add("cint");
colNames.Add("clong");
colNames.Add("cdate");
colNames.Add("cmonth");
colNames.Add("ctime");
colNames.Add("cminute");
colNames.Add("csecond");
colNames.Add("cdatetime");
colNames.Add("ctimestamp");
colNames.Add("cnanotime");
colNames.Add("cnanotimestamp");
colNames.Add("cfloat");
colNames.Add("cdouble");
colNames.Add("csymbol");
colNames.Add("cstring");
List <IVector> cols = new List <IVector>();
//boolean
byte[] vbool = new byte[] { 1, 0 };
BasicBooleanVector bbv = new BasicBooleanVector(vbool);
cols.Add(bbv);
//char
byte[] vchar = new byte[] { (byte)'c', (byte)'a' };
BasicByteVector bcv = new BasicByteVector(vchar);
cols.Add(bcv);
//cshort
short[] vshort = new short[] { 32767, 29 };
BasicShortVector bshv = new BasicShortVector(vshort);
cols.Add(bshv);
//cint
int[] vint = new int[] { 2147483647, 483647 };
BasicIntVector bintv = new BasicIntVector(vint);
cols.Add(bintv);
//clong
long[] vlong = new long[] { 2147483647, 483647 };
BasicLongVector blongv = new BasicLongVector(vlong);
cols.Add(blongv);
//cdate
int[] vdate = new int[] { Utils.countDays(new DateTime(2018, 2, 14)), Utils.countDays(new DateTime(2018, 8, 15)) };
BasicDateVector bdatev = new BasicDateVector(vdate);
cols.Add(bdatev);
//cmonth
int[] vmonth = new int[] { Utils.countMonths(new DateTime(2018, 2, 6)), Utils.countMonths(new DateTime(2018, 8, 11)) };
BasicMonthVector bmonthv = new BasicMonthVector(vmonth);
cols.Add(bmonthv);
//ctime
int[] vtime = new int[] { Utils.countMilliseconds(16, 46, 05, 123), Utils.countMilliseconds(18, 32, 05, 321) };
BasicTimeVector btimev = new BasicTimeVector(vtime);
cols.Add(btimev);
//cminute
int[] vminute = new int[] { Utils.countMinutes(new TimeSpan(16, 30, 00)), Utils.countMinutes(new TimeSpan(9, 30, 00)) };
BasicMinuteVector bminutev = new BasicMinuteVector(vminute);
cols.Add(bminutev);
//csecond
int[] vsecond = new int[] { Utils.countSeconds(new TimeSpan(16, 30, 00)), Utils.countSeconds(new TimeSpan(16, 30, 50)) };
BasicSecondVector bsecondv = new BasicSecondVector(vsecond);
cols.Add(bsecondv);
//cdatetime
int[] vdatetime = new int[] { Utils.countSeconds(new DateTime(2018, 9, 8, 9, 30, 01)), Utils.countSeconds(new DateTime(2018, 11, 8, 16, 30, 01)) };
BasicDateTimeVector bdatetimev = new BasicDateTimeVector(vdatetime);
cols.Add(bdatetimev);
//ctimestamp
long[] vtimestamp = new long[] { Utils.countMilliseconds(2018, 11, 12, 9, 30, 01, 123), Utils.countMilliseconds(2018, 11, 12, 16, 30, 01, 123) };
BasicTimestampVector btimestampv = new BasicTimestampVector(vtimestamp);
cols.Add(btimestampv);
//cnanotime
long[] vnanotime = new long[] { Utils.countNanoseconds(new TimeSpan(9, 30, 05, 1234567)), Utils.countNanoseconds(new TimeSpan(16, 30, 05, 9876543)) };
BasicNanoTimeVector bnanotimev = new BasicNanoTimeVector(vnanotime);
cols.Add(bnanotimev);
//cnanotimestamp
long[] vnanotimestamp = new long[] { Utils.countNanoseconds(new DateTime(2018, 11, 12, 9, 30, 05, 123)), Utils.countNanoseconds(new DateTime(2018, 11, 13, 16, 30, 05, 987)) };
BasicNanoTimestampVector bnanotimestampv = new BasicNanoTimestampVector(vnanotimestamp);
cols.Add(bnanotimestampv);
//cfloat
float[] vfloat = new float[] { 2147.483647f, 483.647f };
BasicFloatVector bfloatv = new BasicFloatVector(vfloat);
cols.Add(bfloatv);
//cdouble
double[] vdouble = new double[] { 214.7483647, 48.3647 };
BasicDoubleVector bdoublev = new BasicDoubleVector(vdouble);
cols.Add(bdoublev);
//csymbol
String[] vsymbol = new String[] { "GOOG", "MS" };
BasicStringVector bsymbolv = new BasicStringVector(vsymbol);
cols.Add(bsymbolv);
//cstring
String[] vstring = new String[] { "", "test string" };
BasicStringVector bstringv = new BasicStringVector(vstring);
cols.Add(bstringv);
BasicTable t1 = new BasicTable(colNames, cols);
return(t1);
}
/**
* If fail to connect to the specified DolphinDB server, this function throw an exception.
*/
public MultithreadedTableWriter(string hostName, int port, string userId, string password,
string dbName, string tableName, bool useSSL, bool enableHighAvailability = false, string[] pHighAvailabilitySites = null,
int batchSize = 1, float throttle = 0.01f, int threadCount = 5, string partitionCol = "", int[] pCompressMethods = null)
{
hostName_ = hostName;
port_ = port;
userId_ = userId;
password_ = password;
useSSL_ = useSSL;
dbName_ = dbName;
tableName_ = tableName;
batchSize_ = batchSize;
throttleMilsecond_ = (int)throttle * 1000;
isExiting_ = false;
if (threadCount < 1)
{
throw new Exception("The parameter threadCount must be greater than or equal to 1.");
}
if (batchSize < 1)
{
throw new Exception("The parameter batchSize must be greater than or equal to 1.");
}
if (throttle < 0)
{
throw new Exception("The parameter throttle must be positive.");
}
if (threadCount > 1 && partitionCol == String.Empty)
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
DBConnection pConn = new DBConnection(false, useSSL_, pCompressMethods != null);
bool ret = pConn.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites);
if (!ret)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
BasicDictionary schema;
if (tableName == "")
{
schema = (BasicDictionary)pConn.run("schema(" + dbName + ")");
}
else
{
schema = (BasicDictionary)pConn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))");
}
IEntity partColNames = null;
if (schema.ContainsKey("partitionColumnName"))
{
partColNames = schema.get(new BasicString("partitionColumnName"));
isPartionedTable_ = true;
}
else
{
isPartionedTable_ = false;
if (tableName != "")
{
if (threadCount > 1)
{//只有多线程的时候需要
throw new Exception("The parameter threadCount must be 1 for a dimension table.");
}
}
}
BasicTable colDefs = (BasicTable)schema.get("colDefs");
BasicIntVector colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt");
BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name");
BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString");
colTypes_ = new List <DATA_TYPE>();
colNames_ = new List <string>();
colTypeString_ = new List <string>();
int columnSize = colDefsName.rows();
if (pCompressMethods != null)
{
if (columnSize != pCompressMethods.Length)
{
throw new Exception(string.Format("The number of elements in parameter compressMethods does not match the column size {0}. ", columnSize));
}
this.compressTypes_ = new int[columnSize];
Array.Copy(pCompressMethods, this.compressTypes_, columnSize);
}
for (int i = 0; i < columnSize; i++)
{
colNames_.Add(colDefsName.getString(i));
colTypes_.Add((DATA_TYPE)colDefsTypeInt.getInt(i));
colTypeString_.Add(colDefsTypeString.getString(i));
if (compressTypes_ != null)
{
AbstractVector.checkCompressedMethod(colTypes_[i], compressTypes_[i]);
}
}
if (threadCount > 1)
{
if (isPartionedTable_)
{
IEntity partitionSchema;
int partitionType;
if (partColNames.isScalar())
{
if (partColNames.getString() != partitionCol)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicInt)schema.get("partitionColumnIndex")).getInt();
partitionSchema = schema.get("partitionSchema");
partitionType = ((BasicInt)schema.get("partitionType")).getInt();
}
else
{
int dims = ((BasicStringVector)partColNames).rows();
if (dims > 1 && partitionCol == "")
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
int index = -1;
for (int i = 0; i < dims; ++i)
{
if (((BasicStringVector)partColNames).getString(i) == partitionCol)
{
index = i;
break;
}
}
if (index < 0)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicIntVector)schema.get("partitionColumnIndex")).getInt(index);
partitionSchema = ((BasicAnyVector)schema.get("partitionSchema")).get(index);
partitionType = ((BasicIntVector)schema.get("partitionType")).getInt(index);
}
DATA_TYPE partitionColType = colTypes_[partitionColumnIdx_];
partitionDomain_ = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema);
}
else
{//isPartionedTable_==false
if (partitionCol != "")
{
int threadcolindex = -1;
for (int i = 0; i < colNames_.Count; i++)
{
if (colNames_[i] == partitionCol)
{
threadcolindex = i;
break;
}
}
if (threadcolindex < 0)
{
throw new Exception(string.Format("No match found for {0}. ", partitionCol));
}
threadByColIndexForNonPartion_ = threadcolindex;
}
}
}
// init done, start thread now.
isExiting_ = false;
threads_ = new List <WriterThread>(threadCount);
for (int i = 0; i < threadCount; i++)
{
WriterThread writerThread = new WriterThread(this, pConn);
if (i == 0)
{
writerThread.conn_ = pConn;
}
else
{
writerThread.conn_ = new DBConnection(useSSL_, false);
if (writerThread.conn_.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites) == false)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
}
threads_.Add(writerThread);
}
}
