public void Test_Upload_DataTable() { DataTable dt = new DataTable(); DataColumn dc = new DataColumn("dt_string", Type.GetType("System.String")); dt.Columns.Add(dc); dc = new DataColumn("dt_short", Type.GetType("System.Int16")); dt.Columns.Add(dc); dc = new DataColumn("dt_int", Type.GetType("System.Int32")); dt.Columns.Add(dc); dc = new DataColumn("dt_long", Type.GetType("System.Int64")); dt.Columns.Add(dc); dc = new DataColumn("dt_double", Type.GetType("System.Int32")); dt.Columns.Add(dc); dc = new DataColumn("dt_datetime", Type.GetType("System.DateTime")); dt.Columns.Add(dc); dc = new DataColumn("dt_bool", Type.GetType("System.Boolean")); dt.Columns.Add(dc); dc = new DataColumn("dt_byte", Type.GetType("System.Byte")); dt.Columns.Add(dc); dc = new DataColumn("dt_string", Type.GetType("System.String")); dt.Columns.Add(dc); DataRow dr = dt.NewRow(); dr["dt_short"] = 1; dr["dt_int"] = 2147483646; dr["dt_long"] = 2147483649; dr["dt_double"] = 3.14159893984; dr["dt_datetime"] = new DateTime(2018, 03, 30, 14, 59, 02, 111); dr["dt_bool"] = false; dr["dt_byte"] = (byte)97; dr["dt_string"] = "test_string"; dt.Rows.Add(dr); DBConnection db = new DBConnection(); db.connect(SERVER, PORT); BasicTable bt = new BasicTable(dt); Dictionary <string, IEntity> obj = new Dictionary <string, IEntity>(); obj.Add("up_datatable", (IEntity)bt); db.upload(obj); BasicIntVector v = (BasicIntVector)db.run("up_datatable.dt_int"); Assert.AreEqual(2147483646, v.get(0)); }
public void Test_WriteLocalTableToDfs() { //=======================prepare data to writing into dfs database ======================= List <string> colNames = new List <string>() { "sym", "dt", "prc", "cnt" }; BasicStringVector symVec = new BasicStringVector(new List <string>() { "MS", "GOOG", "FB" }); BasicDateTimeVector dtVec = new BasicDateTimeVector(new List <int?>() { Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now) }); BasicDoubleVector prcVec = new BasicDoubleVector(new double[] { 101.5, 132.75, 37.96 }); BasicIntVector cntVec = new BasicIntVector(new int[] { 50, 78, 88 }); List <IVector> cols = new List <IVector>() { (IVector)symVec, (IVector)dtVec, (IVector)prcVec, (IVector)cntVec }; BasicTable table1 = new BasicTable(colNames, cols); //====================================================================================== DBConnection db = new DBConnection(); db.connect(SERVER, PORT); db.login("admin", "123456", false);//login //prepare dfs database and table db.run("if(existsDatabase('dfs://testDatabase')){dropDatabase('dfs://testDatabase')}"); db.run("db = database('dfs://testDatabase',VALUE,'MS' 'GOOG' 'FB')"); db.run("tb= table('MS' as sym,datetime(now()) as dt,1.01 as prc,1 as cnt)"); db.run("db.createPartitionedTable(tb,'tb1','sym')"); db.run("def saveQuotes(t){ loadTable('dfs://testDatabase','tb1').append!(t)}"); List <IEntity> args = new List <IEntity>(1); args.Add(table1); db.run("saveQuotes", args); }
public void Test_GetList() { DBConnection db = new DBConnection(); db.connect(SERVER, PORT); BasicIntVector iv = (BasicIntVector)db.run("1 2 3 4"); int[] r = (int[])iv.getList(); Assert.AreEqual(4, r.Length); BasicDoubleVector dv = (BasicDoubleVector)db.run("1.1 2.01 3.25 4.251441 3.32222"); double[] dr = (double[])dv.getList(); Assert.AreEqual(5, dr.Length); Assert.AreEqual(1.1, dr[0]); BasicLongVector lv = (BasicLongVector)db.run("102012522 12345678900 12221114455"); long[] lr = (long[])lv.getList(); Assert.AreEqual(3, lr.Length); Assert.AreEqual(12345678900, lr[1]); }
public autoFitTableAppender(string dbUrl, string tableName, bool asynTask, DBConnection conn) { this.dbUrl = dbUrl; this.tableName = tableName; this.asynTask = asynTask; this.conn = conn; conn.setasynTask(false); BasicDictionary tableInfo = (BasicDictionary)conn.run("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))"); BasicTable colDefs; colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs"))); BasicStringVector names = (BasicStringVector)colDefs.getColumn("name"); BasicIntVector types = (BasicIntVector)colDefs.getColumn("typeInt"); int rows = names.rows(); this.schema = new Dictionary <string, DATA_TYPE>(); for (int i = 0; i < rows; ++i) { schema.Add(names.getString(i), (DATA_TYPE)types.getInt(i)); } conn.setasynTask(asynTask); }
private BasicTable createBasicTable() { List <string> colNames = new List <string>(); colNames.Add("cbool"); colNames.Add("cchar"); colNames.Add("cshort"); colNames.Add("cint"); colNames.Add("clong"); colNames.Add("cdate"); colNames.Add("cmonth"); colNames.Add("ctime"); colNames.Add("cminute"); colNames.Add("csecond"); colNames.Add("cdatetime"); colNames.Add("ctimestamp"); colNames.Add("cnanotime"); colNames.Add("cnanotimestamp"); colNames.Add("cfloat"); colNames.Add("cdouble"); colNames.Add("csymbol"); colNames.Add("cstring"); List <IVector> cols = new List <IVector>(); //boolean byte[] vbool = new byte[] { 1, 0 }; BasicBooleanVector bbv = new BasicBooleanVector(vbool); cols.Add(bbv); //char byte[] vchar = new byte[] { (byte)'c', (byte)'a' }; BasicByteVector bcv = new BasicByteVector(vchar); cols.Add(bcv); //cshort short[] vshort = new short[] { 32767, 29 }; BasicShortVector bshv = new BasicShortVector(vshort); cols.Add(bshv); //cint int[] vint = new int[] { 2147483647, 483647 }; BasicIntVector bintv = new BasicIntVector(vint); cols.Add(bintv); //clong long[] vlong = new long[] { 2147483647, 483647 }; BasicLongVector blongv = new BasicLongVector(vlong); cols.Add(blongv); //cdate int[] vdate = new int[] { Utils.countDays(new DateTime(2018, 2, 14)), Utils.countDays(new DateTime(2018, 8, 15)) }; BasicDateVector bdatev = new BasicDateVector(vdate); cols.Add(bdatev); //cmonth int[] vmonth = new int[] { Utils.countMonths(new DateTime(2018, 2, 6)), Utils.countMonths(new DateTime(2018, 8, 11)) }; BasicMonthVector bmonthv = new BasicMonthVector(vmonth); cols.Add(bmonthv); //ctime int[] vtime = new int[] { Utils.countMilliseconds(16, 46, 05, 123), Utils.countMilliseconds(18, 32, 05, 321) }; BasicTimeVector btimev = new BasicTimeVector(vtime); cols.Add(btimev); //cminute int[] vminute = new int[] { Utils.countMinutes(new TimeSpan(16, 30, 00)), Utils.countMinutes(new TimeSpan(9, 30, 00)) }; BasicMinuteVector bminutev = new BasicMinuteVector(vminute); cols.Add(bminutev); //csecond int[] vsecond = new int[] { Utils.countSeconds(new TimeSpan(16, 30, 00)), Utils.countSeconds(new TimeSpan(16, 30, 50)) }; BasicSecondVector bsecondv = new BasicSecondVector(vsecond); cols.Add(bsecondv); //cdatetime int[] vdatetime = new int[] { Utils.countSeconds(new DateTime(2018, 9, 8, 9, 30, 01)), Utils.countSeconds(new DateTime(2018, 11, 8, 16, 30, 01)) }; BasicDateTimeVector bdatetimev = new BasicDateTimeVector(vdatetime); cols.Add(bdatetimev); //ctimestamp long[] vtimestamp = new long[] { Utils.countMilliseconds(2018, 11, 12, 9, 30, 01, 123), Utils.countMilliseconds(2018, 11, 12, 16, 30, 01, 123) }; BasicTimestampVector btimestampv = new BasicTimestampVector(vtimestamp); cols.Add(btimestampv); //cnanotime long[] vnanotime = new long[] { Utils.countNanoseconds(new TimeSpan(9, 30, 05, 1234567)), Utils.countNanoseconds(new TimeSpan(16, 30, 05, 9876543)) }; BasicNanoTimeVector bnanotimev = new BasicNanoTimeVector(vnanotime); cols.Add(bnanotimev); //cnanotimestamp long[] vnanotimestamp = new long[] { Utils.countNanoseconds(new DateTime(2018, 11, 12, 9, 30, 05, 123)), Utils.countNanoseconds(new DateTime(2018, 11, 13, 16, 30, 05, 987)) }; BasicNanoTimestampVector bnanotimestampv = new BasicNanoTimestampVector(vnanotimestamp); cols.Add(bnanotimestampv); //cfloat float[] vfloat = new float[] { 2147.483647f, 483.647f }; BasicFloatVector bfloatv = new BasicFloatVector(vfloat); cols.Add(bfloatv); //cdouble double[] vdouble = new double[] { 214.7483647, 48.3647 }; BasicDoubleVector bdoublev = new BasicDoubleVector(vdouble); cols.Add(bdoublev); //csymbol String[] vsymbol = new String[] { "GOOG", "MS" }; BasicStringVector bsymbolv = new BasicStringVector(vsymbol); cols.Add(bsymbolv); //cstring String[] vstring = new String[] { "", "test string" }; BasicStringVector bstringv = new BasicStringVector(vstring); cols.Add(bstringv); BasicTable t1 = new BasicTable(colNames, cols); return(t1); }
/** * If fail to connect to the specified DolphinDB server, this function throw an exception. */ public MultithreadedTableWriter(string hostName, int port, string userId, string password, string dbName, string tableName, bool useSSL, bool enableHighAvailability = false, string[] pHighAvailabilitySites = null, int batchSize = 1, float throttle = 0.01f, int threadCount = 5, string partitionCol = "", int[] pCompressMethods = null) { hostName_ = hostName; port_ = port; userId_ = userId; password_ = password; useSSL_ = useSSL; dbName_ = dbName; tableName_ = tableName; batchSize_ = batchSize; throttleMilsecond_ = (int)throttle * 1000; isExiting_ = false; if (threadCount < 1) { throw new Exception("The parameter threadCount must be greater than or equal to 1."); } if (batchSize < 1) { throw new Exception("The parameter batchSize must be greater than or equal to 1."); } if (throttle < 0) { throw new Exception("The parameter throttle must be positive."); } if (threadCount > 1 && partitionCol == String.Empty) { throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1."); } DBConnection pConn = new DBConnection(false, useSSL_, pCompressMethods != null); bool ret = pConn.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites); if (!ret) { throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port)); } BasicDictionary schema; if (tableName == "") { schema = (BasicDictionary)pConn.run("schema(" + dbName + ")"); } else { schema = (BasicDictionary)pConn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))"); } IEntity partColNames = null; if (schema.ContainsKey("partitionColumnName")) { partColNames = schema.get(new BasicString("partitionColumnName")); isPartionedTable_ = true; } else { isPartionedTable_ = false; if (tableName != "") { if (threadCount > 1) {//只有多线程的时候需要 throw new Exception("The parameter threadCount must be 1 for a dimension table."); } } } BasicTable colDefs = (BasicTable)schema.get("colDefs"); BasicIntVector colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt"); BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name"); BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString"); colTypes_ = new List <DATA_TYPE>(); colNames_ = new List <string>(); colTypeString_ = new List <string>(); int columnSize = colDefsName.rows(); if (pCompressMethods != null) { if (columnSize != pCompressMethods.Length) { throw new Exception(string.Format("The number of elements in parameter compressMethods does not match the column size {0}. ", columnSize)); } this.compressTypes_ = new int[columnSize]; Array.Copy(pCompressMethods, this.compressTypes_, columnSize); } for (int i = 0; i < columnSize; i++) { colNames_.Add(colDefsName.getString(i)); colTypes_.Add((DATA_TYPE)colDefsTypeInt.getInt(i)); colTypeString_.Add(colDefsTypeString.getString(i)); if (compressTypes_ != null) { AbstractVector.checkCompressedMethod(colTypes_[i], compressTypes_[i]); } } if (threadCount > 1) { if (isPartionedTable_) { IEntity partitionSchema; int partitionType; if (partColNames.isScalar()) { if (partColNames.getString() != partitionCol) { throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol)); } partitionColumnIdx_ = ((BasicInt)schema.get("partitionColumnIndex")).getInt(); partitionSchema = schema.get("partitionSchema"); partitionType = ((BasicInt)schema.get("partitionType")).getInt(); } else { int dims = ((BasicStringVector)partColNames).rows(); if (dims > 1 && partitionCol == "") { throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1."); } int index = -1; for (int i = 0; i < dims; ++i) { if (((BasicStringVector)partColNames).getString(i) == partitionCol) { index = i; break; } } if (index < 0) { throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol)); } partitionColumnIdx_ = ((BasicIntVector)schema.get("partitionColumnIndex")).getInt(index); partitionSchema = ((BasicAnyVector)schema.get("partitionSchema")).get(index); partitionType = ((BasicIntVector)schema.get("partitionType")).getInt(index); } DATA_TYPE partitionColType = colTypes_[partitionColumnIdx_]; partitionDomain_ = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema); } else {//isPartionedTable_==false if (partitionCol != "") { int threadcolindex = -1; for (int i = 0; i < colNames_.Count; i++) { if (colNames_[i] == partitionCol) { threadcolindex = i; break; } } if (threadcolindex < 0) { throw new Exception(string.Format("No match found for {0}. ", partitionCol)); } threadByColIndexForNonPartion_ = threadcolindex; } } } // init done, start thread now. isExiting_ = false; threads_ = new List <WriterThread>(threadCount); for (int i = 0; i < threadCount; i++) { WriterThread writerThread = new WriterThread(this, pConn); if (i == 0) { writerThread.conn_ = pConn; } else { writerThread.conn_ = new DBConnection(useSSL_, false); if (writerThread.conn_.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites) == false) { throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port)); } } threads_.Add(writerThread); } }