public void testKeepWritingTask()
{
DBConnection conn = new DBConnection();
conn.connect(SERVER, PORT, USER, PASSWORD, WRITE_FUNCTION_SCRIPT, true, HA_SITES);
int count = 0;
Random rnd = new Random();
string funcScript = String.Format("saveData{{{0}}}", TBNAME);
Console.WriteLine(funcScript);
for (int i = 0; i < 86400000; i++)
{
IList <IEntity> args = new List <IEntity>();
DateTime dt = new DateTime(2021, 1, 1);
BasicDateVector bdv = new BasicDateVector(new int[] { Utils.countDays(2021, 4, 7) });
BasicTimeVector btv = new BasicTimeVector(new int[] { i });
BasicStringVector bsv = new BasicStringVector(new string[] { SYMBOLS[rnd.Next(3)] });
BasicIntVector bqv = new BasicIntVector(new int[] { rnd.Next(80, 100) });
BasicDoubleVector bpv = new BasicDoubleVector(new double[] { rnd.NextDouble() });
List <String> colNames = new List <string>()
{
"date", "time", "sym", "qty", "price"
};
List <IVector> cols = new List <IVector>()
{
bdv, btv, bsv, bqv, bpv
};
BasicTable table1 = new BasicTable(colNames, cols);
args.Add(table1);
BasicInt re = (BasicInt)conn.run(funcScript, args);
Console.WriteLine(re.getInt());
}
}
public void createPartitionedTableAppender()
{
IDBConnectionPool pool = new ExclusiveDBConnectionPool("192.168.1.37", 8848, "admin", "123456", 5, true, true);
IDBTask conn = new BasicDBTask("dbPath = \"dfs://demohash\";if(existsDatabase(dbPath)) dropDatabase(dbPath); db = database(dbPath, HASH,[STRING, 2]);t= table(100:0,`id`valuie,[STRING,INT]);pt=db.createPartitionedTable(t,`pt,`id);");
pool.execute(conn);
PartitionedTableAppender appender = new PartitionedTableAppender("dfs://demohash", "pt", "id", pool);
List <String> colNames = new List <String>(2);
colNames.Add("id");
colNames.Add("value");
List <IVector> cols = new List <IVector>(2);
BasicStringVector id = new BasicStringVector(3);
id.setString(0, "ORCA");
id.setString(1, "YHOO");
id.setString(2, "Ford");
cols.Add(id);
BasicIntVector value = new BasicIntVector(3);
value.setInt(0, 10);
value.setInt(1, 11);
value.setInt(2, 12);
cols.Add(value);
int res = appender.append(new BasicTable(colNames, cols));
Assert.AreEqual(3, res);
}
public static Task getTask()
{
return(new Task(() => {
int i = 0;
while (true)
{
if (i == 20000000)
{
break;
}
Row row;
if (RowQueue.queue.TryTake(out row))
{
BasicIntVector v1 = (BasicIntVector)Program.bt.getColumn(0);
BasicStringVector v2 = (BasicStringVector)Program.bt.getColumn(1);
v1.setInt(i, (int)row["aid"]);
v2.setString(i, (string)row["namec"]);
i++;
}
if (i % 100000 == 0 && i > 0)
{
Console.WriteLine("100000 rows parsed! left " + RowQueue.queue.Count().ToString());
}
}
}));
}
public void blob_imemory_table_upload()
{
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
db.run("share table(100:0, `id`memo, [INT, BLOB])as st");
List <String> colNames = new List <String>(2);
colNames.Add("id");
colNames.Add("str");
List <IVector> cols = new List <IVector>(2);
BasicIntVector id = new BasicIntVector(2);
id.setInt(1, 1);
id.setInt(0, 1);
BasicStringVector str = new BasicStringVector(2);
String tmp = "534656 unuk6.太阳能与看见了和和 规划局广告费 hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金dd，/5额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技hudfgdg TYF FEW 867R 8T至于金额及计划一 至于科技";
str.setString(0, tmp);
str.setString(1, tmp);
cols.Add(id);
cols.Add(str);
BasicTable t = new BasicTable(colNames, cols);
List <IEntity> arg = new List <IEntity>()
{
t
};
db.run("tableInsert{st}", arg);
BasicTable st = (BasicTable)db.run("st");
// Console.WriteLine(((BasicTable)st).getColumn(2).get(0).getValue());
}
private void addRow(Row row, int Pos)
{
BasicIntVector v = (BasicIntVector)cols[0];
BasicStringVector v1 = (BasicStringVector)cols[1];
v.setInt(Pos, (int)row[row.Columns[0]]);
v1.setString(Pos, (string)row[row.Columns[1]]);
}
public ExclusiveDBConnectionPool(string host, int port, string uid, string pwd, int count, bool loadBalance, bool highAvaliability)
{
conns = new List <DBConnection>(count);
if (!loadBalance)
{
for (int i = 0; i < count; i++)
{
DBConnection conn = new DBConnection();
if (!conn.connect(host, port, uid, pwd, "", highAvaliability))
{
throw new Exception("Cant't connect to the specified host.");
}
conns.Add(conn);
}
}
else
{
DBConnection entryPoint = new DBConnection();
if (!entryPoint.connect(host, port, uid, pwd))
{
throw new Exception("Can't connect to the specified host.");
}
BasicStringVector nodes = (BasicStringVector)entryPoint.run("rpc(getControllerAlias(), getClusterLiveDataNodes{false})");
int nodeCount = nodes.rows();
string[] hosts = new string[nodeCount];
int[] ports = new int[nodeCount];
for (int i = 0; i < nodeCount; i++)
{
string[] fields = nodes.getString(i).Split(':');
if (fields.Length < 2)
{
throw new Exception("Invalid datanode adress: " + nodes.getString(i));
}
hosts[i] = fields[0];
ports[i] = int.Parse(fields[1]);
}
for (int i = 0; i < count; i++)
{
DBConnection conn = new DBConnection();
if (!(conn.connect(hosts[i % nodeCount], ports[i % nodeCount], uid, pwd, "", highAvaliability)))
{
throw new Exception("Can't connect to the host: " + nodes.getString(i % nodeCount));
}
conns.Add(conn);
}
}
ThreadPool.SetMaxThreads(count, count);
}
private void initialize(IList <long?> longs, BasicAnyVector locations)
{
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (isSorted(longs) == false)
{
throw new Exception("ranges " + longs.ToString() + " not sorted!");
}
if (longs.Count != locations.rows() + 1)
{
throw new Exception("expect # locations == # range values - 1");
}
ranges = longs.ToArray();
this.locations = new List <>();
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
this.locations.Add(location.String);
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
this.locations.Add(new List <>());
for (int j = 0; j < locationVector.rows(); ++j)
{
this.locations[this.locations.Count - 1].Add(((BasicString)locationVector.get(j)).String);
}
}
}
}
public void Test_WriteLocalTableToDfs()
{
//=======================prepare data to writing into dfs database =======================
List <string> colNames = new List <string>()
{
"sym", "dt", "prc", "cnt"
};
BasicStringVector symVec = new BasicStringVector(new List <string>()
{
"MS", "GOOG", "FB"
});
BasicDateTimeVector dtVec = new BasicDateTimeVector(new List <int?>()
{
Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now), Utils.countSeconds(DateTime.Now)
});
BasicDoubleVector prcVec = new BasicDoubleVector(new double[] { 101.5, 132.75, 37.96 });
BasicIntVector cntVec = new BasicIntVector(new int[] { 50, 78, 88 });
List <IVector> cols = new List <IVector>()
{
(IVector)symVec, (IVector)dtVec, (IVector)prcVec, (IVector)cntVec
};
BasicTable table1 = new BasicTable(colNames, cols);
//======================================================================================
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
db.login("admin", "123456", false);//login
//prepare dfs database and table
db.run("if(existsDatabase('dfs://testDatabase')){dropDatabase('dfs://testDatabase')}");
db.run("db = database('dfs://testDatabase',VALUE,'MS' 'GOOG' 'FB')");
db.run("tb= table('MS' as sym,datetime(now()) as dt,1.01 as prc,1 as cnt)");
db.run("db.createPartitionedTable(tb,'tb1','sym')");
db.run("def saveQuotes(t){ loadTable('dfs://testDatabase','tb1').append!(t)}");
List <IEntity> args = new List <IEntity>(1);
args.Add(table1);
db.run("saveQuotes", args);
}
public void Test_ContructBasicTableByDataTable()
{
DataTable dt = new DataTable();
dt.Columns.Add("dt", Type.GetType("System.DateTime"));
dt.Columns.Add("bl", Type.GetType("System.Boolean"));
dt.Columns.Add("sb", Type.GetType("System.Byte"));
dt.Columns.Add("db", Type.GetType("System.Double"));
dt.Columns.Add("ts", Type.GetType("System.TimeSpan"));
dt.Columns.Add("i1", Type.GetType("System.Int16"));
dt.Columns.Add("i3", Type.GetType("System.Int32"));
dt.Columns.Add("i6", Type.GetType("System.Int64"));
dt.Columns.Add("s", Type.GetType("System.String"));
DataRow dr = dt.NewRow();
dr["dt"] = DBNull.Value;
dr["bl"] = DBNull.Value;
dr["sb"] = DBNull.Value;
dr["db"] = DBNull.Value;
dr["ts"] = DBNull.Value;
dr["i1"] = DBNull.Value;
dr["i3"] = DBNull.Value;
dr["i6"] = DBNull.Value;
dr["s"] = DBNull.Value;
dt.Rows.Add(dr);
BasicTable bt = new BasicTable(dt);
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
Dictionary <string, IEntity> obj = new Dictionary <string, IEntity>();
obj.Add("up_datatable", (IEntity)bt);
db.upload(obj);
db.run("share up_datatable as tb1");
BasicStringVector bsv = (BasicStringVector)bt.getColumn(8);
Assert.AreEqual("", bsv.get(0).getString());
}
private void initialize(IList <string> strings, BasicAnyVector locations)
{
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (strings.Count != locations.rows())
{
throw new Exception("expect # locations == # values");
}
map = new Dictionary <>();
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
string val = strings[i];
map[val] = location.String;
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
string val = strings[i];
map[val] = new List <>();
for (int j = 0; j < locationVector.rows(); ++j)
{
map[val].Add(locationVector.getString(j));
}
}
}
}
public autoFitTableAppender(string dbUrl, string tableName, bool asynTask, DBConnection conn)
{
this.dbUrl = dbUrl;
this.tableName = tableName;
this.asynTask = asynTask;
this.conn = conn;
conn.setasynTask(false);
BasicDictionary tableInfo = (BasicDictionary)conn.run("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
BasicTable colDefs;
colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs")));
BasicStringVector names = (BasicStringVector)colDefs.getColumn("name");
BasicIntVector types = (BasicIntVector)colDefs.getColumn("typeInt");
int rows = names.rows();
this.schema = new Dictionary <string, DATA_TYPE>();
for (int i = 0; i < rows; ++i)
{
schema.Add(names.getString(i), (DATA_TYPE)types.getInt(i));
}
conn.setasynTask(asynTask);
}
public void blob_imemory_table_upload_download_bigdata()
{
DBConnection db = new DBConnection();
db.connect(SERVER, PORT);
db.run("share table(blob(`d`f) as blob)as st");
List <String> colNames = new List <String>(1);
colNames.Add("str");
List <IVector> cols = new List <IVector>(1);
BasicStringVector vec = new BasicStringVector(1000);
BasicString str = (BasicString)db.run("mt=table(take(1..1000000,1000) as id,take(`fyf``ftf`ddtjtydtyty`通常都是,1000) as str,take(`fyf``ftf`ddtjtydtyty`通常都是,1000) as str1," +
"take(`fyf``ftf`ddtjtydtyty`通常都是,1000) as str2);mt.toStdJson()");
String tmp = str.getString();
for (int i = 0; i < 1000; i++)
{
vec.setString(i, tmp + i.ToString());
}
cols.Add(vec);
BasicTable tb = new BasicTable(colNames, cols);
List <IEntity> arg = new List <IEntity>()
{
tb
};
// for (int i = 0; i < 100; i++)
// {
db.run("tableInsert{st}", arg);
// }
BasicTable st = (BasicTable)db.run("st");
Assert.AreEqual(1000, tb.rows());
Assert.AreEqual(1, tb.columns());
Assert.AreEqual(tmp + 8.ToString(), ((BasicTable)st).getColumn(0).get(10).getString());
}
public bool switchServer()
{
if (!highAvailability)
{
return(false);
}
if (controllerHost == null)
{
return(false);
}
// 1 find live datanodes and pick one
DBConnection tmp = new DBConnection();
tmp.connect(controllerHost, controllerPort);
BasicStringVector liveDataNodes = (BasicStringVector)tmp.run("getLiveDataNodes(false)");
tmp.close();
// 2 try get connection to new datanode
int size = liveDataNodes.rows();
Console.WriteLine("living node size: " + size);
for (int i = 0; i < size; i++)
{
string[] liveDataNodeSite = liveDataNodes.get(i).getString().Split(':');
hostName = liveDataNodeSite[0];
port = int.Parse(liveDataNodeSite[1]);
Console.WriteLine("Try node " + i + ": " + hostName + ":" + port);
try
{
connect();
return(true);
}
catch (Exception) { }
}
return(false);
}
/**
* Add the name of the database and table that you want to insert data into before actually call insert.
* The parameter partitioned indicates whether the added table is a partitioned table. If this function
* is called to add an in-memory table, the parameter dbName indicates the name of the shared in-memory
* table, and the parameter tableName should be a null string. If error is raised on the server, this
* function throws an exception.
*/
public void addTable(string dbName, string tableName = "", bool partitioned = true)
{
try
{
rwLock_.EnterReadLock();
if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName)))
{
throw new Exception("Failed to add table, the specified table has not been removed yet.");
}
}
finally
{
rwLock_.ExitReadLock();
}
DBConnection conn = new DBConnection(false, false);
bool ret = conn.connect(hostName_, port_, userId_, password_);
if (!ret)
{
throw new Exception("Failed to connect to server.");
}
string tableInsert;
string saveTable = "";
BasicDictionary schema;
string tmpDiskGlobal = "tmpDiskGlobal";
if (tableName == "")
{
tableInsert = "tableInsert{" + dbName + "}";
schema = (BasicDictionary)conn.run("schema(" + dbName + ")");
}
else if (partitioned)
{
tableInsert = "tableInsert{loadTable(\"" + dbName + "\",\"" + tableName + "\")}";
schema = (BasicDictionary)conn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))");
}
else
{
throw new Exception("The target table must be an in-memory table or a table in a distributed database.");
}
BasicTable colDefs = (BasicTable)schema.get(new BasicString("colDefs"));
DestTable destTable;
if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName)))
{
throw new Exception("Failed to add table, the specified table has not been removed yet.");
}
destTable = new DestTable();
destTable.dbName = dbName;
destTable.tableName = tableName;
destTable.conn = conn;
destTable.tableInsert = tableInsert;
destTable.saveTable = saveTable;
destTable.colDefs = colDefs;
destTable.columnNum = colDefs.rows();
destTable.colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt");
destTable.destroy = false;
destTable.writeQueue = new ConcurrentQueue <List <IScalar> >();
destTable.failQueue = new ConcurrentQueue <List <IScalar> >();
List <string> colNames = new List <string>();
List <DATA_TYPE> colTypes = new List <DATA_TYPE>();
BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name");
for (int i = 0; i < destTable.columnNum; i++)
{
colNames.Add(colDefsName.getString(i));
colTypes.Add((DATA_TYPE)destTable.colDefsTypeInt.getInt(i));
}
destTable.colNames = colNames;
destTable.colTypes = colTypes;
if (!partitioned)
{
string colName = "";
string colType = "";
BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString");
for (int i = 0; i < destTable.columnNum; i++)
{
colName = colName + "`" + colDefsName.getString(i);
colType = colType + "`" + colDefsTypeString.getString(i);
}
destTable.createTmpSharedTable = "share table(" + "1000:0," + colName + "," + colType + ") as " + tmpDiskGlobal;
}
try
{
rwLock_.EnterWriteLock();
if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName)))
{
throw new Exception("Failed to add table, the specified table has not been removed yet.");
}
destTables_[Tuple.Create <string, string>(dbName, tableName)] = destTable;
WriteThread writeThreadFuc = new WriteThread(destTable, partitioned);
destTable.writeThread = new Thread(writeThreadFuc.run);
destTable.writeThread.Start();
}
finally
{
rwLock_.ExitWriteLock();
}
}
protected BlockingCollection <List <IMessage> > subscribeInternal(string host, int port, string tableName, string actionName, MessageHandler handler, long offset, bool reconnect, IVector filter, bool createSubInfo)
{
string topic = "";
IEntity re;
BlockingCollection <List <IMessage> > queue;
DBConnection dbConn = new DBConnection();
dbConn.connect(host, port);
try
{
string localIP = listeningHost_;
if (localIP == null || localIP.Equals(String.Empty))
{
localIP = dbConn.LocalAddress;
}
List <IEntity> @params = new List <IEntity>
{
new BasicString(tableName),
new BasicString(actionName)
};
re = dbConn.run("getSubscriptionTopic", @params);
topic = ((BasicAnyVector)re).getEntity(0).getString();
@params.Clear();
@params.Add(new BasicString(localIP));
@params.Add(new BasicInt(listeningPort_));
@params.Add(new BasicString(tableName));
@params.Add(new BasicString(actionName));
@params.Add(new BasicLong(offset));
if (filter != null)
{
@params.Add(filter);
}
re = dbConn.run("publishTable", @params);
lock (subscribeInfos_)
{
if (createSubInfo)
{
Site[] sites;
if (re is BasicAnyVector)
{
BasicStringVector HASiteStrings = (BasicStringVector)((BasicAnyVector)re).getEntity(1);
int HASiteNum = HASiteStrings.rows();
sites = new Site[HASiteNum];
for (int i = 0; i < HASiteNum; ++i)
{
String HASite = HASiteStrings.get(i).getString();
String[] HASiteHostAndPort = HASite.Split(':');
String HASiteHost = HASiteHostAndPort[0];
int HASitePort = int.Parse(HASiteHostAndPort[1]);
String HASiteAlias = HASiteHostAndPort[2];
sites[i] = new Site(HASiteHost, HASitePort);
String HATopic = getTopic(HASiteHost, HASitePort, HASiteAlias, tableName, actionName);
HATopicToTrueTopic_[HATopic] = topic;
}
}
else
{
sites = new Site[] { new Site(host, port) };
lock (HATopicToTrueTopic_)
{
HATopicToTrueTopic_[topic] = topic;
}
}
SubscribeInfo subscribeInfo = new SubscribeInfo(DateTime.Now, new BlockingCollection <List <IMessage> >(), sites, topic, offset, reconnect, filter, handler, tableName, actionName);
subscribeInfo.setConnectState(ConnectState.REQUEST);
queue = subscribeInfo.getQueue();
if (subscribeInfos_.ContainsKey(topic))
{
throw new Exception("Subscription with topic " + topic + " exist. ");
}
else
{
subscribeInfos_.TryAdd(topic, subscribeInfo);
}
Console.WriteLine("Successfully subscribed table " + topic);
}
else
{
SubscribeInfo subscribeInfo = null;
if (!subscribeInfos_.TryGetValue(topic, out subscribeInfo))
{
throw new Exception("Subscription with topic " + topic + " doesn't exist. ");
}
lock (subscribeInfo)
{
if (subscribeInfo.getConnectState() == ConnectState.RECEIVED_SCHEMA)
{
throw new Exception("Subscription with topic " + topic + " the connection has been created. ");
}
subscribeInfo.setConnectState(ConnectState.REQUEST);
}
queue = subscribeInfo.getQueue();
}
}
}
finally
{
dbConn.close();
}
return(queue);
}
public PartitionedTableAppender(string dbUrl, string tableName, string partitionColName, string appendFunction, IDBConnectionPool pool)
{
this.pool = pool;
threadCount = pool.getConnectionCount();
chunkIndices = new List <List <int> >(threadCount);
for (int i = 0; i < threadCount; ++i)
{
chunkIndices.Add(new List <int>());
}
DBConnection conn = new DBConnection();
IEntity partitionSchema;
BasicTable colDefs;
BasicIntVector typeInts;
int partitionType;
DATA_TYPE partitionColType;
try
{
IDBTask task;
if (dbUrl == null || dbUrl.Length == 0)
{
task = new BasicDBTask("schema(" + tableName + ")");
appendScript = "tableInsert{" + tableName + "}";
}
else
{
task = new BasicDBTask("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
appendScript = "tableInsert{loadTable('" + dbUrl + "', '" + tableName + "')}";
}
if (appendFunction != null && appendFunction.Length != 0)
{
appendScript = appendFunction;
}
pool.execute(task);
if (!task.isSuccessful())
{
throw new Exception(task.getErrorMsg());
}
tableInfo = (BasicDictionary)task.getResults();
IEntity partColNames = tableInfo.get(new BasicString("partitionColumnName"));
if (partColNames == null)
{
throw new Exception("Can't find specified partition column name.");
}
if (partColNames.isScalar())
{
if (!((BasicString)partColNames).getString().Equals(partitionColName, StringComparison.OrdinalIgnoreCase))
{
throw new Exception("Can't find specified partition column name.");
}
partitionColumnIdx = ((BasicInt)tableInfo.get(new BasicString("partitionColumnIndex"))).getValue();
partitionSchema = tableInfo.get(new BasicString("partitionSchema"));
partitionType = ((BasicInt)tableInfo.get(new BasicString("partitionType"))).getValue();
//
partitionColType = (DATA_TYPE)((BasicInt)tableInfo.get(new BasicString("partitionColumnType"))).getValue();
}
else
{
BasicStringVector vec = (BasicStringVector)partColNames;
int dims = vec.rows();
int index = -1;
for (int i = 0; i < dims; ++i)
{
if (!vec.getString(i).Equals(partitionColName, StringComparison.OrdinalIgnoreCase))
{
index = i;
break;
}
}
if (index < 0)
{
throw new Exception("Can't find specified partition column name.");
}
partitionColumnIdx = ((BasicIntVector)tableInfo.get(new BasicString("partitionColumnIndex"))).getInt(index);
partitionSchema = ((BasicAnyVector)tableInfo.get(new BasicString("partitionSchema"))).getEntity(index);
partitionType = ((BasicIntVector)tableInfo.get(new BasicString("partitionType"))).getInt(index);
//
partitionColType = (DATA_TYPE)((BasicIntVector)tableInfo.get(new BasicString("partitionColumnType"))).getInt(index);
}
colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs")));
this.cols = colDefs.getColumn(0).rows();
typeInts = (BasicIntVector)colDefs.getColumn("typeInt");
this.columnCategories = new DATA_CATEGORY[this.cols];
this.columnTypes = new DATA_TYPE[this.cols];
for (int i = 0; i < cols; ++i)
{
this.columnTypes[i] = (DATA_TYPE)typeInts.getInt(i);
this.columnCategories[i] = Utils.typeToCategory(this.columnTypes[i]);
}
domain = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema);
}
catch (Exception e)
{
throw e;
}
finally
{
conn.close();
}
}
private BasicTable createBasicTable()
{
List <string> colNames = new List <string>();
colNames.Add("cbool");
colNames.Add("cchar");
colNames.Add("cshort");
colNames.Add("cint");
colNames.Add("clong");
colNames.Add("cdate");
colNames.Add("cmonth");
colNames.Add("ctime");
colNames.Add("cminute");
colNames.Add("csecond");
colNames.Add("cdatetime");
colNames.Add("ctimestamp");
colNames.Add("cnanotime");
colNames.Add("cnanotimestamp");
colNames.Add("cfloat");
colNames.Add("cdouble");
colNames.Add("csymbol");
colNames.Add("cstring");
List <IVector> cols = new List <IVector>();
//boolean
byte[] vbool = new byte[] { 1, 0 };
BasicBooleanVector bbv = new BasicBooleanVector(vbool);
cols.Add(bbv);
//char
byte[] vchar = new byte[] { (byte)'c', (byte)'a' };
BasicByteVector bcv = new BasicByteVector(vchar);
cols.Add(bcv);
//cshort
short[] vshort = new short[] { 32767, 29 };
BasicShortVector bshv = new BasicShortVector(vshort);
cols.Add(bshv);
//cint
int[] vint = new int[] { 2147483647, 483647 };
BasicIntVector bintv = new BasicIntVector(vint);
cols.Add(bintv);
//clong
long[] vlong = new long[] { 2147483647, 483647 };
BasicLongVector blongv = new BasicLongVector(vlong);
cols.Add(blongv);
//cdate
int[] vdate = new int[] { Utils.countDays(new DateTime(2018, 2, 14)), Utils.countDays(new DateTime(2018, 8, 15)) };
BasicDateVector bdatev = new BasicDateVector(vdate);
cols.Add(bdatev);
//cmonth
int[] vmonth = new int[] { Utils.countMonths(new DateTime(2018, 2, 6)), Utils.countMonths(new DateTime(2018, 8, 11)) };
BasicMonthVector bmonthv = new BasicMonthVector(vmonth);
cols.Add(bmonthv);
//ctime
int[] vtime = new int[] { Utils.countMilliseconds(16, 46, 05, 123), Utils.countMilliseconds(18, 32, 05, 321) };
BasicTimeVector btimev = new BasicTimeVector(vtime);
cols.Add(btimev);
//cminute
int[] vminute = new int[] { Utils.countMinutes(new TimeSpan(16, 30, 00)), Utils.countMinutes(new TimeSpan(9, 30, 00)) };
BasicMinuteVector bminutev = new BasicMinuteVector(vminute);
cols.Add(bminutev);
//csecond
int[] vsecond = new int[] { Utils.countSeconds(new TimeSpan(16, 30, 00)), Utils.countSeconds(new TimeSpan(16, 30, 50)) };
BasicSecondVector bsecondv = new BasicSecondVector(vsecond);
cols.Add(bsecondv);
//cdatetime
int[] vdatetime = new int[] { Utils.countSeconds(new DateTime(2018, 9, 8, 9, 30, 01)), Utils.countSeconds(new DateTime(2018, 11, 8, 16, 30, 01)) };
BasicDateTimeVector bdatetimev = new BasicDateTimeVector(vdatetime);
cols.Add(bdatetimev);
//ctimestamp
long[] vtimestamp = new long[] { Utils.countMilliseconds(2018, 11, 12, 9, 30, 01, 123), Utils.countMilliseconds(2018, 11, 12, 16, 30, 01, 123) };
BasicTimestampVector btimestampv = new BasicTimestampVector(vtimestamp);
cols.Add(btimestampv);
//cnanotime
long[] vnanotime = new long[] { Utils.countNanoseconds(new TimeSpan(9, 30, 05, 1234567)), Utils.countNanoseconds(new TimeSpan(16, 30, 05, 9876543)) };
BasicNanoTimeVector bnanotimev = new BasicNanoTimeVector(vnanotime);
cols.Add(bnanotimev);
//cnanotimestamp
long[] vnanotimestamp = new long[] { Utils.countNanoseconds(new DateTime(2018, 11, 12, 9, 30, 05, 123)), Utils.countNanoseconds(new DateTime(2018, 11, 13, 16, 30, 05, 987)) };
BasicNanoTimestampVector bnanotimestampv = new BasicNanoTimestampVector(vnanotimestamp);
cols.Add(bnanotimestampv);
//cfloat
float[] vfloat = new float[] { 2147.483647f, 483.647f };
BasicFloatVector bfloatv = new BasicFloatVector(vfloat);
cols.Add(bfloatv);
//cdouble
double[] vdouble = new double[] { 214.7483647, 48.3647 };
BasicDoubleVector bdoublev = new BasicDoubleVector(vdouble);
cols.Add(bdoublev);
//csymbol
String[] vsymbol = new String[] { "GOOG", "MS" };
BasicStringVector bsymbolv = new BasicStringVector(vsymbol);
cols.Add(bsymbolv);
//cstring
String[] vstring = new String[] { "", "test string" };
BasicStringVector bstringv = new BasicStringVector(vstring);
cols.Add(bstringv);
BasicTable t1 = new BasicTable(colNames, cols);
return(t1);
}
/**
* If fail to connect to the specified DolphinDB server, this function throw an exception.
*/
public MultithreadedTableWriter(string hostName, int port, string userId, string password,
string dbName, string tableName, bool useSSL, bool enableHighAvailability = false, string[] pHighAvailabilitySites = null,
int batchSize = 1, float throttle = 0.01f, int threadCount = 5, string partitionCol = "", int[] pCompressMethods = null)
{
hostName_ = hostName;
port_ = port;
userId_ = userId;
password_ = password;
useSSL_ = useSSL;
dbName_ = dbName;
tableName_ = tableName;
batchSize_ = batchSize;
throttleMilsecond_ = (int)throttle * 1000;
isExiting_ = false;
if (threadCount < 1)
{
throw new Exception("The parameter threadCount must be greater than or equal to 1.");
}
if (batchSize < 1)
{
throw new Exception("The parameter batchSize must be greater than or equal to 1.");
}
if (throttle < 0)
{
throw new Exception("The parameter throttle must be positive.");
}
if (threadCount > 1 && partitionCol == String.Empty)
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
DBConnection pConn = new DBConnection(false, useSSL_, pCompressMethods != null);
bool ret = pConn.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites);
if (!ret)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
BasicDictionary schema;
if (tableName == "")
{
schema = (BasicDictionary)pConn.run("schema(" + dbName + ")");
}
else
{
schema = (BasicDictionary)pConn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))");
}
IEntity partColNames = null;
if (schema.ContainsKey("partitionColumnName"))
{
partColNames = schema.get(new BasicString("partitionColumnName"));
isPartionedTable_ = true;
}
else
{
isPartionedTable_ = false;
if (tableName != "")
{
if (threadCount > 1)
{//只有多线程的时候需要
throw new Exception("The parameter threadCount must be 1 for a dimension table.");
}
}
}
BasicTable colDefs = (BasicTable)schema.get("colDefs");
BasicIntVector colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt");
BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name");
BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString");
colTypes_ = new List <DATA_TYPE>();
colNames_ = new List <string>();
colTypeString_ = new List <string>();
int columnSize = colDefsName.rows();
if (pCompressMethods != null)
{
if (columnSize != pCompressMethods.Length)
{
throw new Exception(string.Format("The number of elements in parameter compressMethods does not match the column size {0}. ", columnSize));
}
this.compressTypes_ = new int[columnSize];
Array.Copy(pCompressMethods, this.compressTypes_, columnSize);
}
for (int i = 0; i < columnSize; i++)
{
colNames_.Add(colDefsName.getString(i));
colTypes_.Add((DATA_TYPE)colDefsTypeInt.getInt(i));
colTypeString_.Add(colDefsTypeString.getString(i));
if (compressTypes_ != null)
{
AbstractVector.checkCompressedMethod(colTypes_[i], compressTypes_[i]);
}
}
if (threadCount > 1)
{
if (isPartionedTable_)
{
IEntity partitionSchema;
int partitionType;
if (partColNames.isScalar())
{
if (partColNames.getString() != partitionCol)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicInt)schema.get("partitionColumnIndex")).getInt();
partitionSchema = schema.get("partitionSchema");
partitionType = ((BasicInt)schema.get("partitionType")).getInt();
}
else
{
int dims = ((BasicStringVector)partColNames).rows();
if (dims > 1 && partitionCol == "")
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
int index = -1;
for (int i = 0; i < dims; ++i)
{
if (((BasicStringVector)partColNames).getString(i) == partitionCol)
{
index = i;
break;
}
}
if (index < 0)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicIntVector)schema.get("partitionColumnIndex")).getInt(index);
partitionSchema = ((BasicAnyVector)schema.get("partitionSchema")).get(index);
partitionType = ((BasicIntVector)schema.get("partitionType")).getInt(index);
}
DATA_TYPE partitionColType = colTypes_[partitionColumnIdx_];
partitionDomain_ = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema);
}
else
{//isPartionedTable_==false
if (partitionCol != "")
{
int threadcolindex = -1;
for (int i = 0; i < colNames_.Count; i++)
{
if (colNames_[i] == partitionCol)
{
threadcolindex = i;
break;
}
}
if (threadcolindex < 0)
{
throw new Exception(string.Format("No match found for {0}. ", partitionCol));
}
threadByColIndexForNonPartion_ = threadcolindex;
}
}
}
// init done, start thread now.
isExiting_ = false;
threads_ = new List <WriterThread>(threadCount);
for (int i = 0; i < threadCount; i++)
{
WriterThread writerThread = new WriterThread(this, pConn);
if (i == 0)
{
writerThread.conn_ = pConn;
}
else
{
writerThread.conn_ = new DBConnection(useSSL_, false);
if (writerThread.conn_.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites) == false)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
}
threads_.Add(writerThread);
}
}
private void initialize(AbstractVector values, BasicAnyVector locations)
{
if (values.DataType != Entity_DATA_TYPE.DT_ANY || values.DataForm != Entity_DATA_FORM.DF_VECTOR)
{
throw new Exception("expect a vector of partitioning lists");
}
if (values.rows() <= 0)
{
throw new Exception("requires at least one partitioning list");
}
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (values.rows() != locations.rows())
{
throw new Exception("expect # locations == # partitioning lists");
}
this.locationMap = new Dictionary <>();
IList <string>[] locationListArray = new IList[locations.rows()];
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
locationListArray[i] = location.String;
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
locationListArray[i] = new List <>();
for (int j = 0; j < locationVector.rows(); ++j)
{
BasicString location = (BasicString)locationVector.get(j);
locationListArray[i].Add(location.String);
}
}
}
for (int i = 0; i < values.rows(); ++i)
{
AbstractVector partitioningList = (AbstractVector)((BasicAnyVector)values).getEntity(i);
if (partitioningList.rows() <= 0)
{
throw new Exception("expect partitioning list to be nonempty");
}
if (partitioningList.DataCategory != Entity_DATA_CATEGORY.INTEGRAL)
{
throw new Exception("expect partitioning column values to be integral-typed.");
}
for (int j = 0; j < partitioningList.rows(); ++j)
{
try
{
long val = partitioningList.get(j).Number.longValue();
locationMap[val] = locationListArray[i];
}
catch (Exception e)
{
throw new Exception(e);
}
}
}
}