public ExclusiveDBConnectionPool(string host, int port, string uid, string pwd, int count, bool loadBalance, bool highAvaliability)
{
conns = new List <DBConnection>(count);
if (!loadBalance)
{
for (int i = 0; i < count; i++)
{
DBConnection conn = new DBConnection();
if (!conn.connect(host, port, uid, pwd, "", highAvaliability))
{
throw new Exception("Cant't connect to the specified host.");
}
conns.Add(conn);
}
}
else
{
DBConnection entryPoint = new DBConnection();
if (!entryPoint.connect(host, port, uid, pwd))
{
throw new Exception("Can't connect to the specified host.");
}
BasicStringVector nodes = (BasicStringVector)entryPoint.run("rpc(getControllerAlias(), getClusterLiveDataNodes{false})");
int nodeCount = nodes.rows();
string[] hosts = new string[nodeCount];
int[] ports = new int[nodeCount];
for (int i = 0; i < nodeCount; i++)
{
string[] fields = nodes.getString(i).Split(':');
if (fields.Length < 2)
{
throw new Exception("Invalid datanode adress: " + nodes.getString(i));
}
hosts[i] = fields[0];
ports[i] = int.Parse(fields[1]);
}
for (int i = 0; i < count; i++)
{
DBConnection conn = new DBConnection();
if (!(conn.connect(hosts[i % nodeCount], ports[i % nodeCount], uid, pwd, "", highAvaliability)))
{
throw new Exception("Can't connect to the host: " + nodes.getString(i % nodeCount));
}
conns.Add(conn);
}
}
ThreadPool.SetMaxThreads(count, count);
}
private void initialize(IList <long?> longs, BasicAnyVector locations)
{
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (isSorted(longs) == false)
{
throw new Exception("ranges " + longs.ToString() + " not sorted!");
}
if (longs.Count != locations.rows() + 1)
{
throw new Exception("expect # locations == # range values - 1");
}
ranges = longs.ToArray();
this.locations = new List <>();
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
this.locations.Add(location.String);
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
this.locations.Add(new List <>());
for (int j = 0; j < locationVector.rows(); ++j)
{
this.locations[this.locations.Count - 1].Add(((BasicString)locationVector.get(j)).String);
}
}
}
}
private void initialize(IList <string> strings, BasicAnyVector locations)
{
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (strings.Count != locations.rows())
{
throw new Exception("expect # locations == # values");
}
map = new Dictionary <>();
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
string val = strings[i];
map[val] = location.String;
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
string val = strings[i];
map[val] = new List <>();
for (int j = 0; j < locationVector.rows(); ++j)
{
map[val].Add(locationVector.getString(j));
}
}
}
}
public autoFitTableAppender(string dbUrl, string tableName, bool asynTask, DBConnection conn)
{
this.dbUrl = dbUrl;
this.tableName = tableName;
this.asynTask = asynTask;
this.conn = conn;
conn.setasynTask(false);
BasicDictionary tableInfo = (BasicDictionary)conn.run("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
BasicTable colDefs;
colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs")));
BasicStringVector names = (BasicStringVector)colDefs.getColumn("name");
BasicIntVector types = (BasicIntVector)colDefs.getColumn("typeInt");
int rows = names.rows();
this.schema = new Dictionary <string, DATA_TYPE>();
for (int i = 0; i < rows; ++i)
{
schema.Add(names.getString(i), (DATA_TYPE)types.getInt(i));
}
conn.setasynTask(asynTask);
}
public bool switchServer()
{
if (!highAvailability)
{
return(false);
}
if (controllerHost == null)
{
return(false);
}
// 1 find live datanodes and pick one
DBConnection tmp = new DBConnection();
tmp.connect(controllerHost, controllerPort);
BasicStringVector liveDataNodes = (BasicStringVector)tmp.run("getLiveDataNodes(false)");
tmp.close();
// 2 try get connection to new datanode
int size = liveDataNodes.rows();
Console.WriteLine("living node size: " + size);
for (int i = 0; i < size; i++)
{
string[] liveDataNodeSite = liveDataNodes.get(i).getString().Split(':');
hostName = liveDataNodeSite[0];
port = int.Parse(liveDataNodeSite[1]);
Console.WriteLine("Try node " + i + ": " + hostName + ":" + port);
try
{
connect();
return(true);
}
catch (Exception) { }
}
return(false);
}
protected BlockingCollection <List <IMessage> > subscribeInternal(string host, int port, string tableName, string actionName, MessageHandler handler, long offset, bool reconnect, IVector filter, bool createSubInfo)
{
string topic = "";
IEntity re;
BlockingCollection <List <IMessage> > queue;
DBConnection dbConn = new DBConnection();
dbConn.connect(host, port);
try
{
string localIP = listeningHost_;
if (localIP == null || localIP.Equals(String.Empty))
{
localIP = dbConn.LocalAddress;
}
List <IEntity> @params = new List <IEntity>
{
new BasicString(tableName),
new BasicString(actionName)
};
re = dbConn.run("getSubscriptionTopic", @params);
topic = ((BasicAnyVector)re).getEntity(0).getString();
@params.Clear();
@params.Add(new BasicString(localIP));
@params.Add(new BasicInt(listeningPort_));
@params.Add(new BasicString(tableName));
@params.Add(new BasicString(actionName));
@params.Add(new BasicLong(offset));
if (filter != null)
{
@params.Add(filter);
}
re = dbConn.run("publishTable", @params);
lock (subscribeInfos_)
{
if (createSubInfo)
{
Site[] sites;
if (re is BasicAnyVector)
{
BasicStringVector HASiteStrings = (BasicStringVector)((BasicAnyVector)re).getEntity(1);
int HASiteNum = HASiteStrings.rows();
sites = new Site[HASiteNum];
for (int i = 0; i < HASiteNum; ++i)
{
String HASite = HASiteStrings.get(i).getString();
String[] HASiteHostAndPort = HASite.Split(':');
String HASiteHost = HASiteHostAndPort[0];
int HASitePort = int.Parse(HASiteHostAndPort[1]);
String HASiteAlias = HASiteHostAndPort[2];
sites[i] = new Site(HASiteHost, HASitePort);
String HATopic = getTopic(HASiteHost, HASitePort, HASiteAlias, tableName, actionName);
HATopicToTrueTopic_[HATopic] = topic;
}
}
else
{
sites = new Site[] { new Site(host, port) };
lock (HATopicToTrueTopic_)
{
HATopicToTrueTopic_[topic] = topic;
}
}
SubscribeInfo subscribeInfo = new SubscribeInfo(DateTime.Now, new BlockingCollection <List <IMessage> >(), sites, topic, offset, reconnect, filter, handler, tableName, actionName);
subscribeInfo.setConnectState(ConnectState.REQUEST);
queue = subscribeInfo.getQueue();
if (subscribeInfos_.ContainsKey(topic))
{
throw new Exception("Subscription with topic " + topic + " exist. ");
}
else
{
subscribeInfos_.TryAdd(topic, subscribeInfo);
}
Console.WriteLine("Successfully subscribed table " + topic);
}
else
{
SubscribeInfo subscribeInfo = null;
if (!subscribeInfos_.TryGetValue(topic, out subscribeInfo))
{
throw new Exception("Subscription with topic " + topic + " doesn't exist. ");
}
lock (subscribeInfo)
{
if (subscribeInfo.getConnectState() == ConnectState.RECEIVED_SCHEMA)
{
throw new Exception("Subscription with topic " + topic + " the connection has been created. ");
}
subscribeInfo.setConnectState(ConnectState.REQUEST);
}
queue = subscribeInfo.getQueue();
}
}
}
finally
{
dbConn.close();
}
return(queue);
}
public PartitionedTableAppender(string dbUrl, string tableName, string partitionColName, string appendFunction, IDBConnectionPool pool)
{
this.pool = pool;
threadCount = pool.getConnectionCount();
chunkIndices = new List <List <int> >(threadCount);
for (int i = 0; i < threadCount; ++i)
{
chunkIndices.Add(new List <int>());
}
DBConnection conn = new DBConnection();
IEntity partitionSchema;
BasicTable colDefs;
BasicIntVector typeInts;
int partitionType;
DATA_TYPE partitionColType;
try
{
IDBTask task;
if (dbUrl == null || dbUrl.Length == 0)
{
task = new BasicDBTask("schema(" + tableName + ")");
appendScript = "tableInsert{" + tableName + "}";
}
else
{
task = new BasicDBTask("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
appendScript = "tableInsert{loadTable('" + dbUrl + "', '" + tableName + "')}";
}
if (appendFunction != null && appendFunction.Length != 0)
{
appendScript = appendFunction;
}
pool.execute(task);
if (!task.isSuccessful())
{
throw new Exception(task.getErrorMsg());
}
tableInfo = (BasicDictionary)task.getResults();
IEntity partColNames = tableInfo.get(new BasicString("partitionColumnName"));
if (partColNames == null)
{
throw new Exception("Can't find specified partition column name.");
}
if (partColNames.isScalar())
{
if (!((BasicString)partColNames).getString().Equals(partitionColName, StringComparison.OrdinalIgnoreCase))
{
throw new Exception("Can't find specified partition column name.");
}
partitionColumnIdx = ((BasicInt)tableInfo.get(new BasicString("partitionColumnIndex"))).getValue();
partitionSchema = tableInfo.get(new BasicString("partitionSchema"));
partitionType = ((BasicInt)tableInfo.get(new BasicString("partitionType"))).getValue();
//
partitionColType = (DATA_TYPE)((BasicInt)tableInfo.get(new BasicString("partitionColumnType"))).getValue();
}
else
{
BasicStringVector vec = (BasicStringVector)partColNames;
int dims = vec.rows();
int index = -1;
for (int i = 0; i < dims; ++i)
{
if (!vec.getString(i).Equals(partitionColName, StringComparison.OrdinalIgnoreCase))
{
index = i;
break;
}
}
if (index < 0)
{
throw new Exception("Can't find specified partition column name.");
}
partitionColumnIdx = ((BasicIntVector)tableInfo.get(new BasicString("partitionColumnIndex"))).getInt(index);
partitionSchema = ((BasicAnyVector)tableInfo.get(new BasicString("partitionSchema"))).getEntity(index);
partitionType = ((BasicIntVector)tableInfo.get(new BasicString("partitionType"))).getInt(index);
//
partitionColType = (DATA_TYPE)((BasicIntVector)tableInfo.get(new BasicString("partitionColumnType"))).getInt(index);
}
colDefs = ((BasicTable)tableInfo.get(new BasicString("colDefs")));
this.cols = colDefs.getColumn(0).rows();
typeInts = (BasicIntVector)colDefs.getColumn("typeInt");
this.columnCategories = new DATA_CATEGORY[this.cols];
this.columnTypes = new DATA_TYPE[this.cols];
for (int i = 0; i < cols; ++i)
{
this.columnTypes[i] = (DATA_TYPE)typeInts.getInt(i);
this.columnCategories[i] = Utils.typeToCategory(this.columnTypes[i]);
}
domain = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema);
}
catch (Exception e)
{
throw e;
}
finally
{
conn.close();
}
}
/**
* If fail to connect to the specified DolphinDB server, this function throw an exception.
*/
public MultithreadedTableWriter(string hostName, int port, string userId, string password,
string dbName, string tableName, bool useSSL, bool enableHighAvailability = false, string[] pHighAvailabilitySites = null,
int batchSize = 1, float throttle = 0.01f, int threadCount = 5, string partitionCol = "", int[] pCompressMethods = null)
{
hostName_ = hostName;
port_ = port;
userId_ = userId;
password_ = password;
useSSL_ = useSSL;
dbName_ = dbName;
tableName_ = tableName;
batchSize_ = batchSize;
throttleMilsecond_ = (int)throttle * 1000;
isExiting_ = false;
if (threadCount < 1)
{
throw new Exception("The parameter threadCount must be greater than or equal to 1.");
}
if (batchSize < 1)
{
throw new Exception("The parameter batchSize must be greater than or equal to 1.");
}
if (throttle < 0)
{
throw new Exception("The parameter throttle must be positive.");
}
if (threadCount > 1 && partitionCol == String.Empty)
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
DBConnection pConn = new DBConnection(false, useSSL_, pCompressMethods != null);
bool ret = pConn.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites);
if (!ret)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
BasicDictionary schema;
if (tableName == "")
{
schema = (BasicDictionary)pConn.run("schema(" + dbName + ")");
}
else
{
schema = (BasicDictionary)pConn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))");
}
IEntity partColNames = null;
if (schema.ContainsKey("partitionColumnName"))
{
partColNames = schema.get(new BasicString("partitionColumnName"));
isPartionedTable_ = true;
}
else
{
isPartionedTable_ = false;
if (tableName != "")
{
if (threadCount > 1)
{//只有多线程的时候需要
throw new Exception("The parameter threadCount must be 1 for a dimension table.");
}
}
}
BasicTable colDefs = (BasicTable)schema.get("colDefs");
BasicIntVector colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt");
BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name");
BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString");
colTypes_ = new List <DATA_TYPE>();
colNames_ = new List <string>();
colTypeString_ = new List <string>();
int columnSize = colDefsName.rows();
if (pCompressMethods != null)
{
if (columnSize != pCompressMethods.Length)
{
throw new Exception(string.Format("The number of elements in parameter compressMethods does not match the column size {0}. ", columnSize));
}
this.compressTypes_ = new int[columnSize];
Array.Copy(pCompressMethods, this.compressTypes_, columnSize);
}
for (int i = 0; i < columnSize; i++)
{
colNames_.Add(colDefsName.getString(i));
colTypes_.Add((DATA_TYPE)colDefsTypeInt.getInt(i));
colTypeString_.Add(colDefsTypeString.getString(i));
if (compressTypes_ != null)
{
AbstractVector.checkCompressedMethod(colTypes_[i], compressTypes_[i]);
}
}
if (threadCount > 1)
{
if (isPartionedTable_)
{
IEntity partitionSchema;
int partitionType;
if (partColNames.isScalar())
{
if (partColNames.getString() != partitionCol)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicInt)schema.get("partitionColumnIndex")).getInt();
partitionSchema = schema.get("partitionSchema");
partitionType = ((BasicInt)schema.get("partitionType")).getInt();
}
else
{
int dims = ((BasicStringVector)partColNames).rows();
if (dims > 1 && partitionCol == "")
{
throw new Exception("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
int index = -1;
for (int i = 0; i < dims; ++i)
{
if (((BasicStringVector)partColNames).getString(i) == partitionCol)
{
index = i;
break;
}
}
if (index < 0)
{
throw new Exception(string.Format("The parameter partionCol must be the partitioning column \"{0}\" in the table. ", partitionCol));
}
partitionColumnIdx_ = ((BasicIntVector)schema.get("partitionColumnIndex")).getInt(index);
partitionSchema = ((BasicAnyVector)schema.get("partitionSchema")).get(index);
partitionType = ((BasicIntVector)schema.get("partitionType")).getInt(index);
}
DATA_TYPE partitionColType = colTypes_[partitionColumnIdx_];
partitionDomain_ = DomainFactory.createDomain((PARTITION_TYPE)partitionType, partitionColType, partitionSchema);
}
else
{//isPartionedTable_==false
if (partitionCol != "")
{
int threadcolindex = -1;
for (int i = 0; i < colNames_.Count; i++)
{
if (colNames_[i] == partitionCol)
{
threadcolindex = i;
break;
}
}
if (threadcolindex < 0)
{
throw new Exception(string.Format("No match found for {0}. ", partitionCol));
}
threadByColIndexForNonPartion_ = threadcolindex;
}
}
}
// init done, start thread now.
isExiting_ = false;
threads_ = new List <WriterThread>(threadCount);
for (int i = 0; i < threadCount; i++)
{
WriterThread writerThread = new WriterThread(this, pConn);
if (i == 0)
{
writerThread.conn_ = pConn;
}
else
{
writerThread.conn_ = new DBConnection(useSSL_, false);
if (writerThread.conn_.connect(hostName_, port_, userId_, password_, "", enableHighAvailability, pHighAvailabilitySites) == false)
{
throw new Exception(string.Format("Failed to connect to server {0}:{1}. ", hostName, port));
}
}
threads_.Add(writerThread);
}
}
private void initialize(AbstractVector values, BasicAnyVector locations)
{
if (values.DataType != Entity_DATA_TYPE.DT_ANY || values.DataForm != Entity_DATA_FORM.DF_VECTOR)
{
throw new Exception("expect a vector of partitioning lists");
}
if (values.rows() <= 0)
{
throw new Exception("requires at least one partitioning list");
}
if (locations.rows() <= 0)
{
throw new Exception("requires at least one location");
}
if (locations.getEntity(0).DataType != Entity_DATA_TYPE.DT_STRING)
{
throw new Exception("location must be a string");
}
if (values.rows() != locations.rows())
{
throw new Exception("expect # locations == # partitioning lists");
}
this.locationMap = new Dictionary <>();
IList <string>[] locationListArray = new IList[locations.rows()];
bool isScalar = locations.getEntity(0).DataForm == Entity_DATA_FORM.DF_SCALAR;
if (isScalar)
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicString location = (BasicString)locations.get(i);
locationListArray[i] = location.String;
}
}
else
{
for (int i = 0; i < locations.rows(); ++i)
{
BasicStringVector locationVector = (BasicStringVector)locations.getEntity(i);
locationListArray[i] = new List <>();
for (int j = 0; j < locationVector.rows(); ++j)
{
BasicString location = (BasicString)locationVector.get(j);
locationListArray[i].Add(location.String);
}
}
}
for (int i = 0; i < values.rows(); ++i)
{
AbstractVector partitioningList = (AbstractVector)((BasicAnyVector)values).getEntity(i);
if (partitioningList.rows() <= 0)
{
throw new Exception("expect partitioning list to be nonempty");
}
if (partitioningList.DataCategory != Entity_DATA_CATEGORY.INTEGRAL)
{
throw new Exception("expect partitioning column values to be integral-typed.");
}
for (int j = 0; j < partitioningList.rows(); ++j)
{
try
{
long val = partitioningList.get(j).Number.longValue();
locationMap[val] = locationListArray[i];
}
catch (Exception e)
{
throw new Exception(e);
}
}
}
}