/** * Release the resouces occupied by the specified table, including write queue and write thread. If this * function is called to add an in-memory table, the parameter dbName indicates the name of the in-memory * table, and the parameter tableName should be a null string. */ public void removeTable(string dbName, string tableName = "") { DestTable destTable = null; try { rwLock_.EnterReadLock(); if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { destTable = destTables_[Tuple.Create <string, string>(dbName, tableName)]; if (destTable.destroy) { return; } else { destTable.destroy = true; } } } finally { rwLock_.ExitReadLock(); } if (destTable != null) { destTable.writeThread.Join(); destTable.conn.close(); destTables_.Remove(Tuple.Create <string, string>(dbName, tableName)); } }
public BasicTable getUnwrittenData(string dbName, string tableName) { try { List <IVector> cols = new List <IVector>(); List <List <IScalar> > data = new List <List <IScalar> >(); rwLock_.EnterReadLock(); if (!destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { throw new Exception("Failed to get unwritten data.Please use addTable to add infomation of database and table first."); } DestTable destTable = destTables_[Tuple.Create <string, string>(dbName, tableName)]; int columns = destTable.colTypes.Count(); lock (destTable.failQueue) { while (destTable.failQueue.Count() != 0) { if (destTable.failQueue.TryDequeue(out List <IScalar> queueData)) { data.Add(queueData); } } } while (destTable.writeQueue.Count() != 0) { if (destTable.writeQueue.TryDequeue(out List <IScalar> queueData)) { data.Add(queueData); } } int size = data.Count(); BasicEntityFactory factory = new BasicEntityFactory(); for (int i = 0; i < destTable.colTypes.Count(); ++i) { IVector vector = factory.createVectorWithDefaultValue(destTable.colTypes[i], size); for (int j = 0; j < size; ++j) { vector.set(j, data[j][i]); } cols.Add(vector); } BasicTable ret = new BasicTable(destTable.colNames, cols); return(ret); } finally { rwLock_.ExitReadLock(); } }
/** * Gets the current size of the specified table write queue and whethre the specified table is removed. If * the specified table is not added first, this function throw and exception. */ public Tuple <int, bool, bool> getStatus(string dbName, string tableName = "") { rwLock_.EnterReadLock(); try { if (!destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { throw new Exception("Failed to get queue depth. Please use addTable to add infomation of database and table first."); } DestTable destTable = destTables_[Tuple.Create <string, string>(dbName, tableName)]; return(Tuple.Create <int, bool, bool>(destTable.writeQueue.Count(), destTable.destroy, destTable.finished)); } finally { rwLock_.ExitReadLock(); } }
void insertRecursive(DestTable destTable, List <IScalar> args) { if (destTable.finished) { throw new Exception("Failed to insert data. Error writing data in backgroud thread. Talbe (" + destTable.dbName + " " + destTable.tableName + ") will be removed."); } int cols = args.Count(); for (int i = 0; i < cols; ++i) { if ((int)args[i].getDataType() != destTable.colDefsTypeInt.getInt(i)) { if (!(args[i].getDataType() == DATA_TYPE.DT_STRING && destTable.colDefsTypeInt.getInt(i) == (int)DATA_TYPE.DT_SYMBOL)) { throw new Exception("Failed to insert data, the type of argument does not match the type of column at column: " + i.ToString()); } } } destTable.writeQueue.Enqueue(args); }
public WriteThread(DestTable destTable, bool partitioned) { this.destTableRawPtr = destTable; this.partitioned = partitioned; }
/** * Add the name of the database and table that you want to insert data into before actually call insert. * The parameter partitioned indicates whether the added table is a partitioned table. If this function * is called to add an in-memory table, the parameter dbName indicates the name of the shared in-memory * table, and the parameter tableName should be a null string. If error is raised on the server, this * function throws an exception. */ public void addTable(string dbName, string tableName = "", bool partitioned = true) { try { rwLock_.EnterReadLock(); if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { throw new Exception("Failed to add table, the specified table has not been removed yet."); } } finally { rwLock_.ExitReadLock(); } DBConnection conn = new DBConnection(false, false); bool ret = conn.connect(hostName_, port_, userId_, password_); if (!ret) { throw new Exception("Failed to connect to server."); } string tableInsert; string saveTable = ""; BasicDictionary schema; string tmpDiskGlobal = "tmpDiskGlobal"; if (tableName == "") { tableInsert = "tableInsert{" + dbName + "}"; schema = (BasicDictionary)conn.run("schema(" + dbName + ")"); } else if (partitioned) { tableInsert = "tableInsert{loadTable(\"" + dbName + "\",\"" + tableName + "\")}"; schema = (BasicDictionary)conn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))"); } else { throw new Exception("The target table must be an in-memory table or a table in a distributed database."); } BasicTable colDefs = (BasicTable)schema.get(new BasicString("colDefs")); DestTable destTable; if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { throw new Exception("Failed to add table, the specified table has not been removed yet."); } destTable = new DestTable(); destTable.dbName = dbName; destTable.tableName = tableName; destTable.conn = conn; destTable.tableInsert = tableInsert; destTable.saveTable = saveTable; destTable.colDefs = colDefs; destTable.columnNum = colDefs.rows(); destTable.colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt"); destTable.destroy = false; destTable.writeQueue = new ConcurrentQueue <List <IScalar> >(); destTable.failQueue = new ConcurrentQueue <List <IScalar> >(); List <string> colNames = new List <string>(); List <DATA_TYPE> colTypes = new List <DATA_TYPE>(); BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name"); for (int i = 0; i < destTable.columnNum; i++) { colNames.Add(colDefsName.getString(i)); colTypes.Add((DATA_TYPE)destTable.colDefsTypeInt.getInt(i)); } destTable.colNames = colNames; destTable.colTypes = colTypes; if (!partitioned) { string colName = ""; string colType = ""; BasicStringVector colDefsTypeString = (BasicStringVector)colDefs.getColumn("typeString"); for (int i = 0; i < destTable.columnNum; i++) { colName = colName + "`" + colDefsName.getString(i); colType = colType + "`" + colDefsTypeString.getString(i); } destTable.createTmpSharedTable = "share table(" + "1000:0," + colName + "," + colType + ") as " + tmpDiskGlobal; } try { rwLock_.EnterWriteLock(); if (destTables_.ContainsKey(Tuple.Create <string, string>(dbName, tableName))) { throw new Exception("Failed to add table, the specified table has not been removed yet."); } destTables_[Tuple.Create <string, string>(dbName, tableName)] = destTable; WriteThread writeThreadFuc = new WriteThread(destTable, partitioned); destTable.writeThread = new Thread(writeThreadFuc.run); destTable.writeThread.Start(); } finally { rwLock_.ExitWriteLock(); } }