/// <summary> /// Copy PRD Annotation Table data to DEV by groups of result ids /// </summary> /// <returns></returns> public static string CopyPrdAnnotationTableDataToDevBy_Rslt_Id() { long minRi = 1; //long minRi = 1310251601; // appro half of rows, allows index on all cols to be built long maxRi = 4331096190; long chunkSize = 100000; DbCommandMx cmd = new DbCommandMx(); cmd.MxConn = DbConnectionMx.GetConnection("DEV857"); cmd.BeginTransaction(); long totalIns = 0; for (long ri = minRi; ri <= maxRi; ri += chunkSize) { string range = (ri + 1).ToString() + " and " + (ri + chunkSize); string sql = @"insert /*+ APPEND */ into mbs_owner.mbs_adw_rslt (select * from mbs_owner.mbs_adw_rslt where rslt_id between " + range + ")"; cmd.PrepareUsingDefinedConnection(sql); int insCnt = cmd.ExecuteNonReader(); cmd.Commit(); totalIns += insCnt; string msg = "Rows copied: " + totalIns + " (" + range + ")"; Mobius.UAL.Progress.Show(msg); DebugLog.Message(msg); } cmd.Dispose(); return("Rows copied: " + totalIns); }
/// <summary> /// Create the temp table for the list if needed /// </summary> /// <param name="schema"></param> /// <param name="name"></param> public static bool CreateListTableIfNeeded( string dsName, string listName) { DbCommandMx cmd = null; bool created = true; string fullTableName; DataSourceMx ds = GetDataSourceInfo(ref dsName, ref listName, out fullTableName); if (Lists.ContainsKey(fullTableName)) { return(false); } string sql = @" create global temporary table <schema.name> ( rowPos integer, intKey integer, stringKey varchar2(256)) on commit preserve rows" ; sql = sql.Replace("<schema.name>", fullTableName); try { DateTime t0 = DateTime.Now; cmd = new DbCommandMx(); DbConnectionMx dbc = DbConnectionMx.GetConnection(dsName); cmd.MxConn = dbc; cmd.Prepare(sql); cmd.ExecuteNonReader(); created = true; int tDelta = (int)TimeOfDay.Delta(t0); //DebugLog.Message("Create Table time: " + tDelta); } catch (Exception ex) { if (!ex.Message.Contains("ORA-xxx")) { throw ex; // if other than already exists then throw exception } created = false; // already exists } finally { if (cmd != null) { cmd.Dispose(); } } TempDbList list = new TempDbList(); list.DataSource = ds; list.Name = listName; Lists[fullTableName] = list; return(created); }
/// <summary> /// Update copies of CMN_ASSY_ATRBTS in other Oracle instances /// </summary> /// <returns></returns> static string UpdateCmnAssyAtrbtsCopies() { DbCommandMx dao; string sql, msg = "", errMsg = ""; int cnt; string[] dbList = new string[] { //"AssayDB1", //"AssayDB2", //"...", }; foreach (string conName in dbList) { try { dao = new DbCommandMx(); dao.MxConn = DbConnectionMx.GetConnection(conName); sql = "delete from MBS_OWNER.CMN_ASSY_ATRBTS"; dao.PrepareUsingDefinedConnection(sql); cnt = dao.ExecuteNonReader(); sql = @" insert into MBS_OWNER.CMN_ASSY_ATRBTS select * from MBS_OWNER.CMN_ASSY_ATRBTS" ; dao.PrepareUsingDefinedConnection(sql); cnt = dao.ExecuteNonReader(); dao.Commit(); dao.Dispose(); if (msg != "") { msg += ", "; } msg += conName; } catch (Exception ex) { errMsg += "Error updating " + conName + ": " + ex.Message + "\r\n"; } } if (msg != "") { msg = "Updated copies of CMN_ASSY_ATRBTS in: " + msg; } if (errMsg != "") { msg += "\r\n" + errMsg; } return(msg); }
/// <summary> /// Get a metatable from a database catalog /// </summary> /// <param name="schemaDotTableName"></param> /// <returns></returns> public static MetaTable GetMetaTableFromDatabaseDictionary( string schemaDotTableName) { string[] sa = schemaDotTableName.Split('.'); if (sa.Length != 2) { return(null); } string schemaName = sa[0]; string tableName = sa[1]; DbSchemaMx schema = DbSchemaMx.GetSchemaInfoFromName(schemaName); if (schema == null) { return(null); // unknown schema } if (Lex.IsDefined(schema.AliasFor)) { schemaName = schema.AliasFor; } string dsName = schema.DataSourceName; DataSourceMx dataSource = DbConnectionMx.GetRootDataSource(schemaDotTableName); if (dataSource == null) { return(null); } DbConnectionMx conn = DbConnectionMx.GetConnection(dataSource.DataSourceName); if (dataSource.DbType == DatabaseType.MySql) { return(MySqlMx.GetMetaTableFromDatabaseDictionary(conn, schemaName, tableName)); } else if (dataSource.DbType == DatabaseType.Oracle) { return(OracleMx.GetMetaTableFromDatabaseDictionary(conn, schemaName, tableName)); } else if (dataSource.DbType == DatabaseType.ODBC) { return(OdbcMx.GetMetaTableFromDatabaseDictionary(conn, schemaName, tableName)); } throw new Exception("Database dictionary not supported for: " + schemaDotTableName); }
/// <summary> /// /// <summary> /// Copy PRD Annotation Table data to DEV reording and compressing the data by method /// </summary> /// <returns></returns> public static string CopyPrdAnnotationTableDataToDev() { string mvidSql = @" select mthd_vrsn_id, count from ( select mthd_vrsn_id, count(mthd_vrsn_id) count FROM MBS_OWNER.mbs_adw_rslt WHERE mthd_vrsn_id > 0 /* = 708070 */ group by mthd_vrsn_id) order by mthd_vrsn_id "; Mobius.UAL.Progress.Show("Getting list of annotation table mtht_vrsn_ids ..."); DbCommandMx c1 = new DbCommandMx(); c1.Prepare(mvidSql); c1.ExecuteReader(); DbCommandMx cmd = new DbCommandMx(); cmd.MxConn = DbConnectionMx.GetConnection("DEV857"); cmd.BeginTransaction(); long totalIns = 0; int vmIdCnt = 0; while (c1.Read()) { long mvId = c1.GetLong(0); string sql = @"insert /*+ APPEND */ into mbs_owner.mbs_adw_rslt select * from mbs_owner.mbs_adw_rslt where mthd_vrsn_id = " + mvId + @" order by ext_cmpnd_id_nbr, rslt_grp_id, rslt_id"; cmd.PrepareUsingDefinedConnection(sql); int insCnt = cmd.ExecuteNonReader(); cmd.Commit(); totalIns += insCnt; vmIdCnt++; string msg = "Mthd_Vrsn_id: " + mvId + ", Vmids: " + vmIdCnt + ", Rows: " + totalIns; Mobius.UAL.Progress.Show(msg); DebugLog.Message(msg); } c1.Dispose(); cmd.Dispose(); return("Rows copied: " + totalIns); }
/// <summary> /// Be sure the table is truncated /// </summary> /// <param name="schema"></param> /// <param name="listName"></param> public static void TruncateListTable( string dsName, string listName) { DbCommandMx cmd = null; string fullTableName; DataSourceMx ds = GetDataSourceInfo(ref dsName, ref listName, out fullTableName); string sql = "truncate table " + fullTableName; try { DateTime t0 = DateTime.Now; cmd = new DbCommandMx(); DbConnectionMx dbc = DbConnectionMx.GetConnection(dsName); cmd.MxConn = dbc; cmd.Prepare(sql); cmd.ExecuteNonReader(); int tDelta = (int)TimeOfDay.Delta(t0); //DebugLog.Message("Truncate Table time: " + tDelta); } catch (Exception ex) // if truncate failed see if need to create { if (!ex.Message.Contains("ORA-00942")) { throw ex; // if other than already exists then throw exception } bool created = CreateListTableIfNeeded(dsName, listName); } finally { if (cmd != null) { cmd.Dispose(); } } return; }
/// <summary> /// Sync the Mobius CorpMoltable replicate used to retrieve Smiles /// Syntax: UpdateCorpDbMoltableMx [ ByDateRange | ByCorpIdRange | LoadMissing | <singleCorpId>] /// </summary> /// <returns></returns> public static string UpdateCorpDbMoltableMx( string args) { DateTime moleculeDateTime = DateTime.MinValue; double mw; string msg = "", sql = "", maxCorpIdSql, mf = "", chime = "", smiles = "", checkPointDate = "", helm = "", sequence = "", svg = ""; object[][] pva = null; int pvaCount = 0, CorpId, lowCorpId = 0, highCorpId = 0, srcMaxCorpId = 0; int SelectChunkSize = 20; // small chunks int InsertBufferSize = 10; //int SelectChunkSize = 100000; // big chunks //int InsertBufferSize = 1000; // Select data from corp_moltable by CorpId range const string SelectByCorpIdRange = @" SELECT m.corp_nbr, chime(m.ctab), m.molformula, m.molweight, null molsmiles, s.helm_txt, s.sequence_txt, m.molecule_date FROM corp_owner.corp_moltable m, corp_owner.corp_substance s where m.corp_nbr > 0 and s.corp_nbr = m.corp_nbr and (s.status_code is null or s.status_code = 'A') ORDER BY corp_nbr"; // Select data from corp_moltable by date range comparing to corp_moltable_mx const string SelectByDateRange = @" select m.corp_nbr, chime(m.ctab), m.molformula, m.molweight, null molsmiles, s.helm_txt, s.sequence_txt, m.molecule_date, m2.molecule_date from corp_owner.corp_moltable m, corp_owner.corp_substance s, corp_moltable_mx m2 where m.molecule_date > to_date('1-jan-1900 000000','DD-MON-YYYY HH24MISS') and s.corp_nbr = M.CORP_NBR and (s.status_code is null or s.status_code = 'A') and m2.corp_nbr (+) = m.corp_nbr and m2.molecule_date (+) != m.molecule_date order by m.molecule_date" ; // Select for missing smiles strings, ex: Update CorpDbMoltableMx LoadMissing mx.molecule_date > '1-jan-2014' const string SelectMissingSmilesFix = @" select /* check for CorpIds in corp_moltable not in corp_moltable_mx */ corp_nbr, chime(ctab), molformula, molweight, null molsmiles, helm_txt, sequence_txt, molecule_date from ( select m.*, s.helm_txt, s.sequence_txt, mx.molsmiles molsmiles_mx from corp_owner.corp_moltable m, corp_owner.corp_substance s, corp_moltable_mx mx where s.corp_nbr = M.CORP_NBR and (s.status_code is null or s.status_code = 'A') and mx.corp_nbr (+) = m.corp_nbr and 1=1 /* condition to substitute */ ) m where molsmiles_mx is null /* extra condition */ order by corp_nbr" ; // Insert missing helm info const string SelectMissingHelmFix = @" select /* check for CorpIds in corp_moltable not in corp_moltable_mx */ corp_nbr, chime(ctab), molformula, molweight, null molsmiles, helm_txt, sequence_txt, molecule_date from ( select m.*, s.helm_txt, s.sequence_txt, mx.molsmiles molsmiles_mx from corp_owner.corp_moltable m, corp_owner.corp_substance s, corp_moltable_mx mx where s.corp_nbr = M.CORP_NBR and (s.status_code is null or s.status_code = 'A') and mx.corp_nbr (+) = m.corp_nbr and 1=1 /* condition to substitute */ ) m where length(helm_txt) > 0 /* extra condition */ order by corp_nbr" ; // Secondary "large" structure table (~5k mols) const string SelectLargeMols = @" select corp_nbr, to_clob(molstructure), to_clob(molformula), molweight, molsmiles, null helm_txt, null sequence_txt, molecule_date from (select corp_srl_nbr corp_nbr, 'CompoundId=' || corp_srl_nbr molstructure, null ctab, mlclr_frml_txt molformula, mlclr_wgt molweight, null molsmiles, null molecule_date from rdm_owner.rdm_sbstnc where rdw_src_cd = 'LRG'" ; // Insert statement const string InsertSql = @" insert into mbs_owner.corp_moltable_mx ( corp_nbr, molstructure, molformula, molweight, molsmiles, molecule_date) values (:0, :1, :2, :3, :4, :5)" ; // Build select sql bool byDateRange = false, byCorpIdRange = false, missingFix = true, deleteExisting = true; string missingFixCriteria = ""; if (Lex.IsUndefined(args) || Lex.Eq(args, "ByDateRange")) { byDateRange = true; } else if (Lex.Eq(args, "ByCorpIdRange")) { byCorpIdRange = true; Progress.Show("Getting range of CorpIds to insert..."); maxCorpIdSql = "select max(corp_nbr) from corp_owner.corp_moltable"; // get highest CorpId in source db srcMaxCorpId = SelectSingleValueDao.SelectInt(maxCorpIdSql); if (srcMaxCorpId < 0) { srcMaxCorpId = 0; } maxCorpIdSql = "select max(corp_nbr) from mbs_owner.corp_moltable_mx"; // get highest CorpId in dest db highCorpId = SelectSingleValueDao.SelectInt(maxCorpIdSql); if (highCorpId < 0) { highCorpId = 0; } } else if (Lex.StartsWith(args, "LoadMissing")) { missingFix = true; if (args.Contains(" ")) { missingFixCriteria = args.Substring(10).Trim(); } } else if (int.TryParse(args, out srcMaxCorpId)) // single CorpId { byCorpIdRange = true; highCorpId = srcMaxCorpId - 1; // say 1 less is the max we have } else { return("Syntax: UpdateCorpDbMoltableMx [ ByDateRange | ByCorpIdRange | LoadMissing | <singleCorpId>]"); } Log("UpdateCorpDbMoltableMx started: " + args); int readCount = 0, insCount = 0, insertCount = 0, updateCount = 0, undefinedStructures = 0, smilesSuccess = 0, smilesFails = 0, helmStructures = 0; List <string> CorpIdList = new List <string>(); for (int chunk = 1; ; chunk++) // loop over chunks { if (byDateRange) // single chunk { if (chunk > 1) { break; // break 2nd time through } checkPointDate = UserObjectDao.GetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", "01-sep-2013 000000"); //UserObjectDao.SetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", checkPointDate); sql = Lex.Replace(SelectByDateRange, "1-jan-1900 000000", checkPointDate); msg = "Reading where date >= " + checkPointDate; } else if (byCorpIdRange) // by CorpId range { if (highCorpId >= srcMaxCorpId) { break; // done } lowCorpId = highCorpId + 1; // start of next chunk highCorpId = lowCorpId + SelectChunkSize; if (highCorpId >= srcMaxCorpId) { highCorpId = srcMaxCorpId; } sql = Lex.Replace(SelectByCorpIdRange, "corp_nbr > 0", "corp_nbr between " + lowCorpId + " and " + highCorpId); msg = "Reading: " + lowCorpId + " to " + highCorpId + ", Reads: " + readCount + ", Inserts: " + insertCount; } else if (missingFix) { if (chunk > 1) { break; // break 2nd time through } sql = SelectMissingHelmFix; if (Lex.IsDefined(missingFixCriteria)) // substitute any criteria { sql = Lex.Replace(sql, "1=1", missingFixCriteria); } msg = "Fixing missing data"; } Progress.Show(msg); DbCommandMx readCmd = new DbCommandMx(); readCmd.MxConn = DbConnectionMx.GetConnection("prd123"); readCmd.PrepareUsingDefinedConnection(sql, null); DbDataReader rdr = readCmd.ExecuteReader(); DbCommandMx insertCmd = new DbCommandMx(); OracleDbType[] pta = new OracleDbType[6]; pta[0] = OracleDbType.Int32; // corp_nbr pta[1] = OracleDbType.Clob; // molstructure pta[2] = OracleDbType.Clob; // molformula pta[3] = OracleDbType.Double; // molweight pta[4] = OracleDbType.Clob; // smiles pta[5] = OracleDbType.Date; // molecule_date insertCmd.Prepare(InsertSql, pta); insertCmd.BeginTransaction(); // be sure we have a transaction going pva = DbCommandMx.NewObjectArrayArray(6, InsertBufferSize); // alloc insert row array object[] vo = new object[6]; while (true) { bool readOk = rdr.Read(); if (readOk) { rdr.GetValues(vo); CorpId = readCmd.GetInt(0); // corp_nbr vo[0] = CorpId; CorpIdList.Add(CorpId.ToString()); if (!readCmd.IsNull(1)) // molstructure { chime = readCmd.GetClob(1); chime = OracleMx.ClearStringIfExceedsMaxStringSize(chime); vo[1] = chime; } else { chime = ""; } if (!readCmd.IsNull(2)) // molformula { mf = readCmd.GetClob(2); mf = OracleMx.ClearStringIfExceedsMaxStringSize(mf); vo[2] = mf; } if (!readCmd.IsNull(3)) // molweight { mw = readCmd.GetDouble(3); vo[3] = mw; } if (Lex.IsDefined(chime)) // molsmiles - calculate from chime string { MoleculeMx cs = new MoleculeMx(MoleculeFormat.Chime, chime); if (cs.AtomCount > 1) // need more than one atom { MoleculeMx cs2 = cs.ConvertTo(MoleculeFormat.Smiles); smiles = cs2.GetSmilesString(); if (Lex.IsDefined(smiles)) { smilesSuccess++; } else { Log("Smiles conversion failure for CorpId: " + CorpId); smilesFails++; } smiles = OracleMx.ClearStringIfExceedsMaxStringSize(smiles); vo[4] = smiles; } else { undefinedStructures++; } } else { undefinedStructures++; } if (!readCmd.IsNull(5)) { helm = readCmd.GetClob(5); if (Lex.IsDefined(helm)) { svg = HelmControl.GetSvg(helm); vo[1] = SvgUtil.CompressSvgString(svg); // store compressed svg in molstructure column for now helmStructures++; } } if (!readCmd.IsNull(6)) { sequence = readCmd.GetClob(6); if (Lex.IsDefined(sequence)) { // nothing yet } } moleculeDateTime = DateTime.MinValue; if (!readCmd.IsNull(7)) // molecule_date { moleculeDateTime = readCmd.GetDateTime(7); vo[5] = moleculeDateTime; } for (int pi = 0; pi < 6; pi++) // invert for insert { pva[pi][pvaCount] = vo[pi]; } if (Debug) { msg = String.Format("CorpId: {0}, mf: {1}, chime: {2}, smiles: {3}", CorpId.ToString(), mf.Length, chime.Length, smiles.Length); Log(msg); } pvaCount++; } if (pvaCount >= InsertBufferSize || (!readOk && pvaCount > 0)) // write if buffer full or at end { try { if (deleteExisting) { int delCount = DoDeletes(CorpIdList); updateCount += delCount; // count deletes as updates insertCount -= delCount; // subtract from inserts } CorpIdList.Clear(); insCount = insertCmd.ExecuteArrayNonReader(pva, ref pvaCount); insertCmd.Commit(); insertCmd.BeginTransaction(); insertCount += insCount; } catch (Exception ex) { throw new Exception(ex.Message, ex); } if (byDateRange) { string checkPointDate2 = String.Format("{0:dd-MMM-yyyy HHmmss}", moleculeDateTime); // format date time that will work with oracle UserObjectDao.SetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", checkPointDate2); msg = "Processing where date >= " + checkPointDate + ", Reads: " + readCount + ", Inserts: " + insertCount + ", Updates: " + updateCount; } else if (byCorpIdRange) // CorpId range { msg = "Processing: " + lowCorpId + " to " + highCorpId + ", Reads: " + readCount + ", Inserts: " + insertCount; } else if (missingFix) { msg = "Fixing missing smiles, Updates: " + updateCount; } msg += String.Format(", Undefined structures: {0} , Smiles failures: {1}, Helms: {2}", undefinedStructures, smilesFails, helmStructures); Progress.Show(msg); } if (!readOk) { break; } readCount++; } readCmd.Dispose(); insertCmd.Dispose(); } // end for select chunk msg = "UpdateCorpDbMoltableMx - Inserts: " + insertCount + ", Updates: " + updateCount; msg += String.Format(", Undefined structures: {0} , Smiles failures: {1}, Helms: {2}", undefinedStructures, smilesFails, helmStructures); Log(msg); return(msg); }
/// <summary> /// Get the next value for the sequence (MySQL) /// </summary> /// <param name="seqName"></param> /// <returns></returns> public static long NextValLongMySQL( string seqName) { string sql; long nextVal; SequenceDao seqDao = Lookup(seqName); Queue <long> seqQueue = seqDao.Queue; if (seqQueue.Count > 0) { nextVal = seqQueue.Dequeue(); return(nextVal); } int count = (seqDao.CacheSize > 0 ? seqDao.CacheSize : 1); int t0 = TimeOfDay.Milliseconds(); DbCommandMx seqCmd = new DbCommandMx(); seqCmd.MxConn = DbConnectionMx.GetConnection("MySql_Mobius"); // "MySql_Mobius_Sequences" sql = String.Format( @"update mbs_owner.mbs_sequences set value = last_insert_id(value) + {0} where name = '{1}'" , count, seqName.ToUpper()); seqCmd.PrepareUsingDefinedConnection(sql); int updCount = seqCmd.ExecuteNonReader(); if (updCount <= 0) { throw new Exception("Error updating sequence (may not exist): " + seqName); } sql = "select last_insert_id()"; // gets value before update above seqCmd.PrepareUsingDefinedConnection(sql, null); DbDataReader rdr = seqCmd.ExecuteReader(); bool readOk = rdr.Read(); AssertMx.IsTrue(readOk, "readOk"); long value = rdr.GetInt64(0); rdr.Close(); seqCmd.CloseReader(); nextVal = value + 1; // return this one now long v2 = value + 2; // next value long vn = value + count; // last value for (long vi = v2; vi <= vn; vi++) { seqQueue.Enqueue(vi); } t0 = TimeOfDay.Milliseconds() - t0; // DebugLog.Message("Read sequence, set size = " + seqQueue.Count.ToString() + ", Time(ms) = " + t0.ToString()); return(nextVal); }
/// <summary> /// Write temporary list /// </summary> /// <param name="dsName">Name of datasource</param> /// <param name="listName">Name of list</param> /// <param name="listItems">The list of items to write</param> /// <param name="numeric">Store list items in numeric column if true, text otherwise</param> public static string Write( string dsName, string listName, List <string> listItems, int firstKeyIdx, int keyCount, bool numeric) { DbCommandMx cmd = null; Oracle.DataAccess.Client.OracleDbType parmType; string fullTableName, sKey; int iKey, i1; object o; DataSourceMx ds = GetDataSourceInfo(ref dsName, ref listName, out fullTableName); TruncateListTable(dsName, listName); if (keyCount == 0) { return(null); // nothing to write } string sql = "insert into <schema.name> (<cols>) values(:0)"; sql = sql.Replace("<schema.name>", fullTableName); if (numeric) { sql = sql.Replace("<cols>", "intKey"); } else { sql = sql.Replace("<cols>", "stringKey"); } object[][] pva = DbCommandMx.NewObjectArrayArray(1, keyCount); // alloc insert row array if (numeric) { for (i1 = 0; i1 < keyCount; i1++) { if (int.TryParse(listItems[firstKeyIdx + i1], out iKey)) { pva[0][i1] = iKey; } else { pva[0][i1] = DBNull.Value; } } parmType = Oracle.DataAccess.Client.OracleDbType.Int32; } else // string keys { for (i1 = 0; i1 < keyCount; i1++) { sKey = listItems[firstKeyIdx + i1]; if (!Lex.IsNullOrEmpty(sKey)) { pva[0][i1] = sKey; } else { pva[0][i1] = DBNull.Value; } } parmType = Oracle.DataAccess.Client.OracleDbType.Varchar2; } try { DateTime t0 = DateTime.Now; cmd = new DbCommandMx(); DbConnectionMx dbc = DbConnectionMx.GetConnection(dsName); cmd.MxConn = dbc; cmd.Prepare(sql, parmType); int insCount = cmd.ExecuteArrayNonReader(pva, ref keyCount); int tDelta = (int)TimeOfDay.Delta(t0); if (Debug) { DebugLog.Message("Write TempDbList for KeyCount: " + insCount + ", Time: " + tDelta); } } catch (Exception ex) { throw ex; } finally { cmd.Dispose(); } return(fullTableName); }