Beispiel #1
0
        /// <summary>
        /// Read next record
        /// </summary>
        /// <returns></returns>

        public AnnotationVo Read()
        {
            if (!DbCmd.Read())
            {
                return(null);
            }

            AnnotationVo vo = new AnnotationVo();

            vo.rslt_id           = DbCmd.GetLong(0);
            vo.rslt_grp_id       = DbCmd.GetLong(1);
            vo.ext_cmpnd_id_txt  = DbCmd.GetString(2);
            vo.ext_cmpnd_id_nbr  = DbCmd.GetInt(3);
            vo.src_db_id         = DbCmd.GetInt(4);
            vo.mthd_vrsn_id      = DbCmd.GetInt(5);
            vo.rslt_typ_id       = DbCmd.GetLong(6);
            vo.rslt_val_prfx_txt = DbCmd.GetString(7);
            vo.rslt_val_nbr      = DbCmd.GetDouble(8);
            vo.uom_id            = DbCmd.GetInt(9);
            vo.rslt_val_txt      = DbCmd.GetString(10);
            vo.rslt_val_dt       = DbCmd.GetDateTime(11);
            vo.cmnt_txt          = DbCmd.GetString(12);
            vo.dc_lnk            = DbCmd.GetString(13);
            vo.chng_op_cd        = DbCmd.GetString(14);
            vo.chng_usr_id       = DbCmd.GetString(15);
            return(vo);
        }
Beispiel #2
0
/// <summary>
/// Get image coordinates
/// </summary>
/// <param name="targetMapXDict"></param>
/// <param name="targetMapYDict"></param>

        public static void GetImageCoords(
            out Dictionary <int, double> targetMapXDict,
            out Dictionary <int, double> targetMapYDict)
        {
            targetMapXDict = new Dictionary <int, double>();
            targetMapYDict = new Dictionary <int, double>();

            string sql = @"
				select entrezgene_id, x, y
				from mdbassay_owner.image_coord"                ;

            DbCommandMx drd = new DbCommandMx();

            drd.Prepare(sql);
            drd.ExecuteReader();

            while (drd.Read())
            {
                int    geneId = drd.GetInt(0);
                double x      = drd.GetDouble(1);
                targetMapXDict[geneId] = x;

                double y = drd.GetDouble(2);
                targetMapYDict[geneId] = y;
            }

            drd.CloseReader();
            drd.Dispose();

            return;
        }
Beispiel #3
0
        /// <summary>
        /// Read in data, pivot & buffer for supplied set of rows.
        /// This is called for retrieval only, not for search
        /// </summary>
        /// <param name="eqp"></param>

        public override void ExecuteQuery(
            ExecuteQueryParms eqp)
        {
            DbCommandMx              drd;
            int                      rowsFetched = 0, vosCreated = 0;
            MetaTable                mt;
            MetaColumn               mc = null;
            DateTime                 dt;
            PivotMetaBroker          mb;
            List <GenericMetaBroker> mbList;

            string cid, pivotKey, tableFilter, s, txt, tok;
            int    fci, mci, pvi, pci, si, i1;

            object[] vo = null;
            object   o;

            if (!PivotInCode)             // let Oracle do the pivoting?
            {
                base.ExecuteQuery(eqp);
                return;
            }

            // Self-pivot. Read & buffer data for all query tables from same Source/TableFilterColumns for key set if we are the first table for Source

            int t0 = TimeOfDay.Milliseconds();
            Dictionary <string, MultiTablePivotBrokerTypeData> mbsi = eqp.Qe.MetaBrokerStateInfo;

            mt = eqp.QueryTable.MetaTable;
            string sourceKey = mt.TableMap + "," + Csv.JoinCsvString(mt.TableFilterColumns);             // grouping based on source sql
            MultiTablePivotBrokerTypeData mpd = (MultiTablePivotBrokerTypeData)mbsi[sourceKey];

            if (mpd.FirstTableName != mt.Name)
            {
                return;                                            // retrieve data for all tables when we see first table
            }
            mpd.ClearBuffers();

            // Build sql

            StringBuilder sb = new StringBuilder();             // build filter to select for desired metatable

            tableFilter = "";
            if (mt.TableFilterColumns.Count == 1)
            {             // build single in list if single filter column
                foreach (string mtName in mpd.MbInstances.Keys)
                {
                    mt = MetaTableCollection.Get(mtName);
                    if (sb.Length > 0)
                    {
                        sb.Append(",");
                    }
                    sb.Append(mt.TableFilterValues[0]);
                    tableFilter = mt.TableFilterColumns[0] + " in (" + sb.ToString() + ")";
                }
            }

            else             // multiple table filter columns, build and/or expressions
            {
                foreach (string mtName in mpd.MbInstances.Keys)
                {
                    mt = MetaTableCollection.Get(mtName);
                    if (sb.Length > 0)
                    {
                        sb.Append(" or ");
                    }
                    tableFilter = "(" + GetTableFilterCriteria(mt) + ")";
                    sb.Append(tableFilter);
                }
                tableFilter = "(" + sb.ToString() + ")";
            }

            string sql = "select * from " + mt.TableMap + " where ";

            if (tableFilter != "")
            {
                sql += tableFilter + " and ";
            }
            sql += mt.KeyMetaColumn.ColumnMap + " in (<list>) ";

            // Read unpivoted data, merge/pivot & buffer pivoted rows

            List <string> keySubset = eqp.SearchKeySubset;

            if (keySubset == null)
            {
                keySubset = GetPreviewSubset();                                // assume previewing of single table if no subset
            }
            List <string> parmList = new List <string>();

            for (i1 = 0; i1 < keySubset.Count; i1++)             // copy keys to parameter array properly normalized
            {
                string key = CompoundId.NormalizeForDatabase((string)keySubset[i1], Qt.MetaTable);
                if (key != null)
                {
                    parmList.Add(key);
                }
            }

            drd = new DbCommandMx();
            drd.PrepareListReader(sql, DbType.String);
            drd.ExecuteListReader(parmList);
            while (drd.Read())
            {
                rowsFetched++;

                string tableFilterKey = "";                 // get column values to identify table
                for (fci = 0; fci < mt.TableFilterColumns.Count; fci++)
                {
                    o = drd.GetObjectByName(mt.TableFilterColumns[fci]);
                    if (o == null)
                    {
                        s = "";
                    }
                    else
                    {
                        s = o.ToString();
                    }
                    if (tableFilterKey != "")
                    {
                        tableFilterKey += ", ";
                    }
                    tableFilterKey += s;
                }
                mt = mpd.TableFilterValuesToMetaTableDict[tableFilterKey];
                if (mt == null)
                {
                    continue;                             // continue if don't know about this table
                }
                if (!mpd.MbInstances.ContainsKey(mt.Name))
                {
                    continue;                                                        // have row hash for broker?
                }
                int mbIdx = 0;
                mb = (PivotMetaBroker)mpd.GetFirstBroker(mt.Name, out mbList);

                while (true)                 // copy out for each metabroker
                {
                    mt = mb.Qt.MetaTable;
                    if (mt == null)
                    {
                        continue;
                    }

                    if (mb.MultipivotRowDict == null)
                    {
                        mb.MultipivotRowDict = new Dictionary <string, object[]>();
                    }

                    string rowKey = "";
                    for (mci = 0; mci < mt.PivotMergeColumns.Count; mci++)
                    {
                        o = drd.GetObjectByName(mt.PivotMergeColumns[mci]);
                        if (o == null)
                        {
                            s = "<null>";
                        }
                        else
                        {
                            s = o.ToString();
                        }
                        rowKey += "<" + s + ">";
                    }

                    if (mb.MultipivotRowDict.ContainsKey(rowKey))                     // have entry for row?
                    {
                        vo = (object[])mb.MultipivotRowDict[rowKey];
                    }

                    else                     // new row, create vo for it & fill in merged column values
                    {
                        vo = new Object[mb.SelectList.Count];
                        for (si = 0; si < mb.SelectList.Count; si++)                         // transfer non-pivoted values
                        {
                            mc = mb.SelectList[si];
                            if (mc.PivotValues != null)
                            {
                                continue;                                                     // skip pivoted cols for now
                            }
                            for (mci = 0; mci < mt.PivotMergeColumns.Count; mci++)
                            {
                                if (Lex.Eq(mc.ColumnMap, mt.PivotMergeColumns[mci]))
                                {
                                    o = drd.GetObjectByName(mt.PivotMergeColumns[mci]);
                                    if (mc.IsKey)                                     // normalize cid adding prefix as needed
                                    {
                                        o = CompoundId.Normalize(o.ToString(), mt);
                                    }
                                    vo[si] = o;
                                    break;
                                }
                            }
                        }

                        mb.MultipivotRowDict[rowKey] = vo;
                        vosCreated++;
                    }

                    // Pivot out data based on pivot column values

                    if (mb.PivotKeys == null)
                    {                     // build set of pivot keys for the pivoted columns in the table if not done yet
                        mb.PivotKeys = new string[mb.SelectList.Count];
                        for (si = 0; si < mb.SelectList.Count; si++)
                        {
                            mc = mb.SelectList[si];
                            if (mc.PivotValues == null)
                            {
                                continue;                                                     // skip non-pivoted cols
                            }
                            pivotKey = "";
                            for (pvi = 0; pvi < mc.PivotValues.Count; pvi++)
                            {
                                pivotKey += "<" + mc.PivotValues[pvi].ToLower() + ">";
                            }
                            mb.PivotKeys[si] = pivotKey;
                        }
                    }

                    pivotKey = "";
                    for (pci = 0; pci < mt.PivotColumns.Count; pci++)
                    {                     // build pivot key for this unpivoted row
                        o = drd.GetObjectByName(mt.PivotColumns[pci]);
                        if (o == null)
                        {
                            s = "<null>";
                        }
                        else
                        {
                            s = o.ToString().ToLower();
                        }
                        pivotKey += "<" + s + ">";
                    }

                    for (si = 0; si < mb.SelectList.Count; si++)                // transfer pivoted values
                    {
                        if (mb.PivotKeys[si] == null ||                         // skip non-pivoted cols
                            pivotKey != mb.PivotKeys[si])
                        {
                            continue;                                                   // and non-matches
                        }
                        mc = mb.SelectList[si];
                        int ci = drd.Rdr.GetOrdinal(mc.ColumnMap);

                        if (mc.DataType == MetaColumnType.Integer)
                        {
                            if (!mc.DetailsAvailable)                             // simple scalar value
                            {
                                vo[si] = drd.GetInt(ci);
                            }

                            else                             // value with possible resultId, linking information
                            {
                                txt    = drd.GetString(ci);  // todo: fix for annotation
                                vo[si] = QueryEngine.ParseScalarValue(txt, Qt, mc);
                            }
                        }

                        else if (mc.DataType == MetaColumnType.Number)
                        {
                            if (!mc.DetailsAvailable)                             // simple scalar value
                            {
                                vo[si] = drd.GetDouble(ci);
                            }

                            else                             // value with possible resultId, linking information
                            {
                                txt    = Dr.GetString(ci);   // todo: fix for annotation
                                vo[si] = QueryEngine.ParseScalarValue(txt, Qt, mc);
                            }
                        }

                        else if (mc.DataType == MetaColumnType.QualifiedNo)
                        {
                            // todo
                        }

                        else if (mc.DataType == MetaColumnType.String)
                        {
                            if (!mc.DetailsAvailable)
                            {
                                vo[si] = drd.GetString(ci);
                            }

                            else                             // value with possible resultId, linking information
                            {
                                txt    = Dr.GetString(ci);   // todo: fix for annotation
                                vo[si] = QueryEngine.ParseScalarValue(txt, Qt, mc);
                            }
                        }

                        else if (mc.DataType == MetaColumnType.Date)
                        {
                            if (!mc.DetailsAvailable)                             // simple scalar value
                            {
                                vo[si] = drd.GetDateTime(ci);
                            }

                            else                             // value with possible resultId, linking information
                            {
                                txt    = Dr.GetString(ci);   // todo: fix for annotation
                                vo[si] = QueryEngine.ParseScalarValue(txt, Qt, mc);
                            }
                        }

                        else if (mc.DataType == MetaColumnType.Structure)
                        {                         // structures come in as compound identifiers (todo: fix for annotation)
                            tok    = Dr.GetValue(si).ToString();
                            cid    = CompoundId.Normalize(tok, Qt.MetaTable);
                            vo[si] = cid;
                        }

                        else if (mc.DataType == MetaColumnType.MolFormula)
                        {
                            vo[si] = drd.GetString(ci);
                        }

                        else if (mc.DataType == MetaColumnType.DictionaryId)
                        {
                            try                             // Id may be string or integer value
                            { vo[si] = drd.GetString(ci); }
                            catch (Exception ex)
                            { vo[si] = drd.GetInt(ci); }
                        }

                        else if (mc.DataType == MetaColumnType.Image)
                        {
                            try                             // Id may be string or integer value
                            { vo[si] = drd.GetString(ci); }
                            catch (Exception ex)
                            { vo[si] = drd.GetInt(ci); }
                        }

                        else if (mc.DataType == MetaColumnType.Hyperlink)
                        {
                            txt = drd.GetString(ci);
                            Hyperlink hlink = new Hyperlink();
                            vo[si]     = hlink;
                            hlink.Text = txt;
                        }
                    }
                    if (mbList == null)
                    {
                        break;                   // single broker
                    }
                    mbIdx++;                     // go to next broker
                    if (mbIdx >= mbList.Count)
                    {
                        break;                                            // at end of brokers?
                    }
                    mb = (PivotMetaBroker)mbList[mbIdx];
                }         // end of broker loop
            }             // end of read loop

            drd.Dispose();
            return;
        }
Beispiel #4
0
        /// <summary>
        /// Sync the Mobius CorpMoltable replicate used to retrieve Smiles
        /// Syntax: UpdateCorpDbMoltableMx [ ByDateRange | ByCorpIdRange | LoadMissing | <singleCorpId>]
        /// </summary>
        /// <returns></returns>

        public static string UpdateCorpDbMoltableMx(
            string args)
        {
            DateTime moleculeDateTime = DateTime.MinValue;
            double   mw;
            string   msg = "", sql = "", maxCorpIdSql, mf = "", chime = "", smiles = "", checkPointDate = "", helm = "", sequence = "", svg = "";

            object[][] pva = null;
            int        pvaCount = 0, CorpId, lowCorpId = 0, highCorpId = 0, srcMaxCorpId = 0;

            int SelectChunkSize  = 20;            // small chunks
            int InsertBufferSize = 10;

            //int SelectChunkSize = 100000; // big chunks
            //int InsertBufferSize = 1000;

            // Select data from corp_moltable by CorpId range

            const string SelectByCorpIdRange = @" 
		SELECT 
        m.corp_nbr,
        chime(m.ctab), 
        m.molformula,
        m.molweight,
        null molsmiles,
        s.helm_txt,
        s.sequence_txt,
        m.molecule_date
    FROM 
        corp_owner.corp_moltable m,
        corp_owner.corp_substance s
    where 
        m.corp_nbr > 0
        and s.corp_nbr = m.corp_nbr
        and (s.status_code is null or s.status_code = 'A')    
    ORDER BY corp_nbr";

            // Select data from corp_moltable by date range comparing to corp_moltable_mx

            const string SelectByDateRange = @"
			select
					m.corp_nbr,
					chime(m.ctab), 
					m.molformula,
					m.molweight,
          null molsmiles,
	        s.helm_txt,
					s.sequence_txt,
					m.molecule_date,
					m2.molecule_date
			from
					corp_owner.corp_moltable m,
					corp_owner.corp_substance s,
					corp_moltable_mx m2
			where
					m.molecule_date > to_date('1-jan-1900 000000','DD-MON-YYYY HH24MISS')
					and s.corp_nbr = M.CORP_NBR
					and (s.status_code is null or s.status_code = 'A')    
					and m2.corp_nbr (+) = m.corp_nbr
					and m2.molecule_date (+) != m.molecule_date
			order by m.molecule_date"            ;

            // Select for missing smiles strings, ex: Update CorpDbMoltableMx LoadMissing mx.molecule_date > '1-jan-2014'

            const string SelectMissingSmilesFix = @"
			select /* check for CorpIds in corp_moltable not in corp_moltable_mx */
					corp_nbr,
					chime(ctab), 
					molformula,
					molweight,
					null molsmiles,
					helm_txt,
          sequence_txt,
          molecule_date
			from 
					(
					select 
						m.*, 
						s.helm_txt,
						s.sequence_txt,
						mx.molsmiles molsmiles_mx
					from
					 corp_owner.corp_moltable m,
					 corp_owner.corp_substance s,
					 corp_moltable_mx mx
					where
					 s.corp_nbr = M.CORP_NBR
					 and (s.status_code is null or s.status_code = 'A')
					 and mx.corp_nbr (+) = m.corp_nbr
					 and 1=1 /* condition to substitute */
					) m
			where molsmiles_mx is null /* extra condition */
			order by corp_nbr"            ;

// Insert missing helm info

            const string SelectMissingHelmFix = @"
			select /* check for CorpIds in corp_moltable not in corp_moltable_mx */
					corp_nbr,
					chime(ctab), 
					molformula,
					molweight,
					null molsmiles,
					helm_txt,
          sequence_txt,
          molecule_date
			from 
					(
					select 
						m.*, 
						s.helm_txt,
						s.sequence_txt,
						mx.molsmiles molsmiles_mx
					from
					 corp_owner.corp_moltable m,
					 corp_owner.corp_substance s,
					 corp_moltable_mx mx
					where
					 s.corp_nbr = M.CORP_NBR
					 and (s.status_code is null or s.status_code = 'A')
					 and mx.corp_nbr (+) = m.corp_nbr
					 and 1=1 /* condition to substitute */
					) m
			where length(helm_txt) > 0 /* extra condition */
			order by corp_nbr"            ;

            // Secondary "large" structure table (~5k mols)

            const string SelectLargeMols = @"
			select 
				corp_nbr, 
				to_clob(molstructure), 
				to_clob(molformula), 
				molweight,
				molsmiles,
				null helm_txt,
				null sequence_txt,
				molecule_date
			from
			(select
				corp_srl_nbr corp_nbr,
				'CompoundId=' || corp_srl_nbr molstructure, 
				null ctab,
				mlclr_frml_txt molformula,
				mlclr_wgt molweight,
				null molsmiles,
				null molecule_date
				from rdm_owner.rdm_sbstnc 
				where rdw_src_cd = 'LRG'"                ;

// Insert statement

            const string InsertSql = @"
			insert into mbs_owner.corp_moltable_mx (
				corp_nbr,
				molstructure,
				molformula,
				molweight,
				molsmiles,
				molecule_date)
			values (:0, :1, :2, :3, :4, :5)"            ;

// Build select sql

            bool   byDateRange = false, byCorpIdRange = false, missingFix = true, deleteExisting = true;
            string missingFixCriteria = "";

            if (Lex.IsUndefined(args) || Lex.Eq(args, "ByDateRange"))
            {
                byDateRange = true;
            }

            else if (Lex.Eq(args, "ByCorpIdRange"))
            {
                byCorpIdRange = true;

                Progress.Show("Getting range of CorpIds to insert...");
                maxCorpIdSql = "select max(corp_nbr) from corp_owner.corp_moltable";                 // get highest CorpId in source db
                srcMaxCorpId = SelectSingleValueDao.SelectInt(maxCorpIdSql);
                if (srcMaxCorpId < 0)
                {
                    srcMaxCorpId = 0;
                }

                maxCorpIdSql = "select max(corp_nbr) from mbs_owner.corp_moltable_mx";                 // get highest CorpId in dest db
                highCorpId   = SelectSingleValueDao.SelectInt(maxCorpIdSql);
                if (highCorpId < 0)
                {
                    highCorpId = 0;
                }
            }

            else if (Lex.StartsWith(args, "LoadMissing"))
            {
                missingFix = true;
                if (args.Contains(" "))
                {
                    missingFixCriteria = args.Substring(10).Trim();
                }
            }

            else if (int.TryParse(args, out srcMaxCorpId))             // single CorpId
            {
                byCorpIdRange = true;
                highCorpId    = srcMaxCorpId - 1;              // say 1 less is the max we have
            }

            else
            {
                return("Syntax: UpdateCorpDbMoltableMx [ ByDateRange | ByCorpIdRange | LoadMissing | <singleCorpId>]");
            }

            Log("UpdateCorpDbMoltableMx started: " + args);

            int           readCount = 0, insCount = 0, insertCount = 0, updateCount = 0, undefinedStructures = 0, smilesSuccess = 0, smilesFails = 0, helmStructures = 0;
            List <string> CorpIdList = new List <string>();

            for (int chunk = 1; ; chunk++)       // loop over chunks
            {
                if (byDateRange)                 // single chunk
                {
                    if (chunk > 1)
                    {
                        break;                                // break 2nd time through
                    }
                    checkPointDate = UserObjectDao.GetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", "01-sep-2013 000000");

                    //UserObjectDao.SetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", checkPointDate);

                    sql = Lex.Replace(SelectByDateRange, "1-jan-1900 000000", checkPointDate);

                    msg = "Reading where date >= " + checkPointDate;
                }

                else if (byCorpIdRange)                 // by CorpId range
                {
                    if (highCorpId >= srcMaxCorpId)
                    {
                        break;                                      // done
                    }
                    lowCorpId  = highCorpId + 1;                    // start of next chunk
                    highCorpId = lowCorpId + SelectChunkSize;
                    if (highCorpId >= srcMaxCorpId)
                    {
                        highCorpId = srcMaxCorpId;
                    }
                    sql = Lex.Replace(SelectByCorpIdRange, "corp_nbr > 0", "corp_nbr between " + lowCorpId + " and " + highCorpId);

                    msg = "Reading: " + lowCorpId + " to " + highCorpId + ", Reads: " + readCount + ", Inserts: " + insertCount;
                }

                else if (missingFix)
                {
                    if (chunk > 1)
                    {
                        break;                                // break 2nd time through
                    }
                    sql = SelectMissingHelmFix;
                    if (Lex.IsDefined(missingFixCriteria))                     // substitute any criteria
                    {
                        sql = Lex.Replace(sql, "1=1", missingFixCriteria);
                    }
                    msg = "Fixing missing data";
                }

                Progress.Show(msg);

                DbCommandMx readCmd = new DbCommandMx();
                readCmd.MxConn = DbConnectionMx.GetConnection("prd123");
                readCmd.PrepareUsingDefinedConnection(sql, null);
                DbDataReader rdr = readCmd.ExecuteReader();

                DbCommandMx insertCmd = new DbCommandMx();

                OracleDbType[] pta = new OracleDbType[6];
                pta[0] = OracleDbType.Int32;                // corp_nbr
                pta[1] = OracleDbType.Clob;                 // molstructure
                pta[2] = OracleDbType.Clob;                 // molformula
                pta[3] = OracleDbType.Double;               // molweight
                pta[4] = OracleDbType.Clob;                 // smiles
                pta[5] = OracleDbType.Date;                 // molecule_date

                insertCmd.Prepare(InsertSql, pta);
                insertCmd.BeginTransaction();                               // be sure we have a transaction going

                pva = DbCommandMx.NewObjectArrayArray(6, InsertBufferSize); // alloc insert row array
                object[] vo = new object[6];

                while (true)
                {
                    bool readOk = rdr.Read();

                    if (readOk)
                    {
                        rdr.GetValues(vo);

                        CorpId = readCmd.GetInt(0);                         // corp_nbr
                        vo[0]  = CorpId;
                        CorpIdList.Add(CorpId.ToString());

                        if (!readCmd.IsNull(1))                         // molstructure
                        {
                            chime = readCmd.GetClob(1);
                            chime = OracleMx.ClearStringIfExceedsMaxStringSize(chime);
                            vo[1] = chime;
                        }
                        else
                        {
                            chime = "";
                        }

                        if (!readCmd.IsNull(2))                         // molformula
                        {
                            mf    = readCmd.GetClob(2);
                            mf    = OracleMx.ClearStringIfExceedsMaxStringSize(mf);
                            vo[2] = mf;
                        }

                        if (!readCmd.IsNull(3))                         // molweight
                        {
                            mw    = readCmd.GetDouble(3);
                            vo[3] = mw;
                        }

                        if (Lex.IsDefined(chime))                         // molsmiles - calculate from chime string
                        {
                            MoleculeMx cs = new MoleculeMx(MoleculeFormat.Chime, chime);
                            if (cs.AtomCount > 1)                             // need more than one atom
                            {
                                MoleculeMx cs2 = cs.ConvertTo(MoleculeFormat.Smiles);
                                smiles = cs2.GetSmilesString();
                                if (Lex.IsDefined(smiles))
                                {
                                    smilesSuccess++;
                                }
                                else
                                {
                                    Log("Smiles conversion failure for CorpId: " + CorpId);
                                    smilesFails++;
                                }
                                smiles = OracleMx.ClearStringIfExceedsMaxStringSize(smiles);

                                vo[4] = smiles;
                            }
                            else
                            {
                                undefinedStructures++;
                            }
                        }
                        else
                        {
                            undefinedStructures++;
                        }

                        if (!readCmd.IsNull(5))
                        {
                            helm = readCmd.GetClob(5);
                            if (Lex.IsDefined(helm))
                            {
                                svg   = HelmControl.GetSvg(helm);
                                vo[1] = SvgUtil.CompressSvgString(svg);                                 // store compressed svg in molstructure column for now
                                helmStructures++;
                            }
                        }

                        if (!readCmd.IsNull(6))
                        {
                            sequence = readCmd.GetClob(6);
                            if (Lex.IsDefined(sequence))
                            {
                                // nothing yet
                            }
                        }

                        moleculeDateTime = DateTime.MinValue;
                        if (!readCmd.IsNull(7))                         // molecule_date
                        {
                            moleculeDateTime = readCmd.GetDateTime(7);
                            vo[5]            = moleculeDateTime;
                        }

                        for (int pi = 0; pi < 6; pi++)                         // invert for insert
                        {
                            pva[pi][pvaCount] = vo[pi];
                        }

                        if (Debug)
                        {
                            msg = String.Format("CorpId: {0}, mf: {1}, chime: {2}, smiles: {3}", CorpId.ToString(), mf.Length, chime.Length, smiles.Length);
                            Log(msg);
                        }

                        pvaCount++;
                    }

                    if (pvaCount >= InsertBufferSize || (!readOk && pvaCount > 0))                     // write if buffer full or at end
                    {
                        try
                        {
                            if (deleteExisting)
                            {
                                int delCount = DoDeletes(CorpIdList);
                                updateCount += delCount;                                 // count deletes as updates
                                insertCount -= delCount;                                 // subtract from inserts
                            }
                            CorpIdList.Clear();

                            insCount = insertCmd.ExecuteArrayNonReader(pva, ref pvaCount);
                            insertCmd.Commit();
                            insertCmd.BeginTransaction();
                            insertCount += insCount;
                        }

                        catch (Exception ex)
                        {
                            throw new Exception(ex.Message, ex);
                        }

                        if (byDateRange)
                        {
                            string checkPointDate2 = String.Format("{0:dd-MMM-yyyy HHmmss}", moleculeDateTime);                             // format date time that will work with oracle
                            UserObjectDao.SetUserParameter("MOBIUS", "UpdateCorpDbMoltableMxCheckpointDate", checkPointDate2);
                            msg = "Processing where date >= " + checkPointDate + ", Reads: " + readCount + ", Inserts: " + insertCount + ", Updates: " + updateCount;
                        }

                        else if (byCorpIdRange)                         // CorpId range
                        {
                            msg = "Processing: " + lowCorpId + " to " + highCorpId + ", Reads: " + readCount + ", Inserts: " + insertCount;
                        }

                        else if (missingFix)
                        {
                            msg = "Fixing missing smiles, Updates: " + updateCount;
                        }

                        msg += String.Format(", Undefined structures: {0} , Smiles failures: {1}, Helms: {2}", undefinedStructures, smilesFails, helmStructures);

                        Progress.Show(msg);
                    }

                    if (!readOk)
                    {
                        break;
                    }

                    readCount++;
                }

                readCmd.Dispose();
                insertCmd.Dispose();
            }             // end for select chunk

            msg  = "UpdateCorpDbMoltableMx - Inserts: " + insertCount + ", Updates: " + updateCount;
            msg += String.Format(", Undefined structures: {0} , Smiles failures: {1}, Helms: {2}", undefinedStructures, smilesFails, helmStructures);
            Log(msg);

            return(msg);
        }