/// <summary> /// Read next fingerprint rec from file /// </summary> /// <param name="fi"></param> /// <returns></returns> public FingerprintRec ReadFingerprintRec( int fi) { if (Rfpa == null || Rfpa.reader != ReadStreams[fi]) // start reading file { Rfpa = new ReadFingerprintRecArgs(); Rfpa.Initialize(ReadStreams[fi], FpLengthInLongs); } if (!ReadRawFingerprintRec(Rfpa)) { return(null); // end of file } FingerprintRec r = new FingerprintRec(); r.molId = Rfpa.uci; r.SrcId = Rfpa.src; string cid = ASCIIEncoding.ASCII.GetString(Rfpa.cidBytes, 0, Rfpa.cidLength); r.Cid = CompoundIdNormalizeForDatabase(cid); r.Cardinality = Rfpa.cardinality; r.Fingerprint = (long[])Rfpa.fingerprint.Clone(); return(r); }
/// <summary> /// ReadFingerprintRec /// </summary> /// <returns></returns> public FingerprintRec ReadFingerprintRec() { if (Rsi < 0) { Rsi = 0; // first file } FingerprintRec r = ReadFingerprintRec(Rsi); while (r == null) // if end of file try going to next { if (Rsi + 1 >= ReadStreams.Length) { return(null); } Rsi++; Rfpa = new ReadFingerprintRecArgs(); Rfpa.Initialize(ReadStreams[Rsi], FpLengthInLongs); r = ReadFingerprintRec(Rsi); } r.molId = Rfpa.uci; r.SrcId = Rfpa.src; string cid = ASCIIEncoding.ASCII.GetString(Rfpa.cidBytes, 0, Rfpa.cidLength); r.Cid = CompoundIdNormalizeForDatabase(cid); r.Cardinality = Rfpa.cardinality; r.Fingerprint = (long[])Rfpa.fingerprint.Clone(); return(r); }
/// <summary> /// WriteFingerprintRec /// </summary> /// <param name="fpr"></param> public void WriteFingerprintRec( FingerprintRec fpr) { if (fpr.SrcId != LastSrcId || fpr.Cid != LastCid) { // keep all FPs for a Cid in the same file Bwi++; if (Bwi >= BinaryWriters.Length) { Bwi = 0; } } WriteFingerprintRec(BinaryWriters[Bwi], fpr); LastSrcId = fpr.SrcId; LastCid = fpr.Cid; return; }
/// <summary> /// GetMaxCorpId /// </summary> /// <returns></returns> public static int GetMaxDestMolId() { int molId, maxMolId = 0; FingerprintDao fpd = new FingerprintDao(Database, FingerprintType); List <string> ids = new List <string>(); if (!fpd.DataFilesExist()) { return(maxMolId); } fpd.OpenReaders(); while (true) { FingerprintRec rec = fpd.ReadFingerprintRec(); if (rec == null) { break; } //CorpIds.Add(rec.Cid); // debug if (CorpDatabase) { int.TryParse(rec.Cid, out molId); } else { molId = rec.molId; } if (molId > maxMolId) { maxMolId = molId; } } fpd.CloseReaders(); return(maxMolId); }
/// <summary> /// Get a set of the existing Cids /// </summary> /// <returns></returns> public HashSet <string> GetExistingCidSet() { HashSet <string> cidSet = new HashSet <string>(); OpenReaders(); while (true) { FingerprintRec r = ReadFingerprintRec(); if (r == null) { break; } cidSet.Add(r.Cid); } CloseReaders(); return(cidSet); }
/// <summary> /// WriteFpRec /// </summary> /// <param name="bw"></param> /// <param name="uci"></param> /// <param name="src"></param> /// <param name="cidString"></param> /// <param name="fingerprint"></param> public void WriteFingerprintRec( BinaryWriter bw, FingerprintRec fpr) { long l; if (bw == null || fpr == null) { throw new InvalidDataException(); } bw.Write(fpr.molId); // int - [4] bw.Write(fpr.SrcId); // int - [4] string cid = CompoundIdNormalizeForDatabase(fpr.Cid); bw.Write((byte)cid.Length); // byte - [1] byte[] cidBytes = ASCIIEncoding.ASCII.GetBytes(cid); bw.Write(cidBytes); // byte - [n] bw.Write(fpr.Cardinality); // number of bits set in fp - int - [4] long[] fp64 = fpr.Fingerprint; // FingerprintMx.ByteArrayToLongrray(fingerprint); bw.Write((byte)FpLengthInLongs); // long count - byte - [1] for (int li = 0; li < FpLengthInLongs; li++) // fingerprint long*3 -[24] (MACCS) { if (li < fp64.Length) { l = fp64[li]; } else { l = 0; // pad to fixed length } bw.Write(l); } return; }
/// <summary> /// Merge existing .bin files with new records /// 1. Copy .bin files to .mrg files filtering out cids that have been updated /// 2. Append new records to .mrg files /// 3. Rename .bin files to .bak files and .mrg files to new .bin files /// </summary> static void MergeRecordsIntoFiles( List <FingerprintRec> fpRecList) { string date1, date2; Progress.Show("Merging existing and new files..."); // Copy existing bin file entries filtering out cids that were updated FpDao.OpenReaders("bin"); // open existing .bin files for input FpDao.OpenWriters("mrg", FileMode.Create); // open new merge files for output HashSet <string> fpRecHash = new HashSet <string>(); // build a hash of cids that shouldn't be copied foreach (FingerprintRec fpr in fpRecList) { fpRecHash.Add(fpr.Cid); } int replacementCnt = 0; for (int fi = 0; fi < FingerprintDao.FingerprintFileCount; fi++) // copy each file { BinaryWriter bw = FpDao.BinaryWriters[fi]; while (true) { FingerprintRec fpr = FpDao.ReadFingerprintRec(fi); if (fpr == null) { break; } if (fpRecHash.Contains(fpr.Cid)) { //Log("Removing " + fpr.Cid); // debug replacementCnt++; continue; // skip if this cid is was updated in the incoming list } FpDao.WriteFingerprintRec(bw, fpr); } } FpDao.CloseReaders(); // Append the new records to the merge files foreach (FingerprintRec fpr in fpRecList) // write out buffered recs { FpDao.WriteFingerprintRec(fpr); } FpDao.CloseWriters(); // Backup old files and activate new files for (int fi = 0; fi < FingerprintDao.FingerprintFileCount; fi++) // check that we can backup all bin files { string fileName = FpDao.GetFpFileName(fi) + ".bin"; bool backupOk = FileUtil.CanRename(fileName); if (!backupOk) { throw new Exception("Unable to rename file: " + fileName + ", aborting update"); } } for (int fi = 0; fi < FingerprintDao.FingerprintFileCount; fi++) { string fileName = FpDao.GetFpFileName(fi); bool backupOk = FileUtil.BackupAndReplaceFile(fileName + ".bin", fileName + ".bak", fileName + ".mrg"); if (!backupOk) { throw new Exception("Error replacing file: " + fileName + ".bin"); } } if (ByCheckpoint) { string checkPointDate = String.Format("{0:dd-MMM-yyyy HHmmss}", MoleculeDateTime); // format last new/updated date time in a format that works with Oracle FpDao.WriteFingerPrintSimMxDataUpdateCheckpointDate(checkPointDate); // update checkpoint } Progress.Show("Merging complete..."); int cnt = fpRecList.Count; string cid1 = fpRecList[0].Cid; string cid2 = fpRecList[cnt - 1].Cid; if (CidUpdateDateDict.ContainsKey(cid1)) { date1 = CidUpdateDateDict[cid1].ToString(); } else { date1 = "Missing"; } if (CidUpdateDateDict.ContainsKey(cid2)) { date2 = CidUpdateDateDict[cid1].ToString(); } else { date2 = "Missing"; } Log("Records stored: " + cnt + ", Adds: " + (cnt - replacementCnt) + ", Replacements: " + replacementCnt + ", Date range: " + date1 + " - " + date2 + ", CIDs: First = " + cid1 + ", Last = " + cid2); return; }
static ICdkMol CdkMolUtil => StaticCdkMol.I; // static molecule shortcut for utility methods /// <summary> /// UpdateCorpFingerprintDatabaseMx /// /////////////////////////////////////////////////////// /// Syntax: Update FingerprintDatabaseMx [Corp | ChEMBL] [MACCS | ECFP4] [Load | ByCidRange | SinceLastCheckpoint | LoadMissing | <SingleCorpId>] /// /// Corp Examples: /// Update FingerprintDatabaseMx Corp MACCS Load /// Update FingerprintDatabaseMx Corp MACCS LoadMissing /// Update FingerprintDatabaseMx Corp MACCS SinceLastCheckpoint /// /// Update FingerprintDatabaseMx Corp ECFP4 Load /// Update FingerprintDatabaseMx Corp ECFP4 LoadMissing /// Update FingerprintDatabaseMx Corp ECFP4 SinceLastCheckpoint /// /// ChEMBL Examples: /// Update FingerprintDatabaseMx Chembl MACCS Load /// Update FingerprintDatabaseMx Chembl MACCS LoadMissing /// /// Update FingerprintDatabaseMx Chembl ECFP4 Load /// Update FingerprintDatabaseMx Chembl ECFP4 LoadMissing /// /////////////////////////////////////////////////////// /// </summary> /// <param name="argString"></param> /// <returns></returns> static public string Update( string argString) { MoleculeMx mol; double mw; string chime, smiles, molString, molFile = ""; string msg = "", sql = "", chemblId, cid = "", maxCorpIdSql, maxIdSql2, mf, missingFixCriteria = "", CorpIdList = ""; int storeChunkCount = 0, CorpId, molregno, molId, lowId = 0, highId = 0, maxDestId = 0, maxSrcId = 0; int readCount = 0, storeCount = 0; ByCheckpoint = ByCidRange = ByCidList = LoadIfMissing = false; ReadChunkSize = DefaultReadChunkSize; WriteChunkSize = DefaultWriteChunkSize; Failures = new Dictionary <string, string>(); NewUndefinedStructureCids = new List <string>(); LastFailure = ""; FailureCount = 0; // global try loop try { //////////////////////// /// Parse Parameters /// //////////////////////// // See which database argString = argString.Trim(); if (Lex.StartsWith(argString, "Corp")) { Database = "Corp"; argString = argString.Substring(5).Trim(); } else if (Lex.StartsWith(argString, "Chembl")) { Database = "ChEMBL"; argString = argString.Substring(6).Trim(); } else { return(SyntaxMsg); } // See which fingerprint type FingerprintType = FingerprintType.MACCS; // default to MACCS if type not defined if (Lex.TryReplace(ref argString, "MACCS", "")) { FingerprintType = FingerprintType.MACCS; argString = argString.Trim(); } else if (Lex.TryReplace(ref argString, "ECFP4", "")) { FingerprintType = FingerprintType.Circular; // (i.e. ECFP4) argString = argString.Trim(); } FpDao = new FingerprintDao(Database, FingerprintType); List <FingerprintRec> fpRecList = new List <FingerprintRec>(); string args = argString.Trim(); string initialMsg = "Update FingerprintDatabase started: " + args; CidList = new List <string>(); // init empty list ////////////////////// /// Corp Database /// ////////////////////// if (CorpDatabase) { if (Lex.Eq(args, "Load")) { ByCidRange = true; ShowProgress("Getting range of CorpIds to insert..."); maxCorpIdSql = SelectMaxCorpId; // get highest id in source db maxSrcId = SelectSingleValueDao.SelectInt(maxCorpIdSql); if (maxSrcId < 0) { maxSrcId = 0; } maxDestId = GetMaxDestMolId(); //maxIdSql2 = "select max(src_compound_id_nbr) from dev_mbs_owner.corp_uc_xref where src_id = 0"; // get highest id in UniChemDb db //highCorpId = SelectSingleValueDao.SelectInt(maxIdSql2); if (maxDestId < 0) { maxDestId = 0; } } else if (Lex.Eq(args, "SinceLastCheckpoint")) { ByCheckpoint = true; ShowProgress("Getting list of CorpIds updated since last checkpoint..."); CidList = GetNewAndModifiedCorpIdList(out CidUpdateDateDict); //CidUpdateList = new List<string>(); // debug with single cmpd //CidUpdateList.Add("03435269"); if (CidList.Count == 0) { return("There have been no updates since the last checkpoint"); } initialMsg += ", CorpIds to add/update: " + CidList.Count; } else if (Lex.StartsWith(args, "ByCorpIdList")) { ByCidList = true; CorpIdList = args.Substring("ByCorpIdList".Length).Trim(); if (Lex.IsUndefined(CorpIdList)) { throw new Exception("Undefined CorpId list"); } } else if (Lex.StartsWith(args, "LoadMissing")) { LoadIfMissing = true; if (args.Contains(" ")) { missingFixCriteria = args.Substring("LoadMissing".Length).Trim(); } ShowProgress("Getting list of missing CorpIds..."); CidList = GetMissingCidList(); if (CidList.Count == 0) { return("There are no missing CorpIds"); } initialMsg += ", Missing CorpIds: " + CidList.Count; } else if (int.TryParse(args, out maxSrcId)) // single CorpId { ByCidRange = true; maxDestId = maxSrcId - 1; // say 1 less is the max we have } else { return(SyntaxMsg); } } /////////////////////// /// ChEMBL Database /// /////////////////////// else if (ChemblDatabase) { if (Lex.Eq(args, "Load")) { ByCidRange = true; ShowProgress("Getting range of MolRegNos to insert..."); sql = "select max(molregno) from chembl_owner.compound_struct_xxxxxx"; maxSrcId = SelectSingleValueDao.SelectInt(sql); if (maxSrcId < 0) { maxSrcId = 0; } maxDestId = GetMaxDestMolId(); if (maxDestId < 0) { maxDestId = 0; } } else if (Lex.StartsWith(args, "LoadMissing")) { LoadIfMissing = true; ShowProgress("Getting list of missing ChEMBL Ids..."); CidList = GetMissingCidList(); if (CidList.Count == 0) { return("There are no missing Ids"); } initialMsg += ", Missing Chembl Ids: " + CidList.Count; } else { return(SyntaxMsg); } } else { return(SyntaxMsg); } CidListOriginalCount = CidList.Count; Log(initialMsg); ///////////////////////////// // Loop over chunks of data ///////////////////////////// for (int chunk = 1; ; chunk++) { ////////////////////// /// Corp Database /// ////////////////////// if (CorpDatabase) { if (ByCheckpoint) // single chunk { string cidList = GetNextListChunk(); if (Lex.IsUndefined(cidList)) { break; } sql = SelectByCorpIdCriteria; sql = Lex.Replace(sql, "<CorpIdCriteria>", "in (" + cidList + ")"); string matchString = "order by m.corp_nbr"; if (!Lex.Contains(sql, matchString)) { throw new Exception(matchString + " not found"); } sql = Lex.Replace(sql, matchString, "order by m.molecule_date"); msg = "Processing " + CidListOriginalCount + " updates since " + CheckpointDateTime; // + " (" + Mobius.Data.CidList.FormatCidListForDisplay(null, chunkCidList) + ")"; } else if (ByCidRange) // by CorpId range { if (maxDestId >= maxSrcId) { break; // done } lowId = maxDestId + 1; // start of next chunk highId = lowId + ReadChunkSize; maxDestId = highId; //lowCorpId = highCorpId = 12345; // debug if (highId >= maxSrcId) { highId = maxSrcId; } sql = SelectByCorpIdCriteria; sql = Lex.Replace(sql, "<CorpIdCriteria>", "between " + lowId + " and " + highId); msg = "Processing CorpId range: " + lowId + " to " + highId; } else if (ByCidList) // by single user-supplied CorpId list { if (chunk > 1) { break; // break 2nd time through } sql = SelectByCorpIdCriteria; sql = Lex.Replace(sql, "<CorpIdCriteria>", "in (" + CorpIdList + ")"); msg = "Processing CorpId list: " + CorpIdList; } else if (LoadIfMissing) { string cidList = GetNextListChunk(); if (Lex.IsUndefined(cidList)) { break; // all done } sql = SelectByCorpIdCriteria; sql = Lex.Replace(sql, "<CorpIdCriteria>", "in (" + cidList + ")"); msg = "Processing missing CorpId Chunk: " + Mobius.Data.CidList.FormatAbbreviatedCidListForDisplay(null, cidList) + ", Total Ids: " + CidListOriginalCount; Log(msg); } else { return(SyntaxMsg); } } /////////////////////// /// ChEMBL Database /// /////////////////////// else if (ChemblDatabase) { if (ByCidRange) // by CID range { if (maxDestId >= maxSrcId) { break; // done } lowId = maxDestId + 1; // start of next chunk highId = lowId + ReadChunkSize; maxDestId = highId; //lowId = highId = 12345; // debug if (maxDestId >= maxSrcId) { maxDestId = maxSrcId; } sql = SelectChemblSql; sql = Lex.Replace(sql, "<molregnoCriteria>", "between " + lowId + " and " + highId); msg = "Processing ChEMBL MolRegNo range: " + lowId + " to " + highId; } else if (LoadIfMissing) { string cidList = GetNextListChunk(); if (Lex.IsUndefined(cidList)) { break; // all done } sql = SelectByCorpIdCriteria; sql = Lex.Replace(sql, "<CorpIdCriteria>", "in (" + cidList + ")"); msg = "Processing missing ChEMBL Id Chunk: " + Mobius.Data.CidList.FormatAbbreviatedCidListForDisplay(null, cidList) + ", Total Ids: " + CidListOriginalCount; } else { return(SyntaxMsg); } } else { return(SyntaxMsg); } ShowProgress(msg); // Execute the SQL to get the rows for the chunk DbCommandMx rdr = DbCommandMx.PrepareAndExecuteReader(sql); DateTime lastShowProgressTime = DateTime.MinValue; /////////////////////////////////////////// /// Loop over rows in the current chunk /// /////////////////////////////////////////// while (true) { // Update progress display if (DateTime.Now.Subtract(lastShowProgressTime).TotalSeconds > 1) // show progress { int storeTotalCount = storeCount + storeChunkCount; string msg2 = msg + "\r\n" + "Reads: " + readCount + "\r\n" + "Undefined: " + NewUndefinedStructureCids.Count + "\r\n" + "Insert/Updates: " + storeTotalCount + "\r\n" + "Failures: " + FailureCount + "\r\n" + "Failure Types: " + Failures.Count + "\r\n" + "Last Failure: " + LastFailure; ShowProgress(msg2); lastShowProgressTime = DateTime.Now; } // Read and process next compound bool readOk = rdr.Read(); if (readOk) { readCount++; try { double t1 = 0, t2 = 0, t3 = 0, t4 = 0; DateTime t0 = DateTime.Now; mol = null; //t2 = TimeOfDay.Delta(ref t0); ////////////////////// /// Corp Database /// ////////////////////// if (CorpDatabase) { CorpId = rdr.GetInt(0); // corp_nbr //Log("CorpId: " + CorpId); // debug molId = CorpId; cid = CorpId.ToString(); cid = CompoundId.NormalizeForDatabase(cid); if (!rdr.IsNull(1)) // be sure chime field isn't null { chime = rdr.GetClob(1); if (Lex.IsDefined(chime)) { molFile = MoleculeMx.ChimeStringToMolfileString(chime); // convert Chime to MolFile mol = new MoleculeMx(MoleculeFormat.Molfile, molFile); } } MoleculeDateTime = rdr.GetDateTimeByName("Molecule_Date"); // Date molecule was updated in the CorpDB cartridge DB } /////////////////////// /// ChEMBL Database /// /////////////////////// else // chembl { molId = molregno = rdr.GetInt(0); cid = chemblId = rdr.GetString(1); smiles = rdr.GetString(2); if (Lex.IsDefined(smiles)) { mol = new MoleculeMx(MoleculeFormat.Smiles, smiles); } } if (MoleculeMx.IsUndefined(mol) || mol.AtomCount <= 1) { NewUndefinedStructureCids.Add(cid); continue; //mol = new AtomContainer(); // write empty structure } bool includeOverallFingerprint = true; List <BitSetFingerprint> fps = CdkMol.BuildBitSetFingerprints(mol.MolfileString, includeOverallFingerprint, FingerprintType); //t3 = TimeOfDay.Delta(ref t0); foreach (BitSetFingerprint fp in fps) { FingerprintRec fpr = new FingerprintRec(); fpr.molId = molId; fpr.SrcId = SrcDbId; fpr.Cid = cid; fpr.Cardinality = fp.cardinality(); fpr.Fingerprint = fp.asBitSet().toLongArray(); fpRecList.Add(fpr); } //t4 = TimeOfDay.Delta(ref t0); t4 = t4; } catch (Exception ex) { if (!Failures.ContainsKey(ex.Message)) { Failures.Add(ex.Message, cid); } else { Failures[ex.Message] += ", " + cid; } LastFailure = "Cid: " + cid + " - " + ex.Message; Log(LastFailure); //ShowProgress(ex.Message + "\r\n" + ex.StackTrace.ToString()); // debug FailureCount++; continue; } storeChunkCount++; } bool commitTransaction = (storeChunkCount >= WriteChunkSize || (!readOk && storeChunkCount > 0)); if (commitTransaction) // end of chunk of data to store? { // if updating by CheckPoint date range then merge existing data with new/updated data if (ByCheckpoint) { if (readCount > 0 && (storeCount > 0 || FailureCount == 0)) // make sure not everything has failed) { MergeRecordsIntoFiles(fpRecList); } } // Simple append of records to files else { FpDao.OpenWriters("bin", FileMode.Append); // open bin files for append foreach (FingerprintRec fpr in fpRecList) // write out buffered recs { FpDao.WriteFingerprintRec(fpr); } FpDao.CloseWriters(); int cnt = fpRecList.Count; if (cnt > 0) { string cid1 = fpRecList[0].Cid; string cid2 = fpRecList[cnt - 1].Cid; Log("Records Appended: " + cnt + ", CIDS: " + cid1 + " - " + cid2); } else { Log("Records Appended: 0"); } } fpRecList.Clear(); storeCount += storeChunkCount; storeChunkCount = 0; } if (!readOk) { break; } } // end of read loop for rows in a chunk rdr.Dispose(); } // end for loop of chunks DeleteTempFiles(); if (LoadIfMissing) // update list of cids with missing structures { ExistingUndefinedStructureCids.UnionWith(NewUndefinedStructureCids); FpDao.WriteUndefinedStructuresCids(ExistingUndefinedStructureCids); } msg = "*** Update Complete ***\r\n\r\n" + msg; ShowProgress(msg); System.Threading.Thread.Sleep(100); string logMsg = "UpdateFingerprintDb - CIDs stored: " + storeCount + ", Undefined structures: " + NewUndefinedStructureCids.Count + ", failures: " + FailureCount + "\r\n"; foreach (string key in Failures.Keys) { logMsg += key + " - CIDs: " + Failures[key] + "\r\n"; } Log(logMsg); return(logMsg); } // end of main try loop catch (Exception ex) { Log(DebugLog.FormatExceptionMessage(ex)); throw new Exception(ex.Message, ex); } }