/// <summary> /// Calculate & persist library stats /// </summary> /// <returns></returns> public static string UpdateLibraryStatistics() { Dictionary <string, MetaTableStats> stats = new Dictionary <string, MetaTableStats>(); int libId; long cnt; DateTime dt; string libName, txt; try { DbCommandMx dao = new DbCommandMx(); string sql = // get count and date (must use crt_timestamp) for each library @" select l.lib_id, count(*), l.crt_timestamp from corp_owner.corp_library l, corp_owner.corp_library_substance ls where ls.lib_id (+) = l.lib_id group by l.lib_id, l.crt_timestamp order by l.lib_id" ; dao.Prepare(sql); dao.ExecuteReader(); while (dao.Read()) { libId = dao.GetInt(0); libName = "LIBRARY_" + libId; if (!stats.ContainsKey(libName)) { stats[libName] = new MetaTableStats(); } cnt = dao.GetLong(1); stats[libName].RowCount = cnt; dt = dao.GetDateTime(2); stats[libName].UpdateDateTime = dt; } dao.CloseReader(); dao.Dispose(); string fileName = MetaTableFactory.MetaTableXmlFolder + @"\LibraryStats"; MetaTableFactory.WriteMetaTableStats(stats, fileName); } catch (Exception ex) { throw ex; } return("Updated statistics for " + stats.Count + " libraries"); }
/// <summary> /// Utility method to write out a metatable stats file for a broker /// </summary> /// <param name="stats"></param> /// <param name="arg"></param> /// <param name="fileName"></param> /// <returns></returns> public static int WriteMetaTableStats( Dictionary <string, MetaTableStats> stats, string singleMtName, string fileName) { string path = fileName + ".new"; StreamWriter sw = new StreamWriter(path); if (!Lex.IsNullOrEmpty(singleMtName)) // if mtName specified read & write all data but this metatable { path = fileName + ".txt"; StreamReader sr = new StreamReader(path); // open existing file while (true) { string rec = sr.ReadLine(); if (rec == null) { break; } if (rec.StartsWith(singleMtName + "\t")) { continue; // skip existing recs } sw.WriteLine(rec); // write all other recs } sr.Close(); } foreach (string mtName0 in stats.Keys) { MetaTableStats mts = stats[mtName0]; string rec = mtName0 + "\t" + mts.RowCount.ToString() + "\t" + mts.UpdateDateTime.ToShortDateString(); sw.WriteLine(rec); } sw.Close(); FileUtil.BackupAndReplaceFile(fileName + ".txt", fileName + ".bak", fileName + ".new"); return(stats.Count); }
/// <summary> /// Utility routine to load a file of stats for a single broker type /// </summary> /// <param name="fileName"></param> /// <param name="metaTableStats"></param> /// <returns></returns> public static int LoadMetaTableStats( string fileName, Dictionary <string, MetaTableStats> metaTableStats) { if (!File.Exists(fileName)) { return(0); } StreamReader sr = new StreamReader(fileName); int cnt = 0; while (true) { string rec = sr.ReadLine(); if (Lex.IsUndefined(rec)) { break; } string[] sa = rec.Split('\t'); if (sa.Length < 3) { continue; } string table = sa[0].ToUpper(); MetaTableStats mts = new MetaTableStats(); mts.RowCount = long.Parse(sa[1]); mts.UpdateDateTime = DateTimeUS.ParseDate(sa[2]); metaTableStats[table] = mts; cnt++; } sr.Close(); return(cnt); }
/// <summary> /// Update Common Assay Attributes table (mbs_owner.cmn_assy_atrbts) /// /// Command: Update AssayAttributesTable /// /// This command builds an entry (or two) in the cmn_assy_atrbts table for each assay /// referenced in the Mobius contents tree that reports an SP or CRC value as determined /// by the associated metafactory and available in each metatable. /// If the gene target associated with an assay can be identified then information on that /// gene is included as well. /// /// Additional gene information may come from the metadata for a source such as the results /// warehouse. /// /// Note that this function must be run under an account that has access to all restricted data so that it /// can properly see what's available and build the table. /// </summary> /// <param name="lex"></param> /// <returns></returns> public static string UpdateAssayAttributesTable(string args) { MetaTable mt; MetaColumn mc; AssayAttributes aa, aa2; Dictionary <int, int> geneIdCounts = new Dictionary <int, int>(); // genes and counts keyed by entrez gene id Dictionary <int, double> targetMapXDict; // dendogram X coord keyed by gene id Dictionary <int, double> targetMapYDict; // dendogram Y coord keyed by gene id Dictionary <string, int> targetTypeCounts = new Dictionary <string, int>(); // target types and counts Dictionary <int, string> assayTypeDict; Dictionary <int, string> assayModeDict; //UnpivotedAssayResult rr, rr2; List <string> toks; string mtName, tableNamePrefix, key, fileName, msg; bool crcExists, spExists, isSummary; int tableId, step, ri, resultTypeId, assayRowCount; Log = new LogFile(ServicesDirs.LogDir + @"\UpdateCommonAssayAttributes.log"); Log.ResetFile(); // Get list of all of the assays in the tree LogMessage("Accumulating assays..."); HashSet <string> mtNameHash = new HashSet <string>(); foreach (MetaTreeNode mtn0 in MetaTree.Nodes.Values) { if (mtn0.Type != MetaTreeNodeType.MetaTable) { continue; } mtName = mtn0.Target; if (AssayMetaData.IsAssayMetaTableName(mtName)) { mtNameHash.Add(mtName); } } bool debug = false; // set to true to debug with limited list of assays from below if (debug) { mtNameHash.Clear(); //assayHash.Add("ASSAY_1"); //assayHash.Add("ASSAY_2"); //assayHash.Add("ASSAY_3"); } // Get other informatin needed from AssayMetadata LogMessage("Reading AssayMetadata ASSAY metadata..."); Dictionary <int, AssayDbMetadata> assayMetadataAssayDict = // get assays and associated target/gene information AssayMetadataDao.GetAssayTargetGeneData(); LogMessage("Getting AssayMetadata result types..."); Dictionary <int, AssayDbResultType> resultTypeDict = AssayMetadataDao.GetResultTypeDict(); LogMessage("Getting assay types and modes..."); AssayMetadataDao.GetAssayTypesAndModes(out assayTypeDict, out assayModeDict); LogMessage("Getting gene dendogram coordinates..."); try { AssayMetadataDao.GetImageCoords(out targetMapXDict, out targetMapYDict); } catch (Exception ex) // may fail if problem with data source { LogMessage(DebugLog.FormatExceptionMessage(ex, true)); targetMapXDict = new Dictionary <int, double>(); targetMapYDict = new Dictionary <int, double>(); } // Process each assay int metatablesFound = 0, metatablesNotFound = 0; int assaysProcessed = 0; int assaysWithGenes = 0; int assaysWithGeneCoords = 0; int assaysWithTargets = 0; int assaysWithSpOnly = 0; int assaysWithCrcOnly = 0; int assaysWithNoCrcSP = 0; int assaysWithOtherTypes = 0; int assaysWithCrcAndSp = 0; int assaysWithNoKeyTypes = 0; int assaysWithProcessingErrors = 0; Dictionary <string, int> CrcAssayCnt = new Dictionary <string, int>() { { "ASSAY_DB1", 0 }, { "ASSAY_DB2", 0 }, { "ASSAY_DB3", 0 } }; Dictionary <string, int> SpAssayCnt = new Dictionary <string, int>() { { "ASSAY_DB1", 0 }, { "ASSAY_DB2", 0 }, { "ASSAY_DB3", 0 } }; Dictionary <string, int> OtherAssayCnt = new Dictionary <string, int>() { { "ASSAY_DB1", 0 }, { "ASSAY_DB2", 0 }, { "ASSAY_DB3", 0 } }; List <AssayAttributes> resultTypeRows = new List <AssayAttributes>(); List <AssayAttributes> geneRows = new List <AssayAttributes>(); List <AssayAttributes> dbRows = new List <AssayAttributes>(); string copyUpdateMsg = ""; foreach (string mtName0 in mtNameHash) { AssayMetaData assayMetaData = null; // metadata for assay bool isAssay = false; int assayIdNbr = NullValue.NullNumber; int assayId = NullValue.NullNumber; //if (assaysProcessed >= 100) break; // debug mtName = mtName0; MetaTable.ParseMetaTableName(mtName, out tableNamePrefix, out tableId, out isSummary); string resultType = ""; string rtId = ""; string assayName, assayDb; mt = MetaTableCollection.Get(mtName); // get metatable if (mt == null) { metatablesNotFound++; LogMessage("MetaTable not found: " + mtName); continue; } metatablesFound++; if (mt.Code == "") { continue; // must be single pivoted assay } assayDb = "ASSAY_DB"; // customize assayIdNbr = -1; if (UalUtil.IClient != null && UalUtil.IClient.Attended) { UAL.Progress.Show((assaysProcessed + 1).ToString() + " / " + mtNameHash.Count + " - " + mt.Name + "\r\n" + mt.Label); } aa = new AssayAttributes(); aa.AssayDatabase = assayDb; aa.AssayIdNbr = assayIdNbr; // data-source-specific assay Id aa.AssayIdTxt = mt.Name; // store ASSAY_1234 type table name aa.AssayId2 = assayId; // any associated assay id if (isAssay) { aa.AssayName = assayMetaData.Name; // name from AssayMetadata } else { aa.AssayName = MetaTable.RemoveSuffixesFromName(mt.Label); // name from metatable } if (mt.SummarizedExists) { aa.SummarizedAvailable = true; } else { aa.SummarizedAvailable = false; } if (isAssay) { if (assayTypeDict.ContainsKey(tableId)) { aa.AssayType = assayTypeDict[tableId]; } if (assayModeDict.ContainsKey(tableId)) { aa.AssayMode = assayModeDict[tableId]; } aa.AssaySource = AssayMetaData.GetAssaySource(tableId); aa.AssociationSource = "TODO"; // customize } aa.AssayStatus = "Active"; // say all active for now MetaTableStats mts = MetaTableFactory.GetStats(mtName); if (mts != null) { aa.ResultCount = (int)mts.RowCount; aa.AssayUpdateDate = mts.UpdateDateTime; } else { aa.ResultCount = 0; // assume no results if no stats } if (mt.DescriptionIsAvailable()) // use description from Mobius { aa.AssayDesc = "Y"; } if (String.IsNullOrEmpty(aa.GeneFamily)) { aa.GeneFamily = "Unknown"; // set these to "Unknown" rather than null } if (String.IsNullOrEmpty(aa.AssayType)) { aa.AssayType = "UNKNOWN"; // upper case UNKNOWN } if (String.IsNullOrEmpty(aa.AssayMode)) { aa.AssayMode = "UNKNOWN"; // upper case UNKNOWN } // Step1: Add a row for primary & any secondary results resultTypeRows.Clear(); MetaColumn firstResultCol = null, firstKeyResultCol = null, firstOtherKeyResultCol = null; string resultTypeConcType; HashSet <string> keyResultTypeCodes = new HashSet <string>(); int spCnt = 0, crcCnt = 0, otherCnt = 0; for (int mci = 0; mci < mt.MetaColumns.Count; mci++) // pick first col with result code (could also check summarization method) { mc = mt.MetaColumns[mci]; if (Lex.IsUndefined(mc.ResultCode)) { continue; // must have code defined } if (keyResultTypeCodes.Contains(mc.ResultCode)) { continue; // and not included so far } if (mc.InitialSelection != ColumnSelectionEnum.Selected) { continue; // selected only } if (firstResultCol == null) { firstResultCol = mc; } if (!IsKeyResultType(mc, out resultTypeConcType)) { continue; } if (firstKeyResultCol == null) { firstKeyResultCol = mc; } keyResultTypeCodes.Add(mc.ResultCode); aa2 = aa.Clone(); if (resultTypeRows.Count == 0) { aa2.TopLevelResult = "Y"; } else { aa2.TopLevelResult = "N"; } aa2.ResultTypeId2 = GetAssayResultTypeId(mc); // AssayMetadata result type id aa2.ResultTypeIdNbr = GetInternalResultTypeId(mc); // Internal database result type id aa2.ResultTypeIdTxt = mc.Name; // Mobius column name if (isAssay && resultTypeDict.ContainsKey(aa2.ResultTypeId2)) { aa2.ResultName = resultTypeDict[aa2.ResultTypeId2].Name; // use name from AssayMetadata result type dict } else { aa2.ResultName = mc.Label; // use label from Mobius } aa2.ResultTypeUnits = mc.Units; // result units if (Lex.Eq(resultTypeConcType, "SP")) { aa2.ResultTypeConcType = "SP"; spCnt++; } else if (Lex.Eq(resultTypeConcType, "CRC")) { aa2.ResultTypeConcType = "CRC"; crcCnt++; } else { aa2.ResultTypeConcType = ""; otherCnt++; if (firstOtherKeyResultCol == null) { firstOtherKeyResultCol = mc; } } aa2.ResultTypeConcUnits = ""; // todo resultTypeRows.Add(aa2); } if (resultTypeRows.Count >= 1) { if (crcCnt > 0) { CrcAssayCnt[assayDb]++; // count primary type by db } else if (spCnt > 0) { SpAssayCnt[assayDb]++; } else { OtherAssayCnt[assayDb]++; } if (crcCnt > 0 && spCnt == 0) { assaysWithCrcOnly++; // count overall primary/secondary types } else if (crcCnt == 0 && spCnt > 0) { assaysWithSpOnly++; } else if (crcCnt > 0 && spCnt > 0) { assaysWithCrcAndSp++; } if (crcCnt == 0 && spCnt == 0) // no SP or CRC result types { assaysWithNoCrcSP++; mc = firstKeyResultCol; LogMessage("Assay with No SP/CRC key results: " + mt.Name + "." + mc.Name + " (" + mc.ResultCode + "), " + mt.Label + "." + mc.Label); } else if (otherCnt > 0) // no SP or CRC result types { assaysWithOtherTypes++; mc = firstOtherKeyResultCol; LogMessage("Non SP/CRC key result: " + mt.Name + "." + mc.Name + " (" + mc.ResultCode + "), " + mt.Label + "." + mc.Label); } } else // no key result type { aa2 = aa.Clone(); resultTypeRows.Add(aa2); // include row for step1 OtherAssayCnt[assayDb]++; assaysWithNoKeyTypes++; LogMessage("No key result type for metatable: " + mt.Name + ", " + mt.Label); } // Build a step2 row for each target/gene geneRows.Clear(); List <AssayTarget> targets = new List <AssayTarget>(); int geneCount = 0; if (isAssay) { targets = assayMetaData.Targets; } if (targets.Count > 0) { assaysWithTargets++; } foreach (AssayTarget target in targets) { aa = new AssayAttributes(); aa.GeneFamily = target.TargetTypeShortName; // count target type occurance if (Lex.IsUndefined(aa.GeneFamily)) { aa.GeneFamily = "Unknown"; } if (!targetTypeCounts.ContainsKey(aa.GeneFamily)) { targetTypeCounts[aa.GeneFamily] = 0; } targetTypeCounts[aa.GeneFamily]++; if (target.Genes == null || target.Genes.Count == 0) // if no genes add a single target row { geneRows.Add(aa); continue; } foreach (AssayGene rg in target.Genes) { if (!Lex.IsDefined(rg.GeneSymbol)) { continue; } aa2 = aa.Clone(); geneRows.Add(aa2); aa2.GeneSymbol = rg.GeneSymbol; int.TryParse(rg.GeneId, out aa2.GeneId); if (aa2.GeneId > 0 && targetMapXDict.ContainsKey(aa2.GeneId)) { aa2.TargetMapX = targetMapXDict[aa2.GeneId]; aa2.TargetMapY = targetMapYDict[aa2.GeneId]; if (geneCount == 0) { assaysWithGeneCoords++; } } if (!geneIdCounts.ContainsKey(aa2.GeneId)) // count gene occurance { geneIdCounts[aa2.GeneId] = 0; } geneIdCounts[aa2.GeneId]++; if (geneCount == 0) { assaysWithGenes++; } geneCount++; } } if (geneRows.Count == 0) // if no step 2 rows (i.e. no targets), create a single step2 row { aa = new AssayAttributes(); geneRows.Add(aa); } // Combine key result types with target/genes for (int i1 = 0; i1 < resultTypeRows.Count; i1++) { AssayAttributes s1aa = resultTypeRows[i1]; for (int i2 = 0; i2 < geneRows.Count; i2++) { AssayAttributes s2aa = geneRows[i2]; aa = s1aa.Clone(); aa.GeneId = s2aa.GeneId; aa.GeneSymbol = s2aa.GeneSymbol; aa.GeneFamily = s2aa.GeneFamily; aa.TargetMapX = s2aa.TargetMapX; aa.TargetMapY = s2aa.TargetMapY; aa.GeneCount = geneCount; if (i2 > 0) { aa.GeneCount = -geneCount; // negative for other than 1st gene } dbRows.Add(aa); } } assaysProcessed++; } // Update table bool updateTable = true; // set to false for debug if (dbRows.Count <= 0) { LogMessage("No rows in new dataset, table not updated"); } else if (updateTable) { LogMessage("Deleting existing data..."); DbCommandMx dao = new DbCommandMx(); string sql = "delete from mbs_owner.cmn_assy_atrbts"; sql = AssayAttributesDao.AdjustAssayAttrsTableName(sql); dao.Prepare(sql); dao.BeginTransaction(); int delCnt = dao.ExecuteNonReader(); LogMessage("Inserting new data..."); int t0 = TimeOfDay.Milliseconds(); for (ri = 0; ri < dbRows.Count; ri++) { aa = dbRows[ri]; aa.Id = ri + 1; //aa.Id += 10000; // debug if (aa.GeneSymbol != null) { aa.GeneSymbol = aa.GeneSymbol.ToUpper(); // be sure key match cols are upper case } if (aa.GeneFamily != null) { aa.GeneFamily = aa.GeneFamily.ToUpper(); } if (aa.GeneFamilyTargetSymbol != null) { aa.GeneFamilyTargetSymbol = aa.GeneFamilyTargetSymbol.ToUpper(); } if (aa.ResultTypeConcType != null) { aa.ResultTypeConcType = aa.ResultTypeConcType.ToUpper(); } if (aa.AssayType != null) { aa.AssayType = aa.AssayType.ToUpper(); } if (aa.AssayMode != null) { aa.AssayMode = aa.AssayMode.ToUpper(); } AssayAttributesDao.InsertCommonAssayAttributes(aa, dao); if (TimeOfDay.Milliseconds() - t0 > 1000) { //Progress.Show("Inserting new data " + (ri + 1) + "/" + rows.Count + "..."); t0 = TimeOfDay.Milliseconds(); } } dao.Commit(); dao.Dispose(); copyUpdateMsg = UpdateCmnAssyAtrbtsCopies(); } string response = "----------------------------------\r\n" + "Assays processed: " + assaysProcessed + "\r\n" + "Assays with processing errors: " + assaysWithProcessingErrors + "\r\n" + "Rows inserted: " + dbRows.Count + "\r\n" + copyUpdateMsg + "----------------------------------\r\n" + "Assays with CRC only: " + assaysWithCrcOnly + "\r\n" + "Assays with SP only: " + assaysWithSpOnly + "\r\n" + "Assays with CRC and SP: " + assaysWithCrcAndSp + "\r\n" + "Assays with no CRC or SP: " + assaysWithNoCrcSP + "\r\n" + "Assays with non CRC/SP key types: " + assaysWithOtherTypes + "\r\n" + "Assays with no key types: " + assaysWithNoKeyTypes + "\r\n" + "----------------------------------\r\n" + "Assays with targets defined: " + assaysWithTargets + "\r\n" + "Assays with genes defined: " + assaysWithGenes + "\r\n" + "Assays with gene map coordinates: " + assaysWithGeneCoords + "\r\n" + "----------------------------------\r\n" + //"CRC Assays: " + CrcAssayCnt["ASSAY"] + "\r\n" + //"SP Assays: " + SpAssayCnt["ASSAY"] + "\r\n" + //"??? Assays: " + OtherAssayCnt["ASSAY"] + "\r\n" + "----------------------------------"; LogMessage("\r\n" + response); UAL.Progress.Hide(); return(response); }
/// <summary> /// Calculate & persist metatable stats for tables belonging to broker /// </summary> /// <returns></returns> public int UpdateMetaTableStatistics() { MetaTableStats mts; DateTime dt; string sql; int mthdId; long rowCount; Dictionary <int, MetaTableStats> stats = new Dictionary <int, MetaTableStats>(); DbCommandMx dao = new DbCommandMx(); // Get row count for each table sql = "select " + " mthd_vrsn_id, count(*) " + "from ( " + " select " + " mthd_vrsn_id, rslt_grp_id" + " from mbs_owner.mbs_adw_rslt " + " where " + " sts_id = 1 " + // " and mthd_vrsn_id = 148411 " + // debug " group by mthd_vrsn_id, rslt_grp_id) " + "group by mthd_vrsn_id"; dao.Prepare(sql); dao.ExecuteReader(); while (dao.Read()) { mthdId = dao.GetInt(0); rowCount = dao.GetLong(1); if (!stats.ContainsKey(mthdId)) { stats[mthdId] = new MetaTableStats(); } stats[mthdId].RowCount = rowCount; } dao.CloseReader(); // Get latest update date for each table sql = "select " + " mthd_vrsn_id, max(updt_dt) " + "from mbs_owner.mbs_adw_rslt " + "where " + " updt_dt <= sysdate and " + " sts_id = 1" + // " and mthd_vrsn_id = 148411 " + // debug "group by mthd_vrsn_id"; dao.Prepare(sql); dao.ExecuteReader(); while (dao.Read()) { mthdId = dao.GetInt(0); dt = dao.GetDateTime(1); if (!stats.ContainsKey(mthdId)) { stats[mthdId] = new MetaTableStats(); } stats[mthdId].UpdateDateTime = dt; } dao.CloseReader(); dao.Dispose(); int updCnt = 0; foreach (int mthdId_ in stats.Keys) { mts = stats[mthdId_]; try { UserObjectDao.UpdateUpdateDateAndCount(mthdId_, mts.UpdateDateTime, (int)mts.RowCount); } catch (Exception ex) { continue; } updCnt++; } return(stats.Count); }
/// <summary> /// Calculate & persist metatable stats for tables belonging to broker /// </summary> /// <returns></returns> public int UpdateMetaTableStatistics() { Dictionary <string, MetaTableStats> stats = new Dictionary <string, MetaTableStats>(); //return stats.Count; // debug foreach (MetaTable mt in MetaTableCollection.TableMap.Values) { if (String.IsNullOrEmpty(mt.RowCountSql) && String.IsNullOrEmpty(mt.UpdateDateTimeSql)) { continue; // if no sql defined for stats then skip } string mtName = mt.Name.ToUpper(); // Get row count string rowCountSql = null; if (Lex.Eq(mt.RowCountSql, "TableMap")) // use native source { rowCountSql = "select count(*) from " + mt.TableMap; } else if (mt.RowCountSql != "") // use table-specific sql { rowCountSql = mt.RowCountSql; } if (rowCountSql != null) { try { int rowCount = SelectSingleValueDao.SelectInt(rowCountSql); if (!stats.ContainsKey(mtName)) { stats[mtName] = new MetaTableStats(); } stats[mtName].RowCount = rowCount; } catch (Exception ex) { continue; } } // Get date string dateSql = null; if (Lex.Eq(mt.UpdateDateTimeSql, "TableMap")) // use native source { for (int mci = mt.MetaColumns.Count - 1; mci >= 0; mci--) { // search backwards for date col MetaColumn mc = mt.MetaColumns[mci]; if (mc.DataType == MetaColumnType.Date) { dateSql = "select max(" + mc.Name + ") from " + mt.TableMap + " where " + mc.Name + " <= current_date"; break; } } } else if (mt.UpdateDateTimeSql != "") // use table-specific sql { dateSql = mt.UpdateDateTimeSql; } if (dateSql != null) { try { if (!stats.ContainsKey(mtName)) { stats[mtName] = new MetaTableStats(); } DateTime dt = SelectSingleValueDao.SelectDateTime(dateSql); stats[mtName].UpdateDateTime = dt; } catch (Exception ex) { continue; } } } string fileName = MetaTableXmlFolder + @"\GenericMetaTableStats"; WriteMetaTableStats(stats, fileName); return(stats.Count); }