public static void QcDIA(IRawDataPlus rawFile, WorkflowParameters parameters) { rawFile.SelectInstrument(Device.MS, 1); rawFile.CheckIfBoxcar(); ScanIndex Index = Extract.ScanIndices(rawFile); MethodDataContainer methodData = Extract.MethodData(rawFile, Index); CentroidStreamCollection centroidStreams = new CentroidStreamCollection(); SegmentScanCollection segmentScans = new SegmentScanCollection(); (centroidStreams, segmentScans) = Extract.MsData(rawFile: rawFile, index: Index); TrailerExtraCollection trailerExtras = Extract.TrailerExtras(rawFile, Index); RetentionTimeCollection retentionTimes = Extract.RetentionTimes(rawFile, Index); ScanMetaDataCollectionDIA metaData = MetaDataProcessingDIA.AggregateMetaDataDIA(centroidStreams, segmentScans, methodData, trailerExtras, retentionTimes, Index); RawMetricsDataDIA metrics = MetaDataProcessingDIA.GetMetricsDataDIA(metaData, methodData, rawFile.FileName, retentionTimes, Index); QcDataContainer qcData = new QcDataContainer(); qcData.DIA = metrics; QcDataCollection qcDataCollection = QC.QcWorkflow.LoadOrCreateQcCollection(parameters); QC.QcWorkflow.UpdateQcCollection(qcDataCollection, qcData, methodData, rawFile.FileName); }
public static ScanEventReactionCollection ScanEvents(IRawDataPlus rawFile, ScanIndex index) { if (index.AnalysisOrder == MSOrderType.Ms) { Console.WriteLine("This is not an Orbitrap file. Skipping reaction event extraction."); return(null); } else { rawFile.SelectMsData(); ScanEventReactionCollection events = new ScanEventReactionCollection(); Log.Information("Extracting scan events"); ProgressIndicator P = new ProgressIndicator(index.ScanEnumerators[MSOrderType.Any].Length, "Extracting reaction events"); P.Start(); foreach (int scan in index.ScanEnumerators[index.AnalysisOrder]) { events.Add(scan, rawFile.GetScanEventForScanNumber(scan).GetReaction(0)); P.Update(); } P.Done(); return(events); } }
DependentsAndPrecursorScansByScanDependents(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting scan dependents/precursor scans"); PrecursorScanCollection precursorScans = new PrecursorScanCollection(); ScanDependentsCollections dependents = new ScanDependentsCollections(); int ms2Scan = -1; int ms3Scan = -1; IEnumerable <int> scans = index.ScanEnumerators[MSOrderType.Ms]; ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Indexing linked scan events"); foreach (int scan in scans) { var scanDependents = rawFile.GetScanDependents(scan, 4); dependents[scan] = scanDependents; // check if the ms1 scan has dependent scans if (scanDependents == null) { continue; } for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++) { if (index.AnalysisOrder == MSOrderType.Ms2) // it is ms2 { ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex; precursorScans[ms2Scan] = new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan); } else // it is ms3 { ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex; var ms2Dependents = rawFile.GetScanDependents(ms2Scan, 4).ScanDependentDetailArray; if (ms2Dependents.Length != 0) // make sure there is ms3 data { ms3Scan = ms2Dependents[0].ScanIndex; precursorScans[ms2Scan] = new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan); precursorScans[ms3Scan] = new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan); } else { // there is no ms3 scan, so we only add the ms2 scan precursorScans[ms2Scan] = new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan); } } } progress.Update(); } progress.Done(); return(precursorScans, dependents); }
public static ScanDependentsCollections ScanDependents(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); ScanDependentsCollections scanDependents = new ScanDependentsCollections(); foreach (int scan in index.ScanEnumerators[MSOrderType.Ms]) { scanDependents[scan] = rawFile.GetScanDependents(scan, 4); } return(scanDependents); }
public static MethodDataContainer MethodData(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting method/instrument information"); MethodDataContainer methodData = new MethodDataContainer(); methodData.CreationDate = rawFile.CreationDate; methodData.Instrument = rawFile.GetInstrumentData().Name; methodData.AnalysisOrder = index.AnalysisOrder; int firstQuantScan = index.ScanEnumerators[index.AnalysisOrder][0]; methodData.QuantAnalyzer = index.allScans[firstQuantScan].MassAnalyzer; int firstMs1Scan = index.ScanEnumerators[MSOrderType.Ms][0]; methodData.MassAnalyzers.Add(MSOrderType.Ms, index.allScans[firstMs1Scan].MassAnalyzer); methodData.MSOrderEnumerator.Add(MSOrderType.Ms); if (methodData.AnalysisOrder == MSOrderType.Ms2) { int firstMs2Scan = index.ScanEnumerators[MSOrderType.Ms2][0]; methodData.MassAnalyzers.Add(MSOrderType.Ms2, index.allScans[firstMs2Scan].MassAnalyzer); methodData.MSOrderEnumerator.Add(MSOrderType.Ms2); } else if (methodData.AnalysisOrder == MSOrderType.Ms3) { int firstMs2Scan = index.ScanEnumerators[MSOrderType.Ms2][0]; methodData.MassAnalyzers.Add(MSOrderType.Ms2, index.allScans[firstMs2Scan].MassAnalyzer); methodData.MSOrderEnumerator.Add(MSOrderType.Ms2); int n = index.ScanEnumerators[MSOrderType.Ms2].Count(); double ms2window = rawFile.GetScanEventForScanNumber(index.ScanEnumerators[MSOrderType.Ms2][n / 2]).GetIsolationWidth(0); int firstMs3Scan = index.ScanEnumerators[MSOrderType.Ms3][0]; methodData.MassAnalyzers.Add(MSOrderType.Ms3, index.allScans[firstMs3Scan].MassAnalyzer); methodData.MSOrderEnumerator.Add(MSOrderType.Ms3); double ms3ms1window = rawFile.GetScanEventForScanNumber(firstMs3Scan).GetIsolationWidth(0); double ms3ms2window = rawFile.GetScanEventForScanNumber(firstMs3Scan).GetIsolationWidth(1); methodData.IsolationWindow = (MS2 : ms2window, (MS1Window : ms3ms1window, MS2Window : ms3ms2window)); } else { Console.WriteLine("This raw file contains only MS1 scans."); } return(methodData); }
public static void WriteSearchMGF(WorkflowParameters parameters, CentroidStreamCollection centroids, SegmentScanCollection segments, RetentionTimeCollection retentionTimes, PrecursorMassCollection precursorMasses, PrecursorScanCollection precursorScans, TrailerExtraCollection trailerExtras, MethodDataContainer methodData, ScanIndex index, string rawFileName, bool fixedScans = false) { var pars = parameters.QcParams.SearchParameters; int[] scans = AdditionalMath.SelectRandomScans(scans: index.ScanEnumerators[MSOrderType.Ms2], num: parameters.QcParams.NumberSpectra, fixedScans: parameters.QcParams.FixedScans); string mgfFile = ReadWrite.GetPathToFile(parameters.QcParams.QcSearchDataDirectory, rawFileName, ".mgf"); MgfWriter.WriteMGF(rawFileName, centroids, segments, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, index, outputFile: mgfFile, scans: scans); }
public static void ParseDIA(IRawFileThreadManager rawFile, WorkflowParameters parameters) { var staticRawFile = rawFile.CreateThreadAccessor(); staticRawFile.SelectInstrument(Device.MS, 1); staticRawFile.CheckIfBoxcar(); ScanIndex Index = Extract.ScanIndices(rawFile.CreateThreadAccessor()); TrailerExtraCollection trailerExtras = Extract.TrailerExtras(rawFile.CreateThreadAccessor(), Index); MethodDataContainer methodData = Extract.MethodData(rawFile.CreateThreadAccessor(), Index); PrecursorScanCollection precursorScans = Extract.PrecursorScansByMasterScanMs2Only(rawFile.CreateThreadAccessor(), trailerExtras, Index); (CentroidStreamCollection centroidStreams, SegmentScanCollection segmentScans) = Extract.MsData(rawFile: rawFile.CreateThreadAccessor(), index: Index); RetentionTimeCollection retentionTimes = Extract.RetentionTimes(rawFile.CreateThreadAccessor(), Index); ScanMetaDataCollectionDIA metaData = MetaDataProcessingDIA.AggregateMetaDataDIA(centroidStreams, segmentScans, methodData, trailerExtras, retentionTimes, Index); RawMetricsDataDIA metrics = null; if (parameters.ParseParams.Metrics) { metrics = MetaDataProcessingDIA.GetMetricsDataDIA(metaData, methodData, staticRawFile.FileName, retentionTimes, Index); MetricsWriter.WriteMatrix(metrics, staticRawFile.FileName, parameters.ParseParams.OutputDirectory); } if (parameters.ParseParams.Parse) { string matrixFileName = ReadWrite.GetPathToFile(parameters.ParseParams.OutputDirectory, staticRawFile.FileName, "._parse.txt"); //MatrixWriter.ParseQuantDIA() //ParseWriter writerDIA = new ParseWriter(matrixFileName, centroidStreams, segmentScans, metaData, // retentionTimes, trailerExtras, precursorScans, Index); //writerDIA.WriteMatrixDIA(); } // I'm not sure what goes into a DIA mgf file, so we aren't making one yet //if (parameters.ParseParams.WriteMgf) //{ // ParseWriter writerMGF = new ParseWriter(centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index); // writerMGF.WriteMGF(staticRawFile.FileName); //} }
public static int NumberOfEsiFlags(Dictionary <int, double> SummedIntensity, ScanIndex index) { int flags = 0; int[] scans = index.ScanEnumerators[MSOrderType.Ms]; for (int i = 2; i < scans.Length; i++) { if (SummedIntensity[scans[i]] / SummedIntensity[scans[i - 1]] < 0.1) { flags += 1; } if (SummedIntensity[scans[i]] / SummedIntensity[scans[i - 1]] > 10) { flags += 1; } } return(flags); }
public static ScanEventReactionCollection ScanEvents(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectMsData(); ScanEventReactionCollection events = new ScanEventReactionCollection(); Log.Information("Extracting scan events"); ProgressIndicator P = new ProgressIndicator(index.ScanEnumerators[MSOrderType.Any].Length, "Extracting reaction events"); P.Start(); foreach (int scan in index.ScanEnumerators[index.AnalysisOrder]) { events.Add(scan, rawFile.GetScanEventForScanNumber(scan).GetReaction(0)); P.Update(); } P.Done(); return(events); }
public static void Quantify(this QuantDataCollection quantData, RawDataCollection rawData, IRawDataPlus rawFile, string labelingReagent) { MassAnalyzerType quantAnalyzer = rawData.methodData.QuantAnalyzer; if (quantAnalyzer == MassAnalyzerType.MassAnalyzerFTMS) { rawData.ExtractCentroidStreams(rawFile, rawData.methodData.AnalysisOrder); } else { rawData.ExtractSegmentScans(rawFile, rawData.methodData.AnalysisOrder); } int[] scans; ScanIndex scanIndex = rawData.scanIndex; Dictionary <int, CentroidStreamData> centroidScans = rawData.centroidStreams; Dictionary <int, SegmentedScanData> segmentScans = rawData.segmentedScans; scans = scanIndex.ScanEnumerators[scanIndex.AnalysisOrder]; ProgressIndicator progress = new ProgressIndicator(scans.Length, "Quantifying reporter ions"); quantData.LabelingReagents = labelingReagent; foreach (int scan in scans) { if (quantAnalyzer == MassAnalyzerType.MassAnalyzerFTMS) { quantData.Add(scan, new QuantifyReporters(centroidScans[scan], labelingReagent).quantData); } else { quantData.Add(scan, new QuantifyReporters(segmentScans[scan], labelingReagent).quantData); } progress.Update(); } progress.Done(); rawData.Performed.Add(Operations.Quantification); }
public static RetentionTimeCollection RetentionTimes(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting retention times"); RetentionTimeCollection retentionTimes = new RetentionTimeCollection(); var scans = index.ScanEnumerators[MSOrderType.Any]; ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Extracting retention times"); foreach (int scan in scans) { retentionTimes[scan] = rawFile.RetentionTimeFromScanNumber(scan); progress.Update(); } progress.Done(); return(retentionTimes); }
public static TrailerExtraCollection TrailerExtras(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting trailer extras"); TrailerExtraCollection trailerExtras = new TrailerExtraCollection(); TrailerExtraIndices trailerIndices = new TrailerExtraIndices(rawFile); Double[] spsMasses; var scans = index.ScanEnumerators[MSOrderType.Any]; ProgressIndicator P = new ProgressIndicator(scans.Count(), "Extracting trailer extras"); P.Start(); foreach (int scan in scans) { trailerExtras[scan] = ExtractOneTrailerExtra(rawFile, scan, trailerIndices); P.Update(); } P.Done(); return(trailerExtras); }
public static MethodDataContainer Ms1MethodData(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting method/instrument information"); MethodDataContainer methodData = new MethodDataContainer(); methodData.CreationDate = rawFile.CreationDate; methodData.Instrument = rawFile.GetInstrumentData().Name; methodData.AnalysisOrder = index.AnalysisOrder; int firstQuantScan = index.ScanEnumerators[index.AnalysisOrder][0]; methodData.QuantAnalyzer = index.allScans[firstQuantScan].MassAnalyzer; int firstMs1Scan = index.ScanEnumerators[MSOrderType.Ms][0]; methodData.MassAnalyzers.Add(MSOrderType.Ms, index.allScans[firstMs1Scan].MassAnalyzer); methodData.MSOrderEnumerator.Add(MSOrderType.Ms); return(methodData); }
/// <summary> /// Gets precursor scans directly from the trailer extra. This is intended for DIA files, and by extension only for Ms2 experiments. /// </summary> /// <param name="rawFile"></param> /// <param name="index"></param> /// <returns></returns> public static PrecursorScanCollection PrecursorScansByMasterScanMs2Only(IRawDataPlus rawFile, TrailerExtraCollection trailerExtras, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting scan dependents/precursor scans"); PrecursorScanCollection precursorScans = new PrecursorScanCollection(); int ms2Scan = -1; int ms3Scan = -1; IEnumerable <int> scans = index.ScanEnumerators[MSOrderType.Ms]; ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Indexing linked scan events"); // if it is DIA it must be MS2 scans = index.ScanEnumerators[MSOrderType.Ms2]; foreach (int scan in scans) { int masterScan = trailerExtras[scan].MasterScan; precursorScans[scan] = new PrecursorScanData(ms2scan: scan, masterScan: masterScan); } progress.Done(); return(precursorScans); }
static Assimilate() { Assimilation = new Assimilation(); ScanIndex = new ScanIndex(); ScanIndex.Start(); }
public static (ScanIndex, PrecursorScanCollection, ScanDependentsCollections) ScanIndicesPrecursorsDependents(IRawFileThreadManager rawFileAccessor) { Log.Information("Extracting scan indices"); ConcurrentDictionary <int, ScanData> allScans; allScans = new ConcurrentDictionary <int, ScanData>(); Dictionary <int, ScanData> orphanScans = new Dictionary <int, ScanData>(); MSOrderType AnalysisOrder; ConcurrentBag <int> ms1 = new ConcurrentBag <int>(); ConcurrentBag <int> ms2 = new ConcurrentBag <int>(); ConcurrentBag <int> ms3 = new ConcurrentBag <int>(); ConcurrentBag <int> msAny = new ConcurrentBag <int>(); ConcurrentDictionary <int, PrecursorScanData> precursorScans = new ConcurrentDictionary <int, PrecursorScanData>(); ConcurrentDictionary <int, IScanDependents> dependents = new ConcurrentDictionary <int, IScanDependents>(); var staticRawFile = rawFileAccessor.CreateThreadAccessor(); staticRawFile.SelectMsData(); // populate the scan indices IEnumerable <int> scans = staticRawFile.GetFilteredScanEnumerator(staticRawFile.GetFilterFromString("")); // get all scans // get ms order of experiment Console.Write("Determing MS analysis order... "); AnalysisOrder = (from x in scans select staticRawFile.GetScanEventForScanNumber(x).MSOrder).Max(); Console.WriteLine("Done!"); object lockTarget = new object(); ProgressIndicator P = new ProgressIndicator(scans.Count(), "Extracting scan indices"); int chunkSize = Constants.MultiThreading.ChunkSize(scans.Count()); var batches = scans.Chunk(chunkSize); Parallel.ForEach(batches, batch => { ScanData ms1ScanData; ScanData ms2ScanData; ScanData ms3ScanData; foreach (int scan in batch) { //var rawFile = rawFileAccessor.CreateThreadAccessor(); using (var rawFile = rawFileAccessor.CreateThreadAccessor()) { rawFile.SelectMsData(); IScanEvent scanEvent = rawFile.GetScanEventForScanNumber(scan); ms1ScanData = new ScanData(); ms2ScanData = new ScanData(); ms3ScanData = new ScanData(); ms1ScanData.MassAnalyzer = scanEvent.MassAnalyzer; ms1ScanData.MSOrder = scanEvent.MSOrder; if (ms1ScanData.MSOrder == MSOrderType.Ms) { ms1.Add(scan); msAny.Add(scan); var scanDependents = rawFile.GetScanDependents(scan, 4); dependents.TryAdd(scan, scanDependents); // check if the ms1 scan has dependent scans if (scanDependents == null) { // there are no scan dependents ms1ScanData.HasDependents = false; allScans.TryAdd(scan, ms1ScanData); return; } else { ms1ScanData.HasDependents = true; allScans.TryAdd(scan, ms1ScanData); } for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++) { int ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex; ms2.Add(ms2Scan); msAny.Add(ms2Scan); if (AnalysisOrder == MSOrderType.Ms2) // it is ms2 { ms2ScanData = new ScanData(); scanEvent = rawFile.GetScanEventForScanNumber(ms2Scan); ms2ScanData.MassAnalyzer = scanEvent.MassAnalyzer; ms2ScanData.MSOrder = scanEvent.MSOrder; ms2ScanData.HasDependents = false; ms2ScanData.HasPrecursors = true; allScans.TryAdd(ms2Scan, ms2ScanData); precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan)); } else // it is ms3 { var ms2Dependents = rawFile.GetScanDependents(ms2Scan, 4).ScanDependentDetailArray; ms2ScanData = new ScanData(); scanEvent = rawFile.GetScanEventForScanNumber(ms2Scan); ms2ScanData.MassAnalyzer = scanEvent.MassAnalyzer; ms2ScanData.MSOrder = scanEvent.MSOrder; ms2ScanData.HasPrecursors = true; if (ms2Dependents.Length != 0) // make sure there is ms3 data { int ms3Scan = ms2Dependents[0].ScanIndex; ms3.Add(ms3Scan); msAny.Add(ms3Scan); scanEvent = rawFile.GetScanEventForScanNumber(ms3Scan); precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan)); precursorScans.TryAdd(ms3Scan, new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan)); ms2ScanData.HasDependents = true; ms3ScanData = new ScanData(); ms3ScanData.HasPrecursors = true; ms3ScanData.MassAnalyzer = scanEvent.MassAnalyzer; ms3ScanData.MSOrder = scanEvent.MSOrder; allScans.TryAdd(ms3Scan, ms3ScanData); } else { // there is no ms3 scan, so we only add the ms2 scan precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan)); ms2ScanData.HasDependents = false; } allScans.TryAdd(ms2Scan, ms2ScanData); } } } lock (lockTarget) { P.Update(); } } } }); P.Done(); HashSet <int> allKeys = new HashSet <int>(allScans.Keys); P = new ProgressIndicator(scans.Count(), "Checking for orphaned scans"); foreach (int scan in scans) { if (allKeys.Contains(scan)) { continue; } else { ScanData scanData = new ScanData(); var scanEvent = staticRawFile.GetScanEventForScanNumber(scan); scanData.MassAnalyzer = scanEvent.MassAnalyzer; scanData.MSOrder = scanEvent.MSOrder; orphanScans.Add(scan, scanData); } P.Update(); } P.Done(); Console.WriteLine(); Console.WriteLine("================ Scan indexing report ================"); Console.WriteLine($"Total scans in file: {staticRawFile.RunHeaderEx.SpectraCount}"); Console.WriteLine($"Scans linked: {allScans.Count()}"); Console.WriteLine(); Console.WriteLine("Orphan scans:"); if (orphanScans.Count() > 0) { foreach (var scan in orphanScans) { Console.WriteLine($"\tScan: {scan.Key}, MSOrder: {scan.Value.MSOrder}"); } Console.WriteLine("\nThe above scans will not be present in the output data. You should"); Console.WriteLine("manually check them to ensure they are not critical to you analysis."); } else { Console.WriteLine("None!"); } Console.WriteLine(); if (staticRawFile.RunHeaderEx.SpectraCount == allScans.Count() + orphanScans.Count()) { Console.WriteLine("All scans accounted for!"); } else { Console.WriteLine($"Number of scans unaccounted for: {staticRawFile.RunHeaderEx.SpectraCount - (allScans.Count() + orphanScans.Count())}"); Console.WriteLine("If this number is alarming, please contact the RawTools authors\n" + "by posting an issue at:\n" + "https://github.com/kevinkovalchik/RawTools/issues"); } Console.WriteLine("======================================================"); Console.WriteLine(); // we need to order the scan enumerators before sending them out var orderedMs1 = ms1.ToList(); var orderedMs2 = ms2.ToList(); var orderedMs3 = ms3.ToList(); var orderedMsAny = msAny.ToList(); orderedMs1.Sort(); orderedMs2.Sort(); orderedMs3.Sort(); orderedMsAny.Sort(); ScanIndex scanIndex = new ScanIndex(); scanIndex.allScans = allScans.ConvertToDictionary(); scanIndex.AnalysisOrder = AnalysisOrder; scanIndex.ScanEnumerators.Add(MSOrderType.Any, orderedMsAny.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms, orderedMs1.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms2, orderedMs2.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms3, orderedMs3.ToArray()); scanIndex.TotalScans = staticRawFile.RunHeaderEx.SpectraCount; return(scanIndex, new PrecursorScanCollection(precursorScans), new ScanDependentsCollections(dependents)); }
DependentsAndPrecursorScansByScanDependentsParallel(IRawFileThreadManager rawFileManager, ScanIndex index) { //rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting scan dependents/precursor scans"); ConcurrentDictionary <int, PrecursorScanData> precursorScans = new ConcurrentDictionary <int, PrecursorScanData>(); ConcurrentDictionary <int, IScanDependents> dependents = new ConcurrentDictionary <int, IScanDependents>(); //Dictionary<int, PrecursorScanData> precursorScans = new Dictionary<int, PrecursorScanData>(); //Dictionary<int, IScanDependents> dependents = new Dictionary<int, IScanDependents>(); //ConcurrentBag<(int scan, PrecursorScanData data)> precursorBag = new ConcurrentBag<(int scan, PrecursorScanData data)>(); //ConcurrentBag<(int scan, IScanDependents data)> dependentsBag = new ConcurrentBag<(int scan, IScanDependents data)>(); var updateProgressLock = new object(); var addLockTarget = new object(); IEnumerable <int> scans = index.ScanEnumerators[MSOrderType.Ms]; ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Indexing linked scan events"); var batches = scans.Chunk(Constants.MultiThreading.ChunkSize(scans.Count())); Parallel.ForEach(batches, Constants.MultiThreading.Options(), batch => { int ms2Scan = -1; int ms3Scan = -1; IScanDependents scanDependents; var rawFile = rawFileManager.CreateThreadAccessor(); rawFile.SelectInstrument(Device.MS, 1); foreach (int scan in batch) { scanDependents = rawFile.GetScanDependents(scan, 2); dependents.AddOrUpdate(scan, scanDependents, (a, b) => b); // check if the ms1 scan has dependent scans if (scanDependents == null) { return; } for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++) { if (index.AnalysisOrder == MSOrderType.Ms2) // it is ms2 { ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex; //precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b); precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan)); } else // it is ms3 { ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex; var ms2Dependents = rawFile.GetScanDependents(ms2Scan, 2).ScanDependentDetailArray; if (ms2Dependents.Length != 0) // make sure there is ms3 data { ms3Scan = ms2Dependents[0].ScanIndex; precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b); precursorScans.AddOrUpdate(ms3Scan, new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan), (a, b) => b); } else { // there is no ms3 scan, so we only add the ms2 scan precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b); } } } lock (updateProgressLock) { progress.Update(); } } }); progress.Done(); var outScans = new PrecursorScanCollection(); var outDependents = new ScanDependentsCollections(); foreach (var item in precursorScans) { outScans.Add(item.Key, item.Value); } foreach (var item in dependents) { outDependents.Add(item.Key, item.Value); } // check for missing precursor information /* * foreach (int scan in index.ScanEnumerators[MSOrderType.Ms2]) * { * if (outScans.Keys.Contains(scan)) * { * outScans.HasPrecursorData.Add(scan, true); * } * else * { * outScans.HasPrecursorData.Add(scan, false); * } * }*/ return(outScans, outDependents); }
public static (CentroidStreamCollection centroids, SegmentScanCollection segments) MsData(IRawDataPlus rawFile, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); CentroidStreamCollection centroids = new CentroidStreamCollection(); SegmentScanCollection segments = new SegmentScanCollection(); CentroidStream centroid; SegmentedScan segment; var scans = index.allScans; //var lockTarget = new object(); // this is so we can keep track of progress in the parallel loop ProgressIndicator P = new ProgressIndicator(scans.Count(), "Extracting scan data"); P.Start(); foreach (int scan in scans.Keys) { // first get out the mass spectrum if (index.allScans[scan].MassAnalyzer == MassAnalyzerType.MassAnalyzerFTMS) { centroid = rawFile.GetCentroidStream(scan, false); centroids[scan] = new CentroidStreamData(centroid); } else { segment = rawFile.GetSegmentedScanFromScanNumber(scan, null); segments[scan] = new SegmentedScanData(segment); } //lock (lockTarget) //{ // P.Update(); //} P.Update(); } P.Done(); return(centroids, segments); }
public static double GetMedianMSFillTime(Dictionary <int, double> FillTimes, ScanIndex Index, MSOrderType MsOrder) { return((from x in Index.ScanEnumerators[MsOrder] select FillTimes[x]).ToArray().Percentile(50)); }
public static (double TimeBefore, double TimeAfter, double FractionAbove) ChromIntensityMetrics(ScanMetaDataCollectionDDA metaData, RetentionTimeCollection retentionTimes, ScanIndex index) { return(ChromIntensityMetrics(metaData.SummedIntensity, retentionTimes, index)); }
public static int NumberOfEsiFlags(ScanMetaDataCollectionDIA metaData, ScanIndex index) { return(NumberOfEsiFlags(metaData.SummedIntensity, index)); }
public static PrecursorMassCollection PrecursorMasses(IRawDataPlus rawFile, PrecursorScanCollection precursorScans, TrailerExtraCollection trailerExtras, ScanIndex index) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting precursor masses"); PrecursorMassCollection precursorMasses = new PrecursorMassCollection(); var scans = index.ScanEnumerators[MSOrderType.Any]; ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Extracting precursor masses"); foreach (int scan in scans) { if (index.allScans[scan].MSOrder == MSOrderType.Ms2) { double parent_mass = rawFile.GetScanEventForScanNumber(scan).GetReaction(0).PrecursorMass; precursorMasses[scan] = new PrecursorMassData(trailerExtras[scan].MonoisotopicMZ, parent_mass); } if (index.allScans[scan].MSOrder == MSOrderType.Ms3) { int ms2scan = precursorScans[scan].MS2Scan; double parent_mass = rawFile.GetScanEventForScanNumber(scan).GetReaction(0).PrecursorMass; precursorMasses[scan] = new PrecursorMassData(trailerExtras[ms2scan].MonoisotopicMZ, parent_mass); } progress.Update(); } progress.Done(); return(precursorMasses); }
public static RawMetricsDataDDA GetMetricsDataDDA(ScanMetaDataCollectionDDA metaData, MethodDataContainer methodData, string rawFileName, RetentionTimeCollection retentionTimes, ScanIndex index, PrecursorPeakCollection peakData, PrecursorScanCollection precursorScans, QuantDataCollection quantData = null) { RawMetricsDataDDA metricsData = new RawMetricsDataDDA(); metricsData.DateAcquired = methodData.CreationDate; metricsData.Instrument = methodData.Instrument; Console.WriteLine("Calculating metrics"); metricsData.RawFileName = rawFileName; metricsData.Instrument = methodData.Instrument; metricsData.MS1Analyzer = methodData.MassAnalyzers[MSOrderType.Ms]; metricsData.MS2Analyzer = methodData.MassAnalyzers[MSOrderType.Ms2]; metricsData.TotalAnalysisTime = retentionTimes[index.ScanEnumerators[MSOrderType.Any].Last()] - retentionTimes[index.ScanEnumerators[MSOrderType.Any].First()]; metricsData.NumberOfEsiFlags = MetricsCalculations.NumberOfEsiFlags(metaData, index); metricsData.TotalScans = index.TotalScans; metricsData.MS1Scans = index.ScanEnumerators[MSOrderType.Ms].Length; metricsData.MS2Scans = index.ScanEnumerators[MSOrderType.Ms2].Length; if (methodData.AnalysisOrder == MSOrderType.Ms3) { metricsData.MS3Analyzer = methodData.MassAnalyzers[MSOrderType.Ms3]; metricsData.MS3Scans = index.ScanEnumerators[MSOrderType.Ms3].Length; } else { metricsData.MS3Analyzer = MassAnalyzerType.Any; metricsData.MS3Scans = 0; } var pickedMs1 = new HashSet <int>((from x in index.ScanEnumerators[methodData.AnalysisOrder] select precursorScans[x].MasterScan)).ToList(); metricsData.MSOrder = methodData.AnalysisOrder; metricsData.MedianSummedMS1Intensity = MetricsCalculations.GetMedianSummedMSIntensity(metaData.SummedIntensity, index, MSOrderType.Ms); metricsData.MedianSummedMS2Intensity = MetricsCalculations.GetMedianSummedMSIntensity(metaData.SummedIntensity, index, MSOrderType.Ms2); metricsData.MedianPrecursorIntensity = (from x in peakData.Keys select peakData[x].ParentIntensity).ToArray().Percentile(50); metricsData.MedianMS1FillTime = MetricsCalculations.GetMedianMSFillTime(metaData.FillTime, index, MSOrderType.Ms); metricsData.MedianMS2FillTime = MetricsCalculations.GetMedianMSFillTime(metaData.FillTime, index, MSOrderType.Ms2); if (methodData.AnalysisOrder == MSOrderType.Ms3) { metricsData.MedianMS3FillTime = MetricsCalculations.GetMedianMSFillTime(metaData.FillTime, index, MSOrderType.Ms3); } metricsData.MeanTopN = MetricsCalculations.GetMeanMs2ScansPerCycle(metaData.MS2ScansPerCycle); metricsData.MeanDutyCycle = MetricsCalculations.GetMedianDutyCycle(metaData.DutyCycle, index); metricsData.MedianMs2FractionConsumingTop80PercentTotalIntensity = MetricsCalculations.GetMedianMs2FractionConsumingTop80PercentTotalIntensity( metaData.FractionConsumingTop80PercentTotalIntensity, index); metricsData.MS1ScanRate = metricsData.MS1Scans / metricsData.TotalAnalysisTime; metricsData.MS2ScanRate = metricsData.MS2Scans / metricsData.TotalAnalysisTime; if (methodData.AnalysisOrder == MSOrderType.Ms3) { metricsData.MS3ScanRate = metricsData.MS3Scans / metricsData.TotalAnalysisTime; } metricsData.MedianBaselinePeakWidth = peakData.PeakShapeMedians.Width.P10; metricsData.MedianHalfHeightPeakWidth = peakData.PeakShapeMedians.Width.P50; // we can't access the instrument method in Linux, so we will assume the gradient length is the length of the MS acquisition metricsData.Gradient = retentionTimes[index.allScans.Keys.Max()]; metricsData.PeakCapacity = metricsData.Gradient / metricsData.MedianHalfHeightPeakWidth; metricsData.MedianAsymmetryFactor = peakData.PeakShapeMedians.Asymmetry.P10; // add isolation interference metricsData.MedianMs1IsolationInterference = (from scan in index.ScanEnumerators[methodData.AnalysisOrder] select metaData.Ms1IsolationInterference[scan]).ToArray().Percentile(50); (double timeBefore, double timeAfter, double fracAbove) = MetricsCalculations.ChromIntensityMetrics(metaData, retentionTimes, index); metricsData.TimeBeforeFirstScanToExceedPoint1MaxIntensity = timeBefore; metricsData.TimeAfterLastScanToExceedPoint1MaxIntensity = timeAfter; metricsData.FractionOfRunAbovePoint1MaxIntensity = fracAbove; metricsData.Ms1FillTimeDistribution = new Distribution((from x in index.ScanEnumerators[MSOrderType.Ms] select metaData.FillTime[x]).ToArray()); metricsData.Ms2FillTimeDistribution = new Distribution((from x in index.ScanEnumerators[MSOrderType.Ms2] select metaData.FillTime[x]).ToArray()); if (methodData.AnalysisOrder == MSOrderType.Ms3) { metricsData.Ms3FillTimeDistribution = new Distribution((from x in index.ScanEnumerators[MSOrderType.Ms3] select metaData.FillTime[x]).ToArray()); } metricsData.PeakShape.Asymmetry.P10 = peakData.PeakShapeMedians.Asymmetry.P10; metricsData.PeakShape.Asymmetry.P50 = peakData.PeakShapeMedians.Asymmetry.P50; metricsData.PeakShape.Width.P10 = peakData.PeakShapeMedians.Width.P10; metricsData.PeakShape.Width.P50 = peakData.PeakShapeMedians.Width.P50; // now add the quant meta data, if quant was performed double medianReporterIntensity = 0; QuantMetaData quantMetaData = new QuantMetaData(); SerializableDictionary <string, double> medianReporterIntensityByChannel = new SerializableDictionary <string, double>(); if (quantData != null) { string reagent = quantData.LabelingReagents; string[] allTags = new LabelingReagents().Reagents[reagent].Labels; List <double> allChannels = new List <double>(); Dictionary <string, List <double> > byChannel = new Dictionary <string, List <double> >(); foreach (string tag in allTags) { byChannel.Add(tag, new List <double>()); } foreach (int scan in index.ScanEnumerators[methodData.AnalysisOrder]) { foreach (string tag in allTags) { byChannel[tag].Add(quantData[scan][tag].Intensity); allChannels.Add(quantData[scan][tag].Intensity); } } medianReporterIntensity = allChannels.ToArray().Percentile(50); foreach (string tag in allTags) { medianReporterIntensityByChannel[tag] = byChannel[tag].ToArray().Percentile(50); } quantMetaData.medianReporterIntensity = medianReporterIntensity; quantMetaData.medianReporterIntensityByChannel = medianReporterIntensityByChannel; quantMetaData.quantTags = allTags; metricsData.QuantMeta = quantMetaData; metricsData.IncludesQuant = true; } return(metricsData); }
public static double GetMedianDutyCycle(Dictionary <int, double> DutyCycles, ScanIndex Index) { return((from x in Index.ScanEnumerators[MSOrderType.Ms] select DutyCycles[x]).ToArray().Percentile(50)); }
public static RawMetricsDataDIA GetMetricsDataDIA(ScanMetaDataCollectionDIA metaData, MethodDataContainer methodData, string rawFileName, RetentionTimeCollection retentionTimes, ScanIndex index) { RawMetricsDataDIA metricsData = new RawMetricsDataDIA(); metricsData.DateAcquired = methodData.CreationDate; metricsData.Instrument = methodData.Instrument; Console.WriteLine("Calculating metrics"); metricsData.RawFileName = rawFileName; metricsData.Instrument = methodData.Instrument; metricsData.MS1Analyzer = methodData.MassAnalyzers[MSOrderType.Ms]; metricsData.MS2Analyzer = methodData.MassAnalyzers[MSOrderType.Ms2]; metricsData.TotalAnalysisTime = retentionTimes[index.ScanEnumerators[MSOrderType.Any].Last()] - retentionTimes[index.ScanEnumerators[MSOrderType.Any].First()]; metricsData.NumberOfEsiFlags = MetricsCalculations.NumberOfEsiFlags(metaData, index); metricsData.TotalScans = index.TotalScans; metricsData.MS1Scans = index.ScanEnumerators[MSOrderType.Ms].Length; metricsData.MS2Scans = index.ScanEnumerators[MSOrderType.Ms2].Length; metricsData.MSOrder = methodData.AnalysisOrder; metricsData.MedianSummedMS1Intensity = MetricsCalculations.GetMedianSummedMSIntensity(metaData.SummedIntensity, index, MSOrderType.Ms); metricsData.MedianSummedMS2Intensity = MetricsCalculations.GetMedianSummedMSIntensity(metaData.SummedIntensity, index, MSOrderType.Ms2); metricsData.MedianMS1FillTime = MetricsCalculations.GetMedianMSFillTime(metaData.FillTime, index, MSOrderType.Ms); metricsData.MedianMS2FillTime = MetricsCalculations.GetMedianMSFillTime(metaData.FillTime, index, MSOrderType.Ms2); metricsData.MeanDutyCycle = MetricsCalculations.GetMedianDutyCycle(metaData.DutyCycle, index); metricsData.MedianMs2FractionConsumingTop80PercentTotalIntensity = MetricsCalculations.GetMedianMs2FractionConsumingTop80PercentTotalIntensity( metaData.FractionConsumingTop80PercentTotalIntensity, index); metricsData.MS1ScanRate = metricsData.MS1Scans / metricsData.TotalAnalysisTime; metricsData.MS2ScanRate = metricsData.MS2Scans / metricsData.TotalAnalysisTime; (double timeBefore, double timeAfter, double fracAbove) = MetricsCalculations.ChromIntensityMetrics(metaData, retentionTimes, index); metricsData.TimeBeforeFirstScanToExceedPoint1MaxIntensity = timeBefore; metricsData.TimeAfterLastScanToExceedPoint1MaxIntensity = timeAfter; metricsData.FractionOfRunAbovePoint1MaxIntensity = fracAbove; metricsData.Ms1FillTimeDistribution = new Distribution((from x in index.ScanEnumerators[MSOrderType.Ms] select metaData.FillTime[x]).ToArray()); metricsData.Ms2FillTimeDistribution = new Distribution((from x in index.ScanEnumerators[MSOrderType.Ms2] select metaData.FillTime[x]).ToArray()); return(metricsData); }
public static ScanMetaDataCollectionDIA AggregateMetaDataDIA(CentroidStreamCollection centroidStreams, SegmentScanCollection segmentScans, MethodDataContainer methodData, TrailerExtraCollection trailerExtras, RetentionTimeCollection retentionTimes, ScanIndex index, int maxProcesses) { //ProgressIndicator progress = new ProgressIndicator(index.ScanEnumerators[MSOrderType.Any].Count(), // "Formatting scan meta data"); ScanMetaDataCollectionDIA metaData = new ScanMetaDataCollectionDIA(); int[] scans = index.ScanEnumerators[MSOrderType.Any]; double isoWindow = MetaDataCalculations.Ms1IsoWindow(methodData); Console.WriteLine("Aggregating meta data"); metaData.FillTime = MetaDataCalculations.FillTimes(trailerExtras, index); metaData.DutyCycle = MetaDataCalculations.DutyCycle(retentionTimes, index); metaData.IntensityDistribution = MetaDataCalculations.IntensityDistributions(centroidStreams, segmentScans, index, maxProcesses); metaData.SummedIntensity = MetaDataCalculations.SummedIntensities(centroidStreams, segmentScans, index, maxProcesses); metaData.FractionConsumingTop80PercentTotalIntensity = MetaDataCalculations.Top80Frac(centroidStreams, segmentScans, index, maxProcesses); //Task.WaitAll(); return(metaData); }
public static double GetMedianMs2FractionConsumingTop80PercentTotalIntensity(Dictionary <int, double> FractionConsumingTop80PercentTotalIntensity, ScanIndex Index) { return((from x in Index.ScanEnumerators[MSOrderType.Ms2] select FractionConsumingTop80PercentTotalIntensity[x]).ToArray().Percentile(50)); }
public static ScanIndex ScanIndices(IRawDataPlus rawFile) { rawFile.SelectInstrument(Device.MS, 1); Log.Information("Extracting scan indices"); Dictionary <int, ScanData> allScans; allScans = new Dictionary <int, ScanData>(); MSOrderType AnalysisOrder; List <int> ms1 = new List <int>(); List <int> ms2 = new List <int>(); List <int> ms3 = new List <int>(); List <int> msAny = new List <int>(); // populate the scan indices IEnumerable <int> scans = rawFile.GetFilteredScanEnumerator(rawFile.GetFilterFromString("")); // get all scans ProgressIndicator P = new ProgressIndicator(scans.Count(), "Extracting scan indices"); foreach (int scan in scans) { IScanEvent scanEvent = rawFile.GetScanEventForScanNumber(scan); ScanData scanData = new ScanData(); scanData.MassAnalyzer = scanEvent.MassAnalyzer; scanData.MSOrder = scanEvent.MSOrder; allScans.Add(scan, scanData); msAny.Add(scan); if (allScans[scan].MSOrder == MSOrderType.Ms) { ms1.Add(scan); } if (allScans[scan].MSOrder == MSOrderType.Ms2) { ms2.Add(scan); } if (allScans[scan].MSOrder == MSOrderType.Ms3) { ms3.Add(scan); } P.Update(); } P.Done(); // determine the msorder of the experiment if ((ms1.Count > 0) & (ms2.Count == 0) & (ms3.Count == 0)) { AnalysisOrder = MSOrderType.Ms; } else { if ((ms1.Count > 0) & (ms2.Count > 0) & (ms3.Count == 0)) { AnalysisOrder = MSOrderType.Ms2; } else { AnalysisOrder = MSOrderType.Ms3; } } ScanIndex scanIndex = new ScanIndex(); scanIndex.allScans = allScans; scanIndex.AnalysisOrder = AnalysisOrder; scanIndex.ScanEnumerators.Add(MSOrderType.Any, msAny.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms, ms1.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms2, ms2.ToArray()); scanIndex.ScanEnumerators.Add(MSOrderType.Ms3, ms3.ToArray()); return(scanIndex); }
public static ScanMetaDataCollectionDDA AggregateMetaDataDDA(CentroidStreamCollection centroidStreams, SegmentScanCollection segmentScans, MethodDataContainer methodData, PrecursorScanCollection precursorScans, TrailerExtraCollection trailerExtras, PrecursorMassCollection precursorMasses, RetentionTimeCollection retentionTimes, ScanDependentsCollections scanDependents, ScanEventReactionCollection reactions, ScanIndex index) { //ProgressIndicator progress = new ProgressIndicator(index.ScanEnumerators[MSOrderType.Any].Count(), // "Formatting scan meta data"); ScanMetaDataCollectionDDA metaData = new ScanMetaDataCollectionDDA(); int[] scans = index.ScanEnumerators[MSOrderType.Any]; double isoWindow = MetaDataCalculations.Ms1IsoWindow(methodData); Console.WriteLine("Calculating meta data"); Console.WriteLine(" MS1 isolation interference"); metaData.Ms1IsolationInterference = MetaDataCalculations.Ms1Interference(centroidStreams, precursorMasses, trailerExtras, precursorScans, reactions, index); Console.WriteLine(" MS2 scan cycle density"); metaData.MS2ScansPerCycle = MetaDataCalculations.MS2ScansPerCycle(scanDependents, index); Console.WriteLine(" Ion injection time"); metaData.FillTime = MetaDataCalculations.FillTimes(trailerExtras, index); Console.WriteLine(" Duty cycle"); metaData.DutyCycle = MetaDataCalculations.DutyCycle(retentionTimes, index); Console.WriteLine(" Intensity distribution"); metaData.IntensityDistribution = MetaDataCalculations.IntensityDistributions(centroidStreams, segmentScans, index); Console.WriteLine(" Summed intensities"); metaData.SummedIntensity = MetaDataCalculations.SummedIntensities(centroidStreams, segmentScans, index); metaData.FractionConsumingTop80PercentTotalIntensity = MetaDataCalculations.Top80Frac(centroidStreams, segmentScans, index); //Task.WaitAll(); return(metaData); }
public static double GetMedianSummedMSIntensity(Dictionary <int, double> SummedIntensities, ScanIndex Index, MSOrderType MsOrder) { return((from x in Index.ScanEnumerators[MsOrder] select SummedIntensities[x]).ToArray().Percentile(50)); }