private void CloseReader()
{
if (rawReaderThreader != null)
{
rawReaderThreader.Dispose();
rawReaderThreader = null;
}
}
public RawFileViz(string rawFile)
{
this.InitializeComponent();
#region initialize fields and containers
nextScanButton.Text = Char.ConvertFromUtf32(0x2192);
previousScanButton.Text = Char.ConvertFromUtf32(0x2190);
rawFileThreadManager = RawFileReaderFactory.CreateThreadManager(rawFile);
RawData = rawFileThreadManager.CreateThreadAccessor();
RawData.SelectMsData();
TotalNumScans = RawData.RunHeaderEx.SpectraCount;
FirstScan = RawData.RunHeader.FirstSpectrum;
LastScan = RawData.RunHeader.LastSpectrum;
currentScan = FirstScan;
TotalTime = RawData.RunHeader.EndTime;
this.Text = "ParseDataViz - " + RawData.FileName;
totalScansLabel.Text = String.Format("/ {0}", TotalNumScans);
splitContainer1.SplitterDistance = this.Size.Width - 300;
UpdateChromatogramData();
InitializeChromatogramPlot();
ChroMsLevelComboBox.SelectedIndex = 0;
initializeMassSpecData();
initializeMassSpectrum();
scanNumber.Text = FirstScan.ToString();
y = 0;
x = 1;
#endregion
//#region initial chromatogram
this.plotViewChromatogram.Model = chromatogram;
UpdateChromatogramPlot();
}
public static void ParseDIA(IRawFileThreadManager rawFile, WorkflowParameters parameters)
{
var staticRawFile = rawFile.CreateThreadAccessor();
staticRawFile.SelectInstrument(Device.MS, 1);
staticRawFile.CheckIfBoxcar();
ScanIndex Index = Extract.ScanIndices(rawFile.CreateThreadAccessor());
TrailerExtraCollection trailerExtras = Extract.TrailerExtras(rawFile.CreateThreadAccessor(), Index);
MethodDataContainer methodData = Extract.MethodData(rawFile.CreateThreadAccessor(), Index);
PrecursorScanCollection precursorScans = Extract.PrecursorScansByMasterScanMs2Only(rawFile.CreateThreadAccessor(), trailerExtras, Index);
(CentroidStreamCollection centroidStreams, SegmentScanCollection segmentScans) =
Extract.MsData(rawFile: rawFile.CreateThreadAccessor(), index: Index);
RetentionTimeCollection retentionTimes = Extract.RetentionTimes(rawFile.CreateThreadAccessor(), Index);
ScanMetaDataCollectionDIA metaData = MetaDataProcessingDIA.AggregateMetaDataDIA(centroidStreams, segmentScans, methodData,
trailerExtras, retentionTimes, Index);
RawMetricsDataDIA metrics = null;
if (parameters.ParseParams.Metrics)
{
metrics = MetaDataProcessingDIA.GetMetricsDataDIA(metaData, methodData, staticRawFile.FileName, retentionTimes, Index);
MetricsWriter.WriteMatrix(metrics, staticRawFile.FileName, parameters.ParseParams.OutputDirectory);
}
if (parameters.ParseParams.Parse)
{
string matrixFileName = ReadWrite.GetPathToFile(parameters.ParseParams.OutputDirectory, staticRawFile.FileName, "._parse.txt");
//MatrixWriter.ParseQuantDIA()
//ParseWriter writerDIA = new ParseWriter(matrixFileName, centroidStreams, segmentScans, metaData,
// retentionTimes, trailerExtras, precursorScans, Index);
//writerDIA.WriteMatrixDIA();
}
// I'm not sure what goes into a DIA mgf file, so we aren't making one yet
//if (parameters.ParseParams.WriteMgf)
//{
// ParseWriter writerMGF = new ParseWriter(centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index);
// writerMGF.WriteMGF(staticRawFile.FileName);
//}
}
DependentsAndPrecursorScansByScanDependentsParallel(IRawFileThreadManager rawFileManager, ScanIndex index)
{
//rawFile.SelectInstrument(Device.MS, 1);
Log.Information("Extracting scan dependents/precursor scans");
ConcurrentDictionary <int, PrecursorScanData> precursorScans = new ConcurrentDictionary <int, PrecursorScanData>();
ConcurrentDictionary <int, IScanDependents> dependents = new ConcurrentDictionary <int, IScanDependents>();
//Dictionary<int, PrecursorScanData> precursorScans = new Dictionary<int, PrecursorScanData>();
//Dictionary<int, IScanDependents> dependents = new Dictionary<int, IScanDependents>();
//ConcurrentBag<(int scan, PrecursorScanData data)> precursorBag = new ConcurrentBag<(int scan, PrecursorScanData data)>();
//ConcurrentBag<(int scan, IScanDependents data)> dependentsBag = new ConcurrentBag<(int scan, IScanDependents data)>();
var updateProgressLock = new object();
var addLockTarget = new object();
IEnumerable <int> scans = index.ScanEnumerators[MSOrderType.Ms];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Indexing linked scan events");
var batches = scans.Chunk(Constants.MultiThreading.ChunkSize(scans.Count()));
Parallel.ForEach(batches, Constants.MultiThreading.Options(), batch =>
{
int ms2Scan = -1;
int ms3Scan = -1;
IScanDependents scanDependents;
var rawFile = rawFileManager.CreateThreadAccessor();
rawFile.SelectInstrument(Device.MS, 1);
foreach (int scan in batch)
{
scanDependents = rawFile.GetScanDependents(scan, 2);
dependents.AddOrUpdate(scan, scanDependents, (a, b) => b);
// check if the ms1 scan has dependent scans
if (scanDependents == null)
{
return;
}
for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++)
{
if (index.AnalysisOrder == MSOrderType.Ms2) // it is ms2
{
ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex;
//precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b);
precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
}
else // it is ms3
{
ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex;
var ms2Dependents = rawFile.GetScanDependents(ms2Scan, 2).ScanDependentDetailArray;
if (ms2Dependents.Length != 0) // make sure there is ms3 data
{
ms3Scan = ms2Dependents[0].ScanIndex;
precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b);
precursorScans.AddOrUpdate(ms3Scan, new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan), (a, b) => b);
}
else
{
// there is no ms3 scan, so we only add the ms2 scan
precursorScans.AddOrUpdate(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan), (a, b) => b);
}
}
}
lock (updateProgressLock)
{
progress.Update();
}
}
});
progress.Done();
var outScans = new PrecursorScanCollection();
var outDependents = new ScanDependentsCollections();
foreach (var item in precursorScans)
{
outScans.Add(item.Key, item.Value);
}
foreach (var item in dependents)
{
outDependents.Add(item.Key, item.Value);
}
// check for missing precursor information
/*
* foreach (int scan in index.ScanEnumerators[MSOrderType.Ms2])
* {
* if (outScans.Keys.Contains(scan))
* {
* outScans.HasPrecursorData.Add(scan, true);
* }
* else
* {
* outScans.HasPrecursorData.Add(scan, false);
* }
* }*/
return(outScans, outDependents);
}
public static (ScanIndex, PrecursorScanCollection, ScanDependentsCollections) ScanIndicesPrecursorsDependents(IRawFileThreadManager rawFileAccessor)
{
Log.Information("Extracting scan indices");
ConcurrentDictionary <int, ScanData> allScans;
allScans = new ConcurrentDictionary <int, ScanData>();
Dictionary <int, ScanData> orphanScans = new Dictionary <int, ScanData>();
MSOrderType AnalysisOrder;
ConcurrentBag <int> ms1 = new ConcurrentBag <int>();
ConcurrentBag <int> ms2 = new ConcurrentBag <int>();
ConcurrentBag <int> ms3 = new ConcurrentBag <int>();
ConcurrentBag <int> msAny = new ConcurrentBag <int>();
ConcurrentDictionary <int, PrecursorScanData> precursorScans = new ConcurrentDictionary <int, PrecursorScanData>();
ConcurrentDictionary <int, IScanDependents> dependents = new ConcurrentDictionary <int, IScanDependents>();
var staticRawFile = rawFileAccessor.CreateThreadAccessor();
staticRawFile.SelectMsData();
// populate the scan indices
IEnumerable <int> scans = staticRawFile.GetFilteredScanEnumerator(staticRawFile.GetFilterFromString("")); // get all scans
// get ms order of experiment
Console.Write("Determing MS analysis order... ");
AnalysisOrder = (from x in scans select staticRawFile.GetScanEventForScanNumber(x).MSOrder).Max();
Console.WriteLine("Done!");
object lockTarget = new object();
ProgressIndicator P = new ProgressIndicator(scans.Count(), "Extracting scan indices");
int chunkSize = Constants.MultiThreading.ChunkSize(scans.Count());
var batches = scans.Chunk(chunkSize);
Parallel.ForEach(batches, batch =>
{
ScanData ms1ScanData;
ScanData ms2ScanData;
ScanData ms3ScanData;
foreach (int scan in batch)
{
//var rawFile = rawFileAccessor.CreateThreadAccessor();
using (var rawFile = rawFileAccessor.CreateThreadAccessor())
{
rawFile.SelectMsData();
IScanEvent scanEvent = rawFile.GetScanEventForScanNumber(scan);
ms1ScanData = new ScanData();
ms2ScanData = new ScanData();
ms3ScanData = new ScanData();
ms1ScanData.MassAnalyzer = scanEvent.MassAnalyzer;
ms1ScanData.MSOrder = scanEvent.MSOrder;
if (ms1ScanData.MSOrder == MSOrderType.Ms)
{
ms1.Add(scan);
msAny.Add(scan);
var scanDependents = rawFile.GetScanDependents(scan, 4);
dependents.TryAdd(scan, scanDependents);
// check if the ms1 scan has dependent scans
if (scanDependents == null)
{
// there are no scan dependents
ms1ScanData.HasDependents = false;
allScans.TryAdd(scan, ms1ScanData);
return;
}
else
{
ms1ScanData.HasDependents = true;
allScans.TryAdd(scan, ms1ScanData);
}
for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++)
{
int ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex;
ms2.Add(ms2Scan);
msAny.Add(ms2Scan);
if (AnalysisOrder == MSOrderType.Ms2) // it is ms2
{
ms2ScanData = new ScanData();
scanEvent = rawFile.GetScanEventForScanNumber(ms2Scan);
ms2ScanData.MassAnalyzer = scanEvent.MassAnalyzer;
ms2ScanData.MSOrder = scanEvent.MSOrder;
ms2ScanData.HasDependents = false;
ms2ScanData.HasPrecursors = true;
allScans.TryAdd(ms2Scan, ms2ScanData);
precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
}
else // it is ms3
{
var ms2Dependents = rawFile.GetScanDependents(ms2Scan, 4).ScanDependentDetailArray;
ms2ScanData = new ScanData();
scanEvent = rawFile.GetScanEventForScanNumber(ms2Scan);
ms2ScanData.MassAnalyzer = scanEvent.MassAnalyzer;
ms2ScanData.MSOrder = scanEvent.MSOrder;
ms2ScanData.HasPrecursors = true;
if (ms2Dependents.Length != 0) // make sure there is ms3 data
{
int ms3Scan = ms2Dependents[0].ScanIndex;
ms3.Add(ms3Scan);
msAny.Add(ms3Scan);
scanEvent = rawFile.GetScanEventForScanNumber(ms3Scan);
precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
precursorScans.TryAdd(ms3Scan, new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan));
ms2ScanData.HasDependents = true;
ms3ScanData = new ScanData();
ms3ScanData.HasPrecursors = true;
ms3ScanData.MassAnalyzer = scanEvent.MassAnalyzer;
ms3ScanData.MSOrder = scanEvent.MSOrder;
allScans.TryAdd(ms3Scan, ms3ScanData);
}
else
{
// there is no ms3 scan, so we only add the ms2 scan
precursorScans.TryAdd(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
ms2ScanData.HasDependents = false;
}
allScans.TryAdd(ms2Scan, ms2ScanData);
}
}
}
lock (lockTarget)
{
P.Update();
}
}
}
});
P.Done();
HashSet <int> allKeys = new HashSet <int>(allScans.Keys);
P = new ProgressIndicator(scans.Count(), "Checking for orphaned scans");
foreach (int scan in scans)
{
if (allKeys.Contains(scan))
{
continue;
}
else
{
ScanData scanData = new ScanData();
var scanEvent = staticRawFile.GetScanEventForScanNumber(scan);
scanData.MassAnalyzer = scanEvent.MassAnalyzer;
scanData.MSOrder = scanEvent.MSOrder;
orphanScans.Add(scan, scanData);
}
P.Update();
}
P.Done();
Console.WriteLine();
Console.WriteLine("================ Scan indexing report ================");
Console.WriteLine($"Total scans in file: {staticRawFile.RunHeaderEx.SpectraCount}");
Console.WriteLine($"Scans linked: {allScans.Count()}");
Console.WriteLine();
Console.WriteLine("Orphan scans:");
if (orphanScans.Count() > 0)
{
foreach (var scan in orphanScans)
{
Console.WriteLine($"\tScan: {scan.Key}, MSOrder: {scan.Value.MSOrder}");
}
Console.WriteLine("\nThe above scans will not be present in the output data. You should");
Console.WriteLine("manually check them to ensure they are not critical to you analysis.");
}
else
{
Console.WriteLine("None!");
}
Console.WriteLine();
if (staticRawFile.RunHeaderEx.SpectraCount == allScans.Count() + orphanScans.Count())
{
Console.WriteLine("All scans accounted for!");
}
else
{
Console.WriteLine($"Number of scans unaccounted for: {staticRawFile.RunHeaderEx.SpectraCount - (allScans.Count() + orphanScans.Count())}");
Console.WriteLine("If this number is alarming, please contact the RawTools authors\n" +
"by posting an issue at:\n" +
"https://github.com/kevinkovalchik/RawTools/issues");
}
Console.WriteLine("======================================================");
Console.WriteLine();
// we need to order the scan enumerators before sending them out
var orderedMs1 = ms1.ToList();
var orderedMs2 = ms2.ToList();
var orderedMs3 = ms3.ToList();
var orderedMsAny = msAny.ToList();
orderedMs1.Sort();
orderedMs2.Sort();
orderedMs3.Sort();
orderedMsAny.Sort();
ScanIndex scanIndex = new ScanIndex();
scanIndex.allScans = allScans.ConvertToDictionary();
scanIndex.AnalysisOrder = AnalysisOrder;
scanIndex.ScanEnumerators.Add(MSOrderType.Any, orderedMsAny.ToArray());
scanIndex.ScanEnumerators.Add(MSOrderType.Ms, orderedMs1.ToArray());
scanIndex.ScanEnumerators.Add(MSOrderType.Ms2, orderedMs2.ToArray());
scanIndex.ScanEnumerators.Add(MSOrderType.Ms3, orderedMs3.ToArray());
scanIndex.TotalScans = staticRawFile.RunHeaderEx.SpectraCount;
return(scanIndex, new PrecursorScanCollection(precursorScans), new ScanDependentsCollections(dependents));
}
public static void UniversalDDA(IRawFileThreadManager rawFileThreadManager, WorkflowParameters parameters, QcDataCollection qcDataCollection)
{
MethodDataContainer methodData;
CentroidStreamCollection centroidStreams;
SegmentScanCollection segmentScans;
TrailerExtraCollection trailerExtras;
PrecursorMassCollection precursorMasses;
RetentionTimeCollection retentionTimes;
ScanEventReactionCollection reactions;
ScanMetaDataCollectionDDA metaData = null;
PrecursorPeakCollection peakData = null;
int nScans;
var staticRawFile = rawFileThreadManager.CreateThreadAccessor();
staticRawFile.SelectInstrument(Device.MS, 1);
var err = staticRawFile.FileError;
if (err.HasError)
{
Console.WriteLine("ERROR: {0} reports error code: {1}. The associated message is: {2}",
Path.GetFileName(staticRawFile.FileName), err.ErrorCode, err.ErrorMessage);
Console.WriteLine("Skipping this file");
Log.Error("{FILE} reports error code: {ERRORCODE}. The associated message is: {ERRORMESSAGE}",
Path.GetFileName(staticRawFile.FileName), err.ErrorCode, err.ErrorMessage);
return;
}
//staticRawFile.CheckIfBoxcar();
(ScanIndex Index, PrecursorScanCollection precursorScans, ScanDependentsCollections scanDependents) =
Extract.ScanIndicesPrecursorsDependents(rawFileThreadManager, MaxProcesses: parameters.MaxProcesses);
nScans = Index.ScanEnumerators[MSOrderType.Ms2].Length;
using (var rawFile = rawFileThreadManager.CreateThreadAccessor())
{
reactions = Extract.ScanEvents(rawFile, Index);
methodData = Extract.MethodData(rawFile, Index);
(centroidStreams, segmentScans) = Extract.MsData(rawFile: rawFile, index: Index);
trailerExtras = Extract.TrailerExtras(rawFile, Index);
precursorMasses = Extract.PrecursorMasses(rawFile, precursorScans, trailerExtras, Index);
retentionTimes = Extract.RetentionTimes(rawFile, Index);
}
if (parameters.ParseParams.Parse | parameters.ParseParams.Quant | parameters.ParseParams.Metrics | parameters.RefineMassCharge | parameters.QcParams.QcDirectory != null)
{
peakData = AnalyzePeaks.AnalyzeAllPeaks(centroidStreams, retentionTimes, precursorMasses, precursorScans, Index, parameters.MaxProcesses);
if (parameters.RefineMassCharge)
{
MonoIsoPredictor.RefineMonoIsoMassChargeValues(parameters, centroidStreams, precursorMasses, trailerExtras, peakData, precursorScans);
}
metaData = MetaDataProcessingDDA.AggregateMetaDataDDA(centroidStreams, segmentScans, methodData, precursorScans,
trailerExtras, precursorMasses, retentionTimes, scanDependents, reactions, Index, parameters.MaxProcesses);
}
QuantDataCollection quantData = null;
if (parameters.ParseParams.Quant)
{
quantData = Quantification.Quantify(centroidStreams, segmentScans, parameters, methodData, Index);
}
RawMetricsDataDDA rawMetrics = null;
if (parameters.ParseParams.Metrics | parameters.QcParams.QcDirectory != null)
{
rawMetrics = MetaDataProcessingDDA.GetMetricsDataDDA(metaData, methodData, staticRawFile.FileName, retentionTimes, Index, peakData, precursorScans, quantData);
}
if (parameters.ParseParams.Metrics)
{
MetricsWriter.WriteMatrix(rawMetrics, null, staticRawFile.FileName, parameters.ParseParams.OutputDirectory);
}
if (parameters.ParseParams.Parse | parameters.ParseParams.Quant)
{
string matrixFileName = ReadWrite.GetPathToFile(parameters.ParseParams.OutputDirectory, staticRawFile.FileName, "_Matrix.txt");
/*
* ParseWriter writerDDA = new ParseWriter(matrixFileName, centroidStreams, segmentScans, metaData, retentionTimes,
* precursorMasses, precursorScans, peakData, trailerExtras, Index, quantData);
* writerDDA.WriteMatrixDDA(methodData.AnalysisOrder);
*/
MatrixWriter.ParseQuantDDA(matrixFileName, centroidStreams, segmentScans, metaData, retentionTimes,
precursorMasses, precursorScans, peakData, trailerExtras, Index, quantData);
}
if (parameters.ParseParams.WriteMgf)
{
//ParseWriter writerMGF = new ParseWriter(centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index);
//writerMGF.WriteMGF(staticRawFile.FileName);
MgfWriter.WriteMGF(staticRawFile.FileName, centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index);
}
if (parameters.ParseParams.Chromatogram != null)
{
ChromatogramWriter.WriteChromatogram(centroidStreams, segmentScans, retentionTimes, methodData, Index, parameters, staticRawFile.FileName);
}
if (parameters.QcParams.QcDirectory != null)
{
qcDataCollection = QC.QcWorkflow.LoadOrCreateQcCollection(parameters);
SearchMetricsContainer searchMetrics = new SearchMetricsContainer(staticRawFile.FileName, staticRawFile.CreationDate, methodData);
// check if the raw file already exists in the QC data with a different name
if (QcWorkflow.CheckIfFilePresentInQcCollection(staticRawFile.FileName, qcDataCollection))
{
Log.Information("A file with the same creation date and time as {File} already exists in the QC data", staticRawFile.FileName);
Console.WriteLine("A file with the same creation date and time as {File} already exists in the QC data. Skipping to next file.",
staticRawFile.FileName);
}
else
{
if (parameters.QcParams.PerformSearch)
{
Search.WriteSearchMGF(parameters, centroidStreams, segmentScans, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData,
Index, staticRawFile.FileName, parameters.QcParams.FixedScans);
Search.RunSearch(parameters, methodData, staticRawFile.FileName);
searchMetrics = SearchQC.ParseSearchResults(searchMetrics, parameters, staticRawFile.FileName, nScans);
}
QcDataContainer qcData = new QcDataContainer();
qcData.DDA = rawMetrics;
qcData.SearchMetrics = searchMetrics;
QC.QcWorkflow.UpdateQcCollection(qcDataCollection, qcData, methodData, staticRawFile.FileName);
}
}
}
static int DoStuff(ArgumentParser.ParseOptions opts)
{
if (!opts.ParseData & !opts.Quant & !opts.Metrics & !opts.WriteMGF & (opts.Chromatogram == null))
{
Console.WriteLine("You have not indicated what output you want (i.e. one or more of -p, -q, -m, -x, --chro). " +
"Are you sure you want to proceed? Nothing will be written to disk.");
Console.Write("(press y to proceed): ");
string proceed = Console.ReadKey().KeyChar.ToString();
Console.WriteLine();
if (proceed != "y")
{
Environment.Exit(0);
}
}
List <string> files = new List <string>();
if (opts.InputFiles.Count() > 0) // did the user give us a list of files?
{
List <string> problems = new List <string>();
files = opts.InputFiles.ToList();
// check if the list provided contains only .raw files
foreach (string file in files)
{
if (!file.EndsWith(".raw", StringComparison.OrdinalIgnoreCase))
{
problems.Add(file);
}
}
if (problems.Count() == 1)
{
Console.WriteLine("\nERROR: {0} does not appear to be a .raw file. Invoke '>RawTools --help' if you need help.", problems.ElementAt(0));
Log.Error("Invalid file provided: {0}", problems.ElementAt(0));
return(1);
}
if (problems.Count() > 1)
{
Console.WriteLine("\nERROR: The following {0} files do not appear to be .raw files. Invoke '>RawTools --help' if you need help." +
"\n\n{1}", problems.Count(), String.Join("\n", problems));
Log.Error("Invalid files provided: {0}", String.Join(" ", problems));
return(1);
}
// if the file location(s) are relative, we need to get the absolute path to them
files.EnsureAbsolutePaths();
Log.Information("Files to be processed, provided as list: {0}", String.Join(" ", files));
}
else // did the user give us a directory?
{
if (Directory.Exists(opts.InputDirectory))
{
files = Directory.GetFiles(opts.InputDirectory, "*.*", SearchOption.TopDirectoryOnly)
.Where(s => s.EndsWith(".raw", StringComparison.OrdinalIgnoreCase)).ToList();
}
else
{
Console.WriteLine("ERROR: The provided directory does not appear to be valid.");
Log.Error("Invalid directory provided: {0}", opts.InputDirectory);
return(1);
}
// if the file location(s) are relative, we need to get the absolute path to them
files.EnsureAbsolutePaths();
Log.Information("Files to be processed, provided as directory: {0}", String.Join(" ", files));
}
if (opts.Quant)
{
List <string> possible = new List <string>()
{
"TMT0", "TMT2", "TMT6", "TMT10", "TMT11", "iTRAQ4", "iTRAQ8"
};
if (!possible.Contains(opts.LabelingReagents))
{
Console.WriteLine("ERROR: For quantification, the labeling reagent must be one of {TMT0, TMT2, TMT6, TMT10, TMT11, iTRAQ4, iTRAQ8}");
Log.Error("Invalid labeling reagent provided: {0}", opts.LabelingReagents);
return(1);
}
}
if (opts.Chromatogram != null)
{
List <string> possible = new List <string>()
{
"1T", "2T", "3T", "1B", "2B", "3B", "1TB", "2TB", "3TB", "1TB", "2TB", "3TB"
};
if (!possible.Contains(opts.Chromatogram))
{
Console.WriteLine("ERROR: Incorrect format for --chro. See help.");
Log.Error("Invalid chromatogram argument provided: {Chro}", opts.Chromatogram);
return(1);
}
}
/*
* // is the experiment type valid?
* if (! new List<string>() { "DDA", "DIA", "PRM" }.Contains(opts.ExperimentType))
* {
* Log.Error("Experiment type of {ExpType} was passed", opts.ExperimentType);
* Console.WriteLine("Experiment type must be one of ['DDA', 'DIA', 'PRM'], not {0}", opts.ExperimentType);
* Environment.Exit(1);
* }*/
System.Diagnostics.Stopwatch singleFileTime = new System.Diagnostics.Stopwatch();
System.Diagnostics.Stopwatch totalTime = new System.Diagnostics.Stopwatch();
totalTime.Start();
WorkflowParameters parameters = new WorkflowParameters(opts);
foreach (string file in files)
{
singleFileTime.Start();
Console.WriteLine("\nProcessing: {0}\n", file);
//using (IRawDataPlus rawFile = RawFileReaderFactory.ReadFile(fileName:file))
using (IRawFileThreadManager rawFile = RawFileReaderFactory.CreateThreadManager(file))
{
if (parameters.ParseParams.OutputDirectory == null)
{
parameters.ParseParams.OutputDirectory = Path.GetDirectoryName(file);
}
if (parameters.ExpType == ExperimentType.DDA)
{
WorkFlowsDDA.ParseDDA(rawFile, parameters);
}
else if (parameters.ExpType == ExperimentType.DIA)
{
WorkFlowsDIA.ParseDIA(rawFile, parameters);
}
}
singleFileTime.Stop();
Console.WriteLine("\nElapsed time: {0} s", Math.Round(Convert.ToDouble(singleFileTime.ElapsedMilliseconds) / 1000.0, 2));
singleFileTime.Reset();
}
totalTime.Stop();
Console.WriteLine("\nTime to process all {0} files: {1}", files.Count(), totalTime.Elapsed);
return(0);
}
static int Run(Dictionary <string, object> opts)
{
if ((bool)opts["ExampleCommands"] == true)
{
Examples.CommandLineUsage();
Environment.Exit(0);
}
if ((bool)opts["ExampleModifications"] == true)
{
Examples.ExampleMods();
Environment.Exit(0);
}
List <string> files = new List <string>();
QcDataCollection qcDataCollection = new QcDataCollection();
WorkflowParameters parameters = new WorkflowParameters(opts);
if (parameters.InputFiles != null) // did the user give us a list of files?
{
List <string> problems = new List <string>();
files = parameters.InputFiles.ToList();
// check if the list provided contains only .raw files
foreach (string file in files)
{
if (!file.EndsWith(".raw", StringComparison.OrdinalIgnoreCase))
{
problems.Add(file);
}
}
if (problems.Count() == 1)
{
Console.WriteLine("\nERROR: {0} does not appear to be a .raw file. Invoke '>RawTools --help' if you need help.", problems.ElementAt(0));
Log.Error("Invalid file provided: {0}", problems.ElementAt(0));
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(1);
}
if (problems.Count() > 1)
{
Console.WriteLine("\nERROR: The following {0} files do not appear to be .raw files. Invoke '>RawTools --help' if you need help." +
"\n\n{1}", problems.Count(), String.Join("\n", problems));
Log.Error("Invalid files provided: {0}", String.Join(" ", problems));
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(1);
}
files = RawFileInfo.RemoveInAcquistionFiles(files);
// if the file location(s) are relative, we need to get the absolute path to them
files.EnsureAbsolutePaths();
Log.Information("Files to be processed, provided as list: {0}", String.Join(" ", files));
}
else if (!String.IsNullOrEmpty(parameters.RawFileDirectory)) // did the user give us a directory?
{
// if QC is being done, use the QC method snf get the qc data collection at the same time
if (parameters.QcParams.QcDirectory != null)
{
(files, qcDataCollection) = QcWorkflow.GetFileListAndQcFile(parameters, parameters.IncludeSubdirectories);
}
// if not, the parse method
else if (Directory.Exists(parameters.RawFileDirectory))
{
files = Directory.GetFiles(parameters.RawFileDirectory, "*.*", SearchOption.TopDirectoryOnly)
.Where(s => s.EndsWith(".raw", StringComparison.OrdinalIgnoreCase)).ToList();
}
else
{
Console.WriteLine("ERROR: The provided directory does not appear to be valid.");
Log.Error("Invalid directory provided: {0}", parameters.RawFileDirectory);
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(1);
}
files = RawFileInfo.RemoveInAcquistionFiles(files);
// if the file location(s) are relative, we need to get the absolute path to them
files.EnsureAbsolutePaths();
Log.Information("Files to be processed, provided as directory: {0}", String.Join(" ", files));
}
else
{
Console.WriteLine("ERROR: At least one of the following arguments is required: -f, -d");
Log.Error("No raw files or directory specified.");
return(1);
}
if (parameters.ParseParams.Quant)
{
List <string> possible = new List <string>()
{
"TMT0", "TMT2", "TMT6", "TMT10", "TMT11", "iTRAQ4", "iTRAQ8"
};
if (!possible.Contains(parameters.ParseParams.LabelingReagents))
{
Console.WriteLine("ERROR: For quantification, the labeling reagent must be one of {TMT0, TMT2, TMT6, TMT10, TMT11, iTRAQ4, iTRAQ8}");
Log.Error("Invalid labeling reagent provided: {0}", parameters.ParseParams.LabelingReagents);
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(1);
}
}
if (parameters.ParseParams.Chromatogram != null)
{
List <string> possible = new List <string>()
{
"1", "2", "3", "T", "B"
};
foreach (var x in parameters.ParseParams.Chromatogram)
{
if (!possible.Contains(x.ToString()))
{
Console.WriteLine("ERROR: Incorrect format for -chro. See help.");
Log.Error("Invalid chromatogram argument provided: {Chro}", parameters.ParseParams.Chromatogram);
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(1);
}
}
}
System.Diagnostics.Stopwatch singleFileTime = new System.Diagnostics.Stopwatch();
System.Diagnostics.Stopwatch totalTime = new System.Diagnostics.Stopwatch();
totalTime.Start();
foreach (string file in files)
{
singleFileTime.Start();
Console.WriteLine("\nProcessing: {0}\n", file);
//using (IRawDataPlus rawFile = RawFileReaderFactory.ReadFile(fileName:file))
using (IRawFileThreadManager rawFile = RawFileReaderFactory.CreateThreadManager(file))
{
if (parameters.ParseParams.OutputDirectory == null)
{
parameters.ParseParams.OutputDirectory = Path.GetDirectoryName(file);
}
WorkFlowsDDA.UniversalDDA(rawFile, parameters, qcDataCollection);
}
singleFileTime.Stop();
Console.WriteLine("\nElapsed time: {0} s", Math.Round(Convert.ToDouble(singleFileTime.ElapsedMilliseconds) / 1000.0, 2));
singleFileTime.Reset();
}
if (parameters.LogDump)
{
Write.LogDump.WriteToDisk(parameters);
}
totalTime.Stop();
Console.WriteLine("\nTime to process all {0} files: {1}", files.Count(), totalTime.Elapsed);
//Console.Write("Press any key to exit...");
//Console.ReadKey();
return(0);
}
public RawReaderMetadata(string rawFilePath)
{
RawFilePath = rawFilePath;
rawReaderThreader = RawFileReaderFactory.CreateThreadManager(rawFilePath);
}
public static void DoQc(WorkflowParameters parameters)
{
//QcDataCollectionDDA qcDataCollection;
dynamic qcDataCollection;
string dataDirectory = parameters.RawFileDirectory;
string qcDirectory = parameters.QcParams.QcDirectory;
string qcSearchDataDirecotry = parameters.QcParams.QcSearchDataDirectory;
List <string> fileList = new List <string>();
string qcFile = Path.Combine(qcDirectory, "QC.xml");
(fileList, qcDataCollection) = GetFileListAndQcFile(parameters);
foreach (string fileName in fileList)
{
Console.WriteLine("\nProcessing {0}", fileName);
if (!RawFileInfo.CheckIfValid(fileName))
{
continue;
}
// okay, it is probably a real raw file, let's do the QC
// check if the raw file already exists in the QC data with a different name
if (CheckIfFilePresentInQcCollection(fileName, qcDataCollection))
{
Log.Information("A file with the same creation date and time as {File} already exists in the QC data", fileName);
Console.WriteLine("A file with the same creation date and time as {File} already exists in the QC data. Skipping to next file.",
fileName);
continue;
}
IRawFileThreadManager rawFileThreadManager = RawFileReaderFactory.CreateThreadManager(fileName);
if (parameters.ExpType == ExperimentType.DDA)
{
WorkFlowsDDA.QcDDA(rawFileThreadManager, parameters);
}
//else if (parameters.ExpType == ExperimentType.DIA)
//{
// WorkFlowsDIA.QcDIA(rawFileThreadManager, parameters);
//}
/*
* using (IRawDataPlus rawFile = RawFileReaderFactory.ReadFile(fileName))
* {
* rawFile.SelectInstrument(Device.MS, 1);
*
* if (parameters.ExpType == ExperimentType.DDA)
* {
* WorkFlowsDDA.QcDDA(rawFile, parameters);
* }
* else if (parameters.ExpType == ExperimentType.DIA)
* {
* WorkFlowsDIA.QcDIA(rawFile, parameters);
* }
* }*/
Log.Information("QC finished: {File}", fileName);
}
Log.Information("QC of all files completed");
Console.WriteLine("QC of all files completed!");
}
public static void ParseDDA(IRawFileThreadManager rawFileThreadManager, WorkflowParameters parameters)
{
MethodDataContainer methodData;
CentroidStreamCollection centroidStreams;
SegmentScanCollection segmentScans;
TrailerExtraCollection trailerExtras;
PrecursorMassCollection precursorMasses;
RetentionTimeCollection retentionTimes;
ScanEventReactionCollection reactions;
ScanMetaDataCollectionDDA metaData = null;
PrecursorPeakCollection peakData = null;
RawMetricsDataDDA rawMetrics = null;
QuantDataCollection quantData = null;
var staticRawFile = rawFileThreadManager.CreateThreadAccessor();
staticRawFile.SelectInstrument(Device.MS, 1);
//staticRawFile.CheckIfBoxcar();
(ScanIndex Index, PrecursorScanCollection precursorScans, ScanDependentsCollections scanDependents) =
Extract.ScanIndicesPrecursorsDependents(rawFileThreadManager);
using (var rawFile = rawFileThreadManager.CreateThreadAccessor())
{
reactions = Extract.ScanEvents(rawFile, Index);
methodData = Extract.MethodData(rawFile, Index);
(centroidStreams, segmentScans) = Extract.MsData(rawFile: rawFile, index: Index);
trailerExtras = Extract.TrailerExtras(rawFile, Index);
precursorMasses = Extract.PrecursorMasses(rawFile, precursorScans, trailerExtras, Index);
retentionTimes = Extract.RetentionTimes(rawFile, Index);
}
if (parameters.ParseParams.Parse | parameters.ParseParams.Quant | parameters.ParseParams.Metrics | parameters.RefineMassCharge)
{
peakData = AnalyzePeaks.AnalyzeAllPeaks(centroidStreams, retentionTimes, precursorMasses, precursorScans, Index);
if (parameters.RefineMassCharge)
{
MonoIsoPredictor.RefineMonoIsoMassChargeValues(centroidStreams, precursorMasses, trailerExtras, peakData, precursorScans);
}
metaData = MetaDataProcessingDDA.AggregateMetaDataDDA(centroidStreams, segmentScans, methodData, precursorScans,
trailerExtras, precursorMasses, retentionTimes, scanDependents, reactions, Index);
}
if (parameters.ParseParams.Quant)
{
quantData = Quantification.Quantify(centroidStreams, segmentScans, parameters, methodData, Index);
}
if (parameters.ParseParams.Metrics)
{
rawMetrics = MetaDataProcessingDDA.GetMetricsDataDDA(metaData, methodData, staticRawFile.FileName, retentionTimes, Index, peakData, precursorScans, quantData);
MetricsWriter.WriteMatrix(rawMetrics, null, staticRawFile.FileName, parameters.ParseParams.OutputDirectory);
}
if (parameters.ParseParams.Parse | parameters.ParseParams.Quant)
{
string matrixFileName = ReadWrite.GetPathToFile(parameters.ParseParams.OutputDirectory, staticRawFile.FileName, "_Matrix.txt");
/*
* ParseWriter writerDDA = new ParseWriter(matrixFileName, centroidStreams, segmentScans, metaData, retentionTimes,
* precursorMasses, precursorScans, peakData, trailerExtras, Index, quantData);
* writerDDA.WriteMatrixDDA(methodData.AnalysisOrder);
*/
MatrixWriter.ParseQuantDDA(matrixFileName, centroidStreams, segmentScans, metaData, retentionTimes,
precursorMasses, precursorScans, peakData, trailerExtras, Index, quantData);
}
if (parameters.ParseParams.WriteMgf)
{
//ParseWriter writerMGF = new ParseWriter(centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index);
//writerMGF.WriteMGF(staticRawFile.FileName);
MgfWriter.WriteMGF(staticRawFile.FileName, centroidStreams, segmentScans, parameters, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData, Index);
}
if (parameters.ParseParams.Chromatogram != null)
{
ChromatogramWriter.WriteChromatogram(centroidStreams, segmentScans, retentionTimes, methodData, Index, parameters, staticRawFile.FileName);
}
}
public static void QcDDA(IRawFileThreadManager rawFileThreadManager, WorkflowParameters parameters)
{
MethodDataContainer methodData;
CentroidStreamCollection centroidStreams;
SegmentScanCollection segmentScans;
TrailerExtraCollection trailerExtras;
PrecursorMassCollection precursorMasses;
RetentionTimeCollection retentionTimes;
ScanEventReactionCollection reactions;
var staticRawFile = rawFileThreadManager.CreateThreadAccessor();
staticRawFile.SelectInstrument(Device.MS, 1);
staticRawFile.CheckIfBoxcar();
(ScanIndex Index, PrecursorScanCollection precursorScans, ScanDependentsCollections scanDependents) =
Extract.ScanIndicesPrecursorsDependents(rawFileThreadManager);
using (var rawFile = rawFileThreadManager.CreateThreadAccessor())
{
methodData = Extract.MethodData(rawFile, Index);
reactions = Extract.ScanEvents(rawFile, Index);
(centroidStreams, segmentScans) = Extract.MsData(rawFile: rawFile, index: Index);
trailerExtras = Extract.TrailerExtras(rawFile, Index);
precursorMasses = Extract.PrecursorMasses(rawFile, precursorScans, trailerExtras, Index);
retentionTimes = Extract.RetentionTimes(rawFile, Index);
}
PrecursorPeakCollection peakData = AnalyzePeaks.AnalyzeAllPeaks(centroidStreams, retentionTimes, precursorMasses, precursorScans, Index);
if (parameters.RefineMassCharge)
{
MonoIsoPredictor.RefineMonoIsoMassChargeValues(centroidStreams, precursorMasses, trailerExtras, peakData, precursorScans);
}
ScanMetaDataCollectionDDA metaData = MetaDataProcessingDDA.AggregateMetaDataDDA(centroidStreams, segmentScans, methodData, precursorScans,
trailerExtras, precursorMasses, retentionTimes, scanDependents, reactions, Index);
RawMetricsDataDDA rawMetrics = MetaDataProcessingDDA.GetMetricsDataDDA(metaData, methodData, staticRawFile.FileName, retentionTimes, Index, peakData, precursorScans);
QcDataCollection qcDataCollection = QC.QcWorkflow.LoadOrCreateQcCollection(parameters);
SearchMetricsContainer searchMetrics = new SearchMetricsContainer(staticRawFile.FileName, staticRawFile.CreationDate, methodData);
if (parameters.QcParams.PerformSearch)
{
Search.WriteSearchMGF(parameters, centroidStreams, segmentScans, retentionTimes, precursorMasses, precursorScans, trailerExtras, methodData,
Index, staticRawFile.FileName, parameters.QcParams.FixedScans);
Search.RunSearch(parameters, methodData, staticRawFile.FileName);
searchMetrics = SearchQC.ParseSearchResults(searchMetrics, parameters, staticRawFile.FileName);
}
QcDataContainer qcData = new QcDataContainer();
qcData.DDA = rawMetrics;
qcData.SearchMetrics = searchMetrics;
QC.QcWorkflow.UpdateQcCollection(qcDataCollection, qcData, methodData, staticRawFile.FileName);
staticRawFile.Dispose();
}
