//add log here
public void Analyze(RawFileName rawfile, UIsettings fixedSettings, SettingsForAnalysis fixedSettingsForAnalysis, IProgress <DataRow> resultRow)
{
if (IsFileLocked(rawfile.FullName))
{
Log.Error("[" + rawfile.FullName + "] CRITICAL problem: file locked by another process! Skipping file");
}
else if (IsNotRawfile(rawfile.FullName))
{
Log.Error("[" + rawfile.FullName + "] CRITICAL problem: raw file corrupt! Skipping file");
}
else
{
Log.Information("[" + rawfile.FullName + "] Main analysis started");
rawfile.SetCategories(fixedSettings.UserList, fixedSettings.LcList, fixedSettings.MsList, fixedSettings.SamplesList);
SetRawFileHistory(rawfile, fixedSettings);//This is a mess with the class lines
if (rawfile.SomethingWrongWithFileName)
{
Log.Warning("[" + rawfile.FullName + "] Please check file naming convention: parse rules and filename do not align.");
}
using (ResultItems results = new ResultItems())
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
MainAnalysisThreadedTask(rawfile, fixedSettingsForAnalysis, fixedSettings, results);
Log.Information("[" + rawfile.FullName + "] Main analysis finished in " + stopwatch.Elapsed.TotalMinutes.ToString("0.00") + " minutes");
DoEmailResults(rawfile, fixedSettings, results);
DoSqlUpdateResults(rawfile, fixedSettings, results);
DoSaveResultsToFile(fixedSettings, results);
AddResultRowToDataTable(results.Names, results.Results, results.Types, fixedSettings, resultRow);
}
}
}
private void ReduceToMetaVariables(SettingsForAnalysis settingName)
{
this.MsCount = msScanList.Count;
List <double> msticlist = msScanList.Select(x => x.Tic).ToList();
this.MsTic = Statistics.Median(msticlist);
this.MsTicHigh = Statistics.Percentile(msticlist, 90);
this.MsS2N = Statistics.Median(msScanList.Select(x => x.SignalToNoise));
this.MsIonCount = Statistics.Median(msScanList.Select(x => x.IonCountFromRawOvFtT));
this.LargeFluctuations = GetLargeInjectionTimeShifts(msScanList, settingName.baseMaxFluctuationsPercent / 100);//read from settings
this.Polymer1 = Statistics.Percentile(msScanList.Select(x => x.Polymer[0]), 99) / 1000000.0;
this.Polymer2 = Statistics.Percentile(msScanList.Select(x => x.Polymer[1]), 99) / 1000000.0;
this.Polymer3 = Statistics.Percentile(msScanList.Select(x => x.Polymer[2]), 99) / 1000000.0;
this.Polymer4 = Statistics.Percentile(msScanList.Select(x => x.Polymer[3]), 99) / 1000000.0;
this.Polymer5 = Statistics.Percentile(msScanList.Select(x => x.Polymer[4]), 99) / 1000000.0;
this.AdductNa = Statistics.Percentile(msScanList.Select(x => x.Adduct[1]), 80);
this.AdductCa = Statistics.Percentile(msScanList.Select(x => x.Adduct[3]), 80);
this.AdductFe = Statistics.Percentile(msScanList.Select(x => x.Adduct[6]), 80);
this.AdductK = Statistics.Percentile(msScanList.Select(x => x.Adduct[4]), 80);
this.AdductFA = Statistics.Percentile(msScanList.Select(x => x.Adduct[2]), 80);
this.AdductNH3 = Statistics.Percentile(msScanList.Select(x => x.Adduct[0]), 80);
this.AdductACN = Statistics.Percentile(msScanList.Select(x => x.Adduct[5]), 80);
this.TopN = msScanList.Select(x => x.ScanNumber).Zip(msScanList.Select(x => x.ScanNumber).Skip(1), (x, y) => y - x).Average();
this.Ms2Count = ms2ScanList.Count;
this.Ms2Tic = Statistics.Median(ms2ScanList.Select(x => x.Tic));
this.Ms2S2N = Statistics.Median(ms2ScanList.Select(x => x.SignalToNoise));
this.Ms2IonCount = Statistics.Median(ms2ScanList.Select(x => x.IonCountFromRawOvFtT));
this.Ms2IsolationWindowTic = Statistics.Median(ms2ScanList.Select(x => x.IsolationWindowTic));
this.Ms2IsolationWindowTicRelative = Statistics.Median(ms2ScanList.Select(x => x.IsolationWindowTicRelative));
this.Ms2ScansAboveIonThreshold = GetNoMS2scansAboveIonThreshold(ms2ScanList, settingName.IonThreshold);
Tuple <double, double, double> ms2charges = GetMS2PrecursorChargeStateDistribution(ms2ScanList);
this.Ms2charge2 = ms2charges.Item1;
this.Ms2charge3 = ms2charges.Item2;
this.Ms2charge4orMore = ms2charges.Item3;
this.Ms2ScansAbovePrecursorPercent = ms2ScanList.Select(x => x.Ms2XicAboveThreshold).Average();
this.Ms2ScansAboveIonThresholdPercent = Ms2ScansAboveIonThreshold / Convert.ToDouble(ms2ScanList.Count) * 100.0;
this.Ms2ScansWithPrecursorBasePeakPercent = ms2ScanList.Select(x => x.IsPrecursorBasePeak).Sum() * 100.0 / Convert.ToDouble(ms2ScanList.Count);
this.PrecursorMass5percent = Statistics.Percentile(ms2ScanList.Select(x => x.PrecursorMass), 5);
this.PrecursorMass95percent = Statistics.Percentile(ms2ScanList.Select(x => x.PrecursorMass), 95);
this.CtcdMin = msScanList.Select(x => x.Ctcd).Min();
this.CtcdMax = msScanList.Select(x => x.Ctcd).Max();
this.LockMassCorrectionMaxAbsolute = msScanList.Select(x => x.LockMassCorr).MaximumAbsolute();
}
private static void MainAnalysisThreadedTask(RawFileName rawfile, SettingsForAnalysis fixedSettingsForAnalysis, UIsettings fixedSettings, ResultItems results)
{
//ResultItems results = new ResultItems();
//this can now be done threaded
List <Task <ResultItems> > taskList = new List <Task <ResultItems> >();
taskList.Add(Task.Run(() =>
{
ResultItems results1 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisMorpheus())
{
results1 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
GC.Collect();//ugly but this solves problem with memory not released
return(results1);
}));
taskList.Add(Task.Run(() =>
{
ResultItems results2 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisBase())
{
results2 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
return(results2);
}));
taskList.Add(Task.Run(() =>
{
ResultItems results3 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisExternal1())
{
results3 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
return(results3);
}));
Task.WaitAll(taskList.ToArray());
foreach (Task <ResultItems> resultTask in taskList)
{
results.Add(resultTask.Result);
}
//return results;
}
public RawFileProcessing(IXRawfile5 raw, string rawFileName, SettingsForAnalysis settingName)
{
//open rawfile (add error check)
raw.Open(rawFileName);
raw.SetCurrentController(0, 1);
raw.GetCreationDate(ref _msStart);
double endtime = 0.0;
raw.GetEndTime(ref endtime);
_msEnd = _msStart.AddMinutes(endtime);
_filecreation = File.GetCreationTime(rawFileName);
raw.GetWarningMessage(ref WarningMessage);
ProcessScans(raw, settingName);
raw.Close();
ReduceToMetaVariables(settingName);
}
/// <summary>
/// Read and parse all spectra, spawns a task.
/// </summary>
/// <param name="raw">Raw file</param>
/// <param name="msScanParameters">Parameters</param>
private void ProcessScans(IXRawfile5 raw, SettingsForAnalysis msScanParameters)
{
BlockingCollection <RawScan> rawScanQueue = new BlockingCollection <RawScan>();
List <Task <Tuple <List <MsScan>, List <Ms2Scan> > > > parseTaskList = new List <Task <Tuple <List <MsScan>, List <Ms2Scan> > > >();
for (int i = 0; i < 1; i++)//option here to increase threading if computations get more complicated - note possible problem with unsorted spectra?
{
parseTaskList.Add(Task.Run(() =>
{
return(ParseSpectra(rawScanQueue, msScanParameters));
}));
}
ReadRawFile(raw, rawScanQueue);
Task.WaitAll(parseTaskList.ToArray());
foreach (Task <Tuple <List <MsScan>, List <Ms2Scan> > > resultTask in parseTaskList)
{
this.msScanList.AddRange(resultTask.Result.Item1);
this.ms2ScanList.AddRange(resultTask.Result.Item2);
}
}
public MsScanParameters(SettingsForAnalysis settingName)
{
signalToNoiseThreshold = settingName.StoNthreshold;
ms1massError = 10; //ppm error for matching ms
ms2massError = 10; //ppm error for matching ms
ms1Targets = new List <double>
{
327.20135,
371.227565,
459.279995,
591.35864,
767.4635,
309.24242,
353.268635,
441.321065,
573.39971,
749.50457,
339.25299,
383.279205,
471.331635,
603.41028,
779.51514,
381.29753,
425.323745,
513.376175,
645.45482,
821.55968,
367.28188,
411.308095,
499.360525,
631.43917,
807.54403
//464.25036,
//582.31897,
//722.32465
};
chargeOfms1Targets = new List <short>
{
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1 //,
//2,
//2,
//2
};
//Int16[] polymerIndex = new Int16[25] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24 };
//Int16[] ms1TargetIndex = new Int16[3] { 25, 26, 27 };
ms1adducts = new double[7]
{
17.026549, 21.981943, 32.026213, 37.946941, 37.955882, 41.026547, 52.911464
};
//double[] tmtTargets = new double[10] { 500.0, 501.0, 502.0, 503.0, 504.0, 505.0, 506.0, 507.0, 508.0, 509.0 };
//double[] tmtIntensity = new double[tmtTargets.Length];
//double[] ms2Targets = new double[3] { 600.0, 601.0, 602.0 };
//double[] ms2Intensity = new double[ms2Targets.Length];
/*ms2Targets =
* new List<double>
* {
* 126.127725,
* 127.124760,
* 127.131079,
* 128.128114,
* 128.134433,
* 129.131468,
* 129.137787,
* 130.134822,
* 130.141141,
* 131.138176,
* 651.346052153,
* 951.478187153,
* 1167.494641153
* };*/
}
public override ResultItems Analyze(RawFileName rawfile, SettingsForAnalysis settingName, UIsettings fixedSettings)
{
//Stopwatch stopwatch = new Stopwatch();
//stopwatch.Start();
ResultItems results = GetResultItems(fixedSettings);
//MsScanParameters msScanParameters = new MsScanParameters(settingName);
//progress.Report("[" + rawfile.FullName + "] Start at " + stopwatch.Elapsed.TotalMinutes.ToString("0.00") + " minutes");
results.SetValue("RawFile", rawfile.BaseName);
results.SetValue("User", rawfile.User);
results.SetValue("MS instrument", rawfile.MsInstrument);
results.SetValue("MS instrument number", rawfile.MsInstrumentDetails);
results.SetValue("LC instrument", rawfile.LcInstrument);
results.SetValue("LC instrument number", rawfile.LcInstrumentDetails);
results.SetValue("Sample type", rawfile.SampleType);
results.SetValue("Sample type details", rawfile.SampleTypeDetails);
results.SetValue("File name convention followed", (!rawfile.SomethingWrongWithFileName).ToString()); // shitty hack, should be bool
results.SetValue("File size (MiB)", rawfile.FileSizeMiB, 1);
results.SetValue("Storage space left (GiB)", rawfile.SpaceLeftGiB, 1);
if (settingName.UseBaseAnalysis)
{
StatusLog statusLog;
using (RawFileProcessing rawfileResults = new RawFileProcessing(raw, rawfile.FullName, settingName))
{
rawfile.SetDates(rawfileResults._filecreation, rawfileResults._msStart, rawfileResults._msEnd);
statusLog = rawfileResults.statusLog;
results.SetValue("Warning log", rawfileResults.WarningMessage);
results.SetValue("MS count", rawfileResults.MsCount, 0);
results.SetValue("MS TIC", rawfileResults.MsTic, 0);
results.SetValue("MS TIC (high percentile)", rawfileResults.MsTicHigh, 0);
results.SetValue("MS signal to noise", rawfileResults.MsS2N, 2);
results.SetValue("MS ion count", rawfileResults.MsIonCount, 0);
results.SetValue("CTCD or PrOSA minimum", rawfileResults.CtcdMin, 3);
results.SetValue("CTCD or PrOSA maximum", rawfileResults.CtcdMax, 3);
results.SetValue("Lock mass correction maximum absolute (ppm)", rawfileResults.LockMassCorrectionMaxAbsolute, 2);
results.SetValue("Large injection time fluctuations", rawfileResults.LargeFluctuations, 1); //set as parameter from usrcontrol
results.SetValue("Polymer01", rawfileResults.Polymer1, 1);
results.SetValue("Polymer02", rawfileResults.Polymer2, 1);
results.SetValue("Polymer03", rawfileResults.Polymer3, 1);
results.SetValue("Polymer04", rawfileResults.Polymer4, 1);
results.SetValue("Polymer05", rawfileResults.Polymer5, 1);
results.SetValue("AdductNa", rawfileResults.AdductNa, 1);
results.SetValue("AdductCa", rawfileResults.AdductCa, 1);
results.SetValue("AdductFe", rawfileResults.AdductFe, 1);
results.SetValue("AdductK", rawfileResults.AdductK, 1);
results.SetValue("AdductFA", rawfileResults.AdductFA, 1);
results.SetValue("AdductNH3", rawfileResults.AdductNH3, 1);
results.SetValue("AdductACN", rawfileResults.AdductACN, 1);
results.SetValue("Precursor 5 percent mass quantile", rawfileResults.PrecursorMass5percent, 1);
results.SetValue("Precursor 95 percent mass quantile", rawfileResults.PrecursorMass95percent, 1);
results.SetValue("MS2 topN", rawfileResults.TopN, 2);
results.SetValue("MS2 count", rawfileResults.Ms2Count, 0);
results.SetValue("MS2 TIC", rawfileResults.Ms2Tic, 0);
results.SetValue("MS2 signal to noise", rawfileResults.Ms2S2N, 2);
results.SetValue("MS2 ion count", rawfileResults.Ms2IonCount, 0);
results.SetValue("MS2 precursor isolation width TIC", rawfileResults.Ms2IsolationWindowTic, 0);
results.SetValue("MS2TIC / MS2 precursor isolation width TIC", rawfileResults.Ms2IsolationWindowTicRelative, 3);
results.SetValue("MS2 count above ion threshold", rawfileResults.Ms2ScansAboveIonThreshold, 0);
results.SetValue("MS2 with charge 2 in percent", rawfileResults.Ms2charge2, 2);
results.SetValue("MS2 with charge 3 in percent", rawfileResults.Ms2charge3, 2);
results.SetValue("MS2 with charge 4 or higher in percent", rawfileResults.Ms2charge4orMore, 2);
results.SetValue("MS2 scans XIC above precursor percentage", rawfileResults.Ms2ScansAbovePrecursorPercent, 2);
results.SetValue("MS2 count above ion threshold percentage", rawfileResults.Ms2ScansAboveIonThresholdPercent, 1);
results.SetValue("MS2 scans with precursor as BP percentage", rawfileResults.Ms2ScansWithPrecursorBasePeakPercent, 2);
}
results.SetValue("LCMS start", rawfile.GetDates(0));
results.SetValue("MS start", rawfile.GetDates(1));
results.SetValue("LCMS end", rawfile.GetDates(2));
results.SetValue("MS time in minutes", rawfile.MsTime, 2);
results.SetValue("LC time in minutes", rawfile.LcTime, 2);
Dictionary <int, Tuple <double, double, double> > usage = rawfile.GetUsagePercent();
foreach (int day in usage.Keys)
{
results.SetValue("Last " + day.ToString() + " day(s) LCMS usage percent", usage[day].Item1, 1);
results.SetValue("Last " + day.ToString() + " day(s) MS usage percent", usage[day].Item2, 1);
results.SetValue("Last " + day.ToString() + " day(s) MS excl QC usage percent", usage[day].Item3, 1);
}
//status log entries
foreach (StatusLogItem item in fixedSettings.statusLogItems)
{
if (item.CalculateMedian)
{
results.SetValue(item.Name, statusLog.GetMedianOf(item.Name));
}
if (item.CalculateMin)
{
results.SetValue(item.Name + " min", statusLog.GetMinOf(item.Name));
}
if (item.CalculateMax)
{
results.SetValue(item.Name + " max", statusLog.GetMaxOf(item.Name));
}
if (item.CalculateIQR)
{
results.SetValue(item.Name + " IQR", statusLog.GetIqrOf(item.Name));
}
}
}
results.SetValue("Analysis tool version", "0.0.1.21");
return(results);
}
public override ResultItems Analyze(RawFileName rawfile, SettingsForAnalysis settingName, UIsettings fixedSettings)
{
ResultItems results = GetResultItems(fixedSettings);
List <double> resultlist = new List <double>();
if (settingName.UseExternal1Analysis)
{
Log.Information("Started external1");
string filename = rawfile.FullName;//"f:/temp2/20190114_QE99_Evo1_DBJ_E7_LFQprot_QC4_HELA_PAC_250ng_7500_01.raw";
//get list of files and column/row names
List <string> resultfiles = new List <string>();
List <string> resultnames = new List <string>();
foreach (ExternalResultParseSetting settings in fixedSettings.ExternalResults)
{
string resultfile = settings.ResultFile;
resultfile = resultfile.Replace("{FILE}", rawfile.FullName);
resultfiles.Add(resultfile);
resultnames.Add(settings.Resultname);
}
//maybe add option to delete resultfile?
try
{
string arguments = fixedSettings.Arguments;
arguments = arguments.Replace("{FILE}", rawfile.FullName);
resultlist = ExecuteProgram.GetResultsFromExternalProgram(
fixedSettings.Executable, //raw.exe
arguments, //"parse -x -f " + filename
resultfiles.ToArray(),
resultnames.ToArray(),
fixedSettings.Timeout,
fixedSettings.ExternalToolSepChar,
!fixedSettings.AsRow);
if (resultlist.Count != fixedSettings.ExternalResults.Count)
{
resultlist.Clear();
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
Log.Error("External1 result was not parsed correctly (are headers correct?)");
}
Log.Information("Finished external1");
}
catch
{
//something went wrong, adding 0s as filler and reporting error
Log.Error("External1 failed");
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
}
}
else
{
//no analysis requested, adding 0s as filler
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
}
//set results
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
results.SetValue(fixedSettings.ExternalResults[i].Resultname, resultlist[i]);
}
return(results);
}
// Relies on assumption that the rawscans are sorted, i.e. the currentFullMS is set for each ms2 scan
private Tuple <List <MsScan>, List <Ms2Scan> > ParseSpectra(BlockingCollection <RawScan> rawScanQueue, SettingsForAnalysis fixedSettings)
{
List <Tuple <int, double, int> > polymerTargetList = ExtractPolymers(fixedSettings.Polymers);
List <MsScan> msScanList = new List <MsScan>(1000);
List <Ms2Scan> ms2ScanList = new List <Ms2Scan>(1000);
RawScan currentFullMS = null;
foreach (RawScan rawScan in rawScanQueue.GetConsumingEnumerable())
{
if (rawScan.IsMS2())
{
ms2ScanList.Add(new Ms2Scan(rawScan, fixedSettings.StoNthreshold, currentFullMS));
}
else
{
currentFullMS = rawScan;
msScanList.Add(new MsScan(rawScan, fixedSettings.StoNthreshold, polymerTargetList, fixedSettings.Polymers.Count, fixedSettings.Adducts, fixedSettings.BaseMassTolerance));
}
}
return(new Tuple <List <MsScan>, List <Ms2Scan> >(msScanList, ms2ScanList));
}
public virtual ResultItems Analyze(RawFileName rawfile, SettingsForAnalysis settingName, UIsettings settings)
{
return(GetResultItems(settings));
}