private void DoSaveResultsToFile(UIsettings fixedSettings, ResultItems results)
{
if (fixedSettings.UseStorageFile)
{
string filePath = Path.GetDirectoryName(fixedSettings.Storagefile);
if (!Directory.Exists(filePath))
{
Log.Error("[" + fixedSettings.Storagefile + "] Something is wrong with the file destination, skipping saving of results");
}
else if (!File.Exists(fixedSettings.Storagefile))
{
Log.Information("[" + fixedSettings.Storagefile + "] Creating file and writing results");
using (StreamWriter fileStream = new StreamWriter(fixedSettings.Storagefile))
{
fileStream.WriteLine(String.Join("\t", results.Names));
fileStream.WriteLine(String.Join("\t", results.Results));
}
}
else if (IsFileLocked(fixedSettings.Storagefile))
{
Log.Information("[" + fixedSettings.Storagefile + "] File is busy, skipping saving of results");
}
else
{
Log.Information("[" + fixedSettings.Storagefile + "] Appending results to file");
using (StreamWriter fileStream = new StreamWriter(fixedSettings.Storagefile, true))
{
fileStream.WriteLine(String.Join("\t", results.Results));
}
}
}
}
//add log here
public void Analyze(RawFileName rawfile, UIsettings fixedSettings, SettingsForAnalysis fixedSettingsForAnalysis, IProgress <DataRow> resultRow)
{
if (IsFileLocked(rawfile.FullName))
{
Log.Error("[" + rawfile.FullName + "] CRITICAL problem: file locked by another process! Skipping file");
}
else if (IsNotRawfile(rawfile.FullName))
{
Log.Error("[" + rawfile.FullName + "] CRITICAL problem: raw file corrupt! Skipping file");
}
else
{
Log.Information("[" + rawfile.FullName + "] Main analysis started");
rawfile.SetCategories(fixedSettings.UserList, fixedSettings.LcList, fixedSettings.MsList, fixedSettings.SamplesList);
SetRawFileHistory(rawfile, fixedSettings);//This is a mess with the class lines
if (rawfile.SomethingWrongWithFileName)
{
Log.Warning("[" + rawfile.FullName + "] Please check file naming convention: parse rules and filename do not align.");
}
using (ResultItems results = new ResultItems())
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
MainAnalysisThreadedTask(rawfile, fixedSettingsForAnalysis, fixedSettings, results);
Log.Information("[" + rawfile.FullName + "] Main analysis finished in " + stopwatch.Elapsed.TotalMinutes.ToString("0.00") + " minutes");
DoEmailResults(rawfile, fixedSettings, results);
DoSqlUpdateResults(rawfile, fixedSettings, results);
DoSaveResultsToFile(fixedSettings, results);
AddResultRowToDataTable(results.Names, results.Results, results.Types, fixedSettings, resultRow);
}
}
}
public List <Tuple <string, Type> > GetResultList(UIsettings settingName)
{
ResultItems results = GetResultItems(settingName);
List <Tuple <string, Type> > nameType = new List <Tuple <string, Type> >();
for (int i = 0; i < results.Count; i++)
{
nameType.Add(new Tuple <string, Type>(results.Names[i], results.Types[i]));
}
return(nameType);
}
public void Add(ResultItems newItems)
{
for (int i = 0; i < newItems.Names.Count; i++)
{
if (!resultDicIndex.ContainsKey(newItems.Names[i]))
{
resultDicIndex.Add(newItems.Names[i], Names.Count);
Names.Add(newItems.Names[i]);
Types.Add(newItems.Types[i]);
Results.Add(newItems.Results[i]);
}
}
}
private static void MainAnalysisThreadedTask(RawFileName rawfile, SettingsForAnalysis fixedSettingsForAnalysis, UIsettings fixedSettings, ResultItems results)
{
//ResultItems results = new ResultItems();
//this can now be done threaded
List <Task <ResultItems> > taskList = new List <Task <ResultItems> >();
taskList.Add(Task.Run(() =>
{
ResultItems results1 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisMorpheus())
{
results1 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
GC.Collect();//ugly but this solves problem with memory not released
return(results1);
}));
taskList.Add(Task.Run(() =>
{
ResultItems results2 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisBase())
{
results2 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
return(results2);
}));
taskList.Add(Task.Run(() =>
{
ResultItems results3 = new ResultItems();
using (AnalysisTemplate curMethod = new AnalysisExternal1())
{
results3 = curMethod.Analyze(rawfile, fixedSettingsForAnalysis, fixedSettings);
}
return(results3);
}));
Task.WaitAll(taskList.ToArray());
foreach (Task <ResultItems> resultTask in taskList)
{
results.Add(resultTask.Result);
}
//return results;
}
protected override ResultItems GetResultItems(UIsettings settingName)
{
ResultItems results = new ResultItems();
foreach (ExternalResultParseSetting item in settingName.ExternalResults)
{
results.Add(item.Resultname, typeof(double));
}
/*
* results.Add("MedianMS1IsolationInterference", typeof(double));
* results.Add("MedianPeakWidthAt10Percent(s)", typeof(double));
* results.Add("MedianPeakWidthAtHalfMax(s)", typeof(double));
* results.Add("MedianAsymmetryFactor", typeof(double));
* results.Add("ColumnCapacity", typeof(double));
*/
return(results);
}
protected override ResultItems GetResultItems(UIsettings settingName)
{
ResultItems results = new ResultItems();
results.Add("RawFile", typeof(string));
results.Add("User", typeof(string));
results.Add("MS instrument", typeof(string));
results.Add("MS instrument number", typeof(string));
results.Add("LC instrument", typeof(string));
results.Add("LC instrument number", typeof(string));
results.Add("Sample type", typeof(string));
results.Add("Sample type details", typeof(string));
results.Add("File name convention followed", typeof(string));
results.Add("LCMS start", typeof(DateTime));
results.Add("MS start", typeof(DateTime));
results.Add("LCMS end", typeof(DateTime));
results.Add("MS time in minutes", typeof(double));
results.Add("LC time in minutes", typeof(double));
results.Add("File size (MiB)", typeof(double));
results.Add("Storage space left (GiB)", typeof(double));
results.Add("MS count", typeof(int));
results.Add("MS2 count", typeof(int));
results.Add("MS TIC", typeof(double));
results.Add("MS TIC (high percentile)", typeof(double));
results.Add("MS2 TIC", typeof(double));
results.Add("MS signal to noise", typeof(double));
results.Add("MS2 signal to noise", typeof(double));
results.Add("MS ion count", typeof(double));
results.Add("MS2 ion count", typeof(double));
results.Add("MS2 topN", typeof(double));
results.Add("Large injection time fluctuations", typeof(double));
results.Add("CTCD or PrOSA minimum", typeof(double));
results.Add("CTCD or PrOSA maximum", typeof(double));
results.Add("Lock mass correction maximum absolute (ppm)", typeof(double));
results.Add("MS2 count above ion threshold", typeof(int));
results.Add("MS2 count above ion threshold percentage", typeof(double));
results.Add("MS2 precursor isolation width TIC", typeof(double));
results.Add("MS2TIC / MS2 precursor isolation width TIC", typeof(double));
results.Add("MS2 scans with precursor as BP percentage", typeof(double));
results.Add("MS2 scans XIC above precursor percentage", typeof(double));
results.Add("MS2 with charge 2 in percent", typeof(double));
results.Add("MS2 with charge 3 in percent", typeof(double));
results.Add("MS2 with charge 4 or higher in percent", typeof(double));
foreach (int day in new int[] { 1, 7, 30, 90 })
{
results.Add("Last " + day.ToString() + " day(s) LCMS usage percent", typeof(double));
results.Add("Last " + day.ToString() + " day(s) MS usage percent", typeof(double));
results.Add("Last " + day.ToString() + " day(s) MS excl QC usage percent", typeof(double));
}
results.Add("Polymer01", typeof(double));
results.Add("Polymer02", typeof(double));
results.Add("Polymer03", typeof(double));
results.Add("Polymer04", typeof(double));
results.Add("Polymer05", typeof(double));
results.Add("AdductNa", typeof(double));
results.Add("AdductCa", typeof(double));
results.Add("AdductFe", typeof(double));
results.Add("AdductK", typeof(double));
results.Add("AdductFA", typeof(double));
results.Add("AdductNH3", typeof(double));
results.Add("AdductACN", typeof(double));
results.Add("Precursor 5 percent mass quantile", typeof(double));
results.Add("Precursor 95 percent mass quantile", typeof(double));
results.Add("Warning log", typeof(string));
results.Add("Analysis tool version", typeof(string));
foreach (StatusLogItem item in settingName.statusLogItems)
{
if (item.CalculateMedian)
{
results.Add(item.Name, typeof(double));
}
if (item.CalculateMin)
{
results.Add(item.Name + " min", typeof(double));
}
if (item.CalculateMax)
{
results.Add(item.Name + " max", typeof(double));
}
if (item.CalculateIQR)
{
results.Add(item.Name + " IQR", typeof(double));
}
}
return(results);
}
public override ResultItems Analyze(RawFileName rawfile, SettingsForAnalysis settingName, UIsettings fixedSettings)
{
//Stopwatch stopwatch = new Stopwatch();
//stopwatch.Start();
ResultItems results = GetResultItems(fixedSettings);
//MsScanParameters msScanParameters = new MsScanParameters(settingName);
//progress.Report("[" + rawfile.FullName + "] Start at " + stopwatch.Elapsed.TotalMinutes.ToString("0.00") + " minutes");
results.SetValue("RawFile", rawfile.BaseName);
results.SetValue("User", rawfile.User);
results.SetValue("MS instrument", rawfile.MsInstrument);
results.SetValue("MS instrument number", rawfile.MsInstrumentDetails);
results.SetValue("LC instrument", rawfile.LcInstrument);
results.SetValue("LC instrument number", rawfile.LcInstrumentDetails);
results.SetValue("Sample type", rawfile.SampleType);
results.SetValue("Sample type details", rawfile.SampleTypeDetails);
results.SetValue("File name convention followed", (!rawfile.SomethingWrongWithFileName).ToString()); // shitty hack, should be bool
results.SetValue("File size (MiB)", rawfile.FileSizeMiB, 1);
results.SetValue("Storage space left (GiB)", rawfile.SpaceLeftGiB, 1);
if (settingName.UseBaseAnalysis)
{
StatusLog statusLog;
using (RawFileProcessing rawfileResults = new RawFileProcessing(raw, rawfile.FullName, settingName))
{
rawfile.SetDates(rawfileResults._filecreation, rawfileResults._msStart, rawfileResults._msEnd);
statusLog = rawfileResults.statusLog;
results.SetValue("Warning log", rawfileResults.WarningMessage);
results.SetValue("MS count", rawfileResults.MsCount, 0);
results.SetValue("MS TIC", rawfileResults.MsTic, 0);
results.SetValue("MS TIC (high percentile)", rawfileResults.MsTicHigh, 0);
results.SetValue("MS signal to noise", rawfileResults.MsS2N, 2);
results.SetValue("MS ion count", rawfileResults.MsIonCount, 0);
results.SetValue("CTCD or PrOSA minimum", rawfileResults.CtcdMin, 3);
results.SetValue("CTCD or PrOSA maximum", rawfileResults.CtcdMax, 3);
results.SetValue("Lock mass correction maximum absolute (ppm)", rawfileResults.LockMassCorrectionMaxAbsolute, 2);
results.SetValue("Large injection time fluctuations", rawfileResults.LargeFluctuations, 1); //set as parameter from usrcontrol
results.SetValue("Polymer01", rawfileResults.Polymer1, 1);
results.SetValue("Polymer02", rawfileResults.Polymer2, 1);
results.SetValue("Polymer03", rawfileResults.Polymer3, 1);
results.SetValue("Polymer04", rawfileResults.Polymer4, 1);
results.SetValue("Polymer05", rawfileResults.Polymer5, 1);
results.SetValue("AdductNa", rawfileResults.AdductNa, 1);
results.SetValue("AdductCa", rawfileResults.AdductCa, 1);
results.SetValue("AdductFe", rawfileResults.AdductFe, 1);
results.SetValue("AdductK", rawfileResults.AdductK, 1);
results.SetValue("AdductFA", rawfileResults.AdductFA, 1);
results.SetValue("AdductNH3", rawfileResults.AdductNH3, 1);
results.SetValue("AdductACN", rawfileResults.AdductACN, 1);
results.SetValue("Precursor 5 percent mass quantile", rawfileResults.PrecursorMass5percent, 1);
results.SetValue("Precursor 95 percent mass quantile", rawfileResults.PrecursorMass95percent, 1);
results.SetValue("MS2 topN", rawfileResults.TopN, 2);
results.SetValue("MS2 count", rawfileResults.Ms2Count, 0);
results.SetValue("MS2 TIC", rawfileResults.Ms2Tic, 0);
results.SetValue("MS2 signal to noise", rawfileResults.Ms2S2N, 2);
results.SetValue("MS2 ion count", rawfileResults.Ms2IonCount, 0);
results.SetValue("MS2 precursor isolation width TIC", rawfileResults.Ms2IsolationWindowTic, 0);
results.SetValue("MS2TIC / MS2 precursor isolation width TIC", rawfileResults.Ms2IsolationWindowTicRelative, 3);
results.SetValue("MS2 count above ion threshold", rawfileResults.Ms2ScansAboveIonThreshold, 0);
results.SetValue("MS2 with charge 2 in percent", rawfileResults.Ms2charge2, 2);
results.SetValue("MS2 with charge 3 in percent", rawfileResults.Ms2charge3, 2);
results.SetValue("MS2 with charge 4 or higher in percent", rawfileResults.Ms2charge4orMore, 2);
results.SetValue("MS2 scans XIC above precursor percentage", rawfileResults.Ms2ScansAbovePrecursorPercent, 2);
results.SetValue("MS2 count above ion threshold percentage", rawfileResults.Ms2ScansAboveIonThresholdPercent, 1);
results.SetValue("MS2 scans with precursor as BP percentage", rawfileResults.Ms2ScansWithPrecursorBasePeakPercent, 2);
}
results.SetValue("LCMS start", rawfile.GetDates(0));
results.SetValue("MS start", rawfile.GetDates(1));
results.SetValue("LCMS end", rawfile.GetDates(2));
results.SetValue("MS time in minutes", rawfile.MsTime, 2);
results.SetValue("LC time in minutes", rawfile.LcTime, 2);
Dictionary <int, Tuple <double, double, double> > usage = rawfile.GetUsagePercent();
foreach (int day in usage.Keys)
{
results.SetValue("Last " + day.ToString() + " day(s) LCMS usage percent", usage[day].Item1, 1);
results.SetValue("Last " + day.ToString() + " day(s) MS usage percent", usage[day].Item2, 1);
results.SetValue("Last " + day.ToString() + " day(s) MS excl QC usage percent", usage[day].Item3, 1);
}
//status log entries
foreach (StatusLogItem item in fixedSettings.statusLogItems)
{
if (item.CalculateMedian)
{
results.SetValue(item.Name, statusLog.GetMedianOf(item.Name));
}
if (item.CalculateMin)
{
results.SetValue(item.Name + " min", statusLog.GetMinOf(item.Name));
}
if (item.CalculateMax)
{
results.SetValue(item.Name + " max", statusLog.GetMaxOf(item.Name));
}
if (item.CalculateIQR)
{
results.SetValue(item.Name + " IQR", statusLog.GetIqrOf(item.Name));
}
}
}
results.SetValue("Analysis tool version", "0.0.1.21");
return(results);
}
public override ResultItems Analyze(RawFileName rawfile, SettingsForAnalysis settingName, UIsettings fixedSettings)
{
ResultItems results = GetResultItems(fixedSettings);
List <double> resultlist = new List <double>();
if (settingName.UseExternal1Analysis)
{
Log.Information("Started external1");
string filename = rawfile.FullName;//"f:/temp2/20190114_QE99_Evo1_DBJ_E7_LFQprot_QC4_HELA_PAC_250ng_7500_01.raw";
//get list of files and column/row names
List <string> resultfiles = new List <string>();
List <string> resultnames = new List <string>();
foreach (ExternalResultParseSetting settings in fixedSettings.ExternalResults)
{
string resultfile = settings.ResultFile;
resultfile = resultfile.Replace("{FILE}", rawfile.FullName);
resultfiles.Add(resultfile);
resultnames.Add(settings.Resultname);
}
//maybe add option to delete resultfile?
try
{
string arguments = fixedSettings.Arguments;
arguments = arguments.Replace("{FILE}", rawfile.FullName);
resultlist = ExecuteProgram.GetResultsFromExternalProgram(
fixedSettings.Executable, //raw.exe
arguments, //"parse -x -f " + filename
resultfiles.ToArray(),
resultnames.ToArray(),
fixedSettings.Timeout,
fixedSettings.ExternalToolSepChar,
!fixedSettings.AsRow);
if (resultlist.Count != fixedSettings.ExternalResults.Count)
{
resultlist.Clear();
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
Log.Error("External1 result was not parsed correctly (are headers correct?)");
}
Log.Information("Finished external1");
}
catch
{
//something went wrong, adding 0s as filler and reporting error
Log.Error("External1 failed");
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
}
}
else
{
//no analysis requested, adding 0s as filler
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
resultlist.Add(0.0);
}
}
//set results
for (int i = 0; i < fixedSettings.ExternalResults.Count; i++)
{
results.SetValue(fixedSettings.ExternalResults[i].Resultname, resultlist[i]);
}
return(results);
}
private static void DoEmailResults(RawFileName rawfile, UIsettings fixedSettings, ResultItems results)
{
if (fixedSettings.UseEmail)
{
using (EmailSystem emailSystem = new EmailSystem(fixedSettings.Mailserveradress, fixedSettings.Mailportname, fixedSettings.Mailusername, fixedSettings.Mailpassword))
{
if (emailSystem.Enabled)
{
if (!emailSystem.CheckEmailAdress(rawfile.UserEmail, rawfile.User))
{
Log.Error("[" + rawfile.FullName + "] " + "Email can't be sent, email >" + rawfile.UserEmail + "< not correct for user: "******"");
string[] compareSymbols = fixedSettings.CompareSymbols;
foreach (InformationKey ik in fixedSettings.informationKey)
{
bool addMessage;
if (ik.Message != "")
{
addMessage = CustomCompare(ik.Name, ik.ComparisonText1, results.Types[ik.ResultItemIndex1], results.Results[ik.ResultItemIndex1], compareSymbols[ik.ComparisonIndex1]);
addMessage = addMessage && CustomCompare(ik.Name, ik.ComparisonText2, results.Types[ik.ResultItemIndex2], results.Results[ik.ResultItemIndex2], compareSymbols[ik.ComparisonIndex2]);
addMessage = addMessage && CustomCompare(ik.Name, ik.ComparisonText3, results.Types[ik.ResultItemIndex3], results.Results[ik.ResultItemIndex3], compareSymbols[ik.ComparisonIndex3]);
addMessage = addMessage && CustomCompare(ik.Name, ik.ComparisonText4, results.Types[ik.ResultItemIndex4], results.Results[ik.ResultItemIndex4], compareSymbols[ik.ComparisonIndex4]);
addMessage = addMessage && CustomCompare(ik.Name, ik.ComparisonText5, results.Types[ik.ResultItemIndex5], results.Results[ik.ResultItemIndex5], compareSymbols[ik.ComparisonIndex5]);
if (addMessage)
{
messageContent.Append(ik.Name);
messageContent.Append(": ");
messageContent.Append(ik.Message);
messageContent.Append("<br />");
}
}
}
StringBuilder bodyContent = new StringBuilder("<table>");
for (int i = 0; i < results.Count; i++)
{
bodyContent.Append("<tr><td>");
bodyContent.Append(results.Names[i]);
bodyContent.Append("</td><td>");
bodyContent.Append(results.Results[i]);
bodyContent.Append("</td></tr>");
}
bodyContent.Append("</table>");
string mailresult = emailSystem.SendMail(rawfile.UserEmail, rawfile.User, rawfile.BaseName, bodyContent.ToString(), messageContent.ToString(), rawfile.CcEmails);
Log.Information("[" + rawfile.FullName + "] " + "UseEmail system reported " + mailresult);
}
}
else
{
Log.Error("[" + rawfile.FullName + "] " + "UseEmail system couldn't initialize, is the server configured correctly?");
}
}
}
}
private static void DoSqlUpdateResults(RawFileName rawfile, UIsettings fixedSettings, ResultItems results)
{
if (fixedSettings.UseSql)
{
Log.Information("[" + rawfile.FullName + "] Uploading results to database");
using (SQL newsql = new SQL(fixedSettings.Sqlserveradress, fixedSettings.Sqldatabase, fixedSettings.Sqltable, fixedSettings.Sqlusername, fixedSettings.Sqlpassword))
{
newsql.Insert(results.Names, results.ResultsWithMaxLength(255));
}
}
}
protected virtual ResultItems GetResultItems(UIsettings settingName)
{
ResultItems results = new ResultItems();
return(results);
}