public static void ExtractTrailerExtra(this RawDataCollection rawData, IRawDataPlus rawFile)
{
if (rawData.Performed.Contains(Operations.TrailerExtras))
{
return;
}
Log.Information("Extracting trailer extras");
rawData.trailerExtras = new Dictionary <int, TrailerExtraData>();
TrailerExtraIndices indices = new TrailerExtraIndices(rawFile);
Double[] spsMasses;
IEnumerable <int> scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Any];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Extracting trailer extra data");
foreach (int scan in scans)
{
try
{
rawData.trailerExtras.Add(scan, ExtractTrailerExtra(rawData, rawFile, scan, indices));
}
catch (Exception e)
{
Log.Error(e, "Failed during extraction of trailer extra {Scan}", scan);
throw e;
}
progress.Update();
}
progress.Done();
rawData.Performed.Add(Operations.TrailerExtras);
}
public static void RunXTandem(this RawDataCollection rawData, SearchParameters searchParameters, string mgfFile, string outputFile, bool genDecoy)
{
XElement customPars, taxonomy;
// make a decoy database, if requested
if (genDecoy)
{
// check if the decoy database already exists
if (!searchParameters.FastaDatabase.EndsWith(".TARGET_DECOY.fasta"))
{
FastaManipulation.ReverseDecoy(searchParameters.FastaDatabase);
searchParameters.FastaDatabase = searchParameters.FastaDatabase + ".TARGET_DECOY.fasta";
}
}
// write out the default input file
XElement.Parse(Properties.Resources.XTandem_default_config).Save(Path.Combine(searchParameters.XTandemDirectory, "RawTools_default_config.xml"));
// set up and write the taxonomy file
taxonomy = XElement.Parse(Properties.Resources.XTandem_taxonomy);
taxonomy.UpdateTaxonomy(searchParameters);
taxonomy.Save(Path.Combine(searchParameters.XTandemDirectory, "RawTools_taxonomy.xml"));
// set up and write the custom input file
customPars = XElement.Parse(Properties.Resources.XTandem_custom_config);
customPars.UpdateCustomXParameters(searchParameters, rawData, mgfFile, outputFile);
string xTandemParameters = Path.Combine(searchParameters.XTandemDirectory, "RawTools_custom_config.xml");
customPars.Save(xTandemParameters);
ConsoleUtils.VoidBash(Path.Combine(searchParameters.XTandemDirectory, "tandem.exe"), xTandemParameters);
}
public static XElement LoadSearchResults(QcParameters qcParameters, RawDataCollection rawData)
{
string QcSearchDataDirectory = qcParameters.QcSearchDataDirectory;
string resultsFile = Path.Combine(QcSearchDataDirectory, Path.GetFileName(rawData.rawFileName) + ".pep.xml");
return(XElement.Load(resultsFile));
}
public static void CalcPeakRetTimesAndInts(this RawDataCollection rawData, IRawDataPlus rawFile)
{
CheckIfDone.Check(rawData, rawFile, new List <Operations> {
Operations.ScanIndex, Operations.PrecursorMasses, Operations.PrecursorScans, Operations.Ms1CentroidStreams, Operations.RetentionTimes
});
if (rawData.Performed.Contains(Operations.PeakRetAndInt))
{
return;
}
int[] scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2];
PrecursorPeakDataCollection peaks = new PrecursorPeakDataCollection();
ProgressIndicator P = new ProgressIndicator(total: scans.Length, message: "Analyzing precursor peaks");
foreach (int scan in scans)
{
peaks.Add(scan, OnePeak(rawData: rawData, monoIsoMass: rawData.precursorMasses[scan].MonoisotopicMZ, parentScan: rawData.precursorScans[scan].MasterScan, ddScan: scan));
P.Update();
}
P.Done();
rawData.peakData = peaks;
rawData.Performed.Add(Operations.PeakRetAndInt);
rawData.Performed.RemoveWhere(x => x == Operations.PeakArea);
}
public static void WriteSearchMGF(QcParameters qcParameters, RawDataCollection rawData, IRawDataPlus rawFile, bool fixedScans = false)
{
var pars = qcParameters.searchParameters;
int[] scans = AdditionalMath.SelectRandomScans(scans: rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2], num: pars.NumSpectra, fixedScans: fixedScans);
MGF.WriteMGF(rawData, rawFile, qcParameters.QcSearchDataDirectory, pars.MgfMassCutoff, scans, pars.MgfIntensityCutoff);
}
private void FillRawDataListBoxWithBookInformation()
{
for (var index = 0; index < RawDataCollection.GetItemsCount(); index++)
{
this.lstRawData.Items.Add(RawDataCollection.GetItemAt(index));
}
}
public static void ExtractRetentionTimes(this RawDataCollection rawData, IRawDataPlus rawFile)
{
if (rawData.Performed.Contains(Operations.RetentionTimes))
{
return;
}
if (!rawData.Performed.Contains(Operations.ScanIndex))
{
rawData.ExtractScanIndex(rawFile);
}
Log.Information("Extracting retention times");
rawData.retentionTimes = new Dictionary <int, double>();
IEnumerable <int> scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Any];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Extracting retention times");
foreach (int scan in scans)
{
try
{
rawData.retentionTimes.Add(scan, rawFile.RetentionTimeFromScanNumber(scan));
}
catch (Exception e)
{
Log.Error(e, "Failed during extraction of retention time for scan {Scan}", scan);
throw e;
}
progress.Update();
}
progress.Done();
rawData.Performed.Add(Operations.RetentionTimes);
}
public static void ExtractPrecursorMasses(this RawDataCollection rawData, IRawDataPlus rawFile)
{
if (rawData.Performed.Contains(Operations.PrecursorMasses))
{
return;
}
CheckIfDone.Check(rawData, rawFile, new List <Operations> {
Operations.ScanIndex, Operations.TrailerExtras, Operations.PrecursorScans
});
Log.Information("Extracting precursor masses");
rawData.precursorMasses = new Dictionary <int, PrecursorMassData>();
IEnumerable <int> scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Any];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Extracting precursor masses");
foreach (int scan in scans)
{
try
{
ExtractPrecursorMasses(rawData, rawFile, scan);
}
catch (Exception e)
{
Log.Error(e, "Failed during extraction of precursor mass for scan {Scan}", scan);
throw e;
}
progress.Update();
}
progress.Done();
rawData.Performed.Add(Operations.PrecursorMasses);
}
public static void ExtractSegmentScans(this RawDataCollection rawData, IRawDataPlus rawFile, MSOrderType MSOrder)
{
if (MSOrder == MSOrderType.Ms)
{
if (rawData.Performed.Contains(Operations.Ms1SegmentedScans))
{
return;
}
}
else
{
if (MSOrder == MSOrderType.Ms2)
{
if (rawData.Performed.Contains(Operations.Ms2SegmentedScans))
{
return;
}
}
else
{
if (rawData.Performed.Contains(Operations.Ms3SegmentedScans))
{
return;
}
}
}
Log.Information("Extracting {MSOrder} segment scans", MSOrder);
IEnumerable <int> scans = rawData.scanIndex.ScanEnumerators[MSOrder];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), string.Format("Extracting {0} segment scans", MSOrder));
try
{
rawData.Extract(rawFile, scans, progress);
}
catch (Exception e)
{
Log.Error(e, "Failed while extracting {MSOrder} segment scans", MSOrder);
throw e;
}
progress.Done();
if (MSOrder == MSOrderType.Ms)
{
rawData.Performed.Add(Operations.Ms1SegmentedScans);
}
else
{
if (MSOrder == MSOrderType.Ms2)
{
rawData.Performed.Add(Operations.Ms2SegmentedScans);
}
else
{
rawData.Performed.Add(Operations.Ms3SegmentedScans);
}
}
}
public static void ExtractSegmentScans(this RawDataCollection rawData, IRawDataPlus rawFile, IEnumerable <int> scans)
{
Log.Information("Extracting custom defined collection of segment scans");
ProgressIndicator progress = new ProgressIndicator(scans.Count(), string.Format("Extracting custom collection of segment scans"));
Extract(rawData, rawFile, scans, progress);
progress.Done();
}
public static void ParseSearchResults(this QcDataContainer qcData, RawDataCollection rawData, IRawDataPlus rawFile, QcParameters qcParameters)
{
XElement results = LoadSearchResults(qcParameters, rawData);
PsmDataCollection Psms = ExtractPsmData(results, qcParameters.searchParameters.SearchAlgorithm);
qcData.ParsePSMs(Psms, qcParameters);
}
private static bool FillBackUpList()
{
{
RawDataCollection.AddItem(value);
}
return(RawDataCollection.GetItemsCount() > 0);
}
public static void ExtractPrecursorScans(this RawDataCollection rawData, IRawDataPlus rawFile)
{
if (rawData.Performed.Contains(Operations.PrecursorScans))
{
return;
}
if (!rawData.Performed.Contains(Operations.ScanIndex))
{
rawData.ExtractScanIndex(rawFile);
}
Log.Information("Extracting scan dependents/precursor scans");
int ms2Scan = -1;
int ms3Scan = -1;
IEnumerable <int> scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms];
ProgressIndicator progress = new ProgressIndicator(scans.Count(), "Indexing linked scan events");
foreach (int scan in scans)
{
var scanDependents = rawFile.GetScanDependents(scan, 4);
// check if the ms1 scan has dependent scans
if (scanDependents == null)
{
continue;
}
for (int i = 0; i < scanDependents.ScanDependentDetailArray.Length; i++)
{
if (rawData.scanIndex.AnalysisOrder == MSOrderType.Ms2) // it is ms2
{
ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex;
rawData.precursorScans.Add(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
}
else // it is ms3
{
ms2Scan = scanDependents.ScanDependentDetailArray[i].ScanIndex;
if (rawFile.GetScanDependents(ms2Scan, 4).ScanDependentDetailArray.Length != 0) // make sure there is ms3 data
{
ms3Scan = rawFile.GetScanDependents(ms2Scan, 4).ScanDependentDetailArray[0].ScanIndex;
rawData.precursorScans.Add(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
rawData.precursorScans.Add(ms3Scan, new PrecursorScanData(ms3scan: ms3Scan, ms2Scan: ms2Scan, masterScan: scan));
}
else
{
// there is no ms3 scan, so we only add the ms2 scan
rawData.precursorScans.Add(ms2Scan, new PrecursorScanData(ms2scan: ms2Scan, masterScan: scan));
}
}
}
progress.Update();
}
progress.Done();
rawData.Performed.Add(Operations.PrecursorScans);
}
private static bool FillRawDataCollection(IEnumerable <string> data)
{
foreach (var value in data)
{
RawDataCollection.AddItem(value);
}
return(RawDataCollection.GetItemsCount() > 0);
}
public static void WriteMatrix(RawDataCollection rawData, MetricsData metrics, string outputDirectory = null)
{
string fileName = ReadWrite.GetPathToFile(outputDirectory, rawData.rawFileName, "_Metrics.txt");
using (StreamWriter f = new StreamWriter(fileName)) //Open a new file
{
f.WriteLine("RawFile:\t" + metrics.RawFileName);
f.WriteLine("Instrument:\t" + metrics.Instrument);
f.WriteLine("ExperimentOrder:\t" + metrics.MSOrder);
f.WriteLine("MS1Analyzer:\t" + metrics.MS1Analyzer);
f.WriteLine("MS2Analyzer:\t" + metrics.MS2Analyzer);
if (metrics.MSOrder == MSOrderType.Ms3)
{
f.WriteLine("MS3Analyzer:\t" + metrics.MS3Analyzer);
}
else
{
f.WriteLine("MS3Analyzer:\tNone");
}
f.WriteLine("TotalAnalysisTime(min):\t" + Math.Round(metrics.TotalAnalysisTime, 4));
f.WriteLine("TotalScans:\t" + metrics.TotalScans);
f.WriteLine("MS1Scans:\t" + metrics.MS1Scans);
f.WriteLine("MS2Scans:\t" + metrics.MS2Scans);
f.WriteLine("MS3Scans:\t" + metrics.MS3Scans);
f.WriteLine("MeanTopN:\t" + Math.Round(metrics.MeanTopN, 4));
f.WriteLine("MS1ScanRate(/sec):\t" + Math.Round(metrics.MS1ScanRate / 60, 4));
f.WriteLine("MS2ScanRate(/sec):\t" + Math.Round(metrics.MS2ScanRate / 60, 4));
f.WriteLine("MeanDutyCycle:\t" + Math.Round(metrics.MeanDutyCycle, 4));
f.WriteLine("MedianMS1FillTime:\t" + Math.Round(metrics.MedianMS1FillTime, 4));
f.WriteLine("MedianMS2FillTime:\t" + Math.Round(metrics.MedianMS2FillTime, 4));
f.WriteLine("MedianMS3FillTime:\t" + Math.Round(metrics.MedianMS3FillTime, 4));
f.WriteLine("MedianMS2Intensity:\t" + Math.Round(metrics.MedianSummedMS2Intensity, 4));
f.WriteLine("MedianMS1IsolationInterference:\t" + Math.Round(metrics.MedianMs1IsolationInterference, 4));
if (!rawData.isBoxCar)
{
f.WriteLine("MedianPeakWidthAt10Percent(s):\t" + Math.Round(metrics.MedianBaselinePeakWidth * 60, 4));
f.WriteLine("MedianPeakWidthAtHalfMax(s):\t" + Math.Round(metrics.MedianHalfHeightPeakWidth * 60, 4));
f.WriteLine("MedianAsymmetryFactor:\t" + Math.Round(metrics.MedianAsymmetryFactor, 4));
f.WriteLine("ColumnCapacity:\t" + Math.Round(metrics.PeakCapacity, 4));
}
if (metrics.IncludesQuant == true)
{
foreach (string tag in metrics.QuantMeta.quantTags)
{
f.WriteLine("{0}MedianReporterIonIntensity:\t{1}", tag, metrics.QuantMeta.medianReporterIntensityByChannel[tag]);
}
f.WriteLine("OverallMedianReporterIonIntensity:\t{0}", metrics.QuantMeta.medianReporterIntensity);
}
}
}
public static void ExtractMethodData(this RawDataCollection rawData, IRawDataPlus rawFile)
{
if (rawData.Performed.Contains(Operations.MethodData))
{
return;
}
if (!rawData.Performed.Contains(Operations.ScanIndex))
{
rawData.ExtractScanIndex(rawFile);
}
Log.Information("Extracting method/instrument information");
rawData.methodData = new Containers.MethodData();
rawData.methodData.AnalysisOrder = rawData.scanIndex.AnalysisOrder;
int firstQuantScan = rawData.scanIndex.ScanEnumerators[rawData.scanIndex.AnalysisOrder][0];
rawData.methodData.QuantAnalyzer = rawData.scanIndex.allScans[firstQuantScan].MassAnalyzer;
int firstMs1Scan = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms][0];
rawData.methodData.MassAnalyzers.Add(MSOrderType.Ms, rawData.scanIndex.allScans[firstMs1Scan].MassAnalyzer);
int firstMs2Scan = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2][0];
rawData.methodData.MassAnalyzers.Add(MSOrderType.Ms2, rawData.scanIndex.allScans[firstMs2Scan].MassAnalyzer);
rawData.methodData.MSOrderEnumerator.Add(MSOrderType.Ms);
rawData.methodData.MSOrderEnumerator.Add(MSOrderType.Ms2);
int n = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2].Count();
double ms2window = rawFile.GetScanEventForScanNumber(rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2][n / 2]).GetIsolationWidth(0);
if (rawData.methodData.AnalysisOrder == MSOrderType.Ms3)
{
int firstMs3Scan = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms3][0];
rawData.methodData.MassAnalyzers.Add(MSOrderType.Ms3, rawData.scanIndex.allScans[firstMs3Scan].MassAnalyzer);
rawData.methodData.MSOrderEnumerator.Add(MSOrderType.Ms3);
double ms3ms1window = rawFile.GetScanEventForScanNumber(firstMs3Scan).GetIsolationWidth(0);
double ms3ms2window = rawFile.GetScanEventForScanNumber(firstMs3Scan).GetIsolationWidth(1);
rawData.methodData.IsolationWindow = (MS2 : ms2window, (MS1Window : ms3ms1window, MS2Window : ms3ms2window));
}
else
{
rawData.methodData.IsolationWindow = (MS2 : ms2window, (MS1Window : -1, MS2Window : -1));
}
rawData.Performed.Add(Operations.MethodData);
}
public static void ExtractPrecursorMasses(this RawDataCollection rawData, IRawDataPlus rawFile, int scan)
{
if (rawData.scanIndex.allScans[scan].MSOrder == MSOrderType.Ms2)
{
double parent_mass = rawFile.GetScanEventForScanNumber(scan).GetReaction(0).PrecursorMass;
rawData.precursorMasses.Add(scan, new PrecursorMassData(rawData.trailerExtras[scan].MonoisotopicMZ, parent_mass));
}
if (rawData.scanIndex.allScans[scan].MSOrder == MSOrderType.Ms3)
{
int ms2scan = rawData.precursorScans[scan].MS2Scan;
double parent_mass = rawFile.GetScanEventForScanNumber(scan).GetReaction(0).PrecursorMass;
rawData.precursorMasses.Add(scan, new PrecursorMassData(rawData.trailerExtras[ms2scan].MonoisotopicMZ, parent_mass));
}
}
public static void RunSearch(QcParameters qcParameters, RawDataCollection rawData, IRawDataPlus rawFile)
{
string mgfFile = Path.Combine(qcParameters.QcSearchDataDirectory, Path.GetFileName(rawData.rawFileName) + ".mgf");
string outputFile = Path.Combine(qcParameters.QcSearchDataDirectory, Path.GetFileName(rawData.rawFileName) + ".pep.xml");
if (qcParameters.searchParameters.SearchAlgorithm == SearchAlgorithm.XTandem)
{
XTandem.RunXTandem(rawData, qcParameters.searchParameters, mgfFile, outputFile, genDecoy: true);
}
if (qcParameters.searchParameters.SearchAlgorithm == SearchAlgorithm.IdentiPy)
{
var pars = qcParameters.searchParameters;
Identipy.RunIdentipy(rawData, rawFile, qcParameters.QcSearchDataDirectory, pars, writeMGF: false);
}
}
private static void Extract(this RawDataCollection rawData, IRawDataPlus rawFile, IEnumerable <int> scans, ProgressIndicator progress)
{
foreach (int scan in scans)
{
try
{
rawData.segmentedScans.Add(scan, new SegmentedScanData(rawFile.GetSegmentedScanFromScanNumber(scan, null)));
}
catch (Exception e)
{
Log.Error(e, "Failed during extraction of segment scan {Scan}", scan);
throw e;
}
progress.Update();
}
}
private static void Extract(this RawDataCollection rawData, IRawDataPlus rawFile, IEnumerable <int> scans, ProgressIndicator progress)
{
foreach (int scan in scans)
{
try
{
rawData.centroidStreams.Add(scan, new CentroidStreamData(rawFile.GetCentroidStream(scan, false)));
}
catch (Exception e)
{
Log.Error(e, "Failed during extraction of centroid stream {Scan}", scan);
throw e;
}
progress.Update();
}
}
public static int NumberOfEsiFlags(RawDataCollection rawData)
{
int flags = 0;
int[] scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms];
for (int i = 2; i < scans.Length; i++)
{
if (rawData.metaData[scans[i]].SummedIntensity / rawData.metaData[scans[i - 1]].SummedIntensity < 0.1)
{
flags += 1;
}
if (rawData.metaData[scans[i]].SummedIntensity / rawData.metaData[scans[i - 1]].SummedIntensity > 10)
{
flags += 1;
}
}
return(flags);
}
public static void Quantify(this QuantDataCollection quantData, RawDataCollection rawData, IRawDataPlus rawFile, string labelingReagent)
{
MassAnalyzerType quantAnalyzer = rawData.methodData.QuantAnalyzer;
if (quantAnalyzer == MassAnalyzerType.MassAnalyzerFTMS)
{
rawData.ExtractCentroidStreams(rawFile, rawData.methodData.AnalysisOrder);
}
else
{
rawData.ExtractSegmentScans(rawFile, rawData.methodData.AnalysisOrder);
}
int[] scans;
ScanIndex scanIndex = rawData.scanIndex;
Dictionary <int, CentroidStreamData> centroidScans = rawData.centroidStreams;
Dictionary <int, SegmentedScanData> segmentScans = rawData.segmentedScans;
scans = scanIndex.ScanEnumerators[scanIndex.AnalysisOrder];
ProgressIndicator progress = new ProgressIndicator(scans.Length, "Quantifying reporter ions");
quantData.LabelingReagents = labelingReagent;
foreach (int scan in scans)
{
if (quantAnalyzer == MassAnalyzerType.MassAnalyzerFTMS)
{
quantData.Add(scan, new QuantifyReporters(centroidScans[scan], labelingReagent).quantData);
}
else
{
quantData.Add(scan, new QuantifyReporters(segmentScans[scan], labelingReagent).quantData);
}
progress.Update();
}
progress.Done();
rawData.Performed.Add(Operations.Quantification);
}
public static void ChromIntMetrics(this QcDataContainer qcData, RawDataCollection rawData, MetricsData metrics)
{
double firstRtToExceed10 = 0;
double lastRtToExceed10 = 0;
double proportionCovered;
var scans = rawData.scanIndex.ScanEnumerators[MSOrderType.Ms];
var reversedScans = scans.Reverse();
var totalIntList = (from x in scans select rawData.metaData[x].SummedIntensity).ToArray();
// get Q1 of total intensity from all scans
double threshold = totalIntList.Max() / 10;
// get first RT which exceeds Q1
for (int i = 0; i < scans.Length; i++)
{
int scan = scans[i];
if (totalIntList.MovingAverage(i, 20) > threshold)
{
firstRtToExceed10 = rawData.retentionTimes[scan];
break;
}
}
for (int i = scans.Length - 1; i >= 0; i--)
{
int scan = scans[i];
if (totalIntList.MovingAverage(i, 20) > threshold)
{
lastRtToExceed10 = rawData.retentionTimes[scan];
break;
}
}
// get proportion of run encompassed by these times
//proportionCovered = (lastRtToExceedQ1 - firstRtToExceedQ1) / metrics.TotalAnalysisTime;
proportionCovered = (lastRtToExceed10 - firstRtToExceed10) / rawData.retentionTimes[rawData.scanIndex.ScanEnumerators[MSOrderType.Ms].Last()];
qcData.TimeBeforeFirstScanToExceedPoint1MaxIntensity = firstRtToExceed10;// - rawData.retentionTimes[1];
qcData.TimeAfterLastScanToExceedPoint1MaxIntensity = rawData.retentionTimes[rawData.scanIndex.ScanEnumerators[MSOrderType.Ms].Last()] - lastRtToExceed10;
qcData.FractionOfRunAbovePoint1MaxIntensity = proportionCovered;
}
public static void RunIdentipy(RawDataCollection rawData, IRawDataPlus rawFile, string QcDataDirectory, SearchParameters idpyPars, bool writeMGF = true)
{
int[] scans;
string pyExec, idpyScript, mgfFile;
// get the path to the python executable and identipy start script if they have not been specified
if (idpyPars.PythonExecutable == null & idpyPars.IdentipyScript == null)
{
string pyDir = GetPythonDir();
pyExec = GetPythonExec(pyDir);
idpyScript = GetIdentipyExec(pyDir);
Log.Information("Found python directory: {PyDir}", pyDir);
Log.Information("Found python executable: {PyExec}", pyExec);
Log.Information("Found identipy start script: {IdpyScript}", idpyScript);
}
else
{
pyExec = idpyPars.PythonExecutable;
idpyScript = idpyPars.IdentipyScript;
}
if (writeMGF)
{
// get a random subset of scans
scans = AdditionalMath.SelectRandomScans(scans: rawData.scanIndex.ScanEnumerators[MSOrderType.Ms2], num: idpyPars.NumSpectra);
// write them to a mgf file
MGF.WriteMGF(rawData, rawFile, QcDataDirectory, scans: scans, cutoff: idpyPars.MgfMassCutoff, intensityCutoff: idpyPars.MgfIntensityCutoff);
}
// recreate the path to the mgf
mgfFile = Path.Combine(QcDataDirectory, Path.GetFileName(rawData.rawFileName) + ".mgf");
// run identipy, supresseing RunTimeWarnings
//IdentipyExecute(idpyExec, idpyPars.FastaDatabase, idpyPars.FixedMods, idpyPars.VariableMods, mgfFile);
Console.WriteLine("Starting Identipy");
Log.Information("Starting Identipy");
IdentipyExecute(pyExec, idpyScript, mgfFile, idpyPars, rawData.methodData.MassAnalyzers[MSOrderType.Ms2]);
}
public static void Check(this RawDataCollection rawData, IRawDataPlus rawFile, List <Operations> operations)
{
if (operations.Contains(Operations.MethodData))
{
if (!rawData.Performed.Contains(Operations.MethodData))
{
rawData.ExtractMethodData(rawFile);
}
}
if (operations.Contains(Operations.PrecursorMasses))
{
if (!rawData.Performed.Contains(Operations.PrecursorMasses))
{
rawData.ExtractPrecursorMasses(rawFile);
}
}
if (operations.Contains(Operations.PrecursorScans))
{
if (!rawData.Performed.Contains(Operations.PrecursorScans))
{
rawData.ExtractPrecursorScans(rawFile);
}
}
if (operations.Contains(Operations.RetentionTimes))
{
if (!rawData.Performed.Contains(Operations.RetentionTimes))
{
rawData.ExtractRetentionTimes(rawFile);
}
}
if (operations.Contains(Operations.ScanIndex))
{
if (!rawData.Performed.Contains(Operations.ScanIndex))
{
rawData.ExtractScanIndex(rawFile);
}
}
if (operations.Contains(Operations.TrailerExtras))
{
if (!rawData.Performed.Contains(Operations.TrailerExtras))
{
rawData.ExtractTrailerExtra(rawFile);
}
}
if (operations.Contains(Operations.Ms1CentroidStreams))
{
if (!rawData.Performed.Contains(Operations.Ms1CentroidStreams))
{
rawData.ExtractCentroidStreams(rawFile, MSOrder: MSOrderType.Ms);
}
}
if (operations.Contains(Operations.Ms2CentroidStreams))
{
if (!rawData.Performed.Contains(Operations.Ms2CentroidStreams))
{
rawData.ExtractCentroidStreams(rawFile, MSOrder: MSOrderType.Ms2);
}
}
if (operations.Contains(Operations.Ms3CentroidStreams))
{
if (!rawData.Performed.Contains(Operations.Ms3CentroidStreams))
{
rawData.ExtractCentroidStreams(rawFile, MSOrder: MSOrderType.Ms3);
}
}
if (operations.Contains(Operations.Ms1SegmentedScans))
{
if (!rawData.Performed.Contains(Operations.Ms1SegmentedScans))
{
rawData.ExtractSegmentScans(rawFile, MSOrder: MSOrderType.Ms);
}
}
if (operations.Contains(Operations.Ms2SegmentedScans))
{
if (!rawData.Performed.Contains(Operations.Ms2SegmentedScans))
{
rawData.ExtractSegmentScans(rawFile, MSOrder: MSOrderType.Ms2);
}
}
if (operations.Contains(Operations.Ms3SegmentedScans))
{
if (!rawData.Performed.Contains(Operations.Ms3SegmentedScans))
{
rawData.ExtractSegmentScans(rawFile, MSOrder: MSOrderType.Ms3);
}
}
if (operations.Contains(Operations.MetaData))
{
if (!rawData.Performed.Contains(Operations.MetaData))
{
rawData.metaData.AggregateMetaData(rawData, rawFile);
}
}
if (operations.Contains(Operations.PeakRetAndInt))
{
if (!rawData.Performed.Contains(Operations.PeakRetAndInt))
{
AnalyzePeaks.CalcPeakRetTimesAndInts(rawData, rawFile);
}
}
if (operations.Contains(Operations.PeakShape))
{
if (!rawData.Performed.Contains(Operations.PeakShape))
{
AnalyzePeaks.CalculatePeakShapes(rawData, rawFile);
}
}
if (operations.Contains(Operations.PeakArea))
{
if (!rawData.Performed.Contains(Operations.PeakArea))
{
AnalyzePeaks.QuantifyPrecursorPeaks(rawData, rawFile);
}
}
}
public ImportStartupForm()
{
InitializeComponent();
InitializeSemesters();
_effortMapper = new EffortMapper();
// 載入預設儲存值
LoadConfigData();
_worker = new BackgroundWorker();
_worker.WorkerReportsProgress = true;
_worker.ProgressChanged += delegate(object sender, ProgressChangedEventArgs e)
{
lblMessage.Text = "" + e.UserState;
};
_worker.DoWork += delegate(object sender, DoWorkEventArgs e)
{
#region Worker DoWork
_worker.ReportProgress(0, "檢查讀卡文字格式…");
#region 檢查文字檔
ValidateTextFiles vtf = new ValidateTextFiles(intStudentNumberLenght.Value);
ValidateTextResult vtResult = vtf.CheckFormat(_files);
if (vtResult.Error)
{
e.Result = vtResult;
return;
}
#endregion
//文字檔轉 RawData
RawDataCollection rdCollection = new RawDataCollection();
rdCollection.ConvertFromFiles(_files);
//RawData 轉 DataRecord
DataRecordCollection drCollection = new DataRecordCollection();
drCollection.ConvertFromRawData(rdCollection);
_rawDataValidator = new DataValidator <RawData>();
_dataRecordValidator = new DataValidator <DataRecord>();
#region 取得驗證需要的資料
JHCourse.RemoveAll();
_worker.ReportProgress(0, "取得學生資料…");
List <JHStudentRecord> studentList = GetInSchoolStudents();
List <string> s_ids = new List <string>();
Dictionary <string, List <string> > studentNumberToStudentIDs = new Dictionary <string, List <string> >();
foreach (JHStudentRecord student in studentList)
{
string sn = SCValidatorCreator.GetStudentNumberFormat(student.StudentNumber);
if (!studentNumberToStudentIDs.ContainsKey(sn))
{
studentNumberToStudentIDs.Add(sn, new List <string>());
}
studentNumberToStudentIDs[sn].Add(student.ID);
}
foreach (var dr in drCollection)
{
if (studentNumberToStudentIDs.ContainsKey(dr.StudentNumber))
{
s_ids.AddRange(studentNumberToStudentIDs[dr.StudentNumber]);
}
}
studentList.Clear();
_worker.ReportProgress(0, "取得課程資料…");
List <JHCourseRecord> courseList = JHCourse.SelectBySchoolYearAndSemester(SchoolYear, Semester);
List <JHAEIncludeRecord> aeList = JHAEInclude.SelectAll();
//List<JHSCAttendRecord> scaList = JHSCAttend.SelectAll();
var c_ids = from course in courseList select course.ID;
_worker.ReportProgress(0, "取得修課資料…");
//List<JHSCAttendRecord> scaList2 = JHSCAttend.SelectByStudentIDAndCourseID(s_ids, c_ids.ToList<string>());
List <JHSCAttendRecord> scaList = new List <JHSCAttendRecord>();
FunctionSpliter <string, JHSCAttendRecord> spliter = new FunctionSpliter <string, JHSCAttendRecord>(300, 3);
spliter.Function = delegate(List <string> part)
{
return(JHSCAttend.Select(part, c_ids.ToList <string>(), null, SchoolYear.ToString(), Semester.ToString()));
};
scaList = spliter.Execute(s_ids);
_worker.ReportProgress(0, "取得試別資料…");
List <JHExamRecord> examList = JHExam.SelectAll();
#endregion
#region 註冊驗證
_worker.ReportProgress(0, "載入驗證規則…");
_rawDataValidator.Register(new SubjectCodeValidator());
_rawDataValidator.Register(new ClassCodeValidator());
_rawDataValidator.Register(new ExamCodeValidator());
SCValidatorCreator scCreator = new SCValidatorCreator(JHStudent.SelectByIDs(s_ids), courseList, scaList);
_dataRecordValidator.Register(scCreator.CreateStudentValidator());
_dataRecordValidator.Register(new ExamValidator(examList));
_dataRecordValidator.Register(scCreator.CreateSCAttendValidator());
_dataRecordValidator.Register(new CourseExamValidator(scCreator.StudentCourseInfo, aeList, examList));
#endregion
#region 進行驗證
_worker.ReportProgress(0, "進行驗證中…");
List <string> msgList = new List <string>();
foreach (RawData rawData in rdCollection)
{
List <string> msgs = _rawDataValidator.Validate(rawData);
msgList.AddRange(msgs);
}
if (msgList.Count > 0)
{
e.Result = msgList;
return;
}
foreach (DataRecord dataRecord in drCollection)
{
List <string> msgs = _dataRecordValidator.Validate(dataRecord);
msgList.AddRange(msgs);
}
if (msgList.Count > 0)
{
e.Result = msgList;
return;
}
#endregion
#region 取得學生的評量成績
_deleteScoreList.Clear();
_addScoreList.Clear();
//var student_ids = from student in scCreator.StudentNumberDictionary.Values select student.ID;
//List<string> course_ids = scCreator.AttendCourseIDs;
var scaIDs = from sca in scaList select sca.ID;
Dictionary <string, JHSCETakeRecord> sceList = new Dictionary <string, JHSCETakeRecord>();
FunctionSpliter <string, JHSCETakeRecord> spliterSCE = new FunctionSpliter <string, JHSCETakeRecord>(300, 3);
spliterSCE.Function = delegate(List <string> part)
{
return(JHSCETake.Select(null, null, null, null, part));
};
foreach (JHSCETakeRecord sce in spliterSCE.Execute(scaIDs.ToList()))
{
string key = GetCombineKey(sce.RefStudentID, sce.RefCourseID, sce.RefExamID);
if (!sceList.ContainsKey(key))
{
sceList.Add(key, sce);
}
}
Dictionary <string, JHExamRecord> examTable = new Dictionary <string, JHExamRecord>();
Dictionary <string, JHSCAttendRecord> scaTable = new Dictionary <string, JHSCAttendRecord>();
foreach (JHExamRecord exam in examList)
{
if (!examTable.ContainsKey(exam.Name))
{
examTable.Add(exam.Name, exam);
}
}
foreach (JHSCAttendRecord sca in scaList)
{
string key = GetCombineKey(sca.RefStudentID, sca.RefCourseID);
if (!scaTable.ContainsKey(key))
{
scaTable.Add(key, sca);
}
}
foreach (DataRecord dr in drCollection)
{
JHStudentRecord student = student = scCreator.StudentNumberDictionary[dr.StudentNumber];
JHExamRecord exam = examTable[dr.Exam];
List <JHCourseRecord> courses = new List <JHCourseRecord>();
foreach (JHCourseRecord course in scCreator.StudentCourseInfo.GetCourses(dr.StudentNumber))
{
if (dr.Subjects.Contains(course.Subject))
{
courses.Add(course);
}
}
foreach (JHCourseRecord course in courses)
{
string key = GetCombineKey(student.ID, course.ID, exam.ID);
if (sceList.ContainsKey(key))
{
_deleteScoreList.Add(sceList[key]);
}
JHSCETakeRecord jh = new JHSCETakeRecord();
KH.JHSCETakeRecord sceNew = new KH.JHSCETakeRecord(jh);
sceNew.RefCourseID = course.ID;
sceNew.RefExamID = exam.ID;
sceNew.RefSCAttendID = scaTable[GetCombineKey(student.ID, course.ID)].ID;
sceNew.RefStudentID = student.ID;
sceNew.Score = dr.Score;
sceNew.Effort = _effortMapper.GetCodeByScore(dr.Score);
_addScoreList.Add(sceNew.AsJHSCETakeRecord());
}
}
#endregion
e.Result = null;
#endregion
};
_worker.RunWorkerCompleted += delegate(object sender, RunWorkerCompletedEventArgs e)
{
#region Worker Completed
if (e.Error == null && e.Result == null)
{
if (!_upload.IsBusy)
{
//如果學生身上已有成績，則提醒使用者
if (_deleteScoreList.Count > 0)
{
_warn.RunWorkerAsync();
}
else
{
lblMessage.Text = "成績上傳中…";
FISCA.Presentation.MotherForm.SetStatusBarMessage("成績上傳中…", 0);
counter = 0;
_upload.RunWorkerAsync();
}
}
}
else
{
ControlEnable = true;
if (e.Error != null)
{
MsgBox.Show("匯入失敗。" + e.Error.Message);
SmartSchool.ErrorReporting.ReportingService.ReportException(e.Error);
}
else if (e.Result != null && e.Result is ValidateTextResult)
{
ValidateTextResult result = e.Result as ValidateTextResult;
ValidationErrorViewer viewer = new ValidationErrorViewer();
viewer.SetTextFileError(result.LineIndexes, result.ErrorFormatLineIndexes, result.DuplicateLineIndexes);
viewer.ShowDialog();
}
else if (e.Result != null && e.Result is List <string> )
{
ValidationErrorViewer viewer = new ValidationErrorViewer();
viewer.SetErrorLines(e.Result as List <string>);
viewer.ShowDialog();
}
}
#endregion
};
_upload = new BackgroundWorker();
_upload.WorkerReportsProgress = true;
_upload.ProgressChanged += new ProgressChangedEventHandler(_upload_ProgressChanged);
//_upload.ProgressChanged += delegate(object sender, ProgressChangedEventArgs e)
//{
// counter += double.Parse("" + e.ProgressPercentage);
// FISCA.Presentation.MotherForm.SetStatusBarMessage("成績上傳中…", (int)(counter * 100f / (double)_addScoreList.Count));
//};
_upload.DoWork += new DoWorkEventHandler(_upload_DoWork);
//_upload.DoWork += delegate
//{
//#region Upload DoWork
//Framework.MultiThreadWorker<JHSCETakeRecord> multi = new Framework.MultiThreadWorker<JHSCETakeRecord>();
//multi.MaxThreads = 3;
//multi.PackageSize = 500;
//multi.PackageWorker += delegate(object sender, Framework.PackageWorkEventArgs<JHSCETakeRecord> e)
//{
// JHSCETake.Delete(e.List);
//};
//multi.Run(_deleteScoreList);
//Framework.MultiThreadWorker<JHSCETakeRecord> multi2 = new Framework.MultiThreadWorker<JHSCETakeRecord>();
//multi2.MaxThreads = 3;
//multi2.PackageSize = 500;
//multi2.PackageWorker += delegate(object sender, Framework.PackageWorkEventArgs<JHSCETakeRecord> e)
//{
// JHSCETake.Insert(e.List);
// lock (_upload)
// {
// _upload.ReportProgress(e.List.Count);
// }
//};
//multi2.Run(_addScoreList);
//#endregion
//};
_upload.RunWorkerCompleted += new RunWorkerCompletedEventHandler(_upload_RunWorkerCompleted);
_warn = new BackgroundWorker();
_warn.WorkerReportsProgress = true;
_warn.DoWork += delegate(object sender, DoWorkEventArgs e)
{
_warn.ReportProgress(0, "產生警告訊息...");
Dictionary <string, string> examDict = new Dictionary <string, string>();
foreach (JHExamRecord exam in JHExam.SelectAll())
{
if (!examDict.ContainsKey(exam.ID))
{
examDict.Add(exam.ID, exam.Name);
}
}
WarningForm form = new WarningForm();
int count = 0;
foreach (JHSCETakeRecord sce in _deleteScoreList)
{
// 當成績資料是空值跳過
if (sce.Score.HasValue == false && sce.Effort.HasValue == false && string.IsNullOrEmpty(sce.Text))
{
continue;
}
count++;
JHStudentRecord student = JHStudent.SelectByID(sce.RefStudentID);
JHCourseRecord course = JHCourse.SelectByID(sce.RefCourseID);
string exam = (examDict.ContainsKey(sce.RefExamID) ? examDict[sce.RefExamID] : "<未知的試別>");
string s = "";
if (student.Class != null)
{
s += student.Class.Name;
}
if (!string.IsNullOrEmpty("" + student.SeatNo))
{
s += " " + student.SeatNo + "號";
}
if (!string.IsNullOrEmpty(student.StudentNumber))
{
s += " (" + student.StudentNumber + ")";
}
s += " " + student.Name;
form.Add(student.ID, s, string.Format("學生在「{0}」課程「{1}」中已有成績。", course.Name, exam));
_warn.ReportProgress((int)(count * 100 / _deleteScoreList.Count), "產生警告訊息...");
}
e.Result = form;
};
_warn.RunWorkerCompleted += delegate(object sender, RunWorkerCompletedEventArgs e)
{
WarningForm form = e.Result as WarningForm;
if (form.ShowDialog() == DialogResult.OK)
{
lblMessage.Text = "成績上傳中…";
FISCA.Presentation.MotherForm.SetStatusBarMessage("成績上傳中…", 0);
counter = 0;
_upload.RunWorkerAsync();
}
else
{
this.DialogResult = DialogResult.Cancel;
}
};
_warn.ProgressChanged += delegate(object sender, ProgressChangedEventArgs e)
{
FISCA.Presentation.MotherForm.SetStatusBarMessage("" + e.UserState, e.ProgressPercentage);
};
_files = new List <FileInfo>();
_addScoreList = new List <JHSCETakeRecord>();
_deleteScoreList = new List <JHSCETakeRecord>();
}
public void AddOffset(long offset, byte[] rawData)
{
DiscOffsets.Add(offset);
RawDataCollection.Add(rawData);
}
internal void ClearOffsets()
{
DiscOffsets.Clear();
RawDataCollection.Clear();
}
public static void ExtractScanIndex(this RawDataCollection rawData, IRawDataPlus rawFile)
{
Log.Information("Extracting scan indices");
Dictionary <int, (MSOrderType MSOrder, MassAnalyzerType MassAnalyzer)> allScans;
allScans = new Dictionary <int, (MSOrderType MSOrder, MassAnalyzerType MassAnalyzer)>();
MSOrderType AnalysisOrder;
List <int> ms1 = new List <int>();
List <int> ms2 = new List <int>();
List <int> ms3 = new List <int>();
List <int> msAny = new List <int>();
// populate the scan indices
IEnumerable <int> scans = rawFile.GetFilteredScanEnumerator(rawFile.GetFilterFromString("")); // get all scans
foreach (int scan in scans)
{
IScanEvent scanEvent = rawFile.GetScanEventForScanNumber(scan);
allScans.Add(scan, (scanEvent.MSOrder, scanEvent.MassAnalyzer));
msAny.Add(scan);
if (allScans[scan].MSOrder == MSOrderType.Ms)
{
ms1.Add(scan);
}
if (allScans[scan].MSOrder == MSOrderType.Ms2)
{
ms2.Add(scan);
}
if (allScans[scan].MSOrder == MSOrderType.Ms3)
{
ms3.Add(scan);
}
}
// determine the msorder of the experiment
if ((ms1.Count > 0) & (ms2.Count == 0) & (ms3.Count == 0))
{
AnalysisOrder = MSOrderType.Ms;
}
else
{
if ((ms1.Count > 0) & (ms2.Count > 0) & (ms3.Count == 0))
{
AnalysisOrder = MSOrderType.Ms2;
}
else
{
AnalysisOrder = MSOrderType.Ms3;
}
}
rawData.scanIndex = new ScanIndex();
rawData.scanIndex.allScans = allScans;
rawData.scanIndex.AnalysisOrder = AnalysisOrder;
rawData.scanIndex.ScanEnumerators.Add(MSOrderType.Any, msAny.ToArray());
rawData.scanIndex.ScanEnumerators.Add(MSOrderType.Ms, ms1.ToArray());
rawData.scanIndex.ScanEnumerators.Add(MSOrderType.Ms2, ms2.ToArray());
rawData.scanIndex.ScanEnumerators.Add(MSOrderType.Ms3, ms3.ToArray());
// we need to check if it is a boxcar file because those have some scan index issues
bool isBoxCar = rawFile.GetScanEventForScanNumber(1).MassRangeCount > 1;
rawData.Performed.Add(Operations.ScanIndex);
if (isBoxCar)
{
Log.Information("Raw file looks like a boxcar run. Scan indices being adjusted to account for missing scan dependents.");
rawData.ExtractPrecursorScans(rawFile);
rawData.scanIndex.ScanEnumerators[rawData.scanIndex.AnalysisOrder] = rawData.precursorScans.Keys.ToArray();
}
}
public static void ExtractAll(this RawDataCollection rawData, IRawDataPlus rawFile)
{
Log.Information("Beginning extraction of all possible data");
rawFile.SelectInstrument(Device.MS, 1);
rawData.ExtractPrecursorScans(rawFile);
ProgressIndicator P = new ProgressIndicator(rawData.scanIndex.allScans.Count(), "Extracting raw data");
TrailerExtraIndices indices = new TrailerExtraIndices(rawFile);
for (int i = 1; i <= rawData.scanIndex.allScans.Count(); i++)
{
try
{
// first get out the mass spectrum
if (rawData.scanIndex.allScans[i].MassAnalyzer == MassAnalyzerType.MassAnalyzerFTMS)
{
rawData.centroidStreams.Add(i, new CentroidStreamData(rawFile.GetCentroidStream(i, false)));
}
else
{
rawData.segmentedScans.Add(i, new SegmentedScanData(rawFile.GetSegmentedScanFromScanNumber(i, null)));
}
// add the trailer extra data
rawData.trailerExtras.Add(i, TrailerExtras.ExtractTrailerExtra(rawData, rawFile, i, indices));
rawData.Performed.Add(Operations.TrailerExtras);
// add the retention time
rawData.retentionTimes.Add(i, rawFile.RetentionTimeFromScanNumber(i));
rawData.Performed.Add(Operations.RetentionTimes);
// add the precursor mass
PrecursorMasses.ExtractPrecursorMasses(rawData, rawFile, i);
rawData.Performed.Add(Operations.PrecursorMasses);
P.Update();
}
catch (Exception e)
{
Log.Error("Extraction failed on scan {Scan}", i);
throw e;
}
}
if (rawData.methodData.AnalysisOrder == MSOrderType.Ms2 | rawData.methodData.AnalysisOrder == MSOrderType.Ms3)
{
rawData.Performed.Add(Operations.Ms1CentroidStreams);
if (rawData.methodData.MassAnalyzers[MSOrderType.Ms2] == MassAnalyzerType.MassAnalyzerFTMS)
{
rawData.Performed.Add(Operations.Ms2CentroidStreams);
}
else
{
rawData.Performed.Add(Operations.Ms2SegmentedScans);
}
}
if (rawData.methodData.AnalysisOrder == MSOrderType.Ms3)
{
if (rawData.methodData.MassAnalyzers[MSOrderType.Ms3] == MassAnalyzerType.MassAnalyzerFTMS)
{
rawData.Performed.Add(Operations.Ms3CentroidStreams);
}
else
{
rawData.Performed.Add(Operations.Ms3SegmentedScans);
}
}
P.Done();
}