protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// TODO: print error messages loading GPTMD mods
List <Modification> gptmdModifications = GlobalVariables.AllModsKnown.OfType <Modification>().Where(b => GptmdParameters.ListOfModsGptmd.Contains((b.ModificationType, b.IdWithMotif))).ToList();
IEnumerable <Tuple <double, double> > combos = LoadCombos(gptmdModifications).ToList();
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, DecoyType.Reverse, localizeableModificationTypes, CommonParameters);
List <PeptideSpectralMatch> allPsms = new List <PeptideSpectralMatch>();
var numRawFiles = currentRawFileList.Count;
// write prose settings
ProseCreatedWhileRunning.Append("The following G-PTM-D settings were used: "); ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("G-PTM-D modifications count = " + gptmdModifications.Count + "; ");
// temporary search type for writing prose
// the actual search type is technically file-specific but we don't allow file-specific notches, so it's safe to do this
MassDiffAcceptor tempSearchMode = new DotMassDiffAcceptor("", GetAcceptableMassShifts(fixedModifications, variableModifications, gptmdModifications, combos), CommonParameters.PrecursorMassTolerance);
ProseCreatedWhileRunning.Append("precursor mass tolerance(s) = {" + tempSearchMode.ToProseString() + "}; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy) + " non-decoy protein entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
// start the G-PTM-D task
Status("Running G-PTM-D...", new List <string> {
taskId
});
MyTaskResults = new MyTaskResults(this)
{
NewDatabases = new List <DbForTask>()
};
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
MyFileManager myFileManager = new MyFileManager(true);
object lock1 = new object();
object lock2 = new object();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
// Stop if canceled
if (GlobalVariables.StopLoops)
{
break;
}
var origDataFile = currentRawFileList[spectraFileIndex];
// mark the file as in-progress
StartingDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MassDiffAcceptor searchMode = new DotMassDiffAcceptor("", GetAcceptableMassShifts(fixedModifications, variableModifications, gptmdModifications, combos), combinedParams.PrecursorMassTolerance);
NewCollection(Path.GetFileName(origDataFile), new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
Status("Loading spectra file...", new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
myFileManager.DoneWithFile(origDataFile);
PeptideSpectralMatch[] allPsmsArray = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
new ClassicSearchEngine(allPsmsArray, arrayOfMs2ScansSortedByMass, variableModifications, fixedModifications, null, null, null, proteinList, searchMode, combinedParams, new List <string> {
taskId, "Individual Spectra Files", origDataFile
}).Run();
allPsms.AddRange(allPsmsArray.Where(p => p != null));
FinishedDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files", origDataFile
}));
}
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files"
}));
allPsms = allPsms.OrderByDescending(b => b.Score)
.ThenBy(b => b.PeptideMonisotopicMass.HasValue ? Math.Abs(b.ScanPrecursorMass - b.PeptideMonisotopicMass.Value) : double.MaxValue)
.GroupBy(b => new Tuple <string, int, double?>(b.FullFilePath, b.ScanNumber, b.PeptideMonisotopicMass))
.Select(b => b.First()).ToList();
new FdrAnalysisEngine(allPsms, tempSearchMode.NumNotches, CommonParameters, new List <string> {
taskId
}).Run();
var writtenFile = Path.Combine(OutputFolder, "GPTMD_Candidates.psmtsv");
WritePsmsToTsv(allPsms, writtenFile, new Dictionary <string, int>());
FinishedWritingFile(writtenFile, new List <string> {
taskId
});
// get file-specific precursor mass tolerances for the GPTMD engine
var filePathToPrecursorMassTolerance = new Dictionary <string, Tolerance>();
for (int i = 0; i < currentRawFileList.Count; i++)
{
string filePath = currentRawFileList[i];
Tolerance fileTolerance = CommonParameters.PrecursorMassTolerance;
if (fileSettingsList[i] != null && fileSettingsList[i].PrecursorMassTolerance != null)
{
fileTolerance = fileSettingsList[i].PrecursorMassTolerance;
}
filePathToPrecursorMassTolerance.Add(filePath, fileTolerance);
}
// run GPTMD engine
var gptmdResults = (GptmdResults) new GptmdEngine(allPsms, gptmdModifications, combos, filePathToPrecursorMassTolerance, CommonParameters, new List <string> {
taskId
}).Run();
// Stop if canceled
if (GlobalVariables.StopLoops)
{
return(MyTaskResults);
}
// write GPTMD databases
if (dbFilenameList.Any(b => !b.IsContaminant))
{
List <string> databaseNames = new List <string>();
foreach (var nonContaminantDb in dbFilenameList.Where(p => !p.IsContaminant))
{
var dbName = Path.GetFileNameWithoutExtension(nonContaminantDb.FilePath);
var theExtension = Path.GetExtension(nonContaminantDb.FilePath).ToLowerInvariant();
bool compressed = theExtension.EndsWith("gz");
databaseNames.Add(compressed ? Path.GetFileNameWithoutExtension(dbName) : dbName);
}
string outputXMLdbFullName = Path.Combine(OutputFolder, string.Join("-", databaseNames) + "GPTMD.xml");
var newModsActuallyWritten = ProteinDbWriter.WriteXmlDatabase(gptmdResults.Mods, proteinList.Where(b => !b.IsDecoy && !b.IsContaminant).ToList(), outputXMLdbFullName);
FinishedWritingFile(outputXMLdbFullName, new List <string> {
taskId
});
MyTaskResults.NewDatabases.Add(new DbForTask(outputXMLdbFullName, false));
MyTaskResults.AddTaskSummaryText("Modifications added: " + newModsActuallyWritten.Select(b => b.Value).Sum());
MyTaskResults.AddTaskSummaryText("Mods types and counts:");
MyTaskResults.AddTaskSummaryText(string.Join(Environment.NewLine, newModsActuallyWritten.OrderByDescending(b => b.Value).Select(b => "\t" + b.Key + "\t" + b.Value)));
}
if (dbFilenameList.Any(b => b.IsContaminant))
{
// do NOT use this code (Path.GetFilenameWithoutExtension) because GPTMD on .xml.gz will result in .xml.xml file type being written
//string outputXMLdbFullNameContaminants = Path.Combine(OutputFolder, string.Join("-", dbFilenameList.Where(b => b.IsContaminant).Select(b => Path.GetFileNameWithoutExtension(b.FilePath))) + "GPTMD.xml");
List <string> databaseNames = new List <string>();
foreach (var contaminantDb in dbFilenameList.Where(p => p.IsContaminant))
{
var dbName = Path.GetFileName(contaminantDb.FilePath);
int indexOfFirstDot = dbName.IndexOf(".");
databaseNames.Add(dbName.Substring(0, indexOfFirstDot));
}
string outputXMLdbFullNameContaminants = Path.Combine(OutputFolder, string.Join("-", databaseNames) + "GPTMD.xml");
var newModsActuallyWritten = ProteinDbWriter.WriteXmlDatabase(gptmdResults.Mods, proteinList.Where(b => !b.IsDecoy && b.IsContaminant).ToList(), outputXMLdbFullNameContaminants);
FinishedWritingFile(outputXMLdbFullNameContaminants, new List <string> {
taskId
});
MyTaskResults.NewDatabases.Add(new DbForTask(outputXMLdbFullNameContaminants, true));
MyTaskResults.AddTaskSummaryText("Contaminant modifications added: " + newModsActuallyWritten.Select(b => b.Value).Sum());
MyTaskResults.AddTaskSummaryText("Mods types and counts:");
MyTaskResults.AddTaskSummaryText(string.Join(Environment.NewLine, newModsActuallyWritten.OrderByDescending(b => b.Value).Select(b => "\t" + b.Key + "\t" + b.Value)));
}
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
if (SearchParameters.DoQuantification)
{
// disable quantification if a .mgf is being used
if (currentRawFileList.Any(x => Path.GetExtension(x).Equals(".mgf", StringComparison.OrdinalIgnoreCase)))
{
SearchParameters.DoQuantification = false;
}
//if we're doing SILAC, assign and add the silac labels to the residue dictionary
else if (SearchParameters.SilacLabels != null || SearchParameters.StartTurnoverLabel != null || SearchParameters.EndTurnoverLabel != null)
{
char heavyLabel = 'a'; //char to assign
//add the Turnoverlabels to the silacLabels list. They weren't there before just to prevent duplication in the tomls
if (SearchParameters.StartTurnoverLabel != null || SearchParameters.EndTurnoverLabel != null)
{
//original silacLabels object is null, so we need to initialize it
SearchParameters.SilacLabels = new List <SilacLabel>();
if (SearchParameters.StartTurnoverLabel != null)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.StartTurnoverLabel, heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
SearchParameters.StartTurnoverLabel = updatedLabel.updatedLabel;
SearchParameters.SilacLabels.Add(SearchParameters.StartTurnoverLabel);
}
if (SearchParameters.EndTurnoverLabel != null)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.EndTurnoverLabel, heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
SearchParameters.EndTurnoverLabel = updatedLabel.updatedLabel;
SearchParameters.SilacLabels.Add(SearchParameters.EndTurnoverLabel);
}
}
else
{
//change the silac residues to lower case amino acids (currently null)
List <SilacLabel> updatedLabels = new List <SilacLabel>();
for (int i = 0; i < SearchParameters.SilacLabels.Count; i++)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.SilacLabels[i], heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
updatedLabels.Add(updatedLabel.updatedLabel);
}
SearchParameters.SilacLabels = updatedLabels;
}
}
}
//if no quant, remove any silac labels that may have been added, because they screw up downstream analysis
if (!SearchParameters.DoQuantification) //using "if" instead of "else", because DoQuantification can change if it's an mgf
{
SearchParameters.SilacLabels = null;
}
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, SearchParameters.SearchTarget, SearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
SanitizeProteinDatabase(proteinList, SearchParameters.TCAmbiguity);
// load spectral libraries
var spectralLibrary = LoadSpectralLibraries(taskId, dbFilenameList);
// write prose settings
ProseCreatedWhileRunning.Append("The following search settings were used: ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("max mods per peptide = " + CommonParameters.DigestionParams.MaxModsForPeptide + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("precursor mass tolerance = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("report PSM ambiguity = " + CommonParameters.ReportAllAmbiguity + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy)
+ " non-decoy protein entries including " + proteinList.Count(p => p.IsContaminant) + " contaminant sequences. ");
// start the search task
MyTaskResults = new MyTaskResults(this);
List <PeptideSpectralMatch> allPsms = new List <PeptideSpectralMatch>();
//generate an array to store category specific fdr values (for speedy semi/nonspecific searches)
int numFdrCategories = (int)(Enum.GetValues(typeof(FdrCategory)).Cast <FdrCategory>().Last() + 1); //+1 because it starts at zero
List <PeptideSpectralMatch>[] allCategorySpecificPsms = new List <PeptideSpectralMatch> [numFdrCategories];
for (int i = 0; i < numFdrCategories; i++)
{
allCategorySpecificPsms[i] = new List <PeptideSpectralMatch>();
}
FlashLfqResults flashLfqResults = null;
MyFileManager myFileManager = new MyFileManager(SearchParameters.DisposeOfFileWhenDone);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
Status("Searching files...", new List <string> {
taskId, "Individual Spectra Files"
});
Dictionary <string, int[]> numMs2SpectraPerFile = new Dictionary <string, int[]>();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
var origDataFile = currentRawFileList[spectraFileIndex];
// mark the file as in-progress
StartingDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MassDiffAcceptor massDiffAcceptor = GetMassDiffAcceptor(combinedParams.PrecursorMassTolerance, SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
numMs2SpectraPerFile.Add(Path.GetFileNameWithoutExtension(origDataFile), new int[] { myMsDataFile.GetAllScansList().Count(p => p.MsnOrder == 2), arrayOfMs2ScansSortedByMass.Length });
myFileManager.DoneWithFile(origDataFile);
PeptideSpectralMatch[] fileSpecificPsms = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
// modern search
if (SearchParameters.SearchType == SearchType.Modern)
{
for (int currentPartition = 0; currentPartition < combinedParams.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count / combinedParams.TotalPartitions,
((currentPartition + 1) * proteinList.Count / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, currentPartition, SearchParameters.DecoyType, combinedParams, FileSpecificParameters,
SearchParameters.MaxFragmentSize, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), SearchParameters.TCAmbiguity, new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
lock (indexLock)
{
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
}
Status("Searching files...", taskId);
new ModernSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, currentPartition,
combinedParams, this.FileSpecificParameters, massDiffAcceptor, SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + combinedParams.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
}
// nonspecific search
else if (SearchParameters.SearchType == SearchType.NonSpecific)
{
PeptideSpectralMatch[][] fileSpecificPsmsSeparatedByFdrCategory = new PeptideSpectralMatch[numFdrCategories][]; //generate an array of all possible locals
for (int i = 0; i < numFdrCategories; i++) //only add if we're using for FDR, else ignore it as null.
{
fileSpecificPsmsSeparatedByFdrCategory[i] = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
}
//create params for N, C, or both if semi
List <CommonParameters> paramsToUse = new List <CommonParameters> {
combinedParams
};
if (combinedParams.DigestionParams.SearchModeType == CleavageSpecificity.Semi) //if semi, we need to do both N and C to hit everything
{
paramsToUse.Clear();
List <FragmentationTerminus> terminiToUse = new List <FragmentationTerminus> {
FragmentationTerminus.N, FragmentationTerminus.C
};
foreach (FragmentationTerminus terminus in terminiToUse) //set both termini
{
paramsToUse.Add(combinedParams.CloneWithNewTerminus(terminus));
}
}
//Compress array of deconvoluted ms2 scans to avoid searching the same ms2 multiple times while still identifying coisolated peptides
List <int>[] coisolationIndex = new List <int>[] { new List <int>() };
if (arrayOfMs2ScansSortedByMass.Length != 0)
{
int maxScanNumber = arrayOfMs2ScansSortedByMass.Max(x => x.OneBasedScanNumber);
coisolationIndex = new List <int> [maxScanNumber + 1];
for (int i = 0; i < arrayOfMs2ScansSortedByMass.Length; i++)
{
int scanNumber = arrayOfMs2ScansSortedByMass[i].OneBasedScanNumber;
if (coisolationIndex[scanNumber] == null)
{
coisolationIndex[scanNumber] = new List <int> {
i
};
}
else
{
coisolationIndex[scanNumber].Add(i);
}
}
coisolationIndex = coisolationIndex.Where(x => x != null).ToArray();
}
//foreach terminus we're going to look at
foreach (CommonParameters paramToUse in paramsToUse)
{
//foreach database partition
for (int currentPartition = 0; currentPartition < paramToUse.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count / paramToUse.TotalPartitions,
((currentPartition + 1) * proteinList.Count / paramToUse.TotalPartitions) - (currentPartition * proteinList.Count / paramToUse.TotalPartitions));
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, currentPartition, SearchParameters.DecoyType, paramToUse, FileSpecificParameters,
SearchParameters.MaxFragmentSize, true, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), SearchParameters.TCAmbiguity, new List <string> {
taskId
});
lock (indexLock)
{
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
}
Status("Searching files...", taskId);
new NonSpecificEnzymeSearchEngine(fileSpecificPsmsSeparatedByFdrCategory, arrayOfMs2ScansSortedByMass, coisolationIndex, peptideIndex, fragmentIndex,
precursorIndex, currentPartition, paramToUse, this.FileSpecificParameters, variableModifications, massDiffAcceptor,
SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + paramToUse.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
}
lock (psmLock)
{
for (int i = 0; i < allCategorySpecificPsms.Length; i++)
{
if (allCategorySpecificPsms[i] != null)
{
allCategorySpecificPsms[i].AddRange(fileSpecificPsmsSeparatedByFdrCategory[i]);
}
}
}
}
// classic search
else
{
Status("Starting search...", thisId);
var newClassicSearchEngine = new ClassicSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, proteinList, massDiffAcceptor, combinedParams, this.FileSpecificParameters, spectralLibrary, thisId, SearchParameters.WriteSpectralLibrary);
newClassicSearchEngine.Run();
ReportProgress(new ProgressEventArgs(100, "Done with search!", thisId));
}
//look for internal fragments
if (SearchParameters.MinAllowedInternalFragmentLength != 0)
{
MatchInternalFragmentIons(fileSpecificPsms, arrayOfMs2ScansSortedByMass, combinedParams, SearchParameters.MinAllowedInternalFragmentLength);
}
// calculate/set spectral angles if there is a spectral library being used
if (spectralLibrary != null)
{
Status("Calculating spectral library similarity...", thisId);
}
SpectralLibrarySearchFunction.CalculateSpectralAngles(spectralLibrary, fileSpecificPsms, arrayOfMs2ScansSortedByMass, combinedParams);
lock (psmLock)
{
allPsms.AddRange(fileSpecificPsms);
}
completedFiles++;
FinishedDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
if (spectralLibrary != null)
{
spectralLibrary.CloseConnections();
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
int numNotches = GetNumNotches(SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
//resolve category specific fdrs (for speedy semi and nonspecific
if (SearchParameters.SearchType == SearchType.NonSpecific)
{
allPsms = NonSpecificEnzymeSearchEngine.ResolveFdrCategorySpecificPsms(allCategorySpecificPsms, numNotches, taskId, CommonParameters, FileSpecificParameters);
}
PostSearchAnalysisParameters parameters = new PostSearchAnalysisParameters
{
SearchTaskResults = MyTaskResults,
SearchTaskId = taskId,
SearchParameters = SearchParameters,
ProteinList = proteinList,
AllPsms = allPsms,
VariableModifications = variableModifications,
FixedModifications = fixedModifications,
ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams)),
CurrentRawFileList = currentRawFileList,
MyFileManager = myFileManager,
NumNotches = numNotches,
OutputFolder = OutputFolder,
IndividualResultsOutputFolder = Path.Combine(OutputFolder, "Individual File Results"),
FlashLfqResults = flashLfqResults,
FileSettingsList = fileSettingsList,
NumMs2SpectraPerFile = numMs2SpectraPerFile,
DatabaseFilenameList = dbFilenameList
};
PostSearchAnalysisTask postProcessing = new PostSearchAnalysisTask
{
Parameters = parameters,
FileSpecificParameters = this.FileSpecificParameters,
CommonParameters = CommonParameters
};
return(postProcessing.Run());
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
List <List <CrosslinkSpectralMatch> > ListOfCsmsPerMS2Scan = new List <List <CrosslinkSpectralMatch> >();
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, XlSearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
CommonParameters.TotalPartitions = proteinList.Count() / 250;
if (CommonParameters.TotalPartitions == 0)
{
CommonParameters.TotalPartitions = 1;
}
MyFileManager myFileManager = new MyFileManager(true);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
ProseCreatedWhileRunning.Append("The following crosslink discovery were used: ");
ProseCreatedWhileRunning.Append("crosslinker name = " + XlSearchParameters.Crosslinker.CrosslinkerName + "; ");
ProseCreatedWhileRunning.Append("crosslinker type = " + XlSearchParameters.Crosslinker.Cleavable + "; ");
ProseCreatedWhileRunning.Append("crosslinker mass = " + XlSearchParameters.Crosslinker.TotalMass + "; ");
ProseCreatedWhileRunning.Append("crosslinker modification site(s) = " + XlSearchParameters.Crosslinker.CrosslinkerModSites + "; ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("parent mass tolerance(s) = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
List <CrosslinkSpectralMatch> AllCsms = new List <CrosslinkSpectralMatch>();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2ScansWrapByScanNum(myMsDataFile, origDataFile, combinedParams, out List <List <(double, int, double)> > precursorss).ToArray();
List <CrosslinkSpectralMatch>[] newCsmsPerMS2ScanPerFile = new List <CrosslinkSpectralMatch> [arrayOfMs2ScansSortedByMass.Length];
myFileManager.DoneWithFile(origDataFile);
//Candidates: currentPartition, peptide Index, score
List <(int, int, int)>[] candidates = new List <(int, int, int)> [arrayOfMs2ScansSortedByMass.Length];
public MyTaskResults Run(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList, List <GlycoSpectralMatch> allPsms, CommonParameters commonParameters, GlycoSearchParameters glycoSearchParameters, List <Protein> proteinList, List <Modification> variableModifications, List <Modification> fixedModifications, List <string> localizeableModificationTypes, MyTaskResults MyTaskResults)
{
if (glycoSearchParameters.GlycoSearchType == GlycoSearchType.NGlycanSearch)
{
var allPsmsSingle = allPsms.Where(p => p.NGlycan == null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsSingle, commonParameters, new List <string> {
taskId
});
var writtenFileSingle = Path.Combine(OutputFolder, "single" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsSingle, writtenFileSingle, 1);
FinishedWritingFile(writtenFileSingle, new List <string> {
taskId
});
var allPsmsGly = allPsms.Where(p => p.NGlycan != null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsGly, commonParameters, new List <string> {
taskId
});
var writtenFileNGlyco = Path.Combine(OutputFolder, "nglyco" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsGly, writtenFileNGlyco, 3);
FinishedWritingFile(writtenFileNGlyco, new List <string> {
taskId
});
return(MyTaskResults);
}
else if (glycoSearchParameters.GlycoSearchType == GlycoSearchType.OGlycanSearch)
{
var allPsmsSingle = allPsms.Where(p => p.Routes == null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsSingle, commonParameters, new List <string> {
taskId
});
var writtenFileSingle = Path.Combine(OutputFolder, "single" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsSingle, writtenFileSingle, 1);
FinishedWritingFile(writtenFileSingle, new List <string> {
taskId
});
var allPsmsGly = allPsms.Where(p => p.Routes != null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsGly, commonParameters, new List <string> {
taskId
});
var writtenFileOGlyco = Path.Combine(OutputFolder, "oglyco" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsGly, writtenFileOGlyco, 2);
FinishedWritingFile(writtenFileOGlyco, new List <string> {
taskId
});
var ProteinLevelLocalization = GlycoProteinParsimony.ProteinLevelGlycoParsimony(allPsmsGly.Where(p => p.ProteinAccession != null && p.OneBasedStartResidueInProtein.HasValue).ToList());
var seen_oglyco_localization_file = Path.Combine(OutputFolder, "seen_oglyco_localization" + ".tsv");
WriteFile.WriteSeenProteinGlycoLocalization(ProteinLevelLocalization, seen_oglyco_localization_file);
FinishedWritingFile(seen_oglyco_localization_file, new List <string> {
taskId
});
var protein_oglyco_localization_file = Path.Combine(OutputFolder, "protein_oglyco_localization" + ".tsv");
WriteFile.WriteProteinGlycoLocalization(ProteinLevelLocalization, protein_oglyco_localization_file);
FinishedWritingFile(protein_oglyco_localization_file, new List <string> {
taskId
});
return(MyTaskResults);
}
else
{
var allPsmsSingle = allPsms.Where(p => p.NGlycan == null && p.Routes == null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsSingle, commonParameters, new List <string> {
taskId
});
var writtenFileSingle = Path.Combine(OutputFolder, "single" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsSingle, writtenFileSingle, 1);
FinishedWritingFile(writtenFileSingle, new List <string> {
taskId
});
var allPsmsNGly = allPsms.Where(p => p.NGlycan != null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsNGly, commonParameters, new List <string> {
taskId
});
var writtenFileNGlyco = Path.Combine(OutputFolder, "nglyco" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsNGly, writtenFileNGlyco, 3);
FinishedWritingFile(writtenFileNGlyco, new List <string> {
taskId
});
var allPsmsOGly = allPsms.Where(p => p.Routes != null).OrderByDescending(p => p.Score).ToList();
SingleFDRAnalysis(allPsmsOGly, commonParameters, new List <string> {
taskId
});
var writtenFileOGlyco = Path.Combine(OutputFolder, "oglyco" + ".psmtsv");
WriteFile.WritePsmGlycoToTsv(allPsmsOGly, writtenFileOGlyco, 2);
FinishedWritingFile(writtenFileOGlyco, new List <string> {
taskId
});
return(MyTaskResults);
}
}
public MyTaskResults Run(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList, List <CrosslinkSpectralMatch> allPsms, CommonParameters commonParameters, XlSearchParameters xlSearchParameters, List <Protein> proteinList, List <Modification> variableModifications, List <Modification> fixedModifications, List <string> localizeableModificationTypes, MyTaskResults MyTaskResults)
{
foreach (var csm in allPsms)
{
csm.ResolveProteinPosAmbiguitiesForXl();
}
var allPsmsXL = allPsms.Where(p => p.CrossType == PsmCrossType.Cross).ToList();
// inter-crosslinks; different proteins are linked
var interCsms = allPsmsXL.Where(p => !p.IsIntraCsm()).ToList();
foreach (var item in interCsms)
{
item.CrossType = PsmCrossType.Inter;
}
// intra-crosslinks; crosslinks within a protein
var intraCsms = allPsmsXL.Where(p => p.IsIntraCsm()).ToList();
foreach (var item in intraCsms)
{
item.CrossType = PsmCrossType.Intra;
}
// calculate FDR
DoCrosslinkFdrAnalysis(interCsms);
DoCrosslinkFdrAnalysis(intraCsms);
SingleFDRAnalysis(allPsms, commonParameters, new List <string> {
taskId
});
// write interlink CSMs
if (interCsms.Any())
{
string file = Path.Combine(OutputFolder, "XL_Interlinks.tsv");
WriteFile.WritePsmCrossToTsv(interCsms, file, 2);
FinishedWritingFile(file, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
if (xlSearchParameters.WriteOutputForPercolator)
{
var interPsmsXLPercolator = interCsms.Where(p => p.Score >= 2 && p.BetaPeptide.Score >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteFile.WriteCrosslinkToTxtForPercolator(interPsmsXLPercolator, OutputFolder, "XL_Interlinks_Percolator", xlSearchParameters.Crosslinker);
FinishedWritingFile(Path.Combine(OutputFolder, "XL_Interlinks_Percolator.txt"), new List <string> {
taskId
});
}
// write intralink CSMs
if (intraCsms.Any())
{
string file = Path.Combine(OutputFolder, "XL_Intralinks.tsv");
WriteFile.WritePsmCrossToTsv(intraCsms, file, 2);
FinishedWritingFile(file, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
if (xlSearchParameters.WriteOutputForPercolator)
{
var intraPsmsXLPercolator = intraCsms.Where(p => p.Score >= 2 && p.BetaPeptide.Score >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteFile.WriteCrosslinkToTxtForPercolator(intraPsmsXLPercolator, OutputFolder, "XL_Intralinks_Percolator", xlSearchParameters.Crosslinker);
FinishedWritingFile(Path.Combine(OutputFolder, "XL_Intralinks_Percolator.txt"), new List <string> {
taskId
});
}
// write single peptides
var singlePsms = allPsms.Where(p => p.CrossType == PsmCrossType.Single).ToList();
if (singlePsms.Any())
{
string writtenFileSingle = Path.Combine(OutputFolder, "SinglePeptides" + ".tsv");
WriteFile.WritePsmCrossToTsv(singlePsms, writtenFileSingle, 1);
FinishedWritingFile(writtenFileSingle, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write loops
var loopPsms = allPsms.Where(p => p.CrossType == PsmCrossType.Loop).ToList();
if (loopPsms.Any())
{
string writtenFileLoop = Path.Combine(OutputFolder, "Looplinks" + ".tsv");
WriteFile.WritePsmCrossToTsv(loopPsms, writtenFileLoop, 1);
FinishedWritingFile(writtenFileLoop, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write deadends
var deadendPsms = allPsms.Where(p => p.CrossType == PsmCrossType.DeadEnd ||
p.CrossType == PsmCrossType.DeadEndH2O ||
p.CrossType == PsmCrossType.DeadEndNH2 ||
p.CrossType == PsmCrossType.DeadEndTris).ToList();
if (deadendPsms.Any())
{
string writtenFileDeadend = Path.Combine(OutputFolder, "Deadends" + ".tsv");
WriteFile.WritePsmCrossToTsv(deadendPsms, writtenFileDeadend, 1);
FinishedWritingFile(writtenFileDeadend, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write pepXML
if (xlSearchParameters.WritePepXml)
{
List <CrosslinkSpectralMatch> writeToXml = new List <CrosslinkSpectralMatch>();
writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml = writeToXml.OrderBy(p => p.ScanNumber).ToList();
foreach (var fullFilePath in currentRawFileList)
{
string fileNameNoExtension = Path.GetFileNameWithoutExtension(fullFilePath);
WriteFile.WritePepXML_xl(writeToXml.Where(p => p.FullFilePath == fullFilePath).ToList(), proteinList, dbFilenameList[0].FilePath, variableModifications, fixedModifications, localizeableModificationTypes, OutputFolder, fileNameNoExtension, commonParameters, xlSearchParameters);
FinishedWritingFile(Path.Combine(OutputFolder, fileNameNoExtension + ".pep.XML"), new List <string> {
taskId
});
}
}
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
List <PsmCross> allPsms = new List <PsmCross>();
var compactPeptideToProteinPeptideMatch = new Dictionary <CompactPeptideBase, HashSet <PeptideWithSetModifications> >();
Status("Loading modifications...", taskId);
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = GlobalVariables.AllModTypesKnown.ToList();
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, XlSearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
List <ProductType> ionTypes = new List <ProductType>();
if (CommonParameters.BIons)
{
ionTypes.Add(ProductType.BnoB1ions);
}
if (CommonParameters.YIons)
{
ionTypes.Add(ProductType.Y);
}
if (CommonParameters.ZdotIons)
{
ionTypes.Add(ProductType.Zdot);
}
if (CommonParameters.CIons)
{
ionTypes.Add(ProductType.C);
}
TerminusType terminusType = ProductTypeMethods.IdentifyTerminusType(ionTypes);
var crosslinker = new CrosslinkerTypeClass();
crosslinker.SelectCrosslinker(XlSearchParameters.CrosslinkerType);
if (XlSearchParameters.CrosslinkerType == CrosslinkerType.UserDefined)
{
crosslinker = GenerateUserDefinedCrosslinker(XlSearchParameters);
}
MyFileManager myFileManager = new MyFileManager(XlSearchParameters.DisposeOfFileWhenDone);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
ProseCreatedWhileRunning.Append("The following crosslink discovery were used: ");
ProseCreatedWhileRunning.Append("crosslinker name = " + crosslinker.CrosslinkerName + "; ");
ProseCreatedWhileRunning.Append("crosslinker type = " + crosslinker.Cleavable + "; ");
ProseCreatedWhileRunning.Append("crosslinker mass = " + crosslinker.TotalMass + "; ");
ProseCreatedWhileRunning.Append("crosslinker modification site(s) = " + crosslinker.CrosslinkerModSites + "; ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("parent mass tolerance(s) = " + XlSearchParameters.XlPrecusorMsTl + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
List <PsmCross> newPsms = new List <PsmCross>();
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams.TopNpeaks, combinedParams.MinRatio, combinedParams.TrimMs1Peaks, combinedParams.TrimMsMsPeaks, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams.DoPrecursorDeconvolution, combinedParams.UseProvidedPrecursorInfo, combinedParams.DeconvolutionIntensityRatio, combinedParams.DeconvolutionMaxAssumedChargeState, combinedParams.DeconvolutionMassTolerance).OrderBy(b => b.PrecursorMass).ToArray();
for (int currentPartition = 0; currentPartition < CommonParameters.TotalPartitions; currentPartition++)
{
List <CompactPeptide> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, ionTypes, currentPartition, UsefulProteomicsDatabases.DecoyType.Reverse, ListOfDigestionParams, combinedParams, 30000.0, new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
lock (indexLock)
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, taskId);
Status("Searching files...", taskId);
new TwoPassCrosslinkSearchEngine(newPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, ionTypes, currentPartition, combinedParams, false, XlSearchParameters.XlPrecusorMsTl, crosslinker, XlSearchParameters.CrosslinkSearchTop, XlSearchParameters.CrosslinkSearchTopNum, XlSearchParameters.XlQuench_H2O, XlSearchParameters.XlQuench_NH2, XlSearchParameters.XlQuench_Tris, XlSearchParameters.XlCharge_2_3, XlSearchParameters.XlCharge_2_3_PrimeFragment, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + CommonParameters.TotalPartitions + "!", thisId));
}
lock (psmLock)
{
allPsms.AddRange(newPsms.Where(p => p != null));
}
completedFiles++;
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
Status("Crosslink analysis engine", taskId);
MetaMorpheusEngineResults allcrosslinkanalysisResults;
allcrosslinkanalysisResults = new CrosslinkAnalysisEngine(allPsms, compactPeptideToProteinPeptideMatch, proteinList, variableModifications, fixedModifications, ionTypes, OutputFolder, crosslinker, terminusType, CommonParameters, new List <string> {
taskId
}).Run();
allPsms = allPsms.ToList();
if (XlSearchParameters.XlOutAll)
{
try
{
WriteAllToTsv(allPsms, OutputFolder, "allPsms", new List <string> {
taskId
});
}
catch (Exception)
{
throw;
}
}
var allPsmsXL = allPsms.Where(p => p.CrossType == PsmCrossType.Cross).Where(p => p.XLBestScore >= CommonParameters.ScoreCutoff && p.BetaPsmCross.XLBestScore >= CommonParameters.ScoreCutoff).ToList();
foreach (var item in allPsmsXL)
{
if (item.OneBasedStartResidueInProtein.HasValue)
{
item.XlProteinPos = item.OneBasedStartResidueInProtein.Value + item.XlPos - 1;
}
if (item.BetaPsmCross.OneBasedStartResidueInProtein.HasValue)
{
item.BetaPsmCross.XlProteinPos = item.BetaPsmCross.OneBasedStartResidueInProtein.Value + item.BetaPsmCross.XlPos - 1;
}
}
//Write Inter Psms FDR
var interPsmsXL = allPsmsXL.Where(p => !p.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First().Contains(p.BetaPsmCross.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First()) &&
!p.BetaPsmCross.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First().Contains(p.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First())).OrderByDescending(p => p.XLQvalueTotalScore).ToList();
foreach (var item in interPsmsXL)
{
item.CrossType = PsmCrossType.Inter;
}
var interPsmsXLFDR = CrosslinkDoFalseDiscoveryRateAnalysis(interPsmsXL).ToList();
//var interPsmsXLFDR = CrosslinkFDRAnalysis(interPsmsXL).ToList();
if (XlSearchParameters.XlOutCrosslink)
{
WriteCrosslinkToTsv(interPsmsXLFDR, OutputFolder, "xl_inter_fdr", new List <string> {
taskId
});
}
if (XlSearchParameters.XlOutPercolator)
{
try
{
var interPsmsXLPercolator = interPsmsXL.Where(p => p.XLBestScore >= 2 && p.BetaPsmCross.XLBestScore >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteCrosslinkToTxtForPercolator(interPsmsXLPercolator, OutputFolder, "xl_inter_perc", crosslinker, new List <string> {
taskId
});
}
catch (Exception)
{
throw;
}
}
//Write Intra Psms FDR
var intraPsmsXL = allPsmsXL.Where(p => p.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First() == p.BetaPsmCross.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First() ||
p.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First().Contains(p.BetaPsmCross.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First()) ||
p.BetaPsmCross.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First().Contains(p.CompactPeptides.First().Value.Item2.Select(b => b.Protein.Accession).First())).OrderByDescending(p => p.XLQvalueTotalScore).ToList();
foreach (var item in intraPsmsXL)
{
item.CrossType = PsmCrossType.Intra;
}
var intraPsmsXLFDR = CrosslinkDoFalseDiscoveryRateAnalysis(intraPsmsXL).ToList();
//var intraPsmsXLFDR = CrosslinkFDRAnalysis(intraPsmsXL).ToList();
if (XlSearchParameters.XlOutCrosslink)
{
WriteCrosslinkToTsv(intraPsmsXLFDR, OutputFolder, "xl_intra_fdr", new List <string> {
taskId
});
}
if (XlSearchParameters.XlOutPercolator)
{
try
{
var intraPsmsXLPercolator = intraPsmsXL.Where(p => p.XLBestScore >= 2 && p.BetaPsmCross.XLBestScore >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteCrosslinkToTxtForPercolator(intraPsmsXLPercolator, OutputFolder, "xl_intra_perc", crosslinker, new List <string> {
taskId
});
}
catch (Exception)
{
throw;
}
}
var singlePsms = allPsms.Where(p => p.CrossType == PsmCrossType.Singe && p.FullSequence != null && !p.FullSequence.Contains("Crosslink")).OrderByDescending(p => p.Score).ToList();
var singlePsmsFDR = SingleFDRAnalysis(singlePsms).ToList();
if (XlSearchParameters.XlOutAll)
{
WriteSingleToTsv(singlePsmsFDR, OutputFolder, "single_fdr", new List <string> {
taskId
});
}
var loopPsms = allPsms.Where(p => p.CrossType == PsmCrossType.Loop).OrderByDescending(p => p.XLTotalScore).ToList();
var loopPsmsFDR = SingleFDRAnalysis(loopPsms).ToList();
if (XlSearchParameters.XlOutAll)
{
WriteSingleToTsv(loopPsmsFDR, OutputFolder, "loop_fdr", new List <string> {
taskId
});
}
var deadendPsms = allPsms.Where(p => p.CrossType == PsmCrossType.DeadEnd || p.CrossType == PsmCrossType.DeadEndH2O || p.CrossType == PsmCrossType.DeadEndNH2 || p.CrossType == PsmCrossType.DeadEndTris).OrderByDescending(p => p.XLTotalScore).ToList();
deadendPsms.AddRange(allPsms.Where(p => p.CrossType == PsmCrossType.Singe && p.FullSequence != null && p.FullSequence.Contains("Crosslink")).ToList());
var deadendPsmsFDR = SingleFDRAnalysis(deadendPsms).ToList();
if (XlSearchParameters.XlOutAll)
{
WriteSingleToTsv(deadendPsmsFDR, OutputFolder, "deadend_fdr", new List <string> {
taskId
});
}
if (XlSearchParameters.XlOutPepXML)
{
List <PsmCross> allPsmsFDR = new List <PsmCross>();
allPsmsFDR.AddRange(intraPsmsXLFDR.Where(p => p.IsDecoy != true && p.BetaPsmCross.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsFDR.AddRange(interPsmsXLFDR.Where(p => p.IsDecoy != true && p.BetaPsmCross.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsFDR.AddRange(singlePsmsFDR.Where(p => p.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsFDR.AddRange(loopPsmsFDR.Where(p => p.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsFDR.AddRange(deadendPsmsFDR.Where(p => p.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsFDR = allPsmsFDR.OrderBy(p => p.ScanNumber).ToList();
foreach (var fullFilePath in currentRawFileList)
{
string fileNameNoExtension = Path.GetFileNameWithoutExtension(fullFilePath);
WritePepXML_xl(allPsmsFDR.Where(p => p.FullFilePath == fullFilePath).ToList(), proteinList, dbFilenameList[0].FilePath, variableModifications, fixedModifications, localizeableModificationTypes, OutputFolder, fileNameNoExtension, new List <string> {
taskId
});
}
}
if (XlSearchParameters.XlOutAll)
{
List <PsmCross> allPsmsXLFDR = new List <PsmCross>();
allPsmsXLFDR.AddRange(intraPsmsXLFDR.Where(p => p.IsDecoy != true && p.BetaPsmCross.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
allPsmsXLFDR.AddRange(interPsmsXLFDR.Where(p => p.IsDecoy != true && p.BetaPsmCross.IsDecoy != true && p.FdrInfo.QValue <= 0.05).ToList());
try
{
allPsmsXLFDR = allPsmsXLFDR.OrderByDescending(p => p.XLQvalueTotalScore).ToList();
var allPsmsXLFDRGroup = FindCrosslinks(allPsmsXLFDR);
WriteCrosslinkToTsv(allPsmsXLFDRGroup, OutputFolder, "allPsmsXLFDRGroup", new List <string> {
taskId
});
}
catch (Exception)
{
throw;
}
}
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
List <List <GlycoSpectralMatch> > ListOfGsmsPerMS2Scan = new List <List <GlycoSpectralMatch> >();
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, _glycoSearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
MyFileManager myFileManager = new MyFileManager(true);
int completedFiles = 0;
Status("Searching files...", taskId);
ProseCreatedWhileRunning.Append("\n");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; \n");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; \n");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; \n");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; \n");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; \n");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; \n");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; \n");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; \n");
ProseCreatedWhileRunning.Append("parent mass tolerance(s) = " + CommonParameters.PrecursorMassTolerance + "; \n");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; \n");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. \n");
if (_glycoSearchParameters.GlycoSearchType == GlycoSearchType.OGlycanSearch)
{
ProseCreatedWhileRunning.Append("The O-glycan database: " + _glycoSearchParameters.OGlycanDatabasefile + "\n");
}
else if (_glycoSearchParameters.GlycoSearchType == GlycoSearchType.NGlycanSearch)
{
ProseCreatedWhileRunning.Append("The N-glycan database: " + _glycoSearchParameters.OGlycanDatabasefile + "\n");
}
else
{
ProseCreatedWhileRunning.Append("The O-glycan database: " + _glycoSearchParameters.OGlycanDatabasefile + "\n");
ProseCreatedWhileRunning.Append("The N-glycan database: " + _glycoSearchParameters.NGlycanDatabasefile + "\n");
}
ProseCreatedWhileRunning.Append("\n");
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
List <GlycoSpectralMatch>[] newCsmsPerMS2ScanPerFile = new List <GlycoSpectralMatch> [arrayOfMs2ScansSortedByMass.Length];
myFileManager.DoneWithFile(origDataFile);
for (int currentPartition = 0; currentPartition < CommonParameters.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
//When partition, the proteinList will be split for each Thread.
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
//Only reverse Decoy for glyco search has been tested and are set as fixed parameter.
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, null, null, null, currentPartition, _glycoSearchParameters.DecoyType, combinedParams, this.FileSpecificParameters, 30000.0, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), TargetContaminantAmbiguity.RemoveContaminant, new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
//The second Fragment index is for 'MS1-HCD_MS1-ETD_MS2s' type of data. If LowCID is used for MS1, ion-index is not allowed to use.
List <int>[] secondFragmentIndex = null;
//if (combinedParams.MS2ChildScanDissociationType != DissociationType.LowCID
//&& !CrosslinkSearchEngine.DissociationTypeGenerateSameTypeOfIons(combinedParams.DissociationType, combinedParams.MS2ChildScanDissociationType))
//{
// //Becuase two different type of dissociation methods are used, the parameters are changed with different dissociation type.
// var secondCombinedParams = CommonParameters.CloneWithNewDissociationType(combinedParams.MS2ChildScanDissociationType);
// var secondIndexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, null, null, null, currentPartition, _glycoSearchParameters.DecoyType, secondCombinedParams, this.FileSpecificParameters, 30000.0, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), new List<string> { taskId });
// GenerateSecondIndexes(indexEngine, secondIndexEngine, dbFilenameList, ref secondFragmentIndex, proteinList, taskId);
//}
Status("Searching files...", taskId);
new GlycoSearchEngine(newCsmsPerMS2ScanPerFile, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, secondFragmentIndex, currentPartition, combinedParams, this.FileSpecificParameters,
_glycoSearchParameters.OGlycanDatabasefile, _glycoSearchParameters.NGlycanDatabasefile, _glycoSearchParameters.GlycoSearchType, _glycoSearchParameters.GlycoSearchTopNum, _glycoSearchParameters.MaximumOGlycanAllowed, _glycoSearchParameters.OxoniumIonFilt, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + CommonParameters.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
ListOfGsmsPerMS2Scan.AddRange(newCsmsPerMS2ScanPerFile.Where(p => p != null).ToList());
completedFiles++;
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
//For every Ms2Scans, each have a list of candidates psms. The allPsms from GlycoSearchEngine is the list (all ms2scans) of list (each ms2scan) of psm (all candidate psm).
//Currently, only keep the first scan for consideration.
List <GlycoSpectralMatch> GsmPerScans = ListOfGsmsPerMS2Scan.Select(p => p.First()).ToList();
var filteredAllPsms = new List <GlycoSpectralMatch>();
//For each ms2scan, try to find the best candidate psm from the psms list. Do the localizaiton analysis. Add it into filteredAllPsms.
foreach (var gsmsPerScan in GsmPerScans.GroupBy(p => p.ScanNumber))
{
var glycos = RemoveSimilarSequenceDuplicates(gsmsPerScan.OrderByDescending(p => p.Score).ToList());
foreach (var glycoSpectralMatch in glycos)
{
if (glycoSpectralMatch.LocalizationGraphs != null)
{
List <Route> localizationCandidates = new List <Route>();
for (int i = 0; i < glycoSpectralMatch.LocalizationGraphs.Count; i++)
{
var allPathWithMaxScore = LocalizationGraph.GetAllHighestScorePaths(glycoSpectralMatch.LocalizationGraphs[i].array, glycoSpectralMatch.LocalizationGraphs[i].ChildModBoxes);
foreach (var path in allPathWithMaxScore)
{
var local = LocalizationGraph.GetLocalizedPath(glycoSpectralMatch.LocalizationGraphs[i], path);
local.ModBoxId = glycoSpectralMatch.LocalizationGraphs[i].ModBoxId;
localizationCandidates.Add(local);
}
}
glycoSpectralMatch.Routes = localizationCandidates;
}
if (glycoSpectralMatch.Routes != null)
{
LocalizationLevel localLevel;
glycoSpectralMatch.LocalizedGlycan = GlycoSpectralMatch.GetLocalizedGlycan(glycoSpectralMatch.Routes, out localLevel);
glycoSpectralMatch.LocalizationLevel = localLevel;
//Localization PValue.
if (localLevel == LocalizationLevel.Level1 || localLevel == LocalizationLevel.Level2)
{
List <Route> allRoutes = new List <Route>();
foreach (var graph in glycoSpectralMatch.LocalizationGraphs)
{
allRoutes.AddRange(LocalizationGraph.GetAllPaths_CalP(graph, glycoSpectralMatch.ScanInfo_p, glycoSpectralMatch.Thero_n));
}
glycoSpectralMatch.SiteSpeciLocalProb = LocalizationGraph.CalSiteSpecificLocalizationProbability(allRoutes, glycoSpectralMatch.LocalizationGraphs.First().ModPos);
}
}
filteredAllPsms.Add(glycoSpectralMatch);
}
}
PostGlycoSearchAnalysisTask postGlycoSearchAnalysisTask = new PostGlycoSearchAnalysisTask();
postGlycoSearchAnalysisTask.FileSpecificParameters = this.FileSpecificParameters;
return(postGlycoSearchAnalysisTask.Run(OutputFolder, dbFilenameList, currentRawFileList, taskId, fileSettingsList, filteredAllPsms.OrderByDescending(p => p.Score).ToList(), CommonParameters, _glycoSearchParameters, proteinList, variableModifications, fixedModifications, localizeableModificationTypes, MyTaskResults));
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
List <CrosslinkSpectralMatch> allPsms = new List <CrosslinkSpectralMatch>();
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, XlSearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
var crosslinker = XlSearchParameters.Crosslinker;
MyFileManager myFileManager = new MyFileManager(true);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
ProseCreatedWhileRunning.Append("The following crosslink discovery were used: ");
ProseCreatedWhileRunning.Append("crosslinker name = " + crosslinker.CrosslinkerName + "; ");
ProseCreatedWhileRunning.Append("crosslinker type = " + crosslinker.Cleavable + "; ");
ProseCreatedWhileRunning.Append("crosslinker mass = " + crosslinker.TotalMass + "; ");
ProseCreatedWhileRunning.Append("crosslinker modification site(s) = " + crosslinker.CrosslinkerModSites + "; ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif) + "; "));
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("parent mass tolerance(s) = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
CrosslinkSpectralMatch[] newPsms = new CrosslinkSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
for (int currentPartition = 0; currentPartition < CommonParameters.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, null, currentPartition, UsefulProteomicsDatabases.DecoyType.Reverse, combinedParams, 30000.0, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
Status("Searching files...", taskId);
new CrosslinkSearchEngine(newPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, currentPartition, combinedParams, crosslinker,
XlSearchParameters.RestrictToTopNHits, XlSearchParameters.CrosslinkSearchTopNum, XlSearchParameters.XlQuench_H2O,
XlSearchParameters.XlQuench_NH2, XlSearchParameters.XlQuench_Tris, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + CommonParameters.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
allPsms.AddRange(newPsms.Where(p => p != null));
completedFiles++;
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
allPsms = allPsms.OrderByDescending(p => p.XLTotalScore).ToList();
var allPsmsXL = allPsms.Where(p => p.CrossType == PsmCrossType.Cross).ToList();
// inter-crosslinks; different proteins are linked
var interCsms = allPsmsXL.Where(p => !p.ProteinAccession.Equals(p.BetaPeptide.ProteinAccession)).ToList();
foreach (var item in interCsms)
{
item.CrossType = PsmCrossType.Inter;
}
// intra-crosslinks; crosslinks within a protein
var intraCsms = allPsmsXL.Where(p => p.ProteinAccession.Equals(p.BetaPeptide.ProteinAccession)).ToList();
foreach (var item in intraCsms)
{
item.CrossType = PsmCrossType.Intra;
}
// calculate FDR
DoCrosslinkFdrAnalysis(interCsms);
DoCrosslinkFdrAnalysis(intraCsms);
SingleFDRAnalysis(allPsms, new List <string> {
taskId
});
// calculate protein crosslink residue numbers
foreach (var csm in allPsmsXL)
{
// alpha peptide crosslink residue in the protein
csm.XlProteinPos = csm.OneBasedStartResidueInProtein.Value + csm.LinkPositions[0] - 1;
// beta crosslink residue in protein
csm.BetaPeptide.XlProteinPos = csm.BetaPeptide.OneBasedStartResidueInProtein.Value + csm.BetaPeptide.LinkPositions[0] - 1;
}
// write interlink CSMs
if (interCsms.Any())
{
string file = Path.Combine(OutputFolder, "XL_Interlinks.tsv");
WritePsmCrossToTsv(interCsms, file, 2);
FinishedWritingFile(file, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
if (XlSearchParameters.WriteOutputForPercolator)
{
var interPsmsXLPercolator = interCsms.Where(p => p.Score >= 2 && p.BetaPeptide.Score >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteCrosslinkToTxtForPercolator(interPsmsXLPercolator, OutputFolder, "XL_Interlinks_Percolator", crosslinker, new List <string> {
taskId
});
}
// write intralink CSMs
if (intraCsms.Any())
{
string file = Path.Combine(OutputFolder, "XL_Intralinks.tsv");
WritePsmCrossToTsv(intraCsms, file, 2);
FinishedWritingFile(file, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
if (XlSearchParameters.WriteOutputForPercolator)
{
var intraPsmsXLPercolator = intraCsms.Where(p => p.Score >= 2 && p.BetaPeptide.Score >= 2).OrderBy(p => p.ScanNumber).ToList();
WriteCrosslinkToTxtForPercolator(intraPsmsXLPercolator, OutputFolder, "XL_Intralinks_Percolator", crosslinker, new List <string> {
taskId
});
}
// write single peptides
var singlePsms = allPsms.Where(p => p.CrossType == PsmCrossType.Single).ToList();
if (singlePsms.Any())
{
string writtenFileSingle = Path.Combine(OutputFolder, "SinglePeptides" + ".tsv");
WritePsmCrossToTsv(singlePsms, writtenFileSingle, 1);
FinishedWritingFile(writtenFileSingle, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write loops
var loopPsms = allPsms.Where(p => p.CrossType == PsmCrossType.Loop).ToList();
if (loopPsms.Any())
{
string writtenFileLoop = Path.Combine(OutputFolder, "Looplinks" + ".tsv");
WritePsmCrossToTsv(loopPsms, writtenFileLoop, 1);
FinishedWritingFile(writtenFileLoop, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write deadends
var deadendPsms = allPsms.Where(p => p.CrossType == PsmCrossType.DeadEnd ||
p.CrossType == PsmCrossType.DeadEndH2O ||
p.CrossType == PsmCrossType.DeadEndNH2 ||
p.CrossType == PsmCrossType.DeadEndTris).ToList();
if (deadendPsms.Any())
{
string writtenFileDeadend = Path.Combine(OutputFolder, "Deadends" + ".tsv");
WritePsmCrossToTsv(deadendPsms, writtenFileDeadend, 1);
FinishedWritingFile(writtenFileDeadend, new List <string> {
taskId
});
}
MyTaskResults.AddNiceText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write pepXML
if (XlSearchParameters.WritePepXml)
{
List <CrosslinkSpectralMatch> writeToXml = new List <CrosslinkSpectralMatch>();
writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml = writeToXml.OrderBy(p => p.ScanNumber).ToList();
foreach (var fullFilePath in currentRawFileList)
{
string fileNameNoExtension = Path.GetFileNameWithoutExtension(fullFilePath);
WritePepXML_xl(writeToXml.Where(p => p.FullFilePath == fullFilePath).ToList(), proteinList, dbFilenameList[0].FilePath, variableModifications, fixedModifications, localizeableModificationTypes, OutputFolder, fileNameNoExtension, new List <string> {
taskId
});
}
}
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
List <List <CrosslinkSpectralMatch> > ListOfCsmsPerMS2Scan = new List <List <CrosslinkSpectralMatch> >();
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, XlSearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
var crosslinker = XlSearchParameters.Crosslinker;
MyFileManager myFileManager = new MyFileManager(true);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
ProseCreatedWhileRunning.Append("The following crosslink discovery were used: ");
ProseCreatedWhileRunning.Append("crosslinker name = " + crosslinker.CrosslinkerName + "; ");
ProseCreatedWhileRunning.Append("crosslinker type = " + crosslinker.Cleavable + "; ");
ProseCreatedWhileRunning.Append("crosslinker mass = " + crosslinker.TotalMass + "; ");
ProseCreatedWhileRunning.Append("crosslinker modification site(s) = " + crosslinker.CrosslinkerModSites + "; ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif) + "; "));
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("parent mass tolerance(s) = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
List <CrosslinkSpectralMatch>[] newCsmsPerMS2ScanPerFile = new List <CrosslinkSpectralMatch> [arrayOfMs2ScansSortedByMass.Length];
for (int currentPartition = 0; currentPartition < CommonParameters.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
//When partition, the proteinList will be split for each Thread.
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
//Only reverse Decoy for crosslink search has been tested and are set as fixed parameter.
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, null, null, null, currentPartition, UsefulProteomicsDatabases.DecoyType.Reverse, combinedParams, 30000.0, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
//The second Fragment index is for 'MS1-HCD_MS1-ETD_MS2s' type of data. If LowCID is used for MS1, ion-index is not allowed to use.
List <int>[] secondFragmentIndex = null;
if (combinedParams.ChildScanDissociationType != DissociationType.LowCID &&
!CrosslinkSearchEngine.DissociationTypeGenerateSameTypeOfIons(combinedParams.DissociationType, combinedParams.ChildScanDissociationType))
{
//Becuase two different type of dissociation methods are used, the parameters are changed with different dissociation type.
var secondCombinedParams = CommonParameters.CloneWithNewDissociationType(combinedParams.ChildScanDissociationType);
var secondIndexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, null, null, null, currentPartition, UsefulProteomicsDatabases.DecoyType.Reverse, secondCombinedParams, 30000.0, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), new List <string> {
taskId
});
GenerateSecondIndexes(indexEngine, secondIndexEngine, dbFilenameList, ref secondFragmentIndex, proteinList, taskId);
}
Status("Searching files...", taskId);
new CrosslinkSearchEngine(newCsmsPerMS2ScanPerFile, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, secondFragmentIndex, currentPartition, combinedParams, crosslinker,
XlSearchParameters.RestrictToTopNHits, XlSearchParameters.CrosslinkSearchTopNum, XlSearchParameters.CrosslinkAtCleavageSite,
XlSearchParameters.XlQuench_H2O, XlSearchParameters.XlQuench_NH2, XlSearchParameters.XlQuench_Tris, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + CommonParameters.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
ListOfCsmsPerMS2Scan.AddRange(newCsmsPerMS2ScanPerFile.Where(p => p != null).ToList());
completedFiles++;
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
List <List <CrosslinkSpectralMatch> > ListOfCsmsPerMS2ScanParsimony = new List <List <CrosslinkSpectralMatch> >();
//For every Ms2Scans, each have a list of candidates psms. The allPsms from CrosslinkSearchEngine is the list (all ms2scans) of list (each ms2scan) of psm (all candidate psm).
//The allPsmsList is same as allPsms after ResolveAmbiguities.
foreach (var csmsPerScan in ListOfCsmsPerMS2Scan)
{
foreach (var csm in csmsPerScan)
{
csm.ResolveAllAmbiguities();
if (csm.BetaPeptide != null)
{
csm.BetaPeptide.ResolveAllAmbiguities();
}
}
ListOfCsmsPerMS2ScanParsimony.Add(RemoveDuplicateFromCsmsPerScan(csmsPerScan));
}
var filteredAllPsms = new List <CrosslinkSpectralMatch>();
//For each ms2scan, try to find the best candidate psm from the psms list. Add it into filteredAllPsms
//This function is for current usage, this can be replaced with PEP value.
foreach (var csmsPerScan in ListOfCsmsPerMS2ScanParsimony)
{
filteredAllPsms.Add(csmsPerScan[0]);
}
PostXLSearchAnalysisTask postXLSearchAnalysisTask = new PostXLSearchAnalysisTask();
return(postXLSearchAnalysisTask.Run(OutputFolder, dbFilenameList, currentRawFileList, taskId, fileSettingsList, filteredAllPsms.OrderByDescending(p => p.XLTotalScore).ToList(), CommonParameters, XlSearchParameters, proteinList, variableModifications, fixedModifications, localizeableModificationTypes, MyTaskResults));
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
// load modifications
Status("Loading modifications...", new List <string> {
taskId
});
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = GlobalVariables.AllModTypesKnown.ToList();
// what types of fragment ions to search for
List <ProductType> ionTypes = new List <ProductType>();
if (CommonParameters.BIons)
{
ionTypes.Add(ProductType.BnoB1ions);
}
if (CommonParameters.YIons)
{
ionTypes.Add(ProductType.Y);
}
if (CommonParameters.ZdotIons)
{
ionTypes.Add(ProductType.Zdot);
}
if (CommonParameters.CIons)
{
ionTypes.Add(ProductType.C);
}
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, DecoyType.Reverse, localizeableModificationTypes, CommonParameters);
// write prose settings
ProseCreatedWhileRunning.Append("The following calibration settings were used: ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("max mods per peptide = " + CommonParameters.DigestionParams.MaxModsForPeptide + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("precursor mass tolerance = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy) + " non-decoy protein entries including " + proteinList.Count(p => p.IsContaminant) + " contaminant sequences. ");
// start the calibration task
Status("Calibrating...", new List <string> {
taskId
});
MyTaskResults = new MyTaskResults(this)
{
NewSpectra = new List <string>(),
NewFileSpecificTomls = new List <string>()
};
object lock1 = new object();
var myFileManager = new MyFileManager(true);
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
// get filename stuff
var originalUncalibratedFilePath = currentRawFileList[spectraFileIndex];
var originalUncalibratedFilenameWithoutExtension = Path.GetFileNameWithoutExtension(originalUncalibratedFilePath);
string calibratedFilePath = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + CalibSuffix + ".mzML");
// mark the file as in-progress
StartingDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MsDataFile myMsDataFile;
// load the file
Status("Loading spectra file...", new List <string> {
taskId, "Individual Spectra Files"
});
lock (lock1)
{
myMsDataFile = myFileManager.LoadFile(originalUncalibratedFilePath, CommonParameters.TopNpeaks, CommonParameters.MinRatio, CommonParameters.TrimMs1Peaks, CommonParameters.TrimMsMsPeaks, CommonParameters);
}
// get datapoints to fit calibration function to
Status("Acquiring calibration data points...", new List <string> {
taskId, "Individual Spectra Files"
});
DataPointAquisitionResults acquisitionResults = null;
for (int i = 1; i <= 5; i++)
{
acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
// enough data points to calibrate?
if (acquisitionResults.Psms.Count >= NumRequiredPsms && acquisitionResults.Ms1List.Count > NumRequiredMs1Datapoints && acquisitionResults.Ms2List.Count > NumRequiredMs2Datapoints)
{
break;
}
if (i == 1) // failed round 1
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(20);
CommonParameters.ProductMassTolerance = new PpmTolerance(50);
}
else if (i == 2) // failed round 2
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(30);
CommonParameters.ProductMassTolerance = new PpmTolerance(100);
}
else if (i == 3) // failed round 3
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(40);
CommonParameters.ProductMassTolerance = new PpmTolerance(150);
}
else // failed round 4
{
if (acquisitionResults.Psms.Count < NumRequiredPsms)
{
Warn("Calibration failure! Could not find enough high-quality PSMs. Required " + NumRequiredPsms + ", saw " + acquisitionResults.Psms.Count);
}
if (acquisitionResults.Ms1List.Count < NumRequiredMs1Datapoints)
{
Warn("Calibration failure! Could not find enough MS1 datapoints. Required " + NumRequiredMs1Datapoints + ", saw " + acquisitionResults.Ms1List.Count);
}
if (acquisitionResults.Ms2List.Count < NumRequiredMs2Datapoints)
{
Warn("Calibration failure! Could not find enough MS2 datapoints. Required " + NumRequiredMs2Datapoints + ", saw " + acquisitionResults.Ms2List.Count);
}
FinishedDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
return(MyTaskResults);
}
Warn("Could not find enough PSMs to calibrate with; opening up tolerances to " +
Math.Round(CommonParameters.PrecursorMassTolerance.Value, 2) + " ppm precursor and " +
Math.Round(CommonParameters.ProductMassTolerance.Value, 2) + " ppm product");
}
// stats before calibration
int prevPsmCount = acquisitionResults.Psms.Count;
double preCalibrationPrecursorErrorIqr = acquisitionResults.PsmPrecursorIqrPpmError;
double preCalibrationProductErrorIqr = acquisitionResults.PsmProductIqrPpmError;
// generate calibration function and shift data points
Status("Calibrating...", new List <string> {
taskId, "Individual Spectra Files"
});
new CalibrationEngine(myMsDataFile, acquisitionResults, CommonParameters, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}).Run();
// do another search to evaluate calibration results
Status("Post-calibration search...", new List <string> {
taskId, "Individual Spectra Files"
});
acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
// stats after calibration
int postCalibrationPsmCount = acquisitionResults.Psms.Count;
double postCalibrationPrecursorErrorIqr = acquisitionResults.PsmPrecursorIqrPpmError;
double postCalibrationProductErrorIqr = acquisitionResults.PsmProductIqrPpmError;
// did the data improve? (not used for anything yet...)
bool improvement = ImprovGlobal(preCalibrationPrecursorErrorIqr, preCalibrationProductErrorIqr, prevPsmCount, postCalibrationPsmCount, postCalibrationPrecursorErrorIqr, postCalibrationProductErrorIqr);
// write toml settings for the calibrated file
var newTomlFileName = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + CalibSuffix + ".toml");
var fileSpecificParams = new FileSpecificParameters();
// carry over file-specific parameters from the uncalibrated file to the calibrated one
if (fileSettingsList[spectraFileIndex] != null)
{
fileSpecificParams = fileSettingsList[spectraFileIndex].Clone();
}
// don't write over ppm tolerances if they've been specified by the user already in the file-specific settings
// otherwise, suggest 4 * interquartile range as the ppm tolerance
if (fileSpecificParams.PrecursorMassTolerance == null)
{
fileSpecificParams.PrecursorMassTolerance = new PpmTolerance((4.0 * postCalibrationPrecursorErrorIqr) + Math.Abs(acquisitionResults.PsmPrecursorMedianPpmError));
}
if (fileSpecificParams.ProductMassTolerance == null)
{
fileSpecificParams.ProductMassTolerance = new PpmTolerance((4.0 * postCalibrationProductErrorIqr) + Math.Abs(acquisitionResults.PsmProductMedianPpmError));
}
Toml.WriteFile(fileSpecificParams, newTomlFileName, tomlConfig);
FinishedWritingFile(newTomlFileName, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
// write the calibrated mzML file
MzmlMethods.CreateAndWriteMyMzmlWithCalibratedSpectra(myMsDataFile, calibratedFilePath, false);
myFileManager.DoneWithFile(originalUncalibratedFilePath);
// finished calibrating this file
FinishedWritingFile(calibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
MyTaskResults.NewSpectra.Add(calibratedFilePath);
MyTaskResults.NewFileSpecificTomls.Add(newTomlFileName);
FinishedDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}));
}
// re-write experimental design (if it has been defined) with new calibrated file names
string assumedPathToExperDesign = Directory.GetParent(currentRawFileList.First()).FullName;
assumedPathToExperDesign = Path.Combine(assumedPathToExperDesign, GlobalVariables.ExperimentalDesignFileName);
List <string> newExperimentalDesignOutput = new List <string>();
if (File.Exists(assumedPathToExperDesign))
{
var lines = File.ReadAllLines(assumedPathToExperDesign);
for (int i = 0; i < lines.Length; i++)
{
// header of experimental design file
if (i == 0)
{
newExperimentalDesignOutput.Add(lines[i]);
}
else
{
var split = lines[i].Split('\t');
string newline = Path.GetFileNameWithoutExtension(split[0]) + CalibSuffix + "\t";
for (int j = 1; j < split.Length; j++)
{
newline += split[j] + "\t";
}
newExperimentalDesignOutput.Add(newline);
}
}
}
File.WriteAllLines(Path.Combine(OutputFolder, GlobalVariables.ExperimentalDesignFileName), newExperimentalDesignOutput);
// finished calibrating all files for the task
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files"
}));
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, DecoyType.Reverse, localizeableModificationTypes, CommonParameters);
// write prose settings
ProseCreatedWhileRunning.Append("The following calibration settings were used: ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("max mods per peptide = " + CommonParameters.DigestionParams.MaxModsForPeptide + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("precursor mass tolerance = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy) + " non-decoy protein entries including " + proteinList.Count(p => p.IsContaminant) + " contaminant sequences. ");
// start the calibration task
Status("Calibrating...", new List <string> {
taskId
});
MyTaskResults = new MyTaskResults(this)
{
NewSpectra = new List <string>(),
NewFileSpecificTomls = new List <string>()
};
var myFileManager = new MyFileManager(true);
List <string> spectraFilesAfterCalibration = new List <string>();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
bool couldNotFindEnoughDatapoints = false;
// get filename stuff
var originalUncalibratedFilePath = currentRawFileList[spectraFileIndex];
var originalUncalibratedFilenameWithoutExtension = Path.GetFileNameWithoutExtension(originalUncalibratedFilePath);
string calibratedFilePath = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + CalibSuffix + ".mzML");
// mark the file as in-progress
StartingDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
// load the file
Status("Loading spectra file...", new List <string> {
taskId, "Individual Spectra Files"
});
var myMsDataFile = myFileManager.LoadFile(originalUncalibratedFilePath, CommonParameters);
// get datapoints to fit calibration function to
Status("Acquiring calibration data points...", new List <string> {
taskId, "Individual Spectra Files"
});
DataPointAquisitionResults acquisitionResults = null;
for (int i = 1; i <= 5; i++)
{
acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
// enough data points to calibrate?
if (acquisitionResults.Psms.Count >= NumRequiredPsms && acquisitionResults.Ms1List.Count > NumRequiredMs1Datapoints && acquisitionResults.Ms2List.Count > NumRequiredMs2Datapoints)
{
break;
}
if (i == 1) // failed round 1
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(20);
CommonParameters.ProductMassTolerance = new PpmTolerance(50);
}
else if (i == 2) // failed round 2
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(30);
CommonParameters.ProductMassTolerance = new PpmTolerance(100);
}
else if (i == 3) // failed round 3
{
CommonParameters.PrecursorMassTolerance = new PpmTolerance(40);
CommonParameters.ProductMassTolerance = new PpmTolerance(150);
}
else // failed round 4
{
if (acquisitionResults.Psms.Count < NumRequiredPsms)
{
Warn("Calibration failure! Could not find enough high-quality PSMs. Required " + NumRequiredPsms + ", saw " + acquisitionResults.Psms.Count);
}
if (acquisitionResults.Ms1List.Count < NumRequiredMs1Datapoints)
{
Warn("Calibration failure! Could not find enough MS1 datapoints. Required " + NumRequiredMs1Datapoints + ", saw " + acquisitionResults.Ms1List.Count);
}
if (acquisitionResults.Ms2List.Count < NumRequiredMs2Datapoints)
{
Warn("Calibration failure! Could not find enough MS2 datapoints. Required " + NumRequiredMs2Datapoints + ", saw " + acquisitionResults.Ms2List.Count);
}
couldNotFindEnoughDatapoints = true;
FinishedDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
break;
}
Warn("Could not find enough PSMs to calibrate with; opening up tolerances to " +
Math.Round(CommonParameters.PrecursorMassTolerance.Value, 2) + " ppm precursor and " +
Math.Round(CommonParameters.ProductMassTolerance.Value, 2) + " ppm product");
}
if (couldNotFindEnoughDatapoints)
{
spectraFilesAfterCalibration.Add(Path.GetFileNameWithoutExtension(currentRawFileList[spectraFileIndex]));
ReportProgress(new ProgressEventArgs(100, "Failed to calibrate!", new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}));
continue;
}
// stats before calibration
int prevPsmCount = acquisitionResults.Psms.Count;
double preCalibrationPrecursorErrorIqr = acquisitionResults.PsmPrecursorIqrPpmError;
double preCalibrationProductErrorIqr = acquisitionResults.PsmProductIqrPpmError;
// generate calibration function and shift data points
Status("Calibrating...", new List <string> {
taskId, "Individual Spectra Files"
});
CalibrationEngine engine = new CalibrationEngine(myMsDataFile, acquisitionResults, CommonParameters, FileSpecificParameters, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
engine.Run();
//update file
myMsDataFile = engine.CalibratedDataFile;
// do another search to evaluate calibration results
Status("Post-calibration search...", new List <string> {
taskId, "Individual Spectra Files"
});
acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
//generate calibration function and shift data points AGAIN because it's fast and contributes new data
Status("Calibrating...", new List <string> {
taskId, "Individual Spectra Files"
});
engine = new CalibrationEngine(myMsDataFile, acquisitionResults, CommonParameters, FileSpecificParameters, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
engine.Run();
//update file
myMsDataFile = engine.CalibratedDataFile;
// write the calibrated mzML file
MzmlMethods.CreateAndWriteMyMzmlWithCalibratedSpectra(myMsDataFile, calibratedFilePath, false);
myFileManager.DoneWithFile(originalUncalibratedFilePath);
// stats after calibration
int postCalibrationPsmCount = acquisitionResults.Psms.Count;
double postCalibrationPrecursorErrorIqr = acquisitionResults.PsmPrecursorIqrPpmError;
double postCalibrationProductErrorIqr = acquisitionResults.PsmProductIqrPpmError;
// did the data improve? (not used for anything yet...)
bool improvement = ImprovGlobal(preCalibrationPrecursorErrorIqr, preCalibrationProductErrorIqr, prevPsmCount, postCalibrationPsmCount, postCalibrationPrecursorErrorIqr, postCalibrationProductErrorIqr);
// write toml settings for the calibrated file
var newTomlFileName = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + CalibSuffix + ".toml");
var fileSpecificParams = new FileSpecificParameters();
// carry over file-specific parameters from the uncalibrated file to the calibrated one
if (fileSettingsList[spectraFileIndex] != null)
{
fileSpecificParams = fileSettingsList[spectraFileIndex].Clone();
}
//suggest 4 * interquartile range as the ppm tolerance
fileSpecificParams.PrecursorMassTolerance = new PpmTolerance((4.0 * postCalibrationPrecursorErrorIqr) + Math.Abs(acquisitionResults.PsmPrecursorMedianPpmError));
fileSpecificParams.ProductMassTolerance = new PpmTolerance((4.0 * postCalibrationProductErrorIqr) + Math.Abs(acquisitionResults.PsmProductMedianPpmError));
Toml.WriteFile(fileSpecificParams, newTomlFileName, tomlConfig);
FinishedWritingFile(newTomlFileName, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
// finished calibrating this file
spectraFilesAfterCalibration.Add(Path.GetFileNameWithoutExtension(calibratedFilePath));
FinishedWritingFile(calibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
MyTaskResults.NewSpectra.Add(calibratedFilePath);
MyTaskResults.NewFileSpecificTomls.Add(newTomlFileName);
FinishedDataFile(originalUncalibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilePath
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}));
}
// re-write experimental design (if it has been defined) with new calibrated file names
string assumedPathToExperDesign = Directory.GetParent(currentRawFileList.First()).FullName;
assumedPathToExperDesign = Path.Combine(assumedPathToExperDesign, GlobalVariables.ExperimentalDesignFileName);
if (File.Exists(assumedPathToExperDesign))
{
WriteNewExperimentalDesignFile(assumedPathToExperDesign, OutputFolder, spectraFilesAfterCalibration);
}
// finished calibrating all files for the task
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files"
}));
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificSettings[] fileSettingsList)
{
myTaskResults = new MyTaskResults(this)
{
newSpectra = new List <string>()
};
Status("Loading modifications...", new List <string> {
taskId
});
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = CommonParameters.LocalizeAll ? GlobalVariables.AllModTypesKnown.ToList() : CommonParameters.ListOfModTypesLocalize.ToList();
Status("Loading proteins...", new List <string> {
taskId
});
var proteinList = dbFilenameList.SelectMany(b => LoadProteinDb(b.FilePath, true, UsefulProteomicsDatabases.DecoyType.Reverse, localizeableModificationTypes, b.IsContaminant, out Dictionary <string, Modification> um)).ToList();
proseCreatedWhileRunning.Append("The following calibration settings were used: ");
proseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
proseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
proseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
if (CommonParameters.DigestionParams.MaxPeptideLength == null)
{
proseCreatedWhileRunning.Append("maximum peptide length = unspecified; ");
}
else
{
proseCreatedWhileRunning.Append("maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
}
proseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
proseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
proseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.id)) + "; ");
proseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.id)) + "; ");
proseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + " Da. ");
proseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
object lock1 = new object();
ParallelOptions parallelOptions = new ParallelOptions();
if (CommonParameters.MaxParallelFilesToAnalyze.HasValue)
{
parallelOptions.MaxDegreeOfParallelism = CommonParameters.MaxParallelFilesToAnalyze.Value;
}
var myFileManager = new MyFileManager(true);
Status("Calibrating...", new List <string> {
taskId
});
Parallel.For(0, currentRawFileList.Count, parallelOptions, spectraFileIndex =>
{
var originalUncalibratedFilePath = currentRawFileList[spectraFileIndex];
var originalUncalibratedFilenameWithoutExtension = Path.GetFileNameWithoutExtension(originalUncalibratedFilePath);
ICommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
string calibratedFilePath = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + "-calib.mzml");
IMsDataFile <IMsDataScan <IMzSpectrum <IMzPeak> > > myMsDataFile;
Status("Loading spectra file...", new List <string> {
taskId, "Individual Spectra Files"
});
// only load one file at a time
lock (lock1)
{
myMsDataFile = myFileManager.LoadFile(originalUncalibratedFilePath, CommonParameters.TopNpeaks, CommonParameters.MinRatio, CommonParameters.TrimMs1Peaks, CommonParameters.TrimMsMsPeaks);
}
Status("Acquiring calibration data points...", new List <string> {
taskId, "Individual Spectra Files"
});
// get datapoints to fit calibration function to
var acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
// stats before calibration
int prevPsmCount = acquisitionResults.Item1.Count;
var preCalibrationPrecursorErrors = acquisitionResults.Item1.Select(p => (p.ScanPrecursorMass - p.PeptideMonisotopicMass.Value) / p.PeptideMonisotopicMass.Value * 1e6).ToList();
double preCalibrationPrecursorIqr = Statistics.InterquartileRange(preCalibrationPrecursorErrors);
var preCalibrationProductErrors = acquisitionResults.Item1.SelectMany(p => p.ProductMassErrorPpm.SelectMany(v => v.Value)).ToList();
double preCalibrationProductIqr = Statistics.InterquartileRange(preCalibrationProductErrors);
// enough data points to calibrate with?
if (acquisitionResults.Item2 == null)
{
Warn("Could not find any datapoints to calibrate with!");
return;
}
if (acquisitionResults.Item2.Ms1List.Count < 4 || acquisitionResults.Item2.Ms2List.Count < 4)
{
Warn("Could not find enough MS1 datapoints to calibrate (" + acquisitionResults.Item2.Ms1List.Count + " found)");
Warn("Could not find enough MS2 datapoints to calibrate (" + acquisitionResults.Item2.Ms2List.Count + " found)");
return;
}
// generate calibration function and shift data points
Status("Calibrating...", new List <string> {
taskId, "Individual Spectra Files"
});
new CalibrationEngine(myMsDataFile, acquisitionResults.Item2, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}).Run();
myFileManager.DoneWithFile(originalUncalibratedFilePath);
// do another search to evaluate calibration results
Status("Post-calibration search...", new List <string> {
taskId, "Individual Spectra Files"
});
acquisitionResults = GetDataAcquisitionResults(myMsDataFile, originalUncalibratedFilePath, variableModifications, fixedModifications, proteinList, taskId, combinedParams, combinedParams.PrecursorMassTolerance, combinedParams.ProductMassTolerance);
// stats after calibration
int postCalibrationPsmCount = acquisitionResults.Item1.Count;
var postCalibrationPrecursorErrors = acquisitionResults.Item1.Select(p => (p.ScanPrecursorMass - p.PeptideMonisotopicMass) / p.PeptideMonisotopicMass * 1e6).ToList();
double postCalibrationPrecursorIqr = Statistics.InterquartileRange(postCalibrationPrecursorErrors);
var postCalibrationProductErrors = acquisitionResults.Item1.SelectMany(p => p.ProductMassErrorPpm.SelectMany(v => v.Value)).ToList();
double postCalibrationProductIqr = Statistics.InterquartileRange(postCalibrationProductErrors);
// did the data improve? (not used for anything yet...)
bool improvement = ImprovGlobal(preCalibrationPrecursorIqr, preCalibrationProductIqr, prevPsmCount, postCalibrationPsmCount, postCalibrationPrecursorIqr, postCalibrationProductIqr);
// write suggested tolerances for this file
var tomlFileName = Path.Combine(OutputFolder, originalUncalibratedFilenameWithoutExtension + "-calib.toml");
FileSpecificTolerances f = new FileSpecificTolerances
{
PrecursorMassTolerance = new PpmTolerance(4.0 * postCalibrationPrecursorIqr),
ProductMassTolerance = new PpmTolerance(4.0 * postCalibrationProductIqr)
};
Toml.WriteFile(f, tomlFileName, tomlConfig);
SucessfullyFinishedWritingFile(tomlFileName, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
// write the calibrated MZML file
MzmlMethods.CreateAndWriteMyMzmlWithCalibratedSpectra(myMsDataFile, calibratedFilePath, false);
// all done
SucessfullyFinishedWritingFile(calibratedFilePath, new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
});
myTaskResults.newSpectra.Add(calibratedFilePath);
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files", originalUncalibratedFilenameWithoutExtension
}));
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files"
}));
return(myTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
MyTaskResults = new MyTaskResults(this);
if (NeoType.Equals(NeoTaskType.AggregateTargetDecoyFiles))
{
//getfolders
if (NeoParameters.DecoyFilePath == null)
{
NeoParameters.DecoyFilePath = new DirectoryInfo(OutputFolder).Name;
string taskString = NeoParameters.DecoyFilePath.Split('-')[0];
int taskNum = Convert.ToInt32(taskString.Substring(4, taskString.Length - 4));
taskNum--;
NeoParameters.DecoyFilePath = OutputFolder.Substring(0, OutputFolder.Length - NeoParameters.DecoyFilePath.Length) + "Task" + taskNum + "-SearchTask\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
if (NeoParameters.TargetFilePath == null)
{
NeoParameters.TargetFilePath = new DirectoryInfo(OutputFolder).Name;
taskNum--;
NeoParameters.TargetFilePath = OutputFolder.Substring(0, OutputFolder.Length - NeoParameters.TargetFilePath.Length) + "Task" + taskNum + "-SearchTask\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
}
}
if (NeoParameters.TargetFilePath == null)
{
NeoParameters.TargetFilePath = new DirectoryInfo(OutputFolder).Name;
string taskString = NeoParameters.TargetFilePath.Split('-')[0];
int taskNum = Convert.ToInt32(taskString.Substring(4, taskString.Length - 4));
taskNum--;
NeoParameters.TargetFilePath = OutputFolder.Substring(0, OutputFolder.Length - NeoParameters.TargetFilePath.Length) + "Task" + taskNum + "-SearchTask\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
}
AggregateSearchFiles.Combine(NeoParameters.TargetFilePath, NeoParameters.DecoyFilePath, OutputFolder + "\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]));
}
else if (NeoType.Equals(NeoTaskType.AggregateNormalSplicedFiles))
{
//reset database
dbFilenameList = StoredDatabases;
string normalPath = "";
string cisPath = new DirectoryInfo(OutputFolder).Name;
string taskString = cisPath.Split('-')[0];
int taskNum = Convert.ToInt32(taskString.Substring(4, taskString.Length - 4));
taskNum -= 2;
string transPath = OutputFolder.Substring(0, OutputFolder.Length - cisPath.Length) + "Task" + (taskNum + 1) + "-SearchTask\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
cisPath = OutputFolder.Substring(0, OutputFolder.Length - cisPath.Length) + "Task" + taskNum + "-SearchTask\\" + Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
AggregateSearchFiles.RecursiveNeoAggregation(normalPath, cisPath, OutputFolder, "CisResults.psmtsv");
AggregateSearchFiles.RecursiveNeoAggregation(normalPath, transPath, OutputFolder, "TransResults.psmtsv");
}
else if (NeoType.Equals(NeoTaskType.GenerateSplicedPeptides))
{
NeoMassCalculator.ImportMasses();
MyFileManager myFileManager = new MyFileManager(true);
//Import Spectra
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
var origDataFile = currentRawFileList[spectraFileIndex];
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams.TopNpeaks, combinedParams.MinRatio, combinedParams.TrimMs1Peaks, combinedParams.TrimMsMsPeaks, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams.DoPrecursorDeconvolution, combinedParams.UseProvidedPrecursorInfo, combinedParams.DeconvolutionIntensityRatio, combinedParams.DeconvolutionMaxAssumedChargeState, combinedParams.DeconvolutionMassTolerance).OrderBy(b => b.PrecursorMass).ToArray();
//Import Database
Status("Loading modifications...", taskId);
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = GlobalVariables.AllModTypesKnown.ToList();
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, true, DecoyType.None, localizeableModificationTypes, combinedParams);
//Read N and C files
string nPath = NeoParameters.NFilePath;
string cPath = NeoParameters.CFilePath;
//if termini input
if (nPath == null || cPath == null)
{
//if no termini input
string taskHeader = "Task";
string[] pathArray = OutputFolder.Split('\\');
string basePath = "";
for (int i = 0; i < pathArray.Length - 1; i++)
{
basePath += pathArray[i] + '\\';
}
string currentTaskNumber = pathArray[pathArray.Length - 1].Split('-')[0];
currentTaskNumber = currentTaskNumber.Substring(taskHeader.Length, currentTaskNumber.Length - taskHeader.Length);
string NHeader = "";
string CHeader = "";
if (cPath == null)
{
CHeader = taskHeader + (Convert.ToInt16(currentTaskNumber) - 1);
if (nPath == null)
{
NHeader = taskHeader + (Convert.ToInt16(currentTaskNumber) - 2);
}
}
else
{
NHeader = taskHeader + (Convert.ToInt16(currentTaskNumber) - 1);
}
foreach (string s in Directory.GetDirectories(basePath))
{
if (s.Contains(NHeader))
{
nPath = s;
}
else if (s.Contains(CHeader))
{
cPath = s;
}
}
string fileName = Path.GetFileNameWithoutExtension(currentRawFileList[0]) + "_PSMs.psmtsv";
nPath += "\\" + fileName;
cPath += "\\" + fileName;
}
Status("Importing Search Results...", taskId);
List <NeoPsm> psms = ImportPsmtsv.ImportNeoPsms(nPath, cPath);
//Splice
Status("Splicing Fragments...", taskId);
List <NeoPsm> candidates = NeoSplicePeptides.SplicePeptides(psms);
//Find Ambiguity
Status("Identifying Ambiguity...", taskId);
NeoFindAmbiguity.FindAmbiguity(candidates, proteinList, arrayOfMs2ScansSortedByMass, dbFilenameList[0].FilePath);
//Export Results
Status("Exporting Results...", taskId);
NeoExport.ExportAll(candidates, arrayOfMs2ScansSortedByMass, OutputFolder);
//Switch databases
string outputFolder = NeoExport.path + NeoExport.folder + @"\" + NeoExport.folder + "FusionDatabaseAppendixNC.fasta";
dbFilenameList = new List <DbForTask>()
{
new DbForTask(outputFolder, false)
};
}
}
else //if SearchTransDb
{
string outputFolder = NeoExport.path + NeoExport.folder + @"\" + NeoExport.folder + "FusionDatabaseAppendixTS.fasta";
dbFilenameList = new List <DbForTask>()
{
new DbForTask(outputFolder, false)
};
}
return(MyTaskResults);
}
public MyTaskResults Run(string outputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, List <CrosslinkSpectralMatch> allPsms, CommonParameters commonParameters, XlSearchParameters xlSearchParameters, List <Protein> proteinList, List <Modification> variableModifications, List <Modification> fixedModifications, List <string> localizeableModificationTypes, MyTaskResults MyTaskResults)
{
// inter-crosslinks; different proteins are linked
List <CrosslinkSpectralMatch> interCsms = allPsms.Where(p => p.CrossType == PsmCrossType.Inter).OrderByDescending(p => p.XLTotalScore).ToList();
// intra-crosslinks; crosslinks within a protein
List <CrosslinkSpectralMatch> intraCsms = allPsms.Where(p => p.CrossType == PsmCrossType.Intra).OrderByDescending(p => p.XLTotalScore).ToList();
var singlePsms = allPsms.Where(p => p.CrossType == PsmCrossType.Single).OrderByDescending(p => p.Score).ToList();
var loopPsms = allPsms.Where(p => p.CrossType == PsmCrossType.Loop).OrderByDescending(p => p.Score).ToList();
var deadendPsms = allPsms.Where(p => p.CrossType == PsmCrossType.DeadEnd ||
p.CrossType == PsmCrossType.DeadEndH2O ||
p.CrossType == PsmCrossType.DeadEndNH2 ||
p.CrossType == PsmCrossType.DeadEndTris).OrderByDescending(p => p.Score).ToList();
ComputeXlinkQandPValues(allPsms, intraCsms, interCsms, commonParameters, taskId);
WriteCsvFiles(outputFolder, interCsms, intraCsms, singlePsms, loopPsms, deadendPsms, taskId, xlSearchParameters);
MyTaskResults.AddTaskSummaryText("Target inter-crosslinks within 1% FDR: " + interCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
MyTaskResults.AddTaskSummaryText("Target intra-crosslinks within 1% FDR: " + intraCsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy && !p.BetaPeptide.IsDecoy));
MyTaskResults.AddTaskSummaryText("Target single peptides within 1% FDR: " + singlePsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
MyTaskResults.AddTaskSummaryText("Target loop-linked peptides within 1% FDR: " + loopPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
MyTaskResults.AddTaskSummaryText("Target deadend peptides within 1% FDR: " + deadendPsms.Count(p => p.FdrInfo.QValue <= 0.01 && !p.IsDecoy));
// write pepXML
if (xlSearchParameters.WritePepXml)
{
List <CrosslinkSpectralMatch> writeToXml = new List <CrosslinkSpectralMatch>();
writeToXml.AddRange(intraCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(interCsms.Where(p => !p.IsDecoy && !p.BetaPeptide.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(singlePsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(loopPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml.AddRange(deadendPsms.Where(p => !p.IsDecoy && p.FdrInfo.QValue <= 0.05));
writeToXml = writeToXml.OrderBy(p => p.ScanNumber).ToList();
foreach (var fullFilePath in currentRawFileList)
{
string fileNameNoExtension = Path.GetFileNameWithoutExtension(fullFilePath);
WriteFile.WritePepXML_xl(writeToXml.Where(p => p.FullFilePath == fullFilePath).ToList(), proteinList, dbFilenameList[0].FilePath, variableModifications, fixedModifications, localizeableModificationTypes, outputFolder, fileNameNoExtension, commonParameters, xlSearchParameters);
FinishedWritingFile(Path.Combine(outputFolder, fileNameNoExtension + ".pep.XML"), new List <string> {
taskId
});
}
}
return(MyTaskResults);
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
// disable quantification if a .mgf is being used
if (SearchParameters.DoQuantification && currentRawFileList.Any(x => Path.GetExtension(x).Equals(".mgf", StringComparison.OrdinalIgnoreCase)))
{
SearchParameters.DoQuantification = false;
}
// load modifications
Status("Loading modifications...", taskId);
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = GlobalVariables.AllModTypesKnown.ToList();
// what types of fragment ions to search for
List <ProductType> ionTypes = new List <ProductType>();
if (CommonParameters.BIons && CommonParameters.AddCompIons)
{
ionTypes.Add(ProductType.B);
}
else if (CommonParameters.BIons)
{
ionTypes.Add(ProductType.BnoB1ions);
}
if (CommonParameters.YIons)
{
ionTypes.Add(ProductType.Y);
}
if (CommonParameters.ZdotIons)
{
ionTypes.Add(ProductType.Zdot);
}
if (CommonParameters.CIons)
{
ionTypes.Add(ProductType.C);
}
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, SearchParameters.SearchTarget, SearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
// write prose settings
ProseCreatedWhileRunning.Append("The following search settings were used: ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.id)) + "; ");
ProseCreatedWhileRunning.Append("max mods per peptide = " + CommonParameters.DigestionParams.MaxModsForPeptide + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("precursor mass tolerance = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("report PSM ambiguity = " + CommonParameters.ReportAllAmbiguity + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy) + " non-decoy protein entries including " + proteinList.Count(p => p.IsContaminant) + " contaminant sequences. ");
// start the search task
MyTaskResults = new MyTaskResults(this);
List <PeptideSpectralMatch> allPsms = new List <PeptideSpectralMatch>();
FlashLFQResults flashLfqResults = null;
MyFileManager myFileManager = new MyFileManager(SearchParameters.DisposeOfFileWhenDone);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
HashSet <DigestionParams> ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
Status("Searching files...", new List <string> {
taskId, "Individual Spectra Files"
});
Dictionary <string, int[]> numMs2SpectraPerFile = new Dictionary <string, int[]>();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
var origDataFile = currentRawFileList[spectraFileIndex];
// mark the file as in-progress
StartingDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MassDiffAcceptor massDiffAcceptor = GetMassDiffAcceptor(combinedParams.PrecursorMassTolerance, SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams.TopNpeaks, combinedParams.MinRatio, combinedParams.TrimMs1Peaks, combinedParams.TrimMsMsPeaks, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams.DoPrecursorDeconvolution, combinedParams.UseProvidedPrecursorInfo, combinedParams.DeconvolutionIntensityRatio, combinedParams.DeconvolutionMaxAssumedChargeState, combinedParams.DeconvolutionMassTolerance).OrderBy(b => b.PrecursorMass).ToArray();
numMs2SpectraPerFile.Add(Path.GetFileNameWithoutExtension(origDataFile), new int[] { myMsDataFile.GetAllScansList().Count(p => p.MsnOrder == 2), arrayOfMs2ScansSortedByMass.Length });
myFileManager.DoneWithFile(origDataFile);
var fileSpecificPsms = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
// modern search
if (SearchParameters.SearchType == SearchType.Modern)
{
for (int currentPartition = 0; currentPartition < combinedParams.TotalPartitions; currentPartition++)
{
List <CompactPeptide> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, ionTypes, currentPartition, SearchParameters.DecoyType, ListOfDigestionParams, combinedParams, SearchParameters.MaxFragmentSize, new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
lock (indexLock)
{
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, taskId);
}
Status("Searching files...", taskId);
new ModernSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, ionTypes, currentPartition, combinedParams, massDiffAcceptor, SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + combinedParams.TotalPartitions + "!", thisId));
}
}
// nonspecific search
else if (SearchParameters.SearchType == SearchType.NonSpecific)
{
List <List <ProductType> > terminusSeparatedIons = ProductTypeMethods.SeparateIonsByTerminus(ionTypes);
foreach (List <ProductType> terminusSpecificIons in terminusSeparatedIons)
{
for (int currentPartition = 0; currentPartition < combinedParams.TotalPartitions; currentPartition++)
{
List <CompactPeptide> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
List <int>[] fragmentIndex = new List <int> [1];
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, terminusSpecificIons, currentPartition, SearchParameters.DecoyType, ListOfDigestionParams, combinedParams, SearchParameters.MaxFragmentSize, new List <string> {
taskId
});
lock (indexLock)
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, taskId);
Status("Getting precursor dictionary...", new List <string> {
taskId
});
List <CompactPeptide> peptideIndexPrecursor = null;
List <Protein> proteinListSubsetPrecursor = proteinList.GetRange(currentPartition * proteinList.Count() / combinedParams.TotalPartitions, ((currentPartition + 1) * proteinList.Count() / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count() / combinedParams.TotalPartitions));
List <int>[] fragmentIndexPrecursor = new List <int> [1];
var indexEnginePrecursor = new PrecursorIndexingEngine(proteinListSubsetPrecursor, variableModifications, fixedModifications, terminusSpecificIons, currentPartition, SearchParameters.DecoyType, ListOfDigestionParams, combinedParams, 0, new List <string> {
taskId
});
lock (indexLock)
GenerateIndexes(indexEnginePrecursor, dbFilenameList, ref peptideIndexPrecursor, ref fragmentIndexPrecursor, taskId);
if (peptideIndex.Count != peptideIndexPrecursor.Count)
{
throw new MetaMorpheusException("peptideIndex not identical between indexing engines");
}
Status("Searching files...", taskId);
new NonSpecificEnzymeSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, fragmentIndexPrecursor, terminusSpecificIons, currentPartition, combinedParams, massDiffAcceptor, SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + combinedParams.TotalPartitions + "!", thisId));
}
}
}
// classic search
else
{
Status("Starting search...", thisId);
new ClassicSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, variableModifications, fixedModifications, proteinList, ionTypes, massDiffAcceptor, combinedParams, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search!", thisId));
}
lock (psmLock)
{
allPsms.AddRange(fileSpecificPsms.Where(p => p != null));
}
completedFiles++;
FinishedDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
PostSearchAnalysisParameters parameters = new PostSearchAnalysisParameters();
parameters.SearchTaskResults = MyTaskResults;
parameters.SearchTaskId = taskId;
parameters.SearchParameters = SearchParameters;
parameters.ProteinList = proteinList;
parameters.IonTypes = ionTypes;
parameters.AllPsms = allPsms;
parameters.FixedModifications = fixedModifications;
parameters.VariableModifications = variableModifications;
parameters.ListOfDigestionParams = ListOfDigestionParams;
parameters.CurrentRawFileList = currentRawFileList;
parameters.MyFileManager = myFileManager;
parameters.NumNotches = GetNumNotches(SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
parameters.OutputFolder = OutputFolder;
parameters.IndividualResultsOutputFolder = Path.Combine(OutputFolder, "Individual File Results");
parameters.FlashLfqResults = flashLfqResults;
parameters.FileSettingsList = fileSettingsList;
parameters.NumMs2SpectraPerFile = numMs2SpectraPerFile;
parameters.DatabaseFilenameList = dbFilenameList;
PostSearchAnalysisTask postProcessing = new PostSearchAnalysisTask();
postProcessing.Parameters = parameters;
postProcessing.CommonParameters = CommonParameters;
return(postProcessing.Run());
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificSettings[] fileSettingsList)
{
myTaskResults = new MyTaskResults(this)
{
newDatabases = new List <DbForTask>()
};
Status("Loading modifications...", new List <string> {
taskId
});
List <ModificationWithMass> variableModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsVariable.Contains((b.modificationType, b.id))).ToList();
List <ModificationWithMass> fixedModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => CommonParameters.ListOfModsFixed.Contains((b.modificationType, b.id))).ToList();
List <string> localizeableModificationTypes = CommonParameters.LocalizeAll ? GlobalVariables.AllModTypesKnown.ToList() : CommonParameters.ListOfModTypesLocalize.ToList();
List <ModificationWithMass> gptmdModifications = GlobalVariables.AllModsKnown.OfType <ModificationWithMass>().Where(b => GptmdParameters.ListOfModsGptmd.Contains((b.modificationType, b.id))).ToList();
IEnumerable <Tuple <double, double> > combos = LoadCombos(gptmdModifications).ToList();
List <PeptideSpectralMatch> allPsms = new List <PeptideSpectralMatch>();
List <ProductType> ionTypes = new List <ProductType>();
if (CommonParameters.BIons)
{
ionTypes.Add(ProductType.B);
}
if (CommonParameters.YIons)
{
ionTypes.Add(ProductType.Y);
}
if (CommonParameters.CIons)
{
ionTypes.Add(ProductType.C);
}
if (CommonParameters.ZdotIons)
{
ionTypes.Add(ProductType.Zdot);
}
Status("Loading proteins...", new List <string> {
taskId
});
Dictionary <string, Modification> um = null;
//Decoys are currently not being searched with DecoyType.None
var proteinList = dbFilenameList.SelectMany(b => LoadProteinDb(b.FilePath, true, DecoyType.Reverse, localizeableModificationTypes, b.IsContaminant, out um)).ToList();
var numRawFiles = currentRawFileList.Count;
proseCreatedWhileRunning.Append("The following G-PTM-D settings were used: "); proseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
proseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
proseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
if (CommonParameters.DigestionParams.MaxPeptideLength == null)
{
proseCreatedWhileRunning.Append("maximum peptide length = unspecified; ");
}
else
{
proseCreatedWhileRunning.Append("maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
}
proseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
proseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
proseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.id)) + "; ");
proseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.id)) + "; ");
proseCreatedWhileRunning.Append("G-PTM-D modifications count = " + gptmdModifications.Count + "; ");
//puppet searchmode for writing files. Actual searchmode is filespecific
MassDiffAcceptor tempSearchMode = new DotMassDiffAcceptor("", GetAcceptableMassShifts(fixedModifications, variableModifications, gptmdModifications, combos), CommonParameters.PrecursorMassTolerance);
proseCreatedWhileRunning.Append("parent mass tolerance(s) = {" + tempSearchMode.ToProseString() + "}; ");
proseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + " Da. ");
proseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count + " total entries including " + proteinList.Where(p => p.IsContaminant).Count() + " contaminant sequences. ");
Status("Running G-PTM-D...", new List <string> {
taskId
});
HashSet <IDigestionParams> ListOfDigestionParams = GetListOfDistinctDigestionParams(CommonParameters, fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b)));
MyFileManager myFileManager = new MyFileManager(true);
object lock1 = new object();
object lock2 = new object();
ParallelOptions parallelOptions = new ParallelOptions();
if (CommonParameters.MaxParallelFilesToAnalyze.HasValue)
{
parallelOptions.MaxDegreeOfParallelism = CommonParameters.MaxParallelFilesToAnalyze.Value;
}
Parallel.For(0, currentRawFileList.Count, parallelOptions, spectraFileIndex =>
{
var origDataFile = currentRawFileList[spectraFileIndex];
ICommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MassDiffAcceptor searchMode = new DotMassDiffAcceptor("", GetAcceptableMassShifts(fixedModifications, variableModifications, gptmdModifications, combos), combinedParams.PrecursorMassTolerance);
NewCollection(Path.GetFileName(origDataFile), new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
StartingDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
Status("Loading spectra file...", new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
IMsDataFile <IMsDataScan <IMzSpectrum <IMzPeak> > > myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams.TopNpeaks, combinedParams.MinRatio, combinedParams.TrimMs1Peaks, combinedParams.TrimMsMsPeaks);
Status("Getting ms2 scans...", new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams.DoPrecursorDeconvolution, combinedParams.UseProvidedPrecursorInfo, combinedParams.DeconvolutionIntensityRatio, combinedParams.DeconvolutionMaxAssumedChargeState, combinedParams.DeconvolutionMassTolerance).OrderBy(b => b.PrecursorMass).ToArray();
myFileManager.DoneWithFile(origDataFile);
PeptideSpectralMatch[] allPsmsArray = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
new ClassicSearchEngine(allPsmsArray, arrayOfMs2ScansSortedByMass, variableModifications, fixedModifications, proteinList, ionTypes, searchMode, false, combinedParams, combinedParams.ProductMassTolerance, new List <string> {
taskId, "Individual Spectra Files", origDataFile
}).Run();
lock (lock2)
{
allPsms.AddRange(allPsmsArray);
}
FinishedDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files", origDataFile
}));
});
ReportProgress(new ProgressEventArgs(100, "Done!", new List <string> {
taskId, "Individual Spectra Files"
}));
// Group and order psms
SequencesToActualProteinPeptidesEngine sequencesToActualProteinPeptidesEngineTest = new SequencesToActualProteinPeptidesEngine(allPsms, proteinList, fixedModifications, variableModifications, ionTypes, ListOfDigestionParams, CommonParameters.ReportAllAmbiguity, new List <string> {
taskId
});
var resTest = (SequencesToActualProteinPeptidesEngineResults)sequencesToActualProteinPeptidesEngineTest.Run();
Dictionary <CompactPeptideBase, HashSet <PeptideWithSetModifications> > compactPeptideToProteinPeptideMatchingTest = resTest.CompactPeptideToProteinPeptideMatching;
foreach (var huh in allPsms)
{
if (huh != null)
{
huh.MatchToProteinLinkedPeptides(compactPeptideToProteinPeptideMatchingTest);
}
}
allPsms = allPsms.Where(b => b != null).OrderByDescending(b => b.Score).ThenBy(b => b.PeptideMonisotopicMass.HasValue ? Math.Abs(b.ScanPrecursorMass - b.PeptideMonisotopicMass.Value) : double.MaxValue).GroupBy(b => new Tuple <string, int, double?>(b.FullFilePath, b.ScanNumber, b.PeptideMonisotopicMass)).Select(b => b.First()).ToList();
new FdrAnalysisEngine(allPsms, tempSearchMode.NumNotches, false, new List <string> {
taskId
}).Run();
var writtenFile = Path.Combine(OutputFolder, "GPTMD_Candidates.psmtsv");
WritePsmsToTsv(allPsms, writtenFile, new Dictionary <string, int>());
SucessfullyFinishedWritingFile(writtenFile, new List <string> {
taskId
});
var gptmdResults = (GptmdResults) new GptmdEngine(allPsms, gptmdModifications, combos, CommonParameters.PrecursorMassTolerance, new List <string> {
taskId
}).Run();
if (dbFilenameList.Any(b => !b.IsContaminant))
{
// do NOT use this code (Path.GetFilenameWithoutExtension) because GPTMD on .xml.gz will result in .xml.xml file type being written
//string outputXMLdbFullName = Path.Combine(OutputFolder, string.Join("-", dbFilenameList.Where(b => !b.IsContaminant).Select(b => Path.GetFileNameWithoutExtension(b.FilePath))) + "GPTMD.xml");
List <string> databaseNames = new List <string>();
foreach (var nonContaminantDb in dbFilenameList.Where(p => !p.IsContaminant))
{
var dbName = Path.GetFileName(nonContaminantDb.FilePath);
int indexOfFirstDot = dbName.IndexOf(".");
databaseNames.Add(dbName.Substring(0, indexOfFirstDot));
}
string outputXMLdbFullName = Path.Combine(OutputFolder, string.Join("-", databaseNames) + "GPTMD.xml");
var newModsActuallyWritten = ProteinDbWriter.WriteXmlDatabase(gptmdResults.Mods, proteinList.Where(b => !b.IsDecoy && !b.IsContaminant).ToList(), outputXMLdbFullName);
SucessfullyFinishedWritingFile(outputXMLdbFullName, new List <string> {
taskId
});
myTaskResults.newDatabases.Add(new DbForTask(outputXMLdbFullName, false));
myTaskResults.AddNiceText("Modifications added: " + newModsActuallyWritten.Select(b => b.Value).Sum());
myTaskResults.AddNiceText("Mods types and counts:");
myTaskResults.AddNiceText(string.Join(Environment.NewLine, newModsActuallyWritten.OrderByDescending(b => b.Value).Select(b => "\t" + b.Key + "\t" + b.Value)));
}
if (dbFilenameList.Any(b => b.IsContaminant))
{
// do NOT use this code (Path.GetFilenameWithoutExtension) because GPTMD on .xml.gz will result in .xml.xml file type being written
//string outputXMLdbFullNameContaminants = Path.Combine(OutputFolder, string.Join("-", dbFilenameList.Where(b => b.IsContaminant).Select(b => Path.GetFileNameWithoutExtension(b.FilePath))) + "GPTMD.xml");
List <string> databaseNames = new List <string>();
foreach (var contaminantDb in dbFilenameList.Where(p => p.IsContaminant))
{
var dbName = Path.GetFileName(contaminantDb.FilePath);
int indexOfFirstDot = dbName.IndexOf(".");
databaseNames.Add(dbName.Substring(0, indexOfFirstDot));
}
string outputXMLdbFullNameContaminants = Path.Combine(OutputFolder, string.Join("-", databaseNames) + "GPTMD.xml");
var newModsActuallyWritten = ProteinDbWriter.WriteXmlDatabase(gptmdResults.Mods, proteinList.Where(b => !b.IsDecoy && b.IsContaminant).ToList(), outputXMLdbFullNameContaminants);
SucessfullyFinishedWritingFile(outputXMLdbFullNameContaminants, new List <string> {
taskId
});
myTaskResults.newDatabases.Add(new DbForTask(outputXMLdbFullNameContaminants, true));
myTaskResults.AddNiceText("Contaminant modifications added: " + newModsActuallyWritten.Select(b => b.Value).Sum());
myTaskResults.AddNiceText("Mods types and counts:");
myTaskResults.AddNiceText(string.Join(Environment.NewLine, newModsActuallyWritten.OrderByDescending(b => b.Value).Select(b => "\t" + b.Key + "\t" + b.Value)));
}
return(myTaskResults);
}
