override public ExecuteResult Execute(HandlerStartInfo startInfo)
{
XmlSerializer ser = new XmlSerializer(typeof(WorkflowParameters));
WorkflowParameters wfp = new WorkflowParameters();
TextReader reader = new StringReader(startInfo.Parameters);
wfp = (WorkflowParameters)ser.Deserialize(reader);
Workflow wf = new Workflow(wfp);
wf.OnLogMessage = this.OnLogMessage;
wf.OnProgress = this.OnProgress;
if (config.Aws != null)
{
wf.InitializeS3Client(config.Aws.AccessKey, config.Aws.SecretKey, config.Aws.AwsRegion);
}
wf.ExecuteAction(startInfo);
return(new ExecuteResult()
{
Status = StatusType.Complete
});
}
public static QcDataCollection LoadOrCreateQcCollection(WorkflowParameters parameters)
{
QcDataCollection qcDataCollection;
string qcFile = Path.Combine(parameters.QcParams.QcDirectory, "QC.xml");
// see if the file exists
if (File.Exists(qcFile))
{
// if so, open it
try
{
qcDataCollection = XmlSerialization.ReadFromXmlFile <QcDataCollection>(qcFile);
Log.Information("QC data file loaded successfully");
}
catch (Exception e)
{
Log.Error(e, "Failed while loading QC data");
throw e;
}
}
else
{
// if not, check if the directory exists
if (!Directory.Exists(parameters.QcParams.QcDirectory))
{
Directory.CreateDirectory(parameters.QcParams.QcDirectory);
}
qcDataCollection = new QcDataCollection(parameters.RawFileDirectory, parameters.QcParams.QcDirectory, parameters.ExpType);
Log.Information("Appears to be a new QC directory. New QC data collection created.");
}
return(qcDataCollection);
}
public static void RunXTandem(WorkflowParameters parameters, MethodDataContainer methodData, string mgfFile, string outputFile, bool genDecoy)
{
XElement customPars, taxonomy;
// make a decoy database, if requested
if (genDecoy)
{
// check if the decoy database already exists
if (!parameters.QcParams.FastaDatabase.EndsWith(".TARGET_DECOY.fasta"))
{
FastaManipulation.ReverseDecoy(parameters.QcParams.FastaDatabase);
parameters.QcParams.FastaDatabase = parameters.QcParams.FastaDatabase + ".TARGET_DECOY.fasta";
}
}
// write out the default input file
XElement.Parse(Properties.Resources.XTandem_default_config).Save(Path.Combine(parameters.QcParams.XTandemDirectory, "RawTools_default_config.xml"));
// set up and write the taxonomy file
taxonomy = XElement.Parse(Properties.Resources.XTandem_taxonomy);
taxonomy.UpdateTaxonomy(parameters);
taxonomy.Save(Path.Combine(parameters.QcParams.XTandemDirectory, "RawTools_taxonomy.xml"));
// set up and write the custom input file
customPars = XElement.Parse(Properties.Resources.XTandem_custom_config);
customPars.UpdateCustomXParameters(parameters, methodData, mgfFile, outputFile);
string xTandemParameters = Path.Combine(parameters.QcParams.XTandemDirectory, "RawTools_custom_config.xml");
customPars.Save(xTandemParameters);
ConsoleUtils.VoidBash(Path.Combine(parameters.QcParams.XTandemDirectory, "tandem.exe"), xTandemParameters);
}
override public ExecuteResult Execute(HandlerStartInfo startInfo)
{
XmlSerializer ser = new XmlSerializer(typeof(WorkflowParameters));
WorkflowParameters wfp = new WorkflowParameters();
TextReader reader = new StringReader(startInfo.Parameters);
wfp = (WorkflowParameters)ser.Deserialize(reader);
Workflow wf = null;
if (wfp.Oracle != null)
{
wf = new OracleWorkflow(wfp);
}
else if (wfp.SQLServer != null)
{
wf = new SQLServerWorkflow(wfp);
}
else
{
wf = new Workflow(wfp);
}
wf.OnLogMessage = this.OnLogMessage;
wf.OnProgress = this.OnProgress;
seqNo = 0;
OnProgress("Execute", "Starting", StatusType.Running, startInfo.InstanceId, seqNo++);
wf.ExecuteAction(startInfo);
return(new ExecuteResult()
{
Status = StatusType.Complete
});
}
public override object GetParametersInstance()
{
WorkflowParameters wfp = new WorkflowParameters();
wfp.Oracle = new OracleType();
wfp.Oracle.User = "******";
wfp.Oracle.Password = "******";
wfp.Oracle.DataSource = @"(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SERVER = DEDICATED)(SERVICE_NAME = XE)))";
wfp.SQLServer = new SQLServerType();
wfp.SQLServer.ConnectionTimeout = 30;
wfp.SQLServer.Database = "SANDBOX";
wfp.SQLServer.DataSource = @"Server=myServerAddress;Database=myDataBase;User Id=myUsername;Password=myPassword;";
wfp.SQLServer.IntegratedSecurity = true;
wfp.SQLServer.Password = "******";
wfp.SQLServer.TrustedConnection = true;
wfp.SQLServer.User = "******";
wfp.Query = "SELECT * from USERS where NAME = @userName";
wfp.StoredProcedure = "dbo.uspGetUsers";
wfp.Parameters = new List <ParameterType>();
ParameterType param = new ParameterType();
param.Name = "userName";
param.Value = "Bill Gates";
param.Direction = System.Data.ParameterDirection.Input;
param.Type = SqlParamterTypes.String;
wfp.Parameters.Add(param);
String xml = wfp.Serialize(false);
xml = xml.Substring(xml.IndexOf("<"));
return(xml);
}
public static XElement LoadSearchResults(WorkflowParameters parameters, string rawFileName)
{
string QcSearchDataDirectory = parameters.QcParams.QcSearchDataDirectory;
string resultsFile = Path.Combine(QcSearchDataDirectory, Path.GetFileName(rawFileName) + ".pep.xml");
return(XElement.Load(resultsFile));
}
public static void RunIdentipy(WorkflowParameters parameters, MethodDataContainer methodData, string mgfFile, string outputFile)
{
string pyExec, idpyScript;
// get the path to the python executable and identipy start script if they have not been specified
if (parameters.QcParams.PythonExecutable == null & parameters.QcParams.IdentipyScript == null)
{
string pyDir = GetPythonDir();
pyExec = GetPythonExec(pyDir);
idpyScript = GetIdentipyExec(pyDir);
Log.Information("Found python directory: {PyDir}", pyDir);
Log.Information("Found python executable: {PyExec}", pyExec);
Log.Information("Found identipy start script: {IdpyScript}", idpyScript);
}
else
{
pyExec = parameters.QcParams.PythonExecutable;
idpyScript = parameters.QcParams.IdentipyScript;
}
// run identipy, supresseing RunTimeWarnings
//IdentipyExecute(idpyExec, idpyPars.FastaDatabase, idpyPars.FixedMods, idpyPars.VariableMods, mgfFile);
Console.WriteLine("Starting Identipy");
Log.Information("Starting Identipy");
IdentipyExecute(pyExec, idpyScript, mgfFile, parameters, methodData.MassAnalyzers[MSOrderType.Ms2]);
}
private ActionResult GetPageViewData(WorkflowParameters parameters, string viewName, Func <IPageWorkflow, PageViewData> getPageViewData)
{
var repositoryFactory = new RepositoryFactory();
var workflowFactory = new WorkflowFactory(repositoryFactory);
var pageWorkflow = workflowFactory.GetPageWorkflow(parameters);
return(View(viewName, getPageViewData.Invoke(pageWorkflow)));
}
public void importParametersTest1()
{
var importedParametersFile = Path.Combine(FileRefs.ImportedData, "importedN14N15WorkflowParameters.xml");
var wp = WorkflowParameters.CreateParameters(importedParametersFile);
Assert.AreEqual("N14N15Targeted1", wp.WorkflowType.ToString());
Assert.IsTrue(wp is N14N15Workflow2Parameters);
}
public static void UpdateTaxonomy(this XElement taxonomy, WorkflowParameters parameters)
{
XElement element = new XElement("file");
element.SetAttributeValue("format", "peptide");
element.SetAttributeValue("URL", parameters.QcParams.FastaDatabase);
//taxonomy.Element("bioml").Element("taxon").Add(element);
taxonomy.Element("taxon").Add(element);
}
public void createParametersObjectTest1()
{
var importedParametersFile = Path.Combine(FileRefs.ImportedData, "importedBasicTargetedWorkflowParameters.xml");
var wp = WorkflowParameters.CreateParameters(importedParametersFile);
Assert.AreEqual("UnlabelledTargeted1", wp.WorkflowType.ToString());
Assert.IsTrue(wp is BasicTargetedWorkflowParameters);
wp.LoadParameters(importedParametersFile);
}
public static void RunSearch(WorkflowParameters parameters, MethodDataContainer methodData, string rawFileName)
{
string mgfFile = Path.Combine(parameters.QcParams.QcSearchDataDirectory, Path.GetFileName(rawFileName) + ".mgf");
string outputFile = Path.Combine(parameters.QcParams.QcSearchDataDirectory, Path.GetFileName(rawFileName) + ".pep.xml");
if (parameters.QcParams.SearchAlgorithm == SearchAlgorithm.XTandem)
{
XTandem.RunXTandem(parameters, methodData, mgfFile, outputFile, genDecoy: true);
}
}
public void SetWorkflowParameterValues()
{
foreach (string parameter in WorkflowParameterList)
{
string value;
if (WorkflowParameters.TryGetValue(parameter, out value))
{
SetWorkflowParameterValue(parameter, value);
}
}
}
public static void UpdateCustomXParameters(this XElement customParameters, WorkflowParameters parameters, MethodDataContainer methodData, string mgfFile, string outputFile)
{
// add fixed modifications
customParameters.AddNoteToXTandemParameters(type: "input", label: "residue, modification mass", value: parameters.QcParams.FixedMods);
// add the variable modifications
var tempMods = from x in (new string[] { parameters.QcParams.NMod, parameters.QcParams.KMod, parameters.QcParams.XMod })
where x != null
select x;
if (tempMods.Count() > 0)
{
string vmods = tempMods.Aggregate((i, j) => i + "," + j);
customParameters.AddNoteToXTandemParameters(type: "input", label: "residue, potential modification mass", value: vmods);
}
// add the parent and fragment mass errors
// we assume the parent scan is in the FTMS
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, parent monoisotopic mass error plus", value: "10");
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, parent monoisotopic mass error minus", value: "10");
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, parent monoisotopic mass isotope error", value: "yes");
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, parent monoisotopic mass error units", value: "ppm");
// need to check where the fragment scan is happening to assign mass error
if (methodData.MassAnalyzers[MSOrderType.Ms2] == MassAnalyzerType.MassAnalyzerFTMS)
{
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, fragment monoisotopic mass error", value: "0.5");
}
else
{
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, fragment monoisotopic mass error", value: "0.05");
}
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, fragment monoisotopic mass error units", value: "Daltons");
// add default parameter file
customParameters.AddNoteToXTandemParameters(type: "input", label: "list path, default parameters",
value: Path.Combine(parameters.QcParams.XTandemDirectory, "RawTools_default_config.xml"));
// add taxonomy file
customParameters.AddNoteToXTandemParameters(type: "input", label: "list path, taxonomy information",
value: Path.Combine(parameters.QcParams.XTandemDirectory, "RawTools_taxonomy.xml"));
// add input and output
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, path", value: mgfFile);
customParameters.AddNoteToXTandemParameters(type: "input", label: "output, path", value: outputFile);
// add number of threads to be used
int numProcessors = Environment.ProcessorCount;
customParameters.AddNoteToXTandemParameters(type: "input", label: "spectrum, threads", value: (numProcessors - 1).ToString());
}
public static XElement LoadSearchResults(WorkflowParameters parameters, string rawFileName)
{
string QcSearchDataDirectory = parameters.QcParams.QcSearchDataDirectory;
string resultsFile = Path.Combine(QcSearchDataDirectory, Path.GetFileName(rawFileName) + ".pep.xml");
if (!File.Exists(resultsFile))
{
Console.WriteLine("ERROR: Database search results file does not exist. Did the search fail?");
Environment.Exit(1);
}
return(XElement.Load(resultsFile));
}
public static void WriteSearchMGF(WorkflowParameters parameters, CentroidStreamCollection centroids, SegmentScanCollection segments, RetentionTimeCollection retentionTimes,
PrecursorMassCollection precursorMasses, PrecursorScanCollection precursorScans, TrailerExtraCollection trailerExtras, MethodDataContainer methodData,
ScanIndex index, string rawFileName, bool fixedScans = false)
{
var pars = parameters.QcParams.SearchParameters;
int[] scans = AdditionalMath.SelectRandomScans(scans: index.ScanEnumerators[MSOrderType.Ms2],
num: parameters.QcParams.NumberSpectra, fixedScans: parameters.QcParams.FixedScans);
string mgfFile = ReadWrite.GetPathToFile(parameters.QcParams.QcSearchDataDirectory, rawFileName, ".mgf");
MgfWriter.WriteMGF(rawFileName, centroids, segments, parameters, retentionTimes, precursorMasses, precursorScans,
trailerExtras, methodData, index, outputFile: mgfFile, scans: scans);
}
public static TargetedResultToTextExporter CreateExporter(WorkflowParameters workflowParameters, string outputFileName)
{
TargetedResultToTextExporter exporter;
switch (workflowParameters.WorkflowType)
{
case Globals.TargetedWorkflowTypes.Undefined:
exporter = new UnlabelledTargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.UnlabelledTargeted1:
exporter = new UnlabelledTargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.O16O18Targeted1:
exporter = new O16O18TargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.SipperTargeted1:
exporter = new SipperResultToLcmsFeatureExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.N14N15Targeted1:
exporter = new N14N15TargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.TopDownTargeted1:
exporter = new TopDownTargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.Deuterated:
exporter = new DeuteratedTargetedResultToTextExporter(outputFileName);
break;
case Globals.TargetedWorkflowTypes.TargetedAlignerWorkflow1:
throw new NotImplementedException("Cannot create exporter for this type of workflow");
case Globals.TargetedWorkflowTypes.PeakDetectAndExportWorkflow1:
throw new NotImplementedException("Cannot create exporter for this type of workflow");
case Globals.TargetedWorkflowTypes.BasicTargetedWorkflowExecutor1:
throw new NotImplementedException("Cannot create exporter for this type of workflow");
default:
exporter = new UnlabelledTargetedResultToTextExporter(outputFileName);
break;
}
return(exporter);
}
/// <summary>
/// Executes the specified context.
/// </summary>
/// <param name="context">The context.</param>
public virtual void Execute(BusinessContext context)
{
WorkflowInstanceEntity entityObject = (WorkflowInstanceEntity)BusinessManager.Load(context.GetTargetMetaClassName(), context.GetTargetPrimaryKeyId().Value);
if (entityObject.State == (int)BusinessProcessState.Closed &&
entityObject.ExecutionResult == (int)BusinessProcessExecutionResult.Accepted)
{
if (entityObject.OwnerDocumentId.HasValue)
{
Document.SetReadOnly(
entityObject.OwnerDocumentId.Value,
WorkflowParameters.GetOwnerReadOnly(entityObject));
}
// TODO: Add New Owner Here
}
}
private static void RegisterRoutes(Type context, RouteCollection routes, string routeAlias)
{
var parameters = new WorkflowParameters
{
AssemblyName = context.Assembly.FullName,
ContextName = context.FullName,
RouteAlias = routeAlias
};
routes.MapRoute(
name: "EasyCrud Index",
url: routeAlias,
defaults: new
{
controller = "EasyCrud",
action = "Index",
parameters
}
);
routes.MapRoute(
name: "EasyCrud Repository List",
url: routeAlias + "/{repositoryName}",
defaults: new
{
controller = "EasyCrud",
action = "RepositoryList",
parameters,
repositoryName = UrlParameter.Optional
}
);
routes.MapRoute(
name: "EasyCrud Repository Edit",
url: routeAlias + "/{repositoryName}/{id}",
defaults: new
{
controller = "EasyCrud",
action = "RepositoryEdit",
parameters,
repositoryName = UrlParameter.Optional,
id = UrlParameter.Optional
}
);
}
public void Test1()
{
var rawdataFile = @"D:\Data\Orbitrap\QC_Shew_09_05-pt5-6_4Jan10_Doc_09-11-08.RAW";
var massTagFile = @"\\protoapps\UserData\Slysz\Data\MassTags\QCShew_Formic_MassTags_Bin10_all.txt";
var parameterFile = @"\\protoapps\UserData\Slysz\Data\QCShew_MassiveTargeted\WorkflowParameterFiles\UnlabelledTargeted_WorkflowParameters_noSum.xml";
var mtimporter = new MassTagFromTextFileImporter(massTagFile);
var mtc = new TargetCollection();
mtc = mtimporter.Import();
var run = new RunFactory().CreateRun(rawdataFile);
//RunUtilities.AlignRunUsingAlignmentInfoInFiles(run);
var expectedPeaksFilename = Path.Combine(run.DataSetPath, run.DatasetName + "_peaks.txt");
var peakImporter = new DeconTools.Backend.Data.PeakImporterFromText(expectedPeaksFilename, null);
peakImporter.ImportPeaks(run.ResultCollection.MSPeakResultList);
var testMassTagID = 3513677;
var massTagChargeState = 2;
run.CurrentMassTag = (from n in mtc.TargetList where n.ID == testMassTagID && n.ChargeState == massTagChargeState select n).First();
var parameters = WorkflowParameters.CreateParameters(parameterFile);
var workflow = TargetedWorkflow.CreateWorkflow(parameters);
workflow.Run = run;
workflow.Execute();
workflow.Result.DisplayToConsole();
RunUtilities.AlignRunUsingAlignmentInfoInFiles(run);
workflow.Execute();
workflow.Result.DisplayToConsole();
}
public TargetedAligner(Run run, DeconToolsTargetedWorkflowParameters workflowParameters)
{
_run = run;
_parameters = workflowParameters;
_netGroupings = createNETGroupings();
_workflow = new BasicTargetedWorkflow(_run, workflowParameters);
IsAlignmentInfoExported = true;
NumberOfDesiredMassTagsFoundPerNETGrouping = 25;
NumberOfChromPeaksWithinToleranceAllowed = 1;
NumberOfMaxAttemptsPerNETGrouping = 200;
this.UpperFitScoreAllowedCriteria = 0.1;
this.MinimumChromPeakIntensityCriteria = 2.5e5f;
this.IScoreAllowedCriteria = 0.15;
this.AreFeaturesSavedToTextFile = true;
}
public static void CheckIdentipyDependencies(WorkflowParameters parameters)
{
string pyExec;
// Check if python is installed
Console.WriteLine("Checking Identipy dependencies");
Log.Information("Checking Identipy dependencies");
//try
//{
if (parameters.QcParams.PythonExecutable == null)
{
pyExec = GetPythonExec(GetPythonDir());
}
else
{
pyExec = parameters.QcParams.PythonExecutable;
}
string pyV = GetPythonVersion(pyExec);
// check if it is python 2.7
if (!pyV.StartsWith("2.7"))
{
Log.Error("Python {Version} detected, but Python 2.7 is requried");
Console.WriteLine("ERROR: {0} detected, but python 2.7 is requried. If Python 2.7 is installed, please provide " +
"the path to the executable using the -p argument.", pyV);
Environment.Exit(1);
}
// if all is okay, proceed
Log.Information("Python {Version} detected, using this to run identipy", pyV);
// check if identipy is installed
try
{
ConsoleUtils.Bash("python", "-c \"import identipy\"");
}
catch (Exception e)
{
Log.Error(e, "Identipy is not installed");
Console.WriteLine("ERROR: Identipy is not installed. See identipy installation instructions at https://bitbucket.org/levitsky/identipy/overview");
Environment.Exit(1);
}
Console.WriteLine("All Identipy dependencies satisfied!");
}
protected void SaveButton_ServerClick(object sender, EventArgs e)
{
WorkflowDefinitionEntity definition = (WorkflowDefinitionEntity)BusinessManager.Load(WorkflowDefinitionEntity.ClassName, PrimaryKeyId.Parse(TemplateList.SelectedValue));
WorkflowInstanceEntity instance = BusinessManager.InitializeEntity <WorkflowInstanceEntity>(WorkflowInstanceEntity.ClassName);
instance.Name = definition.Name;
instance.Description = definition.Description;
if (!String.IsNullOrEmpty(OwnerName) && OwnerId > 0)
{
instance[OwnerName] = OwnerId;
}
instance.SchemaId = definition.SchemaId;
instance.Xaml = definition.Xaml;
if (definition.PlanFinishTimeType == (int)TimeType.Duration)
{
instance.PlanFinishTimeType = (int)TimeType.Duration;
instance.PlanDuration = definition.PlanDuration;
}
else if (definition.PlanFinishTimeType == (int)TimeType.DateTime)
{
instance.PlanFinishTimeType = (int)TimeType.DateTime;
instance.PlanFinishDate = definition.PlanFinishDate;
}
//instance.PlanDuration = definition.PlanDuration;
//if (instance.PlanDuration.HasValue)
// instance.PlanFinishTimeType = (int)TimeType.Duration;
// Lock Library
WorkflowParameters.SetOwnerReadOnly(instance, WorkflowParameters.GetOwnerReadOnly(definition));
// Overdue Action
WorkflowParameters.SetAssignmentOverdueAction(instance, WorkflowParameters.GetAssignmentOverdueAction(definition));
BusinessManager.Create(instance);
Mediachase.Ibn.Web.UI.WebControls.CommandHandler.RegisterCloseOpenedFrameScript(this.Page, string.Empty);
}
override public ExecuteResult Execute(HandlerStartInfo startInfo)
{
XmlSerializer ser = new XmlSerializer(typeof(WorkflowParameters));
WorkflowParameters wfp = new WorkflowParameters();
TextReader reader = new StringReader(startInfo.Parameters);
wfp = (WorkflowParameters)ser.Deserialize(reader);
Workflow wf = new Workflow(wfp);
wf.OnLogMessage = this.OnLogMessage;
wf.OnProgress = this.OnProgress;
wf.ExecuteAction(startInfo);
return(new ExecuteResult()
{
Status = StatusType.Complete
});
}
public override object GetParametersInstance()
{
WorkflowParameters wfp = new WorkflowParameters();
wfp.Files = new List <FileType>();
FileType file = new FileType();
file.Type = ConfigType.KeyValue;
file.RootDirectory = @"C:\Source";
file.Source = @"MyApp\Development\java.properties.Development";
file.Destination = @"MyApp\Development\java.properties";
file.SettingsFile = new SettingFileType();
file.SettingsFile.CreateIfNotFoundStr = "true";
file.SettingsFile.HasEncryptedValuesStr = "false";
file.SettingsFile.Value = @"MyApp\Development\Development.settings.csv";
file.CopySourceStr = "true";
wfp.Files.Add(file);
String xml = wfp.Serialize(false);
xml = xml.Substring(xml.IndexOf("<"));
return(xml);
}
override public ExecuteResult Execute(HandlerStartInfo startInfo)
{
XmlSerializer ser = new XmlSerializer(typeof(WorkflowParameters));
WorkflowParameters wfp = new WorkflowParameters();
TextReader reader = new StringReader(startInfo.Parameters);
wfp = (WorkflowParameters)ser.Deserialize(reader);
Workflow wf = new Workflow(wfp)
{
OnLogMessage = this.OnLogMessage,
OnProgress = this.OnProgress
};
seqNo = 0;
OnProgress("Execute", "Starting", StatusType.Running, startInfo.InstanceId, seqNo++);
wf.Initialize(ref wfp, startInfo);
wf.ExecuteAction();
return(new ExecuteResult()
{
Status = StatusType.Complete
});
}
public static (List <string> fileList, QcDataCollection qcDataCollection) GetFileListAndQcFile(WorkflowParameters parameters, bool subdirectoriesIncluded)
{
QcDataCollection qcDataCollection;
string dataDirectory = parameters.RawFileDirectory;
string qcDirectory = parameters.QcParams.QcDirectory;
string qcSearchDataDirecotry = parameters.QcParams.QcSearchDataDirectory;
SearchParameters searchParameters = parameters.QcParams.SearchParameters;
// our qc file
string qcFile = Path.Combine(qcDirectory, "QC.xml");
// see if the file exists
if (File.Exists(qcFile))
{
// if so, open it
try
{
qcDataCollection = XmlSerialization.ReadFromXmlFile <QcDataCollection>(qcFile);
Log.Information("QC data file loaded successfully");
}
catch (Exception e)
{
Log.Error(e, "Failed while loading QC data");
throw e;
}
}
else
{
// if not, check if the directory exists
if (!Directory.Exists(qcDirectory))
{
Directory.CreateDirectory(qcDirectory);
}
qcDataCollection = new QcDataCollection(dataDirectory, qcDirectory, parameters.ExpType);
Log.Information("Appears to be a new QC directory. New QC data collection created.");
}
// get our list of new raw files. it is every raw file in the directory that is not listed in the qc data
List <string> fileList;
if (subdirectoriesIncluded)
{
fileList = Directory.GetFiles(dataDirectory, "*.*", SearchOption.AllDirectories)
.Where(s => s.EndsWith(".raw", StringComparison.OrdinalIgnoreCase)).ToList();
}
else
{
fileList = Directory.GetFiles(dataDirectory, "*.*", SearchOption.TopDirectoryOnly)
.Where(s => s.EndsWith(".raw", StringComparison.OrdinalIgnoreCase)).ToList();
}
// make sure we've got absolute paths to the files
fileList.EnsureAbsolutePaths();
if (fileList.Count() == 0)
{
Log.Error("No raw files found in {Directory}", dataDirectory);
Console.WriteLine("{0} contains no raw files!", dataDirectory);
Environment.Exit(1);
}
fileList.RemoveAll(s => qcDataCollection.ProcessedRawFiles.Contains(Path.GetFileName(s)));
Log.Information("Raw files in QC queue: {Files}", fileList);
if (fileList.Count() == 0)
{
Log.Information("No new files to QC");
Console.WriteLine("No new files in the directory to QC!");
Environment.Exit(0);
}
Console.WriteLine("{0} file(s) to process", fileList.Count());
return(fileList, qcDataCollection);
}
public static SearchMetricsContainer ParseSearchResults(SearchMetricsContainer searchMetrics, WorkflowParameters parameters, string rawFileName)
{
XElement results = LoadSearchResults(parameters, rawFileName);
PsmDataCollection Psms = ExtractPsmData(results, parameters.QcParams.SearchAlgorithm);
searchMetrics.ParsePSMs(Psms, parameters);
return(searchMetrics);
}
public static void GetModificationFrequency(this SearchMetricsContainer searchMetrics, IEnumerable <PsmData> psms, WorkflowParameters parameters)
{
string nmod = parameters.QcParams.NMod;
string kmod = parameters.QcParams.KMod;
string xmod = parameters.QcParams.XMod;
Dictionary <string, string> Modifications = new Dictionary <string, string>();
Dictionary <string, int> TotalLabelingSites = new Dictionary <string, int>();
Dictionary <string, int> LabelingSitesHit = new Dictionary <string, int>();
Dictionary <string, double> LabelingEfficiency = new Dictionary <string, double>();
List <Modification> mods;
List <string> AminosOfInterest = new List <string>();
string[] Mods = new string[] { nmod, kmod, xmod };
// "Prime" the dictionaries
foreach (var item in Mods)
{
if (item == null)
{
continue;
}
var splitString = item.Split('@');
// add the key: value pairs as mass@AA:AA
Modifications.Add(item, splitString.Last());
// and AA:int
TotalLabelingSites.Add(splitString.Last(), 0);
LabelingSitesHit.Add(splitString.Last(), 0);
AminosOfInterest.Add(splitString.Last());
}
foreach (PsmData psm in psms)
{
mods = psm.Mods;
// check the sequence in two steps. First the n-terminus, then remove the n-terminus and check the rest of it.
// FIRST STEP: N-TERMINUS
if (nmod != null)
{
// check if the first residue is lysine
if (psm.Seq[0] == 'K')
{
// if so, we need to see if it was only labeled once. Skip the psm if that is the case because it is ambiguous
IEnumerable <Modification> nMods = from x in mods
where x.Loc == 0
select x;
int numMods = nMods.Count();
if (numMods == 1)
{
// we can't know which reactive site is modified, so don't include this peptide
continue;
}
if (numMods == 0)
{
// nothing is labeled
TotalLabelingSites["["] += 1;
if (AminosOfInterest.Contains("K"))
{
TotalLabelingSites["K"] += 1;
}
}
if (numMods == 2)
{
TotalLabelingSites["["] += 1;
LabelingSitesHit["["] += 1;
if (AminosOfInterest.Contains("K"))
{
TotalLabelingSites["K"] += 1;
LabelingSitesHit["K"] += 1;
}
}
}
// If the first residue is not lysine
else
{
IEnumerable <Modification> nMods = from x in mods
where x.Loc == 0
select x;
// add 1 to total n-termini, because it is always there
TotalLabelingSites["["] += 1;
// get the aa residue letter
string residue = psm.Seq[0].ToString();
//see if it is of interest
if (AminosOfInterest.Contains(residue))
{
// if so, add 1 to total sites for it
TotalLabelingSites[residue] += 1;
}
// now go through each detected modification
foreach (Modification mod in nMods)
{
if (nmod.Contains(mod.Mass.ToString()))
{
LabelingSitesHit["["] += 1;
}
else
{
if (AminosOfInterest.Contains(mod.AA))
{
LabelingSitesHit[mod.AA] += 1;
}
}
}
}
}
int start;
if (nmod != null)
{
start = 1;
}
else
{
start = 0;
}
// now continue with the rest
for (int i = start; i < psm.Seq.Length; i++)
{
// check if we care about this amino acid
string aa = psm.Seq[i].ToString();
if (AminosOfInterest.Contains(aa))
{
// add one to potential labeling sites
TotalLabelingSites[aa] += 1;
// There should only ever be one modification for each of the rest of the residues, so we can reference it by location to see if it exists
bool hit = (from x in mods
where x.Loc == i
select 1).Count() == 1;
if (hit)
{
LabelingSitesHit[aa] += 1;
}
}
else
{
continue;
}
}
}
// spit out some metrics to the console
foreach (string aa in AminosOfInterest)
{
if (aa == "[")
{
Console.WriteLine("Total N-term sites: {0}", TotalLabelingSites["["]);
}
else
{
Console.WriteLine("Total {0} sites: {1}", aa, TotalLabelingSites[aa]);
}
}
foreach (string aa in AminosOfInterest)
{
if (aa == "[")
{
Console.WriteLine("Missed modifications at N-term: {0}", TotalLabelingSites["["] - LabelingSitesHit["["]);
}
else
{
Console.WriteLine("Missed modifications at {0}: {1}", aa, TotalLabelingSites[aa] - LabelingSitesHit[aa]);
}
}
// calculate labelling efficiency for each site
foreach (var aa in AminosOfInterest)
{
double efficiency = (double)LabelingSitesHit[aa] / TotalLabelingSites[aa];
LabelingEfficiency.Add(aa, efficiency);
if (aa == "[")
{
Console.WriteLine("Modification frequency at N-term: {0}", efficiency);
}
else
{
Console.WriteLine("Modification frequency at {0}: {1}", aa, efficiency);
}
// if the sites are n-term or K add them to their own attributes
if (aa == "[")
{
searchMetrics.LabelingEfficiencyAtNTerm = efficiency;
searchMetrics.SearchData.NLabelSites = TotalLabelingSites[aa];
searchMetrics.SearchData.NLabelSitesHit = LabelingSitesHit[aa];
}
else
{
if (aa == "K")
{
searchMetrics.LabelingEfficiencyAtK = efficiency;
searchMetrics.SearchData.KLabelSites = TotalLabelingSites[aa];
searchMetrics.SearchData.KLabelSitesHit = LabelingSitesHit[aa];
}
// if not, then add it to xmod attributes
else
{
searchMetrics.LabelingEfficiencyAtX = efficiency;
searchMetrics.LabelX = aa;
searchMetrics.SearchData.XLabelSites = TotalLabelingSites[aa];
searchMetrics.SearchData.XLabelSitesHit = LabelingSitesHit[aa];
}
}
}
}
public static void ParsePSMs(this SearchMetricsContainer searchMetrics, PsmDataCollection psmCollection, WorkflowParameters parameters)
{
int numGoodPSMs, pepsWithNoMissedCleavages;
IEnumerable <int> charges;
double IdRate, chargeRatio3to2, chargeRatio4to2;
double digestionEfficiency, topDecoyScore;
double missedCleavageRate;
Dictionary <int, int> numCharges = new Dictionary <int, int>();
List <PsmData> psms;
IEnumerable <PsmData> goodPsms, nonDecoys;
int numSearched = parameters.QcParams.NumberSpectra;
// convert the dictionary to a list for easy parsing
psms = psmCollection.Values.ToList();
// get the top decoy score
topDecoyScore = (from x in psms
where x.Decoy
select x.Hyperscore)
.ToArray().Percentile(95);
// get the non-decoys
nonDecoys = from x in psms
where !x.Decoy
select x;
// and select the non-decoy hits which are above the top decoy score
goodPsms = from x in psms
where !x.Decoy & x.Hyperscore > topDecoyScore
select x;
Console.WriteLine("Total hits: {0}", psms.Count());
Console.WriteLine("Top decoy score: {0}", topDecoyScore);
Console.WriteLine("Non-decoy hits: {0}", nonDecoys.Count());
Console.WriteLine("Non-decoy hits above top decoy score: {0}", goodPsms.Count());
// parse out the charges
charges = from x in goodPsms
select x.Charge;
// get the number of each charge, add to a dictionary
foreach (int charge in new List <int>()
{
2, 3, 4
})
{
numCharges.Add(charge, (from x in charges where x == charge select 1).Count());
}
// calculate charge ratios
chargeRatio3to2 = Convert.ToDouble(numCharges[3]) / Convert.ToDouble(numCharges[2]);
chargeRatio4to2 = Convert.ToDouble(numCharges[4]) / Convert.ToDouble(numCharges[2]);
// parse out the missed cleavage data
pepsWithNoMissedCleavages = (from x in goodPsms
where x.MissedCleavages == 0
select 1).Sum();
// number of PSMs is the length of this collection
numGoodPSMs = goodPsms.Count();
// missed cleavages per PSM
digestionEfficiency = (double)pepsWithNoMissedCleavages / numGoodPSMs;
Console.WriteLine("Digestion efficiency: {0}", digestionEfficiency);
// get missed cleavage rate, i.e. number of missed cleavages per psm
missedCleavageRate = (double)(from x in goodPsms select x.MissedCleavages).Sum() / numGoodPSMs;
Console.WriteLine("Missed cleavage rate (/PSM): {0}", missedCleavageRate);
// calculate ID rate
IdRate = (double)numGoodPSMs / numSearched;
Console.WriteLine("IDrate: {0}", IdRate);
// get labeling efficiency metrics
if ((parameters.QcParams.NMod != null) | (parameters.QcParams.KMod != null) | (parameters.QcParams.XMod != null))
{
searchMetrics.GetModificationFrequency(goodPsms, parameters);
}
// get median mass drift
searchMetrics.MedianMassDrift = (from x in goodPsms
select x.MassDrift)
.ToArray().Percentile(50);
searchMetrics.SearchData.PSMsWithNoMissedCleavages = pepsWithNoMissedCleavages;
searchMetrics.SearchData.TotalNumGoodPSMs = numGoodPSMs;
searchMetrics.SearchData.NumCharge2 = numCharges[2];
searchMetrics.SearchData.NumCharge3 = numCharges[3];
searchMetrics.SearchData.NumCharge4 = numCharges[4];
searchMetrics.IdentificationRate = IdRate;
searchMetrics.MissedCleavageRate = missedCleavageRate;
searchMetrics.DigestionEfficiency = digestionEfficiency;
searchMetrics.ChargeRatio3to2 = chargeRatio3to2;
searchMetrics.ChargeRatio4to2 = chargeRatio4to2;
}
