/// <summary>
/// saves the output from the resource usage object to an output file
/// </summary>
private void SaveResourceUsage(DrmaaResourceUsage rUsage, UnitOfWork unitOfWork)
{
// derive the resource usage filename
string resourcesPath = string.Format("{0}.cluster", Path.Combine(unitOfWork.LoggingFolder, unitOfWork.LoggingStub));
// save the resource usage to the output file
try
{
using (StreamWriter writer = new StreamWriter(resourcesPath))
{
writer.WriteLine(rUsage);
}
}
catch (Exception)
{
// this is a non-fatal error
Console.WriteLine("ERROR: Unable to write the resource usage file ({0}).", resourcesPath);
}
}
/// <summary>
/// waits for the specified job to finish
/// </summary>
private bool WaitForJob(UnitOfWork unitOfWork, out IntPtr resourceUsage)
{
byte[] errorMessageBuffer = new byte[DrmaaErrorStringBuffer + 1];
int status = -1;
resourceUsage = IntPtr.Zero;
// convert the job ID
ASCIIEncoding encoding = new ASCIIEncoding();
byte[] jobNameBuffer = encoding.GetBytes(unitOfWork.JobID + char.MinValue);
byte[] jobNameOutBuffer = new byte[DrmaaJobnameBuffer + 1];
int errnum = SafeNativeMethods.drmaa_wait(jobNameBuffer, jobNameOutBuffer, DrmaaJobnameBuffer, ref status,
DrmaaTimeoutWaitForever, ref resourceUsage, errorMessageBuffer,
DrmaaErrorStringBuffer);
if (errnum != DrmaaErrnoSuccess)
{
string error = ConvertByteArrayToString(errorMessageBuffer);
onError(string.Format("Could not wait for job: {0}", error));
return false;
}
// Use the DRMAA function call to resolve the value of status
int isExited = 0;
SafeNativeMethods.drmaa_wifexited(ref isExited, status, errorMessageBuffer, DrmaaErrorStringBuffer);
// isExited evaluates to zero if the job did not terminate normally
if (isExited == 0)
{
string error = ConvertByteArrayToString(errorMessageBuffer);
if (error == null || error == "") error = "<MISSING SGE ERROR>";
onError(string.Format("Job exited abnormally: {0}.\n** Failed command:\n{1} {2}\n** Stderr output (if any) is at: {3}\n",
error, unitOfWork.ExecutablePath, unitOfWork.CommandLine, unitOfWork.ErrorLogPath));
return false;
}
// Job finished normally, but maybe exited with a non-zero exit code,
// we'll check for that now
unitOfWork.ExitCode = 0;
SafeNativeMethods.drmaa_wexitstatus(ref unitOfWork.ExitCode, status, errorMessageBuffer, DrmaaErrorStringBuffer);
if (unitOfWork.ExitCode != 0)
{
string error = ConvertByteArrayToString(errorMessageBuffer);
onError(string.Format("Job returned non-zero exit code {0}: {1}.\n** Failed command:\n{2} {3}\n** Stderr output (if any) is at: {4}\n",
unitOfWork.ExitCode, error, unitOfWork.ExecutablePath, unitOfWork.CommandLine, unitOfWork.ErrorLogPath));
return false;
}
onLog(string.Format("Successfully finished command:\n{0} {1}", unitOfWork.ExecutablePath, unitOfWork.CommandLine));
return true;
}
/// <summary>
/// processes a single job
/// </summary>
public void ProcessJob(UnitOfWork job, bool redirectOutput, bool checkExitCodes, bool throwExceptionOnBadExitCode)
{
ProcessJobs(new List<UnitOfWork> { job }, 1, redirectOutput, checkExitCodes, throwExceptionOnBadExitCode);
}
/// <summary>
/// processes a single job
/// </summary>
private void RunJob(UnitOfWork job, bool redirectOutput, bool throwExceptionOnBadExitCode,
ref bool cancelJobs, ref long maxVMem, ref TimeSpan maxWallTime)
{
// replace the command-line tokens
job.CommandLine = ConvertCommandLineTokens(job.CommandLine);
string executableFilename = Path.GetFileName(job.ExecutablePath);
OnLog(string.Format("Launch process: {0} {1}", job.ExecutablePath, job.CommandLine));
// set the working directory to a drive we can write to (NOT a network drive)
string workingDirectory = string.IsNullOrEmpty(job.WorkingDirectory) ? _alternativeWorkingDirectory : job.WorkingDirectory;
foreach (string key in job.Environment.Keys)
{
job.CommandLine = job.CommandLine.Replace("%" + key + "%", job.Environment[key]);
}
ProcessStartInfo startInfo = new ProcessStartInfo(job.ExecutablePath, job.CommandLine)
{
WorkingDirectory = workingDirectory,
CreateNoWindow = true,
UseShellExecute = false
};
foreach (string key in job.EnvironmentVariables.Keys)
{
if (job.EnvironmentVariables[key] == null && startInfo.EnvironmentVariables.ContainsKey(key))
startInfo.EnvironmentVariables.Remove(key);
else
startInfo.EnvironmentVariables[key] = job.EnvironmentVariables[key];
}
Process jobProcess = new Process { StartInfo = startInfo };
// handle output redirection
OutputRedirector outputDirector = null;
OutputRedirector errorDirector = null;
DeleteLog(job.ErrorLogPath);
DeleteLog(job.OutputLogPath);
//DeleteLog(job.CommandLogPath);
if (redirectOutput)
{
startInfo.RedirectStandardError = true;
startInfo.RedirectStandardOutput = true;
outputDirector = new OutputRedirector(job.OutputLogPath);
errorDirector = new OutputRedirector(job.ErrorLogPath);
jobProcess.OutputDataReceived += outputDirector.LineHandler;
jobProcess.ErrorDataReceived += errorDirector.LineHandler;
}
lock (_commandLogLock)
{
if (!string.IsNullOrEmpty(job.CommandLogPath))
using (var sw = new StreamWriter(job.CommandLogPath, true))
{
sw.WriteLine(job.LoggingStub + "\t" + job.ExecutablePath + " " + job.CommandLine);
}
}
// start the process
bool stopMonitorThread = false;
Thread monitorThread = null;
try
{
jobProcess.Start();
try
{
if (Utilities.RunProcessesWithLowPriority) jobProcess.PriorityClass = ProcessPriorityClass.Idle;
}
catch (Exception)
{
// the process may have completed so quickly we were unable to set the priority on it
}
// handle output redirection
if (redirectOutput)
{
jobProcess.BeginOutputReadLine();
jobProcess.BeginErrorReadLine();
}
// start the process monitor thread
// todo monitoring could probably be done on this thread instead, slightly simplifying the code
long procVMem = 0; //Max. mem used by this process (bytes)
monitorThread = new Thread(o => MonitorProcess(jobProcess, ref stopMonitorThread, ref procVMem));
monitorThread.Name = string.Format("Monitor thread {0}", executableFilename);
monitorThread.Start();
// wait for things to finish
jobProcess.WaitForExit();
lock (_rUsageLock)
{
TimeSpan? wTime = GetProcessWallTime(jobProcess);
if (wTime.HasValue && wTime.Value > maxWallTime)
maxWallTime = wTime.Value;
if (procVMem > maxVMem) maxVMem = procVMem;
}
job.ExitCode = GetProcessExitCode(jobProcess);
}
catch (Exception e)
{
if (!throwExceptionOnBadExitCode) OnError(string.Format("Unable to start the process ({0}): {1}", executableFilename, e.Message));
job.ThrownException = e;
job.ExitCode = -1;
}
// cleanup
stopMonitorThread = true;
jobProcess.Close();
jobProcess.Dispose();
if (redirectOutput)
{
outputDirector.Dispose();
errorDirector.Dispose();
// Cleanup: Tidy up any dummy files!
CleanDummyLogFile(job.OutputLogPath);
CleanDummyLogFile(job.ErrorLogPath);
}
// prevent new jobs from running if we're going to throw an exception
if (throwExceptionOnBadExitCode && (job.ExitCode != 0)) cancelJobs = true;
if (job.ExitCode != 0) OnLog(string.Format("Process exited with code {2}: {0} {1}", job.ExecutablePath, job.CommandLine, job.ExitCode));
// update the thread status
if (Interlocked.Decrement(ref _jobsRemaining) <= 0) _doneEvent.Set();
_jobPool.Release();
}
private int GetBinSize(CanvasCallset callset, string bamPath, List<string> intermediateDataPaths,
string canvasReferencePath, string canvasBedPath)
{
string canvasBinPath = Path.Combine(_canvasFolder, "CanvasBin.exe");
string executablePath = canvasBinPath;
if (CrossPlatform.IsThisMono())
executablePath = Utilities.GetMonoPath();
StringBuilder commandLine = new StringBuilder();
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", canvasBinPath);
}
commandLine.AppendFormat("-b \"{0}\" ", bamPath);
commandLine.AppendFormat("-p "); // Paired-end input mode (Isaac or BWA output)
commandLine.AppendFormat("-r \"{0}\" ", canvasReferencePath);
foreach (string path in intermediateDataPaths)
{
commandLine.AppendFormat("-i \"{0}\" ", path);
}
commandLine.AppendFormat("-y "); // bin size only
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath)) { NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath); }
commandLine.AppendFormat("-t \"{0}\" ", callset.TempManifestPath);
}
string outputStub = Path.Combine(Path.GetDirectoryName(callset.BinSizePath), Path.GetFileNameWithoutExtension(callset.BinSizePath));
commandLine.AppendFormat("-f \"{0}\" -d {1} -o \"{2}\"", canvasBedPath, _countsPerBin, outputStub);
UnitOfWork binJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileNameWithoutExtension(callset.BinSizePath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasBin"))
{
binJob.CommandLine = Utilities.MergeCommandLineOptions(binJob.CommandLine, _customParameters["CanvasBin"], true);
}
_workManager.DoWorkSingleThread(binJob);
int binSize;
using (StreamReader reader = new StreamReader(callset.BinSizePath))
{
binSize = int.Parse(reader.ReadLine());
}
return binSize;
}
protected void RunGermlineCalling(IFileLocation partitionedPath, CanvasCallset callset, string ploidyBedPath)
{
StringBuilder commandLine = new StringBuilder();
////////////////////////////////////////////////////////
// CanvasDiploidCaller:
commandLine.Length = 0;
string executablePath = Path.Combine(_canvasFolder, "CanvasDiploidCaller.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", partitionedPath);
commandLine.AppendFormat("-v \"{0}\" ", callset.VfSummaryPath);
var cnvVcfPath = callset.OutputVcfPath;
commandLine.AppendFormat("-o \"{0}\" ", cnvVcfPath);
commandLine.AppendFormat("-n \"{0}\" ", callset.SampleName);
commandLine.AppendFormat("-r \"{0}\" ", callset.WholeGenomeFastaFolder);
if (!string.IsNullOrEmpty(ploidyBedPath))
{
commandLine.AppendFormat("-p \"{0}\" ", ploidyBedPath);
}
if (callset.IsDbSnpVcf) // a dbSNP VCF file is used in place of the normal VCF file
commandLine.AppendFormat("-d ");
UnitOfWork callJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = cnvVcfPath.Name,
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasDiploidCaller"))
{
callJob.CommandLine = Utilities.MergeCommandLineOptions(callJob.CommandLine, _customParameters["CanvasDiploidCaller"], true);
}
_workManager.DoWorkSingleThread(callJob);
}
private CanvasCleanOutput InvokeCanvasClean(CanvasCallset callset, IFileLocation binnedPath)
{
StringBuilder commandLine = new StringBuilder();
commandLine.Length = 0;
string executablePath = Path.Combine(_canvasFolder, "CanvasClean.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", binnedPath);
string cleanedPath = Path.Combine(callset.TempFolder, string.Format("{0}.cleaned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-g");
string ffpePath = null;
// TruSight Cancer has 1,737 targeted regions. The cut-off 2000 is somewhat arbitrary.
// TruSight One has 62,309 targeted regions.
// Nextera Rapid Capture v1.1 has 411,513 targeted regions.
if (!callset.IsEnrichment || callset.Manifest.Regions.Count > 2000)
{
ffpePath = Path.Combine(callset.TempFolder, "FilterRegions.txt");
commandLine.AppendFormat(" -s -r -f \"{0}\"", ffpePath);
}
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath))
{
NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath);
}
commandLine.AppendFormat(" -t \"{0}\"", callset.TempManifestPath);
}
UnitOfWork cleanJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(cleanedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasClean"))
{
cleanJob.CommandLine = Utilities.MergeCommandLineOptions(cleanJob.CommandLine, _customParameters["CanvasClean"], true);
}
_workManager.DoWorkSingleThread(cleanJob);
var canvasCleanOutput = new CanvasCleanOutput(new FileLocation(cleanedPath), new FileLocation(ffpePath));
return canvasCleanOutput;
}
protected void RunSomaticCalling(IFileLocation partitionedPath, CanvasCallset callset, string canvasBedPath,
string ploidyBedPath, IFileLocation ffpePath)
{
// get somatic SNV output:
string somaticSnvPath = callset.SomaticVcfPath?.FullName;
// Prepare and run CanvasSomaticCaller job:
UnitOfWork callerJob = new UnitOfWork();
var cnvVcfPath = callset.OutputVcfPath;
callerJob.ExecutablePath = Path.Combine(this._canvasFolder, "CanvasSomaticCaller.exe");
if (CrossPlatform.IsThisMono())
{
callerJob.CommandLine = callerJob.ExecutablePath;
callerJob.ExecutablePath = Utilities.GetMonoPath();
}
callerJob.CommandLine += string.Format(" -v {0}", callset.VfSummaryPath);
callerJob.CommandLine += string.Format(" -i {0}", partitionedPath);
callerJob.CommandLine += string.Format(" -o {0}", cnvVcfPath);
callerJob.CommandLine += string.Format(" -b {0}", canvasBedPath);
if (!string.IsNullOrEmpty(ploidyBedPath))
callerJob.CommandLine += string.Format(" -p {0}", ploidyBedPath);
callerJob.CommandLine += string.Format(" -n {0}", callset.SampleName);
if (callset.IsEnrichment)
callerJob.CommandLine += " -e";
if (callset.IsDbSnpVcf) // a dbSNP VCF file is used in place of the normal VCF file
callerJob.CommandLine += " -d";
// get localSD metric:
if (ffpePath != null)
{
// Sanity-check: CanvasClean does not always write this file.
// If it's not present, just carry on:
if (ffpePath.Exists)
{
callerJob.CommandLine += string.Format(" -f \"{0}\"", ffpePath);
}
else
{
_logger.Info("Note: SD file not found at '{0}'", ffpePath);
}
}
if (!string.IsNullOrEmpty(somaticSnvPath))
callerJob.CommandLine += string.Format(" -s {0}", somaticSnvPath);
callerJob.CommandLine += string.Format(" -r \"{0}\" ", callset.WholeGenomeFastaFolder);
if (_customParameters.ContainsKey("CanvasSomaticCaller"))
{
callerJob.CommandLine = Utilities.MergeCommandLineOptions(callerJob.CommandLine, _customParameters["CanvasSomaticCaller"], true);
}
callerJob.LoggingFolder = _workManager.LoggingFolder.FullName;
callerJob.LoggingStub = string.Format("SomaticCNV-{0}", callset.Id);
_workManager.DoWorkSingleThread(callerJob);
}
private IFileLocation InvokeCanvasPartition(CanvasCallset callset, IFileLocation cleanedPath, string canvasBedPath)
{
StringBuilder commandLine = new StringBuilder();
string executablePath = Path.Combine(_canvasFolder, "CanvasPartition.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-b \"{0}\" ", canvasBedPath);
string partitionedPath = Path.Combine(callset.TempFolder, string.Format("{0}.partitioned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", partitionedPath);
if (!_isSomatic)
commandLine.AppendFormat(" -g");
UnitOfWork partitionJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(partitionedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasPartition"))
{
partitionJob.CommandLine = Utilities.MergeCommandLineOptions(partitionJob.CommandLine, _customParameters["CanvasPartition"], true);
}
_workManager.DoWorkSingleThread(partitionJob);
return new FileLocation(partitionedPath);
}
/// <summary>
/// Invoke CanvasSNV. Return null if this fails and we need to abort CNV calling for this sample.
/// </summary>
protected void InvokeCanvasSnv(CanvasCallset callset)
{
List<UnitOfWork> jobList = new List<UnitOfWork>();
List<string> outputPaths = new List<string>();
GenomeMetadata genomeMetadata = callset.GenomeMetadata;
string tumorBamPath = callset.Bam.BamFile.FullName;
string normalVcfPath = callset.NormalVcfPath.FullName;
foreach (GenomeMetadata.SequenceMetadata chromosome in genomeMetadata.Sequences)
{
// Only invoke for autosomes + allosomes;
// don't invoke it for mitochondrial chromosome or extra contigs or decoys
if (chromosome.Type != GenomeMetadata.SequenceType.Allosome && !chromosome.IsAutosome())
continue;
UnitOfWork job = new UnitOfWork();
job.ExecutablePath = Path.Combine(_canvasFolder, "CanvasSNV.exe");
if (CrossPlatform.IsThisMono())
{
job.CommandLine = job.ExecutablePath;
job.ExecutablePath = Utilities.GetMonoPath();
}
string outputPath = Path.Combine(callset.TempFolder, string.Format("{0}-{1}.SNV.txt.gz", chromosome.Name, callset.Id));
outputPaths.Add(outputPath);
job.CommandLine += $" {chromosome.Name} {normalVcfPath} {tumorBamPath} {outputPath}";
if (_customParameters.ContainsKey("CanvasSNV"))
{
job.CommandLine = Utilities.MergeCommandLineOptions(job.CommandLine, _customParameters["CanvasSNV"], true);
}
job.LoggingFolder = _workManager.LoggingFolder.FullName;
job.LoggingStub = string.Format("CanvasSNV-{0}-{1}", callset.Id, chromosome.Name);
jobList.Add(job);
}
Console.WriteLine("Invoking {0} processor jobs...", jobList.Count);
// Invoke CanvasSNV jobs:
Console.WriteLine(">>>CanvasSNV start...");
_workManager.DoWorkParallelThreads(jobList);
Console.WriteLine(">>>CanvasSNV complete!");
// Concatenate CanvasSNV results:
using (GzipWriter writer = new GzipWriter(callset.VfSummaryPath))
{
bool headerWritten = false;
foreach (string outputPath in outputPaths)
{
if (!File.Exists(outputPath))
{
Console.WriteLine("Error: Expected output file not found at {0}", outputPath);
continue;
}
using (GzipReader reader = new GzipReader(outputPath))
{
while (true)
{
string fileLine = reader.ReadLine();
if (fileLine == null) break;
if (fileLine.Length > 0 && fileLine[0] == '#')
{
if (headerWritten) continue;
headerWritten = true;
}
writer.WriteLine(fileLine);
}
}
}
}
}
/// <summary>
/// Invoke CanvasNormalize.
/// </summary>
/// <param name="callset"></param>
/// <returns>path to the bin ratio bed file</returns>
protected string InvokeCanvasNormalize(CanvasCallset callset, string tumorBinnedPath, Dictionary<string, string> bamToBinned,
string ploidyBedPath, string mode = "weightedaverage")
{
string ratioBinnedPath = Path.Combine(callset.TempFolder, string.Format("{0}.ratio.binned", callset.Id));
string canvasNormalizePath = Path.Combine(_canvasFolder, "CanvasNormalize.exe");
string executablePath = canvasNormalizePath;
if (CrossPlatform.IsThisMono())
executablePath = Utilities.GetMonoPath();
StringBuilder commandLine = new StringBuilder();
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", canvasNormalizePath);
}
commandLine.AppendFormat("-t {0} ", tumorBinnedPath.WrapWithShellQuote()); // tumor bed
if (callset.IsEnrichment && callset.Manifest.CanvasControlAvailable)
{
commandLine.AppendFormat("-n {0} ", callset.Manifest.CanvasControlBinnedPath.WrapWithShellQuote()); // normal bed
}
else
{
foreach (string normalBinnedPath in callset.NormalBamPaths.Select(path => bamToBinned[path.BamFile.FullName]))
{
commandLine.AppendFormat("-n {0} ", normalBinnedPath.WrapWithShellQuote()); // normal bed
}
}
commandLine.AppendFormat("-w {0} ", callset.NormalBinnedPath.WrapWithShellQuote()); // weighted average normal bed
commandLine.AppendFormat("-o {0} ", ratioBinnedPath.WrapWithShellQuote()); // ratio bed
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath)) { NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath); }
commandLine.AppendFormat("-f {0} ", callset.TempManifestPath.WrapWithShellQuote());
}
commandLine.AppendFormat("-m {0} ", mode.WrapWithShellQuote());
if (!string.IsNullOrEmpty(ploidyBedPath))
{
commandLine.AppendFormat("-p {0} ", ploidyBedPath.WrapWithShellQuote());
}
UnitOfWork normalizeJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(ratioBinnedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasNormalize"))
{
normalizeJob.CommandLine = Utilities.MergeCommandLineOptions(normalizeJob.CommandLine, _customParameters["CanvasNormalize"], true);
}
_workManager.DoWorkSingleThread(normalizeJob);
return ratioBinnedPath;
}
/// <summary>
/// Invoke CanvasBin. Return null if this fails and we need to abort CNV calling for this sample.
/// </summary>
protected IFileLocation InvokeCanvasBin(CanvasCallset callset, string canvasReferencePath, string canvasBedPath, string ploidyBedPath)
{
StringBuilder commandLine = new StringBuilder();
string canvasBinPath = Path.Combine(_canvasFolder, "CanvasBin.exe");
string executablePath = canvasBinPath;
if (CrossPlatform.IsThisMono())
executablePath = Utilities.GetMonoPath();
//use bam as input
if (callset.Bam == null)
{
Console.WriteLine("Input bam file not seen for sample {0}_{1} - no CNV calls", callset.SampleName, callset.Id);
return null;
}
List<string> bamPaths = new List<string>();
bamPaths.Add(callset.Bam.BamFile.FullName);
if (!(callset.IsEnrichment && callset.Manifest.CanvasControlAvailable)) // do not add normal BAMs if Canvas Control is available
{
bamPaths.AddRange(callset.NormalBamPaths.Select(bam => bam.BamFile.FullName));
}
// loop over the reference sequences in that genome
GenomeMetadata genomeMetadata = callset.GenomeMetadata;
List<UnitOfWork> binJobs = new List<UnitOfWork>();
Dictionary<string, List<string>> intermediateDataPathsByBamPath = new Dictionary<string, List<string>>();
foreach (string bamPath in bamPaths) { intermediateDataPathsByBamPath[bamPath] = new List<string>(); }
for (int bamIndex = 0; bamIndex < bamPaths.Count; bamIndex++)
{
foreach (GenomeMetadata.SequenceMetadata sequenceMetadata in genomeMetadata.Sequences.OrderByDescending(sequence => sequence.Length))
{
// Only invoke CanvasBin for autosomes + allosomes;
// don't invoke it for mitochondrial chromosome or extra contigs or decoys
if (sequenceMetadata.Type != GenomeMetadata.SequenceType.Allosome && !sequenceMetadata.IsAutosome())
continue;
string bamPath = bamPaths[bamIndex];
commandLine.Clear();
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", canvasBinPath);
}
commandLine.AppendFormat("-b \"{0}\" ", bamPath);
if (callset.Bam.IsPairedEnd) commandLine.AppendFormat("-p ");
commandLine.AppendFormat("-r \"{0}\" ", canvasReferencePath);
commandLine.AppendFormat("-c {0} ", sequenceMetadata.Name);
commandLine.AppendFormat("-m {0} ", _coverageMode);
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath)) { NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath); }
commandLine.AppendFormat("-t \"{0}\" ", callset.TempManifestPath);
}
string intermediateDataPath = Path.Combine(callset.TempFolder, string.Format("{0}_{1}_{2}.dat",
callset.Id, bamIndex, sequenceMetadata.Name));
intermediateDataPathsByBamPath[bamPath].Add(intermediateDataPath);
commandLine.AppendFormat("-f \"{0}\" -d {1} -o \"{2}\" ", canvasBedPath, _countsPerBin, intermediateDataPath);
UnitOfWork binJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(intermediateDataPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasBin"))
{
binJob.CommandLine = Utilities.MergeCommandLineOptions(binJob.CommandLine, _customParameters["CanvasBin"], true);
}
binJobs.Add(binJob);
}
}
_workManager.DoWorkParallelThreads(binJobs);
// get bin size (of the smallest BAM) if normal BAMs are given
int binSize = -1;
if (bamPaths.Count > 1)
{
string smallestBamPath = SmallestFile(bamPaths);
binSize = GetBinSize(callset, smallestBamPath, intermediateDataPathsByBamPath[smallestBamPath],
canvasReferencePath, canvasBedPath);
}
else if (callset.IsEnrichment && callset.Manifest.CanvasControlAvailable)
{
binSize = callset.Manifest.CanvasBinSize.Value;
}
Dictionary<string, string> bamToBinned = new Dictionary<string, string>();
List<UnitOfWork> finalBinJobs = new List<UnitOfWork>();
for (int bamIdx = 0; bamIdx < bamPaths.Count; bamIdx++)
{
string bamPath = bamPaths[bamIdx];
// finish up CanvasBin step by merging intermediate data and finally binning
string binnedPath = Path.Combine(callset.TempFolder, string.Format("{0}_{1}.binned", callset.Id, bamIdx));
bamToBinned[bamPath] = binnedPath;
commandLine.Clear();
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", canvasBinPath);
}
commandLine.AppendFormat("-b \"{0}\" ", bamPath);
if (callset.Bam.IsPairedEnd) commandLine.AppendFormat("-p ");
commandLine.AppendFormat("-r \"{0}\" ", canvasReferencePath);
commandLine.AppendFormat("-f \"{0}\" -d {1} -o \"{2}\" ", canvasBedPath, _countsPerBin, binnedPath);
if (binSize != -1)
{
commandLine.AppendFormat("-z \"{0}\" ", binSize);
}
foreach (string path in intermediateDataPathsByBamPath[bamPath])
{
commandLine.AppendFormat("-i \"{0}\" ", path);
}
commandLine.AppendFormat("-m {0} ", _coverageMode);
UnitOfWork finalBinJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(binnedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasBin"))
{
finalBinJob.CommandLine = Utilities.MergeCommandLineOptions(finalBinJob.CommandLine, _customParameters["CanvasBin"], true);
}
finalBinJobs.Add(finalBinJob);
}
_workManager.DoWorkParallel(finalBinJobs, new TaskResourceRequirements(8, 25)); // CanvasBin itself is multi-threaded
string tumorBinnedPath = bamToBinned[callset.Bam.BamFile.FullName]; // binned tumor sample
string outputPath = tumorBinnedPath;
if (callset.NormalBamPaths.Any() || (callset.IsEnrichment && callset.Manifest.CanvasControlAvailable))
{
outputPath = InvokeCanvasNormalize(callset, tumorBinnedPath, bamToBinned, ploidyBedPath);
}
return new FileLocation(outputPath);
}
/// <summary>
/// Germline workflow:
/// - Run CanvasBin, CanvasClean, CanvasPartition, CanvasDiploidCaller
///
/// Somatic workflow:
/// - Run CanvasBin, CanvasClean, CanvasPartition, CanvasSNV, CanvasSomaticCaller
/// </summary>
public void CallSample(CanvasCallset callset)
{
Directory.CreateDirectory(callset.TempFolder);
string canvasReferencePath = callset.KmerFasta.FullName;
string canvasBedPath = callset.FilterBed.FullName;
if (!File.Exists(canvasReferencePath))
{
throw new ApplicationException(string.Format("Error: Missing reference fasta file required for CNV calling at '{0}'", canvasReferencePath));
}
if (!File.Exists(canvasBedPath))
{
throw new ApplicationException(string.Format("Error: Missing filter bed file required for CNV calling at '{0}'", canvasBedPath));
}
// Prepare ploidy file:
GenomeMetadata genomeMetadata = callset.GenomeMetadata;
string ploidyBedPath = callset.PloidyBed?.FullName;
// CanvasBin:
string binnedPath = InvokeCanvasBin(callset, canvasReferencePath, canvasBedPath, ploidyBedPath);
if (string.IsNullOrEmpty(binnedPath)) return;
// CanvasClean:
StringBuilder commandLine = new StringBuilder();
commandLine.Length = 0;
string executablePath = Path.Combine(_canvasFolder, "CanvasClean.exe");
if (Utilities.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", binnedPath);
string cleanedPath = Path.Combine(callset.TempFolder, string.Format("{0}.cleaned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-g");
string ffpePath = null;
// TruSight Cancer has 1,737 targeted regions. The cut-off 2000 is somewhat arbitrary.
// TruSignt One has 62,309 targeted regions.
// Nextera Rapid Capture v1.1 has 411,513 targeted regions.
if (!callset.IsEnrichment || callset.Manifest.Regions.Count > 2000)
{
ffpePath = Path.Combine(callset.TempFolder, "FilterRegions.txt");
commandLine.AppendFormat(" -s -r -f \"{0}\"", ffpePath);
}
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath)) { NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath); }
commandLine.AppendFormat(" -t \"{0}\"", callset.TempManifestPath);
}
UnitOfWork cleanJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(cleanedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasClean"))
{
cleanJob.CommandLine = Utilities.MergeCommandLineOptions(cleanJob.CommandLine, _customParameters["CanvasClean"], true);
}
_workManager.DoWorkSingleThread(cleanJob);
////////////////////////////////////////////////////////
// CanvasPartition:
commandLine.Length = 0;
executablePath = Path.Combine(_canvasFolder, "CanvasPartition.exe");
if (Utilities.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-b \"{0}\" ", canvasBedPath);
string partitionedPath = Path.Combine(callset.TempFolder, string.Format("{0}.partitioned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", partitionedPath);
if (!_isSomatic)
commandLine.AppendFormat(" -g");
UnitOfWork partitionJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(partitionedPath),
CommandLine = commandLine.ToString()
};
_workManager.DoWorkSingleThread(partitionJob);
////////////////////////////////////////////////////////
// CanvasSNV
// Prepare and run CanvasSNV jobs. First create list of jobs:
InvokeCanvasSnv(callset);
////////////////////////////////////////////////////////
// Variant calling:
if (callset.IsEnrichment)
{
partitionedPath = IntersectBinsWithTargetedRegions(callset, partitionedPath); // Intersect bins with manifest
}
if (_isSomatic)
{
RunSomaticCalling(partitionedPath, callset, canvasBedPath, ploidyBedPath, ffpePath);
}
else
{
RunGermlineCalling(partitionedPath, callset, ploidyBedPath);
}
}
