public override IParsingResult <ExclusiveFileOptionResult> Parse(SuccessfulResultCollection parseInput)
{
var result1 = parseInput.Get(_option1);
var result2 = parseInput.Get(_option2);
if (result1 != null && result2 != null)
{
return(ParsingResult <ExclusiveFileOptionResult> .FailedResult($"Please specify either option {_option1.Info.Name} or option {_option2.Info.Name}, but not both"));
}
if (_isRequired && result1 == null && result2 == null)
{
return(ParsingResult <ExclusiveFileOptionResult> .FailedResult($"Either option {_option1.Info.Name} or option {_option2.Info.Name} must be specified"));
}
IFileLocation result = null;
FileOption option = null;
if (result1 != null)
{
result = result1;
option = _option1;
}
if (result2 != null)
{
result = result2;
option = _option2;
}
return(ParsingResult <ExclusiveFileOptionResult> .SuccessfulResult(new ExclusiveFileOptionResult(result, option)));
}
public StringBuilder GetMultiSampleCommandLine(SampleSet <CanvasPedigreeSample> samples, GenomeMetadata genomeMetadata, Vcf vcf, IDirectoryLocation sampleSandbox)
{
StringBuilder commandLine = new StringBuilder();
foreach (var sampleKvp in samples)
{
var sampleId = sampleKvp.Key.Id;
var sample = sampleKvp.Value;
commandLine.Append($" --bam \"{sample.Bam.BamFile}\"");
if (sample.SampleType != SampleType.Other)
{
commandLine.Append($" --{sample.SampleType.GetOptionName()} {sampleId}");
}
}
IFileLocation kmerFasta = _annotationFileProvider.GetKmerFasta(genomeMetadata);
commandLine.Append($" --reference \"{kmerFasta}\"");
IDirectoryLocation wholeGenomeFasta = new FileLocation(genomeMetadata.Sequences.First().FastaPath).Directory;
commandLine.Append($" --genome-folder \"{wholeGenomeFasta}\"");
IFileLocation filterBed = _annotationFileProvider.GetFilterBed(genomeMetadata);
commandLine.Append($" --filter-bed \"{filterBed}\"");
commandLine.Append($" --output \"{sampleSandbox}\"");
return(commandLine);
}
public CanvasOutputNamingConvention(IFileLocation fileNameStub, ICanvasAnnotationFileProvider annotationFileProvider, bool includeIntermediateResults, Func <IFileLocation, bool, TCanvasOutput> getFromStub)
{
_fileNameStub = fileNameStub;
_annotationFileProvider = annotationFileProvider;
_includeIntermediateResults = includeIntermediateResults;
_getFromStub = getFromStub;
}
public IntermediateOutput(IFileLocation coverageAndVariantFrequencies, IFileLocation variantFrequencies, IFileLocation variantFrequenciesBaf, IFileLocation partitioned)
{
CoverageAndVariantFrequencies = coverageAndVariantFrequencies;
VariantFrequencies = variantFrequencies;
VariantFrequenciesBaf = variantFrequenciesBaf;
Partitioned = partitioned;
}
public void HandleVcfHeaderInfo(TextWriter outputWriter, IFileLocation vcfPath)
{
var headerLines = GetVcfHeaderLines(vcfPath);
HandleHeaderLine(outputWriter, headerLines, "EstimatedTumorPurity", LogPurity);
HandleHeaderLine(outputWriter, headerLines, "OverallPloidy", LogPloidy);
}
public TumorNormalEnrichmentRunner(CommonOptions commonOptions, TumorNormalOptions tumorNormalOptions, IFileLocation normalBam, IFileLocation manifest)
{
_tumorNormalOptions = tumorNormalOptions;
_normalBam = normalBam;
_manifest = manifest;
CommonOptions = commonOptions;
}
/// <summary>
/// Write out the ploidy bed file if ploidy information is available from the vcf header
/// Only create the normal XX or XY ploidy bed file so that Canvas can properly classify any abnormalities as variant.
/// If ploidy Y is > 1 produce the XY ploidy bed file, otherwise produce the XX ploidy bed file
/// </summary>
public IFileLocation CreateGermlinePloidyBed(Vcf vcf, GenomeMetadata genomeMetadata, IDirectoryLocation sampleSandbox)
{
string sexKaryotype = PloidyCorrector.GetSexChromosomeKaryotypeFromVcfHeader(vcf.VcfFile.FullName);
if (sexKaryotype == null)
{
_logger.Warn($"Sex chromosome ploidy not found in {vcf.VcfFile} header. No ploidy will be provided to Canvas.");
return(null);
}
_logger.Info($"Found sex chromosome ploidy {PloidyCorrector.PrintPloidy(sexKaryotype)} in {vcf.VcfFile}");
var ploidyInfo = new SamplePloidyInfo();
IFileLocation ploidyBed = sampleSandbox.GetFileLocation("ploidy.bed.gz");
if (sexKaryotype.ToLower().Contains("y"))
{
ploidyInfo.ProvidedPloidy = SexPloidyInfo.NormalMale;
_logger.Info($"Creating male ploidy bed file at {ploidyBed}.");
}
else
{
ploidyInfo.ProvidedPloidy = SexPloidyInfo.NormalFemale;
_logger.Info($"Creating female ploidy bed file at {ploidyBed}.");
}
string headerLine = $"{PloidyCorrector.ReferenceSexChromosomeKaryotype}={PloidyCorrector.PrettyPrintPloidy(ploidyInfo.ProvidedPloidyX.Value, ploidyInfo.ProvidedPloidyY.Value)}";
_ploidyFixer.WritePloidyBedFile(ploidyInfo, genomeMetadata, _ploidyFixer.GetParRegions(genomeMetadata),
ploidyBed.FullName, headerLine, ploidy => true);
return(ploidyBed);
}
/// <summary>
/// common options for workflows with a primary sample
/// </summary>
/// <param name="bAlleleSites"></param>
/// <param name="isDbSnpVcf"></param>
/// <param name="ploidyBed"></param>
/// <param name="sampleName"></param>
public SingleSampleCommonOptions(IFileLocation bAlleleSites, bool isDbSnpVcf, IFileLocation ploidyVcf, string sampleName)
{
BAlleleSites = bAlleleSites;
IsDbSnpVcf = isDbSnpVcf;
PloidyVcf = ploidyVcf;
SampleName = sampleName;
}
private IFileLocation CreateDbSnpVcfForManifest(IFileLocation fullDbSnpVcf, NexteraManifest manifest, IDirectoryLocation sandBox)
{
IFileLocation targetedDbSnpVcf = sandBox.GetFileLocation($"{manifest.Name}_{fullDbSnpVcf.Name}");
Isas.Manifests.NexteraManifest.VcfUtilities.IntersectVcfWithManifest(fullDbSnpVcf.FullName, targetedDbSnpVcf.FullName, manifest);
return(targetedDbSnpVcf);
}
private IFileLocation InvokeCanvasPartition(CanvasCallset callset, IFileLocation cleanedPath, string canvasBedPath)
{
StringBuilder commandLine = new StringBuilder();
string executablePath = Path.Combine(_canvasFolder, "CanvasPartition.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-b \"{0}\" ", canvasBedPath);
string partitionedPath = Path.Combine(callset.TempFolder, string.Format("{0}.partitioned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", partitionedPath);
if (!_isSomatic)
{
commandLine.AppendFormat(" -g");
}
UnitOfWork partitionJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(partitionedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasPartition"))
{
partitionJob.CommandLine = Utilities.MergeCommandLineOptions(partitionJob.CommandLine, _customParameters["CanvasPartition"], true);
}
_workManager.DoWorkSingleThread(partitionJob);
return(new FileLocation(partitionedPath));
}
public CanvasCallset(
IFileLocation bam,
string sampleName,
IDirectoryLocation wholeGenomeFastaFolder,
IDirectoryLocation outputFolder,
IFileLocation kmerFasta,
IFileLocation filterBed,
IFileLocation ploidyBed,
IFileLocation normalVcfPath,
bool isDbSnpVcf,
IEnumerable<IFileLocation> normalBamPaths,
NexteraManifest manifest,
IFileLocation somaticVcfPath,
IFileLocation outputVcfPath)
{
Bam = new Bam(bam);
SampleName = sampleName;
WholeGenomeFastaFolder = wholeGenomeFastaFolder;
OutputFolder = outputFolder;
KmerFasta = kmerFasta;
FilterBed = filterBed;
PloidyBed = ploidyBed;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
Manifest = manifest;
SomaticVcfPath = somaticVcfPath;
OutputVcfPath = outputVcfPath;
NormalBamPaths = normalBamPaths.Select(file => new Bam(file));
var genomeSizeXml = WholeGenomeFastaFolder.GetFileLocation("GenomeSize.xml");
GenomeMetadata = new GenomeMetadata();
GenomeMetadata.Deserialize(genomeSizeXml.FullName);
}
private static List <string> GetVcfHeaderLines(IFileLocation vcfPath)
{
using (var reader = new VcfReader(vcfPath.FullName, false))
{
return(reader.HeaderLines);
}
}
public CanvasCallset(
IFileLocation bam,
string sampleName,
IDirectoryLocation wholeGenomeFastaFolder,
IDirectoryLocation outputFolder,
IFileLocation kmerFasta,
IFileLocation filterBed,
IFileLocation ploidyBed,
IFileLocation normalVcfPath,
bool isDbSnpVcf,
IEnumerable <IFileLocation> normalBamPaths,
NexteraManifest manifest,
IFileLocation somaticVcfPath,
IFileLocation outputVcfPath)
{
Bam = new Bam(bam);
SampleName = sampleName;
WholeGenomeFastaFolder = wholeGenomeFastaFolder;
OutputFolder = outputFolder;
KmerFasta = kmerFasta;
FilterBed = filterBed;
PloidyBed = ploidyBed;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
Manifest = manifest;
SomaticVcfPath = somaticVcfPath;
OutputVcfPath = outputVcfPath;
NormalBamPaths = normalBamPaths.Select(file => new Bam(file));
var genomeSizeXml = WholeGenomeFastaFolder.GetFileLocation("GenomeSize.xml");
GenomeMetadata = new GenomeMetadata();
GenomeMetadata.Deserialize(genomeSizeXml.FullName);
}
public TumorNormalEnrichmentRunner(CommonOptions commonOptions, TumorNormalOptions tumorNormalOptions, IFileLocation normalBam, IFileLocation manifest)
{
_tumorNormalOptions = tumorNormalOptions;
_normalBam = normalBam;
_manifest = manifest;
CommonOptions = commonOptions;
}
public TumorNormalEnrichmentRunner(TumorNormalEnrichmentInput input)
{
_tumorNormalOptions = input.TumorNormalOptions;
_normalBam = input.NormalBam;
_manifest = input.Manifest;
CommonOptions = input.CommonOptions;
SingleSampleCommonOptions = input.SingleSampleCommonOptions;
}
public TumorNormalEnrichmentInput(CommonOptions commonOptions, SingleSampleCommonOptions singleSampleCommonOptions, TumorNormalOptions tumorNormalOptions, IFileLocation normalBam, IFileLocation manifest)
{
CommonOptions = commonOptions;
SingleSampleCommonOptions = singleSampleCommonOptions;
TumorNormalOptions = tumorNormalOptions;
NormalBam = normalBam;
Manifest = manifest;
}
public SmallPedigreeOptions(List <SmallPedigreeSampleOptions> samples, IFileLocation commonCnvsBed, IFileLocation bAlleleSites, bool isPopulationBAlleleSites, IFileLocation multiSamplePloidyVcf)
{
Samples = samples;
CommonCnvsBed = commonCnvsBed;
BAlleleSites = bAlleleSites;
MultiSamplePloidyVcf = multiSamplePloidyVcf;
IsPopulationBAlleleSites = isPopulationBAlleleSites;
}
public void Run(IFileLocation outputFile)
{
int bestNormalSampleIndex = 0;
int normalSampleCount = _controlBinnedFiles.Count();
if (normalSampleCount > 1) // find the best normal
{
List <double[]> binCountsByNormalSample = new List <double[]>();
for (int normalSampleIndex = 0; normalSampleIndex < normalSampleCount; normalSampleIndex++)
{
var controlBinnedFile = _controlBinnedFiles.ElementAt(normalSampleIndex);
var binCounts = new BinCounts(controlBinnedFile.FullName, manifest: _manifest);
List <double> counts = binCounts.OnTargetCounts;
double median = binCounts.OnTargetMedianBinCount;
// If a manifest is available, get the median of bins overlapping the targeted regions only.
// For small panels, there could be a lot of bins with zero count and the median would be 0 if taken over all the bins, resulting in division by zero.
double weight = median > 0 ? 1.0 / median : 0;
binCountsByNormalSample.Add(counts.Select(cnt => cnt * weight).ToArray());
}
double[] tumorBinCounts;
{
var binCounts = new BinCounts(_sampleBinnedFile.FullName, manifest: _manifest);
List <double> counts = binCounts.OnTargetCounts;
double tumorMedian = binCounts.OnTargetMedianBinCount;
double tumorWeight = tumorMedian > 0 ? 1.0 / tumorMedian : 0;
tumorBinCounts = counts.Select(cnt => cnt * tumorWeight).ToArray();
}
// Find the best normal sample
bestNormalSampleIndex = -1;
double minMeanSquaredLogRatios = double.PositiveInfinity;
for (int normalSampleIndex = 0; normalSampleIndex < normalSampleCount; normalSampleIndex++)
{
// Get the sum of squared log ratios
var result = GetMeanSquaredLogRatios(tumorBinCounts, binCountsByNormalSample[normalSampleIndex]);
double meanSquaredLogRatios = result.Item1;
int ignoredBinCount = result.Item2;
// TODO: Skip a (bad) normal sample if too many bins were ignored.
// Donavan's script skips a normal sample if more than 100 log ratios is NA.
// The cut-off is likely panel-dependent.
if (meanSquaredLogRatios < minMeanSquaredLogRatios)
{
minMeanSquaredLogRatios = meanSquaredLogRatios;
bestNormalSampleIndex = normalSampleIndex;
}
}
}
// copy file
var srcBinnedFile = _controlBinnedFiles.ElementAt(bestNormalSampleIndex);
if (outputFile.Exists)
{
outputFile.Delete();
}
srcBinnedFile.CopyTo(outputFile);
}
public static Dictionary <string, List <Balleles> > ReadFrequenciesWrapper(ILogger logger,
IFileLocation variantFrequencyFile, IReadOnlyDictionary <string, List <BedInterval> > intervalsByChromosome)
{
using (var reader = new GzipOrTextReader(variantFrequencyFile.FullName))
{
logger.Info($"Load variant frequencies from {variantFrequencyFile}");
return(ReadFrequencies(reader, intervalsByChromosome));
}
}
public SingleSampleCallset(Bam bam, string sampleName, IFileLocation normalVcfPath, bool isDbSnpVcf, IDirectoryLocation analysisOutputFolder, IFileLocation outputVcfPath)
{
_analysisOutputFolder = analysisOutputFolder;
Bam = bam;
SampleName = sampleName;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
OutputVcfPath = outputVcfPath;
}
public CanvasRunnerFactory(ILogger logger, ICheckpointRunner checkpointRunner,
IWorkDoer workDoer, IFileLocation runtimeExecutable, Func <string, ICommandFactory> runtimeCommandPrefix)
{
_logger = logger;
_checkpointRunner = checkpointRunner;
_workDoer = workDoer;
_runtimeExecutable = runtimeExecutable;
_runtimeCommandPrefix = runtimeCommandPrefix;
}
public void Move(SampleSet <T> source, Action <IFileLocation, IFileLocation> move)
{
foreach (var sampleInfo in source.Keys)
{
IFileLocation stub = _convention(sampleInfo);
ILoadingConvention <TIn, T> namingConvention = _loadingConventionGenerator(stub);
namingConvention.Move(source[sampleInfo], move);
}
}
public void Move(IFileLocation fileNameStub, bool includeIntermediateResults, Action <IFileLocation, IFileLocation> move)
{
CanvasOutput.Move(fileNameStub, includeIntermediateResults, move);
CanvasEnrichmentOutput destination = GetFromStub(fileNameStub, includeIntermediateResults);
BinSize.MoveIfNotNull(destination.BinSize, move);
NormalBinned.MoveIfNotNull(destination.NormalBinned, move);
UnsmoothedCnd.MoveIfNotNull(destination.UnsmoothedCnd, move);
}
public PCAModel(IFileLocation pcaModelFile)
{
List <SampleGenomicBin> mu;
List <double[]> axes;
LoadModel(pcaModelFile, out mu, out axes);
Mu = mu.ToArray();
Axes = axes.Select(a => a.ToArray()).ToArray();
}
public static CanvasEnrichmentOutput GetFromStub(IFileLocation stub)
{
var canvasOutput = CanvasOutput.GetFromStub(stub);
IFileLocation binSize = stub.AppendName(".binsize");
IFileLocation normalBinned = stub.AppendName(".binned");
IFileLocation unsmoothedCnd = stub.AppendName(".unsmoothed.cnd");
return(new CanvasEnrichmentOutput(canvasOutput, binSize, normalBinned, unsmoothedCnd));
}
public SomaticEnrichmentOptions(IFileLocation bam, IFileLocation manifest, IEnumerable <IFileLocation> controlBams, IFileLocation controlBinned, int controlBinSize, IFileLocation controlPloidy)
{
Bam = bam;
Manifest = manifest;
ControlBams = controlBams;
ControlBinned = controlBinned;
ControlBinSize = controlBinSize;
ControlPloidy = controlPloidy;
}
// general common options
/// <summary>
/// common options for all workflows
/// </summary>
/// <param name="outputDirectory"></param>
/// <param name="wholeGenomeFasta"></param>
/// <param name="kmerFasta"></param>
/// <param name="filterBed"></param>
/// <param name="customParams"></param>
/// <param name="startCheckpoint"></param>
/// <param name="stopCheckpoint"></param>
public CommonOptions(IDirectoryLocation outputDirectory, IDirectoryLocation wholeGenomeFasta, IFileLocation kmerFasta, IFileLocation filterBed, Dictionary <string, string> customParams, string startCheckpoint, string stopCheckpoint)
{
OutputDirectory = outputDirectory;
WholeGenomeFasta = wholeGenomeFasta;
KmerFasta = kmerFasta;
FilterBed = filterBed;
CustomParams = customParams;
StartCheckpoint = startCheckpoint;
StopCheckpoint = stopCheckpoint;
}
public static Segments ReadSegments(ILogger logger, IFileLocation partitionedFile)
{
using (var reader = new GzipOrTextReader(partitionedFile.FullName))
{
logger.Info($"Read segments from {partitionedFile}");
var segments = ReadSegments(reader);
logger.Info($"Loaded {segments.AllSegments.Count} segments");
return(segments);
}
}
public CanvasEnrichmentOutput(CanvasOutput canvasOutput,
IFileLocation binSize = null,
IFileLocation normalBinned = null,
IFileLocation unsmoothedCnd = null)
{
CanvasOutput = canvasOutput;
BinSize = binSize;
NormalBinned = normalBinned;
UnsmoothedCnd = unsmoothedCnd;
}
public PCAReferenceGenerator(IFileLocation sampleBinnedFile, IFileLocation pcaModelFile,
NexteraManifest manifest, double minBinCount = 1, double maxBinCount = double.PositiveInfinity)
{
_sampleBinnedFile = sampleBinnedFile;
_model = new PCAModel(pcaModelFile);
_manifest = manifest;
_minBinCount = minBinCount;
_maxBinCount = maxBinCount;
_ratioCalculator = new RawRatioCalculator(manifest, _minBinCount, _maxBinCount);
}
public void Move(IFileLocation fileNameStub, bool includeIntermediateResults, Action <IFileLocation, IFileLocation> move)
{
CanvasOutput destination = GetFromStub(fileNameStub, includeIntermediateResults);
CnvVcf.Move(destination.CnvVcf, move);
move(CoverageAndVariantFrequencies, destination.CoverageAndVariantFrequencies);
VariantFrequencies.MoveIfNotNull(destination.VariantFrequencies, move);
VariantFrequenciesBaf.MoveIfNotNull(destination.VariantFrequenciesBaf, move);
Partitioned.MoveIfNotNull(destination.Partitioned, move);
}
public void Run(IFileLocation outputFile)
{
int sampleCount = _controlBinnedFiles.Count();
if (sampleCount == 1) // copy file
{
if (outputFile.Exists)
{
outputFile.Delete();
}
_controlBinnedFiles.First().CopyTo(outputFile);
}
else // merge normal samples
{
double[] weights = new double[sampleCount];
List <double>[] binCountsBySample = new List <double> [sampleCount];
for (int sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++)
{
var binnedFile = _controlBinnedFiles.ElementAt(sampleIndex);
var binCounts = new BinCounts(binnedFile.FullName, manifest: _manifest);
List <double> counts = binCounts.AllCounts;
// If a manifest is available, get the median of bins overlapping the targeted regions only.
// For small panels, there could be a lot of bins with zero count and the median would be 0 if taken over all the bins, resulting in division by zero.
double median = binCounts.OnTargetMedianBinCount;
weights[sampleIndex] = median > 0 ? 1.0 / median : 0;
binCountsBySample[sampleIndex] = counts;
}
double weightSum = weights.Sum();
for (int i = 0; i < sampleCount; i++)
{
weights[i] /= weightSum;
} // so weights sum to 1
// Computed weighted average of bin counts across samples
using (GzipReader reader = new GzipReader(_controlBinnedFiles.First().FullName))
using (GzipWriter writer = new GzipWriter(outputFile.FullName))
{
string line;
string[] toks;
int lineIdx = 0;
while ((line = reader.ReadLine()) != null)
{
toks = line.Split('\t');
double weightedBinCount = 0;
for (int i = 0; i < sampleCount; i++)
{
weightedBinCount += weights[i] * binCountsBySample[i][lineIdx];
}
toks[3] = String.Format("{0}", weightedBinCount);
writer.WriteLine(String.Join("\t", toks));
lineIdx++;
}
}
}
}
public CommonOptions(IFileLocation bAlleleSites, bool isDbSnpVcf, IFileLocation ploidyBed, IDirectoryLocation outputDirectory, IDirectoryLocation wholeGenomeFasta, IFileLocation kmerFasta, IFileLocation filterBed, string sampleName, Dictionary<string, string> customParams, string startCheckpoint, string stopCheckpoint)
{
BAlleleSites = bAlleleSites;
IsDbSnpVcf = isDbSnpVcf;
PloidyBed = ploidyBed;
OutputDirectory = outputDirectory;
WholeGenomeFasta = wholeGenomeFasta;
KmerFasta = kmerFasta;
FilterBed = filterBed;
SampleName = sampleName;
CustomParams = customParams;
StartCheckpoint = startCheckpoint;
StopCheckpoint = stopCheckpoint;
}
protected void RunSomaticCalling(IFileLocation partitionedPath, CanvasCallset callset, string canvasBedPath,
string ploidyBedPath, IFileLocation ffpePath)
{
// get somatic SNV output:
string somaticSnvPath = callset.SomaticVcfPath?.FullName;
// Prepare and run CanvasSomaticCaller job:
UnitOfWork callerJob = new UnitOfWork();
var cnvVcfPath = callset.OutputVcfPath;
callerJob.ExecutablePath = Path.Combine(this._canvasFolder, "CanvasSomaticCaller.exe");
if (CrossPlatform.IsThisMono())
{
callerJob.CommandLine = callerJob.ExecutablePath;
callerJob.ExecutablePath = Utilities.GetMonoPath();
}
callerJob.CommandLine += string.Format(" -v {0}", callset.VfSummaryPath);
callerJob.CommandLine += string.Format(" -i {0}", partitionedPath);
callerJob.CommandLine += string.Format(" -o {0}", cnvVcfPath);
callerJob.CommandLine += string.Format(" -b {0}", canvasBedPath);
if (!string.IsNullOrEmpty(ploidyBedPath))
callerJob.CommandLine += string.Format(" -p {0}", ploidyBedPath);
callerJob.CommandLine += string.Format(" -n {0}", callset.SampleName);
if (callset.IsEnrichment)
callerJob.CommandLine += " -e";
if (callset.IsDbSnpVcf) // a dbSNP VCF file is used in place of the normal VCF file
callerJob.CommandLine += " -d";
// get localSD metric:
if (ffpePath != null)
{
// Sanity-check: CanvasClean does not always write this file.
// If it's not present, just carry on:
if (ffpePath.Exists)
{
callerJob.CommandLine += string.Format(" -f \"{0}\"", ffpePath);
}
else
{
_logger.Info("Note: SD file not found at '{0}'", ffpePath);
}
}
if (!string.IsNullOrEmpty(somaticSnvPath))
callerJob.CommandLine += string.Format(" -s {0}", somaticSnvPath);
callerJob.CommandLine += string.Format(" -r \"{0}\" ", callset.WholeGenomeFastaFolder);
if (_customParameters.ContainsKey("CanvasSomaticCaller"))
{
callerJob.CommandLine = Utilities.MergeCommandLineOptions(callerJob.CommandLine, _customParameters["CanvasSomaticCaller"], true);
}
callerJob.LoggingFolder = _workManager.LoggingFolder.FullName;
callerJob.LoggingStub = string.Format("SomaticCNV-{0}", callset.Id);
_workManager.DoWorkSingleThread(callerJob);
}
public IFileLocation HardlinkTo(IFileLocation destinationFile, bool copyOnFail = true)
{
return (Illumina.SecondaryAnalysis.Utilities.CreateHardLink(destinationFile.FullName, FullName, copyOnFail))
? destinationFile : null;
}
/// <summary>
/// Creates a relative symlink at <paramref name="destFile"/> pointing to <paramref name="sourceFile"/>
/// Symlinks in results should be relative to allow browsing with different mount-point (e.g. windows)
/// </summary>
/// <param name="sourceFile">Source file</param>
/// <param name="destFile">Symbolic link to be created.</param>
/// <returns>IFileLocation representing the new symlink.</returns>
public IFileLocation RelativeSymlinkTo(IFileLocation destFile)
{
var destDir = new Uri(destFile.FullName);
var relSrcPath = destDir.MakeRelativeUri(new Uri(FullName));
if (!Illumina.SecondaryAnalysis.Utilities.CreateSymbolicLink(destFile.FullName, relSrcPath.ToString(), false))
CopyTo(destFile);
return destFile;
}
/// <summary>
/// Copies an existing file to a new file, overwriting any existing file.
/// </summary>
/// <param name="destination">The location of the new file to copy to.</param>
/// <returns>
/// The location of the new file
/// </returns>
public IFileLocation CopyTo(IFileLocation destination)
{
if (FullName.Equals(destination.FullName)) // Doesn't handle symlinks...
return this;
destination.Delete();
File.Copy(FullName, destination.FullName);
return destination;
}
/// <summary>
/// Moves an existing file to a new location, overwriting any existing file.
/// If the new location is on a different volume, this is equivalent to CopyTo + Delete
/// </summary>
/// <param name="destination">The new location of the file.</param>
/// <returns>
/// The new location of the file
/// </returns>
public IFileLocation MoveTo(IFileLocation destination)
{
if (FullName.Equals(destination.FullName)) // Doesn't handle symlinks...
return this;
if (destination.Directory != null)
destination.Directory.Create();
if (!Equals(destination))
destination.Delete();
File.Move(FullName, destination.FullName);
return destination;
}
public Logger(IFileLocation workflowLog, IFileLocation workflowError, bool alsoWriteToStandardOutput = true, LogEventType lowestEventTypeForWorkflowLog = LogEventType.Information,
LogEventType lowestEventTypeForWorkflowError = LogEventType.Warning) : this(alsoWriteToStandardOutput, lowestEventTypeForWorkflowLog, lowestEventTypeForWorkflowError)
{
_logWriters.Add(InitLog(workflowLog, out _workflowLogFileStream, out _workflowLog));
_errorWriters.Add(InitLog(workflowError, out _workflowErrorFileStream, out _workflowError));
}
public static void HardlinkTo(IFileLocation source, IFileLocation destination)
{
source.HardlinkTo(destination);
}
public CanvasCleanOutput(IFileLocation cleanedPath, IFileLocation ffpePath)
{
CleanedPath = cleanedPath;
FfpePath = ffpePath;
}
private static TextWriter InitLog(IFileLocation log, out FileStream fileStream, out StreamWriter streamWriter)
{
fileStream = new FileStream(log.FullName, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
streamWriter = new StreamWriter(fileStream);
return streamWriter;
}
public GermlineWgsRunner(CommonOptions commonOptions, IFileLocation bam)
{
_bam = bam;
CommonOptions = commonOptions;
}
public Vcf MoveWithStub(IFileLocation newStub, Action<IFileLocation, IFileLocation> move)
{
var newVcf = GetVcfFromStub(newStub);
Move(newVcf, move);
return newVcf;
}
public Vcf(IFileLocation vcfFile)
{
if (!vcfFile.Name.EndsWith(Extension))
throw new ArgumentException($"Invalid vcf path {vcfFile}. Path must end with .vcf.gz");
VcfFile = vcfFile;
}
private CanvasCleanOutput InvokeCanvasClean(CanvasCallset callset, IFileLocation binnedPath)
{
StringBuilder commandLine = new StringBuilder();
commandLine.Length = 0;
string executablePath = Path.Combine(_canvasFolder, "CanvasClean.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", binnedPath);
string cleanedPath = Path.Combine(callset.TempFolder, string.Format("{0}.cleaned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-g");
string ffpePath = null;
// TruSight Cancer has 1,737 targeted regions. The cut-off 2000 is somewhat arbitrary.
// TruSight One has 62,309 targeted regions.
// Nextera Rapid Capture v1.1 has 411,513 targeted regions.
if (!callset.IsEnrichment || callset.Manifest.Regions.Count > 2000)
{
ffpePath = Path.Combine(callset.TempFolder, "FilterRegions.txt");
commandLine.AppendFormat(" -s -r -f \"{0}\"", ffpePath);
}
if (callset.IsEnrichment) // manifest
{
if (!File.Exists(callset.TempManifestPath))
{
NexteraManifestUtils.WriteNexteraManifests(callset.Manifest, callset.TempManifestPath);
}
commandLine.AppendFormat(" -t \"{0}\"", callset.TempManifestPath);
}
UnitOfWork cleanJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(cleanedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasClean"))
{
cleanJob.CommandLine = Utilities.MergeCommandLineOptions(cleanJob.CommandLine, _customParameters["CanvasClean"], true);
}
_workManager.DoWorkSingleThread(cleanJob);
var canvasCleanOutput = new CanvasCleanOutput(new FileLocation(cleanedPath), new FileLocation(ffpePath));
return canvasCleanOutput;
}
/// <summary>
/// void return type
/// </summary>
public static void MoveAndLinkAction(this IFileLocation source, IFileLocation destination)
{
source.MoveAndLink(destination);
}
/// <summary>
/// Intersect bins with the targeted regions defined in callset.Manifest.
/// Assumes that the targeted regions don't intersect, the bins are sorted by genomic location and the bins don't intersect.
/// </summary>
/// <param name="callset"></param>
/// <param name="partitionedPath">Output of CanvasPartition. Bins are assumed to be sorted</param>
/// <returns></returns>
private IFileLocation IntersectBinsWithTargetedRegions(CanvasCallset callset, IFileLocation partitionedPath)
{
if (!partitionedPath.Exists) { return partitionedPath; }
var rawPartitionedPath = partitionedPath.AppendName(".raw");
if (rawPartitionedPath.Exists) { rawPartitionedPath.Delete(); }
partitionedPath.MoveTo(rawPartitionedPath);
//callset.Manifest
Dictionary<string, List<NexteraManifest.ManifestRegion>> manifestRegionsByChrom = callset.Manifest.GetManifestRegionsByChromosome();
// CanvasPartition output file is in the BED format
// start: 0-based, inclusive
// end: 0-based, exclusive
// Manifest
// start: 1-based, inclusive
// end: 1-based, inclusive
using (GzipReader reader = new GzipReader(rawPartitionedPath.FullName))
using (GzipWriter writer = new GzipWriter(partitionedPath.FullName))
{
string currentChrom = null;
int manifestRegionIdx = 0;
string line;
string[] toks;
while ((line = reader.ReadLine()) != null)
{
toks = line.Split('\t');
string chrom = toks[0];
int start = int.Parse(toks[1]) + 1; // 1-based, inclusive
int end = int.Parse(toks[2]); // 1-based, inclusive
if (chrom != currentChrom)
{
currentChrom = chrom;
manifestRegionIdx = 0;
}
if (!manifestRegionsByChrom.ContainsKey(currentChrom)) { continue; }
while (manifestRegionIdx < manifestRegionsByChrom[currentChrom].Count
&& manifestRegionsByChrom[currentChrom][manifestRegionIdx].End < start) // |- manifest region -| |- bin -|
{
manifestRegionIdx++;
}
if (manifestRegionIdx >= manifestRegionsByChrom[currentChrom].Count || // |- last manifest region -| |- bin -|
end < manifestRegionsByChrom[currentChrom][manifestRegionIdx].Start) // |- bin -| |- manifest region -|
{
continue; // skip bin
}
// |- bin -|
// |- manifest region -|
while (manifestRegionIdx < manifestRegionsByChrom[currentChrom].Count &&
end >= manifestRegionsByChrom[currentChrom][manifestRegionIdx].Start)
{
// calculate intersection
int intersectionStart = Math.Max(start, manifestRegionsByChrom[currentChrom][manifestRegionIdx].Start); // 1-based, inclusive
int intersectionEnd = Math.Min(end, manifestRegionsByChrom[currentChrom][manifestRegionIdx].End); // 1-based, inclusive
// start/end in BED format
toks[1] = String.Format("{0}", intersectionStart - 1); // 0-based, inclusive
toks[2] = String.Format("{0}", intersectionEnd); // 0-based, exclusive
// write intersected bin
writer.WriteLine(String.Join("\t", toks));
manifestRegionIdx++;
}
}
}
return partitionedPath;
}
/// <summary>
/// Move a file to the destination, then put a symlink in the original location
/// pointing to the destination copy.
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static IFileLocation MoveAndLink(this IFileLocation source, IFileLocation destination)
{
// If the destination is null, return null
if (destination == null)
return null;
// Make sure you don't try to copy over yourself
if (source.FullName.Equals(destination.FullName))
return source;
// If you're a symbolic link, don't move anything, as you don't have anything real to move.
// The symbolic link is probably there because this method was called previously
// and if you move the link, it will break, accomplishing nothing useful.
// If you're an actual file, move it to the destination
if (!source.Attributes.HasFlag(FileAttributes.ReparsePoint))
source.MoveTo(destination);
// Then put a symlink in the place of the source pointing to the destination
destination.RelativeSymlinkTo(source);
return destination;
}
private IFileLocation InvokeCanvasPartition(CanvasCallset callset, IFileLocation cleanedPath, string canvasBedPath)
{
StringBuilder commandLine = new StringBuilder();
string executablePath = Path.Combine(_canvasFolder, "CanvasPartition.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", cleanedPath);
commandLine.AppendFormat("-b \"{0}\" ", canvasBedPath);
string partitionedPath = Path.Combine(callset.TempFolder, string.Format("{0}.partitioned", callset.Id));
commandLine.AppendFormat("-o \"{0}\" ", partitionedPath);
if (!_isSomatic)
commandLine.AppendFormat(" -g");
UnitOfWork partitionJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = Path.GetFileName(partitionedPath),
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasPartition"))
{
partitionJob.CommandLine = Utilities.MergeCommandLineOptions(partitionJob.CommandLine, _customParameters["CanvasPartition"], true);
}
_workManager.DoWorkSingleThread(partitionJob);
return new FileLocation(partitionedPath);
}
/// <summary>
/// Not making this an extension method so that it doesn't collide with the instance method
/// We still want to be able to refer to it statically though
/// </summary>
/// <param name="source"></param>
/// <param name="destination"></param>
public static void MoveTo(IFileLocation source, IFileLocation destination)
{
source.MoveTo(destination);
}
public Vcf Move(IFileLocation newPath, Action<IFileLocation, IFileLocation> move)
{
var newVcf = new Vcf(newPath);
Move(newVcf, move);
return newVcf;
}
protected void RunGermlineCalling(IFileLocation partitionedPath, CanvasCallset callset, string ploidyBedPath)
{
StringBuilder commandLine = new StringBuilder();
////////////////////////////////////////////////////////
// CanvasDiploidCaller:
commandLine.Length = 0;
string executablePath = Path.Combine(_canvasFolder, "CanvasDiploidCaller.exe");
if (CrossPlatform.IsThisMono())
{
commandLine.AppendFormat("{0} ", executablePath);
executablePath = Utilities.GetMonoPath();
}
commandLine.AppendFormat("-i \"{0}\" ", partitionedPath);
commandLine.AppendFormat("-v \"{0}\" ", callset.VfSummaryPath);
var cnvVcfPath = callset.OutputVcfPath;
commandLine.AppendFormat("-o \"{0}\" ", cnvVcfPath);
commandLine.AppendFormat("-n \"{0}\" ", callset.SampleName);
commandLine.AppendFormat("-r \"{0}\" ", callset.WholeGenomeFastaFolder);
if (!string.IsNullOrEmpty(ploidyBedPath))
{
commandLine.AppendFormat("-p \"{0}\" ", ploidyBedPath);
}
if (callset.IsDbSnpVcf) // a dbSNP VCF file is used in place of the normal VCF file
commandLine.AppendFormat("-d ");
UnitOfWork callJob = new UnitOfWork()
{
ExecutablePath = executablePath,
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = cnvVcfPath.Name,
CommandLine = commandLine.ToString()
};
if (_customParameters.ContainsKey("CanvasDiploidCaller"))
{
callJob.CommandLine = Utilities.MergeCommandLineOptions(callJob.CommandLine, _customParameters["CanvasDiploidCaller"], true);
}
_workManager.DoWorkSingleThread(callJob);
}
bool intersect(uint offset, uint length, IFileLocation location)
{
return intersect(offset, length, location.Offset, location.Length);
}
public static Vcf GetVcfFromStub(IFileLocation stub)
{
return new Vcf(stub.AppendName(Extension));
}
