private ICheckpointRunner GetCheckpointRunner(ILogger logger, IDirectoryLocation outputDirectory, string startCheckpoint, string stopCheckpoint, IDirectoryLocation wholeGenomeFastaFolder)
{
var parentDirectories = new Dictionary<string, IDirectoryLocation>
{
{"Output", outputDirectory},
};
// Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta
IDirectoryLocation genomeRoot = wholeGenomeFastaFolder?.Parent?.Parent?.Parent?.Parent?.Parent;
if (genomeRoot != null) parentDirectories.Add("Genome", genomeRoot);
JsonConverter[] converters = { new FileSystemLocationConverter(parentDirectories) };
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto,
MissingMemberHandling = MissingMemberHandling.Error,
Converters = converters,
ContractResolver = new WritablePropertiesOnlyResolver(),
ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor,
};
ICheckpointSerializer serializer = new CheckpointJsonSerializer(settings, IsasFilePaths.GetCheckpointFolder(outputDirectory), logger);
ICheckpointManager manager = new CheckpointManager(logger, startCheckpoint, stopCheckpoint);
return new CheckpointRunner(logger, outputDirectory, manager, serializer, true);
}
public CanvasCallset(
IFileLocation bam,
string sampleName,
IDirectoryLocation wholeGenomeFastaFolder,
IDirectoryLocation outputFolder,
IFileLocation kmerFasta,
IFileLocation filterBed,
IFileLocation ploidyBed,
IFileLocation normalVcfPath,
bool isDbSnpVcf,
IEnumerable<IFileLocation> normalBamPaths,
NexteraManifest manifest,
IFileLocation somaticVcfPath,
IFileLocation outputVcfPath)
{
Bam = new Bam(bam);
SampleName = sampleName;
WholeGenomeFastaFolder = wholeGenomeFastaFolder;
OutputFolder = outputFolder;
KmerFasta = kmerFasta;
FilterBed = filterBed;
PloidyBed = ploidyBed;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
Manifest = manifest;
SomaticVcfPath = somaticVcfPath;
OutputVcfPath = outputVcfPath;
NormalBamPaths = normalBamPaths.Select(file => new Bam(file));
var genomeSizeXml = WholeGenomeFastaFolder.GetFileLocation("GenomeSize.xml");
GenomeMetadata = new GenomeMetadata();
GenomeMetadata.Deserialize(genomeSizeXml.FullName);
}
public ICheckpointRunner GetCheckpointManager(IDirectoryLocation analysisFolder,
string startingCheckpointName, string stopCheckpointName)
{
// Checkpoint Manager
var parentDirectories = new Dictionary<string, IDirectoryLocation>
{
{"AnalysisFolder", analysisFolder},
{"GenomesRoot", _genomesRoot}
};
JsonConverter[] converters =
{
new FileSystemLocationConverter(parentDirectories),
new EnumerableConverter()
};
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto,
MissingMemberHandling = MissingMemberHandling.Error,
Converters = converters,
ContractResolver = new WritablePropertiesOnlyResolver(),
ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor,
};
ICheckpointSerializer serializer = new CheckpointJsonSerializer(settings, IsasFilePaths.GetCheckpointFolder(analysisFolder), _logger);
//todo: remove this increment logic once the legacy checkpointer is no longer used
int tempStartCheckpoint;
if (int.TryParse(startingCheckpointName, out tempStartCheckpoint))
{
tempStartCheckpoint++;
startingCheckpointName = tempStartCheckpoint.ToString();
}
ICheckpointManager manager = new CheckpointManager(_logger, startingCheckpointName, stopCheckpointName);
return new CheckpointRunner(_logger, analysisFolder, manager, serializer, _retainTemps);
}
private IFileLocation WriteManifest(NexteraManifest manifest, IDirectoryLocation sandbox)
{
var path = sandbox.GetFileLocation(manifest.Name);
NexteraManifestUtils.WriteNexteraManifests(manifest, path.FullName);
return(path);
}
public CanvasSmallPedigreeOutput Run(CanvasSmallPedigreeInput input, IDirectoryLocation sandbox, IFileMover fileMover)
{
var output = _wrapper.Run(input, sandbox);
_move(output, fileMover);
return(_load(input));
}
public static SGEWorkManager GetSGEWorkManager(ILogger logger, IDirectoryLocation analysisFolder,
ISampleSettings settings, IsasConfiguration config)
{
LocalWorkManager local = GetLocalWorkManager(logger, analysisFolder, settings, config);
return new SGEWorkManager(logger, local,
config.ClusterQueueName, config.ClusterParallelEnvironmentName);
}
public CanvasCallset(
IFileLocation bam,
string sampleName,
IDirectoryLocation wholeGenomeFastaFolder,
IDirectoryLocation outputFolder,
IFileLocation kmerFasta,
IFileLocation filterBed,
IFileLocation ploidyBed,
IFileLocation normalVcfPath,
bool isDbSnpVcf,
IEnumerable <IFileLocation> normalBamPaths,
NexteraManifest manifest,
IFileLocation somaticVcfPath,
IFileLocation outputVcfPath)
{
Bam = new Bam(bam);
SampleName = sampleName;
WholeGenomeFastaFolder = wholeGenomeFastaFolder;
OutputFolder = outputFolder;
KmerFasta = kmerFasta;
FilterBed = filterBed;
PloidyBed = ploidyBed;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
Manifest = manifest;
SomaticVcfPath = somaticVcfPath;
OutputVcfPath = outputVcfPath;
NormalBamPaths = normalBamPaths.Select(file => new Bam(file));
var genomeSizeXml = WholeGenomeFastaFolder.GetFileLocation("GenomeSize.xml");
GenomeMetadata = new GenomeMetadata();
GenomeMetadata.Deserialize(genomeSizeXml.FullName);
}
public RnaReferenceGenome(IDirectoryLocation genomeBuildDirectory,
IDirectoryLocation genesDir, bool validate = true)
: base(genomeBuildDirectory)
{
GenesFolder = genesDir;
_validate = validate;
}
public static LocalWorkManager GetLocalWorkManager(ILogger logger, IDirectoryLocation analysisFolder,
ISampleSettings settings, IsasConfiguration config)
{
int maxThreadCount = settings.GetIntegerSetting(SampleSheetUtils.MaximumThreadCount, 0);
float maxMemGB = settings.GetFloatSetting(SampleSheetUtils.MaximumMemoryGB, config.MaximumMemoryGB);
var loggingFolder = IsasFilePaths.GetLoggingFolder(analysisFolder);
return new LocalWorkManager(logger, loggingFolder, maxThreadCount, maxMemGB, config.MaximumHoursPerProcess);
}
public SingleSampleCallset(Bam bam, string sampleName, IFileLocation normalVcfPath, bool isDbSnpVcf, IDirectoryLocation analysisOutputFolder, IFileLocation outputVcfPath)
{
_analysisOutputFolder = analysisOutputFolder;
Bam = bam;
SampleName = sampleName;
NormalVcfPath = normalVcfPath;
IsDbSnpVcf = isDbSnpVcf;
OutputVcfPath = outputVcfPath;
}
public CanvasSmallPedigreeOutput Run(CanvasSmallPedigreeInput input, IDirectoryLocation sampleSandbox)
{
if (!_annotationFileProvider.IsSupported(input.GenomeMetadata))
{
_logger.Info($"Skipping Canvas: unsupported reference genome '{input.GenomeMetadata.Name}'");
return(null);
}
if (!_annotationFileProvider.CustomDbSnpVcf(input.GenomeMetadata) && input.Vcf == null)
{
_logger.Info($"Skipping Canvas. A dbSNP VCF file was not provided and no small variant VCF file is available");
return(null);
}
var commandLine = new StringBuilder("SmallPedigree-WGS");
commandLine.Append(GetMultiSampleCommandLine(input.Samples, input.GenomeMetadata, input.Vcf, sampleSandbox));
// use sample vcf by default (performance could be similar with dbSNP vcf though)
var bAlleleVcf = input.Vcf.VcfFile;
var bAlleleVcfOptionName = SingleSampleCommonOptionsParser.SampleBAlleleVcfOptionName;
if (_annotationFileProvider.CustomDbSnpVcf(input.GenomeMetadata))
{
bAlleleVcf = _annotationFileProvider.GetDbSnpVcf(input.GenomeMetadata);
bAlleleVcfOptionName = SingleSampleCommonOptionsParser.PopulationBAlleleVcfOptionName;
}
commandLine.Append($" --{bAlleleVcfOptionName} \"{bAlleleVcf}\"");
var ploidyInfos = input.Samples.SelectData(sample => sample.PloidyInfo);
var ploidyVcf = _canvasPloidyVcfCreator.CreatePloidyVcf(ploidyInfos, input.GenomeMetadata, sampleSandbox);
if (ploidyVcf != null)
{
commandLine.Append($" --{SmallPedigreeOptionsParser.PloidyVcfOptionName} \"{ploidyVcf.VcfFile}\"");
}
var moreCustomParameters = new Dictionary <string, string>();
// common cnv feature disabled by default due to CANV-397
// string canvasPedigreeCallerParam = $@"--commoncnvs {_annotationFileProvider.GetCanvasAnnotationFile(input.GenomeMetadata, "commoncnvs.bed")}";
// moreCustomParameters["CanvasPedigreeCaller"] = canvasPedigreeCallerParam;
commandLine.Append(_singleSampleInputCommandLineBuilder.GetCustomParameters(moreCustomParameters));
commandLine = _singleSampleInputCommandLineBuilder.MergeCustomCanvasParameters(commandLine);
// use Proband or, when proband is not available, first sample as pedigree id
var pedigreeId = input.Samples.Where(x => x.Value.SampleType == SampleType.Proband).Select(x => x.Key.Id).FirstOrDefault();
if (pedigreeId.IsNullOrEmpty())
{
pedigreeId = input.Samples.First().Key.Id;
}
var job = new JobInfo(_runtimeExecutable.FullName, _canvasExe + " " + commandLine, "Canvas_" + pedigreeId);
return(_workDoer.DoWork(WorkResourceRequest.CreateExact(1, 8), job, GetCanvasOutput(input.Samples, sampleSandbox)).Await());
}
// general common options
/// <summary>
/// common options for all workflows
/// </summary>
/// <param name="outputDirectory"></param>
/// <param name="wholeGenomeFasta"></param>
/// <param name="kmerFasta"></param>
/// <param name="filterBed"></param>
/// <param name="customParams"></param>
/// <param name="startCheckpoint"></param>
/// <param name="stopCheckpoint"></param>
public CommonOptions(IDirectoryLocation outputDirectory, IDirectoryLocation wholeGenomeFasta, IFileLocation kmerFasta, IFileLocation filterBed, Dictionary <string, string> customParams, string startCheckpoint, string stopCheckpoint)
{
OutputDirectory = outputDirectory;
WholeGenomeFasta = wholeGenomeFasta;
KmerFasta = kmerFasta;
FilterBed = filterBed;
CustomParams = customParams;
StartCheckpoint = startCheckpoint;
StopCheckpoint = stopCheckpoint;
}
public BclRunFolder(IDirectoryLocation runFolder, string flowcellId, int flowcellNumber, Adapters adapters, ReadStructure readStructure, TileSelection tileSelection, BclFormat bclFormat)
{
RunFolder = runFolder;
Adapters = adapters;
ReadStructure = readStructure;
TileSelection = tileSelection;
_bclFormat = bclFormat;
FlowcellId = flowcellId;
FlowcellNumber = flowcellNumber;
}
public SampleSet <CanvasOutput> Run(SampleSet <CanvasTumorNormalWgsInput> inputs, IDirectoryLocation sandbox)
{
var outputs = new SampleSet <CanvasOutput>();
foreach (var input in inputs)
{
IDirectoryLocation sampleSandbox = sandbox.CreateSubdirectory(input.Key.Id);
outputs.Add(input.Key, RunSingleSample(input.Key.Id, input.Value, sampleSandbox));
}
return(outputs);
}
private CanvasOutput GetCanvasOutput(string sampleId, IDirectoryLocation sampleSandbox)
{
var cnvVcf = new Vcf(sampleSandbox.GetFileLocation("CNV.vcf.gz"));
var tempCnvDirectory = sampleSandbox.GetDirectoryLocation($"TempCNV_{sampleId}");
var variantFrequencies = tempCnvDirectory.GetFileLocation($"VFResults{sampleId}.txt.gz");
var variantFrequenciesBaf = tempCnvDirectory.GetFileLocation($"VFResults{sampleId}.txt.gz.baf");
IFileLocation coverageAndVariantFrequencies = sampleSandbox.GetFileLocation("CNV.CoverageAndVariantFrequency.txt");
IFileLocation tempStub = tempCnvDirectory.GetFileLocation($"{sampleId}");
IFileLocation partitioned = tempStub.AppendName(".partitioned");
return(new CanvasOutput(cnvVcf, coverageAndVariantFrequencies, variantFrequencies,
variantFrequenciesBaf, partitioned));
}
public CommonOptions(IFileLocation bAlleleSites, bool isDbSnpVcf, IFileLocation ploidyBed, IDirectoryLocation outputDirectory, IDirectoryLocation wholeGenomeFasta, IFileLocation kmerFasta, IFileLocation filterBed, string sampleName, Dictionary<string, string> customParams, string startCheckpoint, string stopCheckpoint)
{
BAlleleSites = bAlleleSites;
IsDbSnpVcf = isDbSnpVcf;
PloidyBed = ploidyBed;
OutputDirectory = outputDirectory;
WholeGenomeFasta = wholeGenomeFasta;
KmerFasta = kmerFasta;
FilterBed = filterBed;
SampleName = sampleName;
CustomParams = customParams;
StartCheckpoint = startCheckpoint;
StopCheckpoint = stopCheckpoint;
}
public AnalysisDetails(IDirectoryLocation outputFolder, IDirectoryLocation wholeGenomeFastaFolder,
IFileLocation kmerFasta, IFileLocation filterBed, IFileLocation ploidyVcf, IFileLocation commonCnvsBed)
{
WholeGenomeFastaFolder = wholeGenomeFastaFolder;
OutputFolder = outputFolder;
KmerFasta = kmerFasta;
FilterBed = filterBed;
PloidyVcf = ploidyVcf;
CommonCnvsBed = commonCnvsBed;
var genomeSizeXml = WholeGenomeFastaFolder.GetFileLocation("GenomeSize.xml");
GenomeMetadata = new GenomeMetadata();
GenomeMetadata.Deserialize(genomeSizeXml);
}
public CommonOptions(IFileLocation bAlleleSites, bool isDbSnpVcf, IFileLocation ploidyBed, IDirectoryLocation outputDirectory, IDirectoryLocation wholeGenomeFasta, IFileLocation kmerFasta, IFileLocation filterBed, string sampleName, Dictionary <string, string> customParams, string startCheckpoint, string stopCheckpoint)
{
BAlleleSites = bAlleleSites;
IsDbSnpVcf = isDbSnpVcf;
PloidyBed = ploidyBed;
OutputDirectory = outputDirectory;
WholeGenomeFasta = wholeGenomeFasta;
KmerFasta = kmerFasta;
FilterBed = filterBed;
SampleName = sampleName;
CustomParams = customParams;
StartCheckpoint = startCheckpoint;
StopCheckpoint = stopCheckpoint;
}
public SampleSet <CanvasEnrichmentOutput> Run(SampleSet <CanvasEnrichmentInput> inputs, IDirectoryLocation sandbox)
{
var targetedDbSnpVcfs = _enrichmentInputCreator.CreateDbSnpVcfForManifests(inputs, sandbox, _annotationFileProvider);
var manifests = _enrichmentInputCreator.WriteManifests(inputs, sandbox);
var outputs = new SampleSet <CanvasEnrichmentOutput>();
foreach (var input in inputs)
{
IDirectoryLocation sampleSandbox = sandbox.CreateSubdirectory(input.Key.Id);
var dbSnpVcf = targetedDbSnpVcfs[input.Value.NexteraManifest];
var manifest = manifests[input.Value.NexteraManifest];
outputs.Add(input.Key, RunSingleSample(input.Key.Id, input.Value, dbSnpVcf, manifest, sampleSandbox));
}
return(outputs);
}
public static IWorkManager GetWorkManager(ILogger logger, IDirectoryLocation analysisFolder,
ISampleSettings settings, IsasConfiguration config)
{
bool useSGE = settings.GetBooleanSetting(SampleSheetUtils.UseSGE, config.UseCluster);
config.UseCluster = useSGE;
if (config.UseCluster)
{
return GetSGEWorkManager(logger, analysisFolder, settings, config);
}
else
{
return GetLocalWorkManager(logger, analysisFolder, settings, config);
}
}
public SampleSet <CanvasOutput> Run(SampleSet <CanvasTumorNormalEnrichmentInput> inputs, IDirectoryLocation sandbox)
{
var bAlleleVcfs = GetBAlleleVcfs(inputs, sandbox);
var manifests = _enrichmentInputCreator.WriteManifests(inputs, sandbox);
var outputs = new SampleSet <CanvasOutput>();
foreach (var input in inputs)
{
IDirectoryLocation sampleSandbox = sandbox.CreateSubdirectory(input.Key.Id);
var bAlleleVcf = bAlleleVcfs[input.Key].Item1;
var isDbSnpVcf = bAlleleVcfs[input.Key].Item2;
var manifest = manifests[input.Value.NexteraManifest];
outputs.Add(input.Key, RunSingleSample(input.Key.Id, input.Value, bAlleleVcf, isDbSnpVcf, manifest, sampleSandbox));
}
return(outputs);
}
/// <summary>
/// Write out the ploidy bed file if ploidy information is available from the vcf header
/// </summary>
public IFileLocation CreatePloidyBed(Vcf vcf, GenomeMetadata genomeMetadata, IDirectoryLocation sampleSandbox)
{
IFileLocation ploidyBed = sampleSandbox.GetFileLocation("ploidy.bed.gz");
string fastaPath = genomeMetadata.Sequences.First().FastaPath;
if (_ploidyFixer.GeneratePloidyBedFileFromVcf(
genomeMetadata,
fastaPath,
vcf.VcfFile.FullName,
ploidyBed.FullName, sampleSandbox.FullName, _logger, _workManager))
{
return(ploidyBed);
}
_logger.Warn($"Sex chromosome ploidy not found in {vcf.VcfFile} header. No ploidy will be provided to Canvas.");
return(null);
}
public int Run(string[] args, TextWriter standardOutput, TextWriter standardError)
{
if (IsMissingMode(args))
{
return(HandleMissingMode(args, standardOutput, standardError));
}
var commonOptionsResult = ParseCommonOptions(args);
if (!commonOptionsResult.Success)
{
return(Parse(args, standardOutput.WriteLine, standardError.WriteLine));
}
var commonOptions = commonOptionsResult.Result;
if (AnyBaseOptions(args, out BaseOptions baseOptions))
{
HandleBaseOptions(baseOptions, standardOutput.WriteLine, GetMode(args));
return(0);
}
IDirectoryLocation outFolder = commonOptions.OutputDirectory;
var log = outFolder.GetFileLocation("CanvasLog.txt");
var error = outFolder.GetFileLocation("CanvasError.txt");
IDirectoryLocation genomeRoot = commonOptions.WholeGenomeFasta?.Parent?.Parent?.Parent?.Parent?.Parent;
int returnValue = 0;
IsasFrameworkFactory.RunWithLogger(log, error, logger =>
{
ISettings settings = new ProgrammaticSettingsBuilder()
.WithSetting(IsasConfigurationSettings.GenomeRepositoryPath, genomeRoot)
.ToSettings();
settings = new NestedSettingsProcessor(settings, IsasConfigurationSettings.GetConfigSettings());
IsasFrameworkFactory.RunWithWorkDoer(logger, settings, outFolder, workDoer =>
{
IsasFrameworkFactory.RunWithCheckpointer(logger, outFolder, settings, commonOptions.StartCheckpoint,
commonOptions.StopCheckpoint, checkpointer =>
{
var result = Parse(logger, settings, checkpointer, workDoer, args, standardOutput, standardError);
returnValue = result.Success ? result.Result.Launch() : -1;
});
});
});
return(returnValue);
}
private CheckpointRunnerAsync GetCheckpointRunner(ILogger logger, IDirectoryLocation outputDirectory, string startCheckpoint, string stopCheckpoint, IDirectoryLocation wholeGenomeFastaFolder)
{
var parentDirectories = new Dictionary <string, IDirectoryLocation>
{
{ "Output", outputDirectory },
};
// Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta
IDirectoryLocation genomeRoot = wholeGenomeFastaFolder?.Parent?.Parent?.Parent?.Parent?.Parent;
if (genomeRoot != null)
{
parentDirectories.Add("Genome", genomeRoot);
}
JsonConverter[] converters = { new FileSystemLocationConverter(parentDirectories) };
ICheckpointSerializerAsync serializer = new CheckpointJsonSerializerAsync(CheckpointManagerFactory.GetCheckpointFolder(outputDirectory), logger, converters);
return(CheckpointRunnerAsync.Create(serializer, logger, outputDirectory, startCheckpoint, stopCheckpoint, true));
}
private static void ComputeCallability(ILogger logger, Dictionary <string, List <CnvCall> > callsByContig,
EvaluateCnvOptions options, IDirectoryLocation output)
{
var kmerFasta = new FileLocation(options.KmerFa);
var canvasAnnotationDir = kmerFasta.Directory;
var filterBed = canvasAnnotationDir.GetFileLocation("filter13.bed");
if (!filterBed.Exists)
{
throw new ArgumentException($"Missing file at {filterBed}");
}
var annotationDir = canvasAnnotationDir.Parent;
var buildDir = annotationDir.Parent;
var genome = new ReferenceGenome(buildDir).GenomeMetadata;
var computer = CallabilityMetricsComputer.Create(logger, genome, filterBed, options.PloidyInfo.SexPloidyInfo.PloidyY == 0);
var callability = computer.CalculateMetric(callsByContig.SelectValues(calls => calls.Where(call => call.PassFilter).ToList()));
var callabilityFile = output.GetFileLocation($"{options.BaseFileName}_callability.txt");
File.WriteAllLines(callabilityFile.FullName, callability.GetMetrics().Select(metric => metric.ToCsv().Replace(",", "\t")));
logger.Info($"Callability: {callability.Callability}. Called bases: {callability.CalledBases}. Total bases: {callability.TotalBases}.");
}
private CanvasEnrichmentOutput GetCanvasOutput(string sampleId, IDirectoryLocation sampleSandbox)
{
var cnvVcf = new Vcf(sampleSandbox.GetFileLocation("CNV.vcf.gz"));
var tempCnvDirectory = sampleSandbox.GetDirectoryLocation($"TempCNV_{sampleId}");
var variantFrequencies = tempCnvDirectory.GetFileLocation($"VFResults{sampleId}.txt.gz");
var variantFrequenciesBaf = tempCnvDirectory.GetFileLocation($"VFResults{sampleId}.txt.gz.baf");
IFileLocation coverageAndVariantFrequencies = sampleSandbox.GetFileLocation("CNV.CoverageAndVariantFrequency.txt");
IFileLocation tempStub = tempCnvDirectory.GetFileLocation($"{sampleId}");
IFileLocation partitioned = tempStub.AppendName(".partitioned");
var canvasOutput = new CanvasOutput(cnvVcf, coverageAndVariantFrequencies, variantFrequencies,
variantFrequenciesBaf, partitioned);
IFileLocation binSize = tempStub.AppendName(".binsize");
IFileLocation normalBinned = tempStub.AppendName(".normal.binned");
IFileLocation unsmoothedCnd = tempStub.AppendName(".ratio.binned.cnd");
if (!binSize.Exists)
{
binSize = null;
}
return(new CanvasEnrichmentOutput(canvasOutput, binSize, normalBinned, unsmoothedCnd));
}
public IFileLocation Write(IReadOnlyList <CanvasSegment> segments, IDirectoryLocation output,
double normalizationFactor)
{
_logger.Info($"Begin writing bedgraph file at '{output}'");
var benchmark = new Benchmark();
var bedGraph = output.GetFileLocation("coverage.bedgraph");
_writer.Write(segments, bedGraph, normalizationFactor);
_logger.Info($"Finished writing bedgraph file at '{bedGraph}'. Elapsed time: {benchmark.GetElapsedTime()}");
_logger.Info($"Begin conversion of '{bedGraph}' to bigwig file");
benchmark = new Benchmark();
var bigWigConverterOutput = output.CreateSubdirectory("BigWigConverter");
var bigwigFile = _converter.Convert(bedGraph, _genome, bigWigConverterOutput);
if (bigwigFile != null)
{
_logger.Info(
$"Finished conversion from bedgraph file at '{bedGraph}' to bigwig file at '{bigwigFile}'. Elapsed time: {benchmark.GetElapsedTime()}");
}
return(bigwigFile);
}
public int Launch()
{
CommonOptions commonOptions = _modeRunner.CommonOptions;
IDirectoryLocation outFolder = commonOptions.OutputDirectory;
var log = outFolder.GetFileLocation("CanvasLog.txt");
var error = outFolder.GetFileLocation("CanvasError.txt");
using (ILogger logger = new Logger(log, error))
{
try
{
logger.Info($"Running Canvas {_mode} {_version}");
logger.Info($"Command-line arguments: {string.Join(" ", _args)}");
var checkpointRunner =
GetCheckpointRunner(
logger,
outFolder,
commonOptions.StartCheckpoint,
commonOptions.StopCheckpoint,
commonOptions.WholeGenomeFasta);
using (var manager = checkpointRunner.Manager)
{
IDirectoryLocation loggingFolder = outFolder.CreateSubdirectory("Logging");
IsasConfiguration config = IsasConfiguration.GetConfiguration();
IWorkManager workManager = new LocalWorkManager(logger, loggingFolder, 0, config.MaximumMemoryGB, config.MaximumHoursPerProcess);
_modeRunner.Run(logger, checkpointRunner, workManager);
manager.CheckFinalState();
}
}
catch (StopCheckpointFoundException) { }
catch (Exception e)
{
logger.Error($"Canvas workflow error: {e}");
return(-1);
}
}
return(0);
}
public int Launch()
{
CommonOptions commonOptions = _modeRunner.CommonOptions;
IDirectoryLocation outFolder = commonOptions.OutputDirectory;
var log = outFolder.GetFileLocation("CanvasLog.txt");
var error = outFolder.GetFileLocation("CanvasError.txt");
IsasConfiguration config = IsasConfiguration.GetConfiguration();
IDirectoryLocation genomeRoot = commonOptions.WholeGenomeFasta?.Parent?.Parent?.Parent?.Parent?.Parent;
int returnValue = 0;
IsasFrameworkFactory.RunWithIsasFramework(outFolder, log, error, commonOptions.StartCheckpoint, commonOptions.StopCheckpoint, 0,
config.MaximumMemoryGB, config.MaximumHoursPerProcess, false, genomeRoot,
frameworkServices =>
{
var logger = frameworkServices.Logger;
try
{
var executableProcessor = new ExecutableProcessor(new NullSampleSettings(), logger);
#if DotNetCore
var runtimeExecutable = new FileLocation(executableProcessor.GetEnvironmentExecutablePath("dotnet"));
#else
var runtimeExecutable = CrossPlatform.IsThisLinux() ? new FileLocation(executableProcessor.GetMonoPath()) : null;
#endif
frameworkServices.Logger.Info($"Running Canvas {_mode} {_version}");
logger.Info($"Command-line arguments: {string.Join(" ", _args)}");
_modeRunner.Run(logger, frameworkServices.Checkpointer, frameworkServices.WorkManager, runtimeExecutable);
returnValue = 0;
}
catch (Exception e)
{
logger.Error($"Canvas workflow error: {e}");
returnValue = -1;
}
});
return(returnValue);
}
private CanvasOutput RunSingleSample(string sampleId, CanvasTumorNormalEnrichmentInput input, IFileLocation bAlleleVcf, bool isDbSnpVcf, IFileLocation manifest, IDirectoryLocation sampleSandbox)
{
if (!_annotationFileProvider.IsSupported(input.GenomeMetadata))
{
_logger.Info($"Skipping Canvas for sample {sampleId}: unsupported reference genome '{input.GenomeMetadata.Name}'");
return(null);
}
StringBuilder commandLine = new StringBuilder("Tumor-normal-enrichment");
commandLine.Append(_singleSampleInputCommandLineBuilder.GetSingleSampleCommandLine(sampleId, input.TumorBam, input.GenomeMetadata, sampleSandbox));
commandLine.Append($" --normal-bam \"{input.NormalBam.BamFile}\"");
commandLine.Append($" --manifest \"{manifest}\"");
var bAlleleVcfOptionName = isDbSnpVcf ?
SingleSampleCommonOptionsParser.PopulationBAlleleVcfOptionName :
SingleSampleCommonOptionsParser.SampleBAlleleVcfOptionName;
commandLine.Append($" --{bAlleleVcfOptionName} {bAlleleVcf.WrapWithShellQuote()}");
commandLine.Append($" --somatic-vcf \"{input.SomaticVcf.VcfFile}\"");
IFileLocation ploidyBed = _canvasPloidyBedCreator.CreatePloidyBed(input.NormalVcf, input.GenomeMetadata, sampleSandbox);
if (ploidyBed != null)
{
commandLine.Append($" --{SingleSampleCommonOptionsParser.PloidyBedOptionName} \"{ploidyBed}\"");
}
commandLine.Append(_singleSampleInputCommandLineBuilder.GetCustomParameters());
commandLine = _singleSampleInputCommandLineBuilder.MergeCustomCanvasParameters(commandLine);
UnitOfWork singleSampleJob = new UnitOfWork
{
ExecutablePath = CrossPlatform.IsThisLinux() ? _runtimeExecutable.FullName : _canvasExe.FullName,
CommandLine = CrossPlatform.IsThisLinux() ? _canvasExe + " " + commandLine : commandLine.ToString(),
LoggingFolder = _workManager.LoggingFolder.FullName,
LoggingStub = "Canvas_" + sampleId,
};
_workManager.DoWorkSingleThread(singleSampleJob);
return(GetCanvasOutput(sampleId, sampleSandbox));
}
/// <summary>
/// Copy a directory to a new location. Input is the directory location for the resulting copied directory and not the parent directory.
/// If newDirectoryLocation already exists it is first removed.
/// Returns newDirectoryLocation to enable chaining
/// </summary>
public IDirectoryLocation CopyTo(IDirectoryLocation newDirectoryLocation)
{
if (newDirectoryLocation.FullName == FullName) return this;
newDirectoryLocation.CreateClean();
foreach (IFileLocation file in EnumerateFiles())
{
file.CopyTo(newDirectoryLocation.GetFileLocation(file.Name));
}
foreach (IDirectoryLocation subdir in EnumerateDirectories())
{
subdir.CopyInto(newDirectoryLocation);
}
return newDirectoryLocation;
}
public SampleSet <Tuple <IFileLocation, bool> > GetBAlleleVcfs(SampleSet <CanvasTumorNormalEnrichmentInput> inputs, IDirectoryLocation sandbox)
{
var inputsUsingDbSnpVcf = inputs.WhereData(input => _annotationFileProvider.CustomDbSnpVcf(input.GenomeMetadata));
var targetedDbSnpVcfs = _enrichmentInputCreator.CreateDbSnpVcfForManifests(inputsUsingDbSnpVcf, sandbox, _annotationFileProvider);
var bAlleleVcfs = new SampleSet <Tuple <IFileLocation, bool> >();
foreach (var input in inputs)
{
IFileLocation bAlleleVcf;
bool isDbSnpVcf = true;
if (!targetedDbSnpVcfs.TryGetValue(input.Value.NexteraManifest, out bAlleleVcf))
{
bAlleleVcf = input.Value.NormalVcf.VcfFile;
isDbSnpVcf = false;
}
bAlleleVcfs[input.Key] = Tuple.Create(bAlleleVcf, isDbSnpVcf);
}
return(bAlleleVcfs);
}
public static IFileLocation GetCoverageAndVariantFrequencyOutput(IDirectoryLocation analysisOutputFolder, string sampleName)
{
return(GetCoverageAndVariantFrequencyOutput(GetSampleOutputFolder(analysisOutputFolder, sampleName).GetFileLocation("CNV")));
}
public static IDirectoryLocation GetCheckpointFolder(IDirectoryLocation analysisFolder)
{
return analysisFolder.CreateSubdirectory(CheckpointFolderName);
}
public static IDirectoryLocation GetSampleOutputFolder(IDirectoryLocation analysisOutputFolder, string sampleName)
{
return(analysisOutputFolder.GetDirectoryLocation($"TempCNV_{sampleName}"));
}
public static IFileLocation GetVfSummaryBafPath(IDirectoryLocation analysisOutputFolder, string sampleName)
{
return(GetVfSummaryPath(analysisOutputFolder, sampleName).AppendName(".baf"));
}
/// <summary>
/// Moves an existing file into a parent directory, overwriting any existing file.
/// </summary>
/// <param name="destination">The parent directory for the new location of the file.</param>
/// <returns>
/// The new location of the file
/// </returns>
public IFileLocation MoveInto(IDirectoryLocation destination)
{
if (Directory.FullName.Equals(destination.FullName)) // Doesn't handle symlinks...
return this;
var result = destination.GetFileLocation(Name);
MoveTo(result);
return result;
}
/// <summary>
/// Moves a directory into a parent directory. If paths are not on the same filesystem volume this is equivalent to CopyInto() then Delete()
/// If the parentDirectoryLocation already contains a directory with the same it is first removed
/// Returns the directory in its new location
/// </summary>
public IDirectoryLocation MoveInto(IDirectoryLocation parentDirectoryLocation)
{
return MoveTo(parentDirectoryLocation.CreateSubdirectory(Name));
}
/// <summary>
/// Move a folder to the destination, then put a symlink in the original location
/// pointing to the destination copy.
/// </summary>
/// <param name="sourcePath"></param>
/// <param name="destinationPath"></param>
public static IDirectoryLocation MoveAndLink(this IDirectoryLocation source, IDirectoryLocation destination)
{
// Move the file to the destination
source.MoveTo(destination);
// then symlink the original file to the destination
destination.SymlinkTo(source);
return destination;
}
public static IFileLocation SymlinkInto(this IFileLocation sourceFile, IDirectoryLocation destFolder)
{
IFileLocation dest = destFolder.GetFileLocation(sourceFile.Name);
return sourceFile.SymlinkTo(dest);
}
public static IFileLocation MoveAndLinkInto(this IFileLocation source, IDirectoryLocation destination)
{
return source.MoveAndLink(destination.GetFileLocation(source.Name));
}
/// <summary>
/// Moves a directory to a new location. If paths are not on the same filesystem volume this is equivalent to CopyTo() then Delete()
/// Input is the directory in its new location and not the parent directory
/// If the newDirectoryLocation already exists it is first removed
/// Returns newDirectoryLocation to enable chaining
/// </summary>
public IDirectoryLocation MoveTo(IDirectoryLocation newDirectoryLocation)
{
if (newDirectoryLocation.FullName == FullName) return this;
if (newDirectoryLocation.Parent != null)
newDirectoryLocation.Parent.Create();
newDirectoryLocation.Delete();
try
{
Directory.Move(FullName, newDirectoryLocation.FullName);
}
catch (IOException)
{
CopyTo(newDirectoryLocation);
Delete();
}
return newDirectoryLocation;
}
/// <summary>
/// Copy a directory into some parent directory.
/// If a directory with the same name exists in the parent directory it is first removed.
/// Returns the copied directory
/// </summary>
public IDirectoryLocation CopyInto(IDirectoryLocation parentDirectoryLocation)
{
return CopyTo(parentDirectoryLocation.CreateSubdirectory(Name));
}
public static IDirectoryLocation GetLoggingFolder(IDirectoryLocation analysisFolder)
{
return analysisFolder.CreateSubdirectory(LoggingFolderName);
}
/* Requires Mono.Unix assembly
[DllImport("kernel32.dll")]
private static extern bool CreateSymbolicLink(string lpSymlinkFileName, string lpTargetFileName, int dwFlags);
/// <summary>
/// Symlinks an DirectoryLocation instance and its contents to a new path.
/// </summary>
/// <param name="destDirName">The name and path to which to symlink this directory.</param>
/// <returns>
/// A symlinked directory
/// </returns>
public IDirectoryLocation SymlinkTo(string destDirName)
{
if (Type.GetType("Mono.Runtime") != null)
{
var directory = new UnixDirectoryInfo(_instance.FullName).CreateSymbolicLink(destDirName);
return new DirectoryLocation(directory.FullName);
}
CreateSymbolicLink(destDirName, _instance.FullName, 1);
return new DirectoryLocation(destDirName);
}
*/
/// <summary>
/// Symlinks an existing file to a new location.
/// If the symlink cannot be created (Windows may require elevated privileges to do this),
/// then a copy is created instead if <paramref name="copyOnFail"/> is true.
/// </summary>
/// <param name="sourceFile">The instance of IDirectoryLocation this method extends</param>
/// <param name="destinationFolder">The IDirectoryLocation representing the symlink,
/// which points to <paramref name="sourceFile"/></param>
/// <param name="copyOnFail">If true, copies the file if a symlink cannot be created.</param>
/// <returns>An IDirectoryLocation of the destination file, or null if the symlink failed and <paramref name="copyOnFail"/> is false.</returns>
public IDirectoryLocation SymlinkTo(IDirectoryLocation destinationFolder, bool copyOnFail = true)
{
return (Illumina.SecondaryAnalysis.Utilities.CreateSymbolicLink(destinationFolder.FullName, FullName, copyOnFail))
? destinationFolder : null;
}
public static IFileLocation GetSingleSamplePedigreeVcfOutput(IDirectoryLocation analysisOutputFolder, string sampleName)
{
return(GetSingleSamplePedigreeVcfOutput(GetSampleOutputFolder(analysisOutputFolder, sampleName).GetFileLocation("CNV")));
}
public CheckpointManagerFactory(ILogger logger, IDirectoryLocation genomesRoot, bool retainTemps)
{
_logger = logger;
_genomesRoot = genomesRoot;
_retainTemps = retainTemps;
}
public static IFileLocation GetVfSummaryPath(IDirectoryLocation analysisOutputFolder, string sampleName)
{
var stub = GetSampleOutputFolder(analysisOutputFolder, sampleName).GetFileLocation($"VFResults{sampleName}");
return(GetVfSummaryPathExtension(stub));
}
public ReferenceGenome(IDirectoryLocation buildDirectory)
{
BuildDirectory = buildDirectory;
}
public static IFileLocation GetPartitionedPath(IDirectoryLocation analysisOutputFolder, string sampleName)
{
var stub = GetSampleOutputFolder(analysisOutputFolder, sampleName).GetFileLocation(sampleName);
return(GetPartitionedPath(stub));
}
public IReferenceGenome GetReferenceGenome(IDirectoryLocation wholeGenomeFasta)
{
return new ReferenceGenome(wholeGenomeFasta.Parent.Parent);
}
public SampleSet <TCanvasOutput> Run(SampleSet <TCanvasInput> inputs, IDirectoryLocation sandbox)
{
return(_canvasCnvCaller.Run(inputs, sandbox));
}
public static IDirectoryLocation GetFastqFolder(IDirectoryLocation analysisFolder)
{
return analysisFolder.CreateSubdirectory(FastqFolderName);
}
public StringBuilder GetSingleSampleCommandLine(string sampleId, Bam bam, GenomeMetadata genomeMetadata, IDirectoryLocation sampleSandbox)
{
StringBuilder commandLine = new StringBuilder();
commandLine.Append($" --bam \"{bam.BamFile}\"");
commandLine.Append($" --sample-name \"{sampleId}\"");
IFileLocation kmerFasta = _annotationFileProvider.GetKmerFasta(genomeMetadata);
commandLine.Append($" --reference \"{kmerFasta}\"");
IDirectoryLocation wholeGenomeFasta = new FileLocation(genomeMetadata.Sequences.First().FastaPath).Directory;
commandLine.Append($" --genome-folder \"{wholeGenomeFasta}\"");
IFileLocation filterBed = _annotationFileProvider.GetFilterBed(genomeMetadata);
commandLine.Append($" --filter-bed \"{filterBed}\"");
commandLine.Append($" --output \"{sampleSandbox}\"");
return(commandLine);
}
private CheckpointRunnerAsync GetCheckpointRunner(ILogger logger, IDirectoryLocation outputDirectory, string startCheckpoint, string stopCheckpoint, IDirectoryLocation wholeGenomeFastaFolder)
{
var parentDirectories = new Dictionary<string, IDirectoryLocation>
{
{"Output", outputDirectory},
};
// Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta
IDirectoryLocation genomeRoot = wholeGenomeFastaFolder?.Parent?.Parent?.Parent?.Parent?.Parent;
if (genomeRoot != null) parentDirectories.Add("Genome", genomeRoot);
JsonConverter[] converters = { new FileSystemLocationConverter(parentDirectories) };
ICheckpointSerializerAsync serializer = new CheckpointJsonSerializerAsync(CheckpointManagerFactory.GetCheckpointFolder(outputDirectory), logger, converters);
return CheckpointRunnerAsync.Create(serializer, logger, outputDirectory, startCheckpoint, stopCheckpoint, true);
}