/// <summary> /// Creates recalibration table and recalibrates reads. /// </summary> /// <param name="spritzDirectory"></param> /// <param name="genomeFasta"></param> /// <param name="bam"></param> /// <param name="recalibrationTablePath"></param> /// <param name="knownSitesVcf"></param> public List <string> BaseRecalibration(string spritzDirectory, string analysisDirectory, string genomeFasta, string bam, string knownSitesVcf) { RecalibrationTablePath = Path.Combine(Path.GetDirectoryName(bam), Path.GetFileNameWithoutExtension(bam) + ".recaltable"); RecalibratedBamPath = Path.Combine(Path.GetDirectoryName(bam), Path.GetFileNameWithoutExtension(bam) + ".recal.bam"); return(new List <string> { WrapperUtility.ChangeToToolsDirectoryCommand(spritzDirectory), SamtoolsWrapper.GenomeFastaIndexCommand(genomeFasta), GenomeDictionaryIndexCommand(genomeFasta), // check that reference VCF is indexed "if [ ! -f " + WrapperUtility.ConvertWindowsPath(knownSitesVcf) + ".idx ]; then " + Gatk(Workers) + " IndexFeatureFile -F " + WrapperUtility.ConvertWindowsPath(knownSitesVcf) + "; fi", "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(RecalibrationTablePath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(RecalibrationTablePath) + " ) ]]; then " + Gatk(Workers) + " BaseRecalibrator" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -I " + WrapperUtility.ConvertWindowsPath(bam) + (knownSitesVcf != "" ? " --known-sites " + WrapperUtility.ConvertWindowsPath(knownSitesVcf) : "") + " -O " + WrapperUtility.ConvertWindowsPath(RecalibrationTablePath) + "; fi", "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(RecalibratedBamPath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(RecalibratedBamPath) + " ) ]]; then " + Gatk(Workers) + " ApplyBQSR" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -I " + WrapperUtility.ConvertWindowsPath(bam) + " --bqsr-recal-file " + WrapperUtility.ConvertWindowsPath(RecalibrationTablePath) + " -O " + WrapperUtility.ConvertWindowsPath(RecalibratedBamPath) + "; fi", SamtoolsWrapper.IndexBamCommand(RecalibratedBamPath), }); }
/// <summary> /// Groups (I'm just using one group, so it's more a formality) and sorts reads. Marks duplicates. /// </summary> /// <param name="spritzDirectory"></param> /// <param name="threads"></param> /// <param name="bam"></param> /// <param name="genomeFasta"></param> /// <param name="reference"></param> /// <param name="newBam"></param> /// <param name="convertToUCSC"></param> public List <string> PrepareBamAndFasta(string spritzDirectory, string analysisDirectory, int threads, string bam, string genomeFasta, string reference) { string sortedCheckPath = Path.Combine(Path.GetDirectoryName(bam), Path.GetFileNameWithoutExtension(bam) + ".headerSorted"); string readGroupedCheckfile = Path.Combine(Path.GetDirectoryName(bam), Path.GetFileNameWithoutExtension(bam) + ".headerReadGrouped"); string sortedBam = Path.Combine(Path.GetDirectoryName(bam), Path.GetFileNameWithoutExtension(bam) + ".sorted.bam"); string groupedBamPath = Path.Combine(Path.GetDirectoryName(sortedBam), Path.GetFileNameWithoutExtension(sortedBam) + ".grouped.bam"); string markedDuplicatesBamPath = Path.Combine(Path.GetDirectoryName(groupedBamPath), Path.GetFileNameWithoutExtension(groupedBamPath) + ".marked.bam"); string markedDuplicateMetrics = Path.Combine(Path.GetDirectoryName(groupedBamPath), Path.GetFileNameWithoutExtension(groupedBamPath) + ".marked.metrics"); string tmpDir = Path.Combine(spritzDirectory, "tmp"); Directory.CreateDirectory(tmpDir); List <string> commands = new List <string> { WrapperUtility.ChangeToToolsDirectoryCommand(spritzDirectory), SamtoolsWrapper.GenomeFastaIndexCommand(genomeFasta), GenomeDictionaryIndexCommand(genomeFasta), "samtools view -H " + WrapperUtility.ConvertWindowsPath(bam) + " | grep SO:coordinate > " + WrapperUtility.ConvertWindowsPath(sortedCheckPath), "samtools view -H " + WrapperUtility.ConvertWindowsPath(bam) + " | grep '^@RG' > " + WrapperUtility.ConvertWindowsPath(readGroupedCheckfile), // group and sort (note, using picard-tools works, but picard.jar somehow is trucating the BAM files) "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(groupedBamPath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(groupedBamPath) + " ) && " + " ( ! -f " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " ) ]]; then " + Gatk(Workers, 2) + " AddOrReplaceReadGroups" + " -PU platform -PL illumina -SM sample -LB library" + " -I " + WrapperUtility.ConvertWindowsPath(bam) + " -O " + WrapperUtility.ConvertWindowsPath(groupedBamPath) + " -SO coordinate" + " --TMP_DIR " + WrapperUtility.ConvertWindowsPath(tmpDir) + "; fi", SamtoolsWrapper.IndexBamCommand(groupedBamPath), // mark duplicates (AS means assume sorted; note, using picard-tools works, but picard.jar somehow is trucating the BAM files) "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " ) ]]; then " + Gatk(Workers) + " MarkDuplicates" + // formerly picard " -I " + WrapperUtility.ConvertWindowsPath(groupedBamPath) + " -O " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " -M " + WrapperUtility.ConvertWindowsPath(markedDuplicateMetrics) + " --TMP_DIR " + WrapperUtility.ConvertWindowsPath(tmpDir) + " -AS true" + "; fi", SamtoolsWrapper.IndexBamCommand(markedDuplicatesBamPath), // clean up "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " || ! -s " + WrapperUtility.ConvertWindowsPath(markedDuplicatesBamPath) + " ) ]]; then " + "rm " + WrapperUtility.ConvertWindowsPath(groupedBamPath) + "; fi", }; PreparedBamPath = markedDuplicatesBamPath; return(commands); }
/// <summary> /// HaplotypeCaller for calling variants on each RNA-Seq BAM file individually. /// </summary> /// <param name="spritzDirectory"></param> /// <param name="threads"></param> /// <param name="genomeFasta"></param> /// <param name="splitTrimBam"></param> /// <param name="dbsnpReferenceVcfPath"></param> /// <param name="newVcf"></param> public List <string> VariantCalling(string spritzDirectory, ExperimentType experimentType, int threads, string genomeFasta, string splitTrimBam, string dbsnpReferenceVcfPath) { HaplotypeCallerGvcfPath = Path.Combine(Path.GetDirectoryName(splitTrimBam), Path.GetFileNameWithoutExtension(splitTrimBam) + ".g.vcf.gz"); HaplotypeCallerVcfPath = Path.Combine(Path.GetDirectoryName(splitTrimBam), Path.GetFileNameWithoutExtension(splitTrimBam) + ".g.gt.vcf"); FilteredHaplotypeCallerVcfPath = Path.Combine(Path.GetDirectoryName(splitTrimBam), Path.GetFileNameWithoutExtension(splitTrimBam) + ".g.gt.NoIndels.vcf"); var vcftools = new VcfToolsWrapper(); List <string> commands = new List <string> { WrapperUtility.ChangeToToolsDirectoryCommand(spritzDirectory), SamtoolsWrapper.GenomeFastaIndexCommand(genomeFasta), GenomeDictionaryIndexCommand(genomeFasta), // check that reference VCF is indexed "if [ ! -f " + WrapperUtility.ConvertWindowsPath(dbsnpReferenceVcfPath) + ".idx ]; then " + Gatk(Workers) + " IndexFeatureFile -F " + WrapperUtility.ConvertWindowsPath(dbsnpReferenceVcfPath) + "; fi", // call variants "if [ ! -f " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " ]; then " + Gatk(Workers, 2) + " HaplotypeCaller" + " --native-pair-hmm-threads " + threads.ToString() + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -I " + WrapperUtility.ConvertWindowsPath(splitTrimBam) + " --min-base-quality-score 20" + (experimentType == ExperimentType.RNASequencing ? " --dont-use-soft-clipped-bases true" : "") + " --dbsnp " + WrapperUtility.ConvertWindowsPath(dbsnpReferenceVcfPath) + " -O " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " -ERC GVCF" + // this prompts phasing! " --max-mnp-distance 3" + // note: this can't be used for joint genotyping here, but this setting is available in mutect2 for doing tumor vs normal calls "; fi", // index compressed gvcf file $"if [ ! -f {WrapperUtility.ConvertWindowsPath($"{HaplotypeCallerGvcfPath}.tbi")} ]; then {Gatk(Workers)} IndexFeatureFile -F {WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath)}; fi", // genotype the gvcf file into a traditional vcf file "if [ ! -f " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ]; then " + Gatk(Workers, 2) + " GenotypeGVCFs" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -V " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " -O " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + "; fi", $"if [ ! -f {WrapperUtility.ConvertWindowsPath($"{HaplotypeCallerVcfPath}.idx")} ]; then {Gatk(Workers)} IndexFeatureFile -F {WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath)}; fi", // filter out indels "if [ ! -f " + WrapperUtility.ConvertWindowsPath(FilteredHaplotypeCallerVcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(FilteredHaplotypeCallerVcfPath) + " ]; then " + Gatk(Workers, 2) + " SelectVariants" + " --select-type-to-exclude INDEL" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -V " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " -O " + WrapperUtility.ConvertWindowsPath(FilteredHaplotypeCallerVcfPath) + "; fi", $"if [ ! -f {WrapperUtility.ConvertWindowsPath($"{FilteredHaplotypeCallerVcfPath}.idx")} ]; then {Gatk(Workers)} IndexFeatureFile -F {WrapperUtility.ConvertWindowsPath(FilteredHaplotypeCallerVcfPath)}; fi", // filter variants (RNA-Seq specific params... need to check out recommendations before using DNA-Seq) //"if [ ! -f " + WrapperUtility.ConvertWindowsPath(newVcf) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(newVcf) + " ]; then " + // Gatk() + // " -T VariantFiltration" + // " -nct " + threads.ToString() + // " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + // " -V " + WrapperUtility.ConvertWindowsPath(unfliteredVcf) + // " -window 35 -cluster 3" + // filter out clusters of 3 snps within 35 bases (https://software.broadinstitute.org/gatk/documentation/topic?name=methods) // " -filterName FS -filter \"FS > 30.0\"" + // " -filterName QD -filter \"QD < 2.0\"" + // " -o " + WrapperUtility.ConvertWindowsPath(newVcf) + // "; fi", }; return(commands); }
public List <string> CombineAndGenotypeGvcfs(string spritzDirectory, string genomeFasta, List <string> gvcfPaths) { if (gvcfPaths == null || gvcfPaths.Count <= 1) { throw new ArgumentException("CombineAndGenotypeGvcfs exception: no gvcfs were specified to combine"); } int uniqueSuffix = 1; foreach (string f in gvcfPaths) { uniqueSuffix = uniqueSuffix ^ f.GetHashCode(); } HaplotypeCallerGvcfPath = Path.Combine(Path.GetDirectoryName(gvcfPaths.First()), $"combined{uniqueSuffix}.g.vcf.gz"); HaplotypeCallerVcfPath = Path.Combine(Path.GetDirectoryName(HaplotypeCallerGvcfPath), $"{Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(HaplotypeCallerGvcfPath))}.gt.vcf"); FilteredHaplotypeCallerVcfPath = Path.Combine(Path.GetDirectoryName(HaplotypeCallerGvcfPath), $"{Path.GetFileNameWithoutExtension(HaplotypeCallerGvcfPath)}.NoIndels.vcf"); List <string> commands = new List <string> { WrapperUtility.ChangeToToolsDirectoryCommand(spritzDirectory), SamtoolsWrapper.GenomeFastaIndexCommand(genomeFasta), GenomeDictionaryIndexCommand(genomeFasta) }; foreach (string gvcf in gvcfPaths) { // double check that the compressed gvcf file is indexed commands.Add($"if [ ! -f {WrapperUtility.ConvertWindowsPath(gvcf)}.idx ]; then {Gatk(Workers)} IndexFeatureFile -F {WrapperUtility.ConvertWindowsPath(gvcf)}; fi"); } // combine GVCFs string combineCommand = "if [ ! -f " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " ]; then " + Gatk(Workers) + " CombineGVCFs" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -V " + string.Join(" -V ", gvcfPaths.Select(gvcf => WrapperUtility.ConvertWindowsPath(gvcf))) + " -O " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + "; fi"; commands.Add(combineCommand); commands.Add($"if [ ! -f {WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath)}.idx ]; then {Gatk(Workers)} IndexFeatureFile -F {WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath)}; fi"); // genotype the gvcf file into a traditional vcf file string genotypeCommand = "if [ ! -f " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ]; then " + Gatk(Workers) + " GenotypeGVCFs" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -V " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerGvcfPath) + " -O " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + "; fi"; commands.Add(genotypeCommand); // filter out indels string filterIndelsCommand = "if [ ! -f " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ] || [ " + " ! -s " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " ]; then " + Gatk(Workers) + " SelectVariants" + " --select-type-to-exclude INDEL" + " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -V " + WrapperUtility.ConvertWindowsPath(HaplotypeCallerVcfPath) + " -O " + WrapperUtility.ConvertWindowsPath(FilteredHaplotypeCallerVcfPath) + "; fi"; commands.Add(filterIndelsCommand); return(commands); }
/// <summary> /// Splits and trims reads splice junction reads with SplitNCigarReads. /// Apparently cigars are genomic intervals, and splice junctions are represented by a bunch of N's (unkonwn nucleotide), HaplotypeCaller requires splitting them in the BAM file. /// /// It's tempting to want to run a few of these at the same time because it's not well parallelized. It's just not worth it. It uses quite a bit of RAM and racks the I/O at the beginning when reading the BAM files. /// Could possibly do 4 at a time on 128 GB RAM and 28 processors. /// </summary> /// <param name="spritzDirectory"></param> /// <param name="genomeFasta"></param> /// <param name="dedupedBam"></param> /// <param name="splitTrimBam"></param> /// <returns></returns> public List <string> SplitNCigarReads(string spritzDirectory, string genomeFasta, string dedupedBam) { string fixedQualsBam = Path.Combine(Path.GetDirectoryName(dedupedBam), Path.GetFileNameWithoutExtension(dedupedBam) + ".fixedQuals.bam"); SplitTrimBamPath = Path.Combine(Path.GetDirectoryName(fixedQualsBam), Path.GetFileNameWithoutExtension(fixedQualsBam) + ".split.bam"); // This also filters malformed reads string fixMisencodedQualsCmd = Gatk(Workers) + " FixMisencodedBaseQualityReads" + " -I " + WrapperUtility.ConvertWindowsPath(dedupedBam) + " -O " + WrapperUtility.ConvertWindowsPath(fixedQualsBam); string splitNCigarReadsCmd1 = Gatk(Workers) + " SplitNCigarReads" + //" --num_threads " + threads.ToString() + // not supported " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -I " + WrapperUtility.ConvertWindowsPath(fixedQualsBam) + " -O " + WrapperUtility.ConvertWindowsPath(SplitTrimBamPath) //" -rf ReassignOneMappingQuality" + // doing this with STAR //" -RMQF 255" + //" -RMQT 60" + // default mapping quality is 60; required for RNA-Seq aligners //" -U ALLOW_N_CIGAR_READS" ; string splitNCigarReadsCmd2 = Gatk(Workers) + " SplitNCigarReads" + //" --num_threads " + threads.ToString() + // not supported " -R " + WrapperUtility.ConvertWindowsPath(genomeFasta) + " -I " + WrapperUtility.ConvertWindowsPath(dedupedBam) + " -O " + WrapperUtility.ConvertWindowsPath(SplitTrimBamPath) //" -rf ReassignOneMappingQuality" + // doing this with STAR //" -RMQF 255" + //" -RMQT 60" + // default mapping quality is 60; required for RNA-Seq aligners //" -U ALLOW_N_CIGAR_READS" ; List <string> commands = new List <string> { WrapperUtility.ChangeToToolsDirectoryCommand(spritzDirectory), SamtoolsWrapper.GenomeFastaIndexCommand(genomeFasta), GenomeDictionaryIndexCommand(genomeFasta), // split and trim reads (some datasets are probably going to have misencoded quality scores; -fixMisencodedQuals just subtracts 31 from all quality scores if possible...) // exit code of 2 means that the FixMisencodedQualityBaseReads errored out because there were correctly encode base quality scores SamtoolsWrapper.IndexBamCommand(dedupedBam), "if [[ ( ! -f " + WrapperUtility.ConvertWindowsPath(fixedQualsBam) + " || ! -s " + WrapperUtility.ConvertWindowsPath(fixedQualsBam) + " ) ]]; then", " " + fixMisencodedQualsCmd, " if [ $? -ne 2 ]; then", " " + splitNCigarReadsCmd1, " else", " " + splitNCigarReadsCmd2, " fi", "fi", SamtoolsWrapper.IndexBamCommand(SplitTrimBamPath), }; return(commands); }