private static void CreateAndExecuteProcessor(bool threadByChr, MockFactoryWithDefaults factory, Genome genome)
{
if (threadByChr)
{
var processor = new GenomeProcessor(factory, genome);
processor.Execute(1);
}
else
{
var processor = new GenomeProcessor(factory, genome);
processor.Execute(1);
}
}
private void CallVariantsWithMockData(string vcfOutputPath, ApplicationOptions options, AmpliconTestFactory atf)
{
var appFactory = new MockFactoryWithDefaults(options);
using (var vcfWriter = appFactory.CreateVcfWriter(vcfOutputPath, new VcfWriterInputContext()))
{
var svc = CreateMockVariantCaller(vcfWriter, options, atf.ChrInfo, atf._MAE);
vcfWriter.WriteHeader();
svc.Execute();
}
Assert.True(File.Exists(vcfOutputPath));
}
private MockFactoryWithDefaults GetMockFactory(int numBams = 1)
{
var factory = new MockFactoryWithDefaults(new HygeaOptions()
{
BAMPaths = numBams == 1 ? new[] { "bamfile" } : new[] { "bamfile", "bamfile2" },
GenomePaths = new[] { "someGenome" }
});
factory.MockWriter = new Mock <IRealignmentWriter>();
factory.MockAlignmentExtractor = new Mock <IAlignmentExtractor>();
factory.MockChrRealigner = new Mock <IChrRealigner>();
return(factory);
}
public static void CallStrandedVariantsWithMockData(string vcfOutputPath, ApplicationOptions options, AmpliconTestFactory testFactory)
{
var appFactory = new MockFactoryWithDefaults(options);
using (var vcfWriter = appFactory.CreateVcfWriter(vcfOutputPath, new VcfWriterInputContext()))
{
using (var biasWriter = new StrandBiasFileWriter(vcfOutputPath))
{
var svc = CreateMockVariantCaller(vcfWriter, options, testFactory.ChrInfo, testFactory.AlignmentExtractor, biasWriter);
vcfWriter.WriteHeader();
biasWriter.WriteHeader();
svc.Execute();
biasWriter.Dispose();
}
}
Assert.True(File.Exists(vcfOutputPath));
}
private MockFactoryWithDefaults GetMockedFlowFactory(int numIterations)
{
var currentIteration = 0;
var factory = new MockFactoryWithDefaults(new ApplicationOptions());
// alignment source
var mockAlignmentSource = new Mock <IAlignmentSource>();
mockAlignmentSource.Setup(s => s.GetNextAlignmentSet()).Returns(() =>
currentIteration < numIterations ? new AlignmentSet(TestHelper.CreateRead(_chrReference.Name, "AAA", 1 + currentIteration++), null) : null);
mockAlignmentSource.Setup(s => s.LastClearedPosition).Returns(() => currentIteration);
mockAlignmentSource.Setup(s => s.ChromosomeFilter).Returns(_chrReference.Name);
factory.MockAlignmentSource = mockAlignmentSource;
// state manager
_candidateList = new List <CandidateAllele>()
{
new CandidateAllele("chr1", 100, "A", "G", AlleleCategory.Snv)
};
_batch = new CandidateBatch(_candidateList);
var mockStateManager = new Mock <IStateManager>();
mockStateManager.Setup(s => s.GetCandidatesToProcess(It.IsAny <int?>(), _chrReference)).Returns(_batch);
factory.MockStateManager = mockStateManager;
// variant finder
var mockVariantFinder = new Mock <ICandidateVariantFinder>();
mockVariantFinder.Setup(v => v.FindCandidates(It.IsAny <AlignmentSet>(), _chrReference.Sequence, _chrReference.Name)).Returns(_candidateList);
factory.MockVariantFinder = mockVariantFinder;
// variant caller
var mockVariantCaller = new Mock <IAlleleCaller>();
mockVariantCaller.Setup(v => v.Call(_batch, mockStateManager.Object)).Returns(_calledList);
factory.MockVariantCaller = mockVariantCaller;
// region mapper
var mockRegionMapper = new Mock <IRegionPadder>();
factory.MockRegionMapper = mockRegionMapper;
return(factory);
}
private MockFactoryWithDefaults GetMockFactory(ApplicationOptions options, List<AlignmentSet> readSets)
{
var currentReadIndex = -1;
var factory = new MockFactoryWithDefaults(options);
// alignment source
var mockAlignmentSource = new Mock<IAlignmentSource>();
mockAlignmentSource.Setup(s => s.GetNextAlignmentSet()).Returns(() =>
{
currentReadIndex ++;
return currentReadIndex < readSets.Count() ? readSets[currentReadIndex] : null;
});
mockAlignmentSource.Setup(s => s.LastClearedPosition).Returns(() => 0);
mockAlignmentSource.Setup(s => s.ChromosomeFilter).Returns(_chrName);
factory.MockAlignmentSource = mockAlignmentSource;
return factory;
}
private MockFactoryWithDefaults GetMockFactory(PiscesApplicationOptions options, List <Read> readSets)
{
var currentReadIndex = -1;
var factory = new MockFactoryWithDefaults(options);
// alignment source
var mockAlignmentSource = new Mock <IAlignmentSource>();
mockAlignmentSource.Setup(s => s.GetNextRead()).Returns(() =>
{
currentReadIndex++;
return(currentReadIndex < readSets.Count() ? readSets[currentReadIndex] : null);
});
mockAlignmentSource.Setup(s => s.LastClearedPosition).Returns(() => 0);
factory.MockAlignmentSource = mockAlignmentSource;
return(factory);
}
private void ExecuteChromosomeThreadingTest(int numberOfThreads, int expectedNumberOfThreads, string outDir)
{
var bamFilePath = Path.Combine(TestPaths.LocalTestDataDirectory, "Chr17Chr19.bam");
var vcfFilePath = Path.Combine(outDir, "Chr17Chr19.vcf");
var genomePath = Path.Combine(TestPaths.SharedGenomesDirectory, "chr17chr19");
var options = new PiscesApplicationOptions
{
BAMPaths = new[] { bamFilePath },
GenomePaths = new[] { genomePath },
VcfWritingParameters = new VcfWritingParameters()
{
OutputGvcfFile = false
},
OutputDirectory = outDir
};
options.SetIODirectories("Pisces");
var logFile = Path.Combine(options.LogFolder, options.LogFileName);
if (File.Exists(logFile))
{
File.Delete(logFile);
}
Logger.OpenLog(options.LogFolder, options.LogFileName);
var factory = new MockFactoryWithDefaults(options);
factory.MockSomaticVariantCaller = new Mock <ISmallVariantCaller>();
factory.MockSomaticVariantCaller.Setup(s => s.Execute()).Callback(() =>
{
Thread.Sleep(500);
});
var processor = new GenomeProcessor(factory, factory.GetReferenceGenome(genomePath), false);
processor.Execute(numberOfThreads);
Assert.False(File.Exists(vcfFilePath + "_chr17"));
Assert.False(File.Exists(vcfFilePath + "_chr19"));
Assert.True(File.Exists(vcfFilePath));
Logger.CloseLog();
//var threadsSpawnedBeforeFirstCompleted = 0;
/* dont worry about logging
* using (var reader = new StreamReader(new FileStream(logFile, FileMode.Open, FileAccess.Read)))
* {
* string line;
* while ((line = reader.ReadLine()) != null)
* {
* if (string.IsNullOrEmpty(line)) continue;
*
* if (line.Contains("Completed processing chr")) break;
*
* if (line.Contains("Start processing chr"))
* threadsSpawnedBeforeFirstCompleted++;
* }
* }*/
//Assert.Equal(expectedNumberOfThreads, threadsSpawnedBeforeFirstCompleted);
}
private void ExecuteChromosomeThreadingTest(int numberOfThreads, int expectedNumberOfThreads)
{
var bamFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "Chr17Chr19.bam");
var vcfFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "Chr17Chr19.vcf");
var genomePath = Path.Combine(UnitTestPaths.TestGenomesDirectory, "chr17chr19");
var options = new ApplicationOptions
{
BAMPaths = new[] { bamFilePath },
GenomePaths = new[] { genomePath },
};
var logFile = Path.Combine(options.LogFolder, options.LogFileName);
if (File.Exists(logFile))
{
File.Delete(logFile);
}
Logger.TryOpenLog(options.LogFolder, options.LogFileName);
var factory = new MockFactoryWithDefaults(options);
factory.MockSomaticVariantCaller = new Mock <ISomaticVariantCaller>();
factory.MockSomaticVariantCaller.Setup(s => s.Execute()).Callback(() =>
{
Thread.Sleep(500);
});
var processor = new GenomeProcessor(factory, factory.GetReferenceGenome(genomePath), false);
processor.Execute(numberOfThreads);
Assert.False(File.Exists(vcfFilePath + "_chr17"));
Assert.False(File.Exists(vcfFilePath + "_chr19"));
Assert.True(File.Exists(vcfFilePath));
Logger.TryCloseLog();
var threadsSpawnedBeforeFirstCompleted = 0;
using (var reader = new StreamReader(logFile))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
if (line.Contains("Completed processing chr"))
{
break;
}
if (line.Contains("Start processing chr"))
{
threadsSpawnedBeforeFirstCompleted++;
}
}
}
Assert.Equal(expectedNumberOfThreads, threadsSpawnedBeforeFirstCompleted);
}
private void ExecuteTest(int numberOfThreads, int expectedNumberOfThreads)
{
var bamFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "var123var35.bam");
var bamFilePath2 = Path.Combine(UnitTestPaths.TestDataDirectory, "var123var35_removedSQlines.bam");
var vcfFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "var123var35.vcf");
var vcfFilePath2 = Path.Combine(UnitTestPaths.TestDataDirectory, "var123var35_removedSQlines.vcf");
var genomePath = Path.Combine(UnitTestPaths.TestGenomesDirectory, "chr17chr19");
var options = new ApplicationOptions
{
BAMPaths = new[] { bamFilePath, bamFilePath2 },
GenomePaths = new[] { genomePath },
};
var logFile = Path.Combine(options.LogFolder, ApplicationOptions.LogFileName);
if (File.Exists(logFile))
{
File.Delete(logFile);
}
Logger.TryOpenLog(options.LogFolder, ApplicationOptions.LogFileName);
var factory = new MockFactoryWithDefaults(options);
factory.MockSomaticVariantCaller = new Mock <ISomaticVariantCaller>();
factory.MockSomaticVariantCaller.Setup(s => s.Execute()).Callback(() =>
{
Thread.Sleep(500);
});
var processor = new GenomeProcessor(factory, factory.GetReferenceGenome(genomePath));
processor.Execute(numberOfThreads);
Assert.True(File.Exists(vcfFilePath));
Assert.True(File.Exists(vcfFilePath2));
Logger.TryCloseLog();
var chrCheck = new Dictionary <string, Tuple <int, bool> >();
chrCheck["chr17"] = new Tuple <int, bool>(0, false);
chrCheck["chr19"] = new Tuple <int, bool>(0, false);
var startedChr19 = false;
using (var reader = new StreamReader(logFile))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
foreach (var chr in chrCheck.Keys.ToList())
{
if (line.Contains("Start processing chr " + chr))
{
var chrState = chrCheck[chr];
chrCheck[chr] = new Tuple <int, bool>(chrState.Item1 + 1, true);
}
}
foreach (var chr in chrCheck.Keys.ToList())
{
if (line.Contains("Completed processing chr " + chr) && chrCheck[chr].Item2)
{
var chrState = chrCheck[chr];
Assert.Equal(expectedNumberOfThreads, chrState.Item1);
chrCheck[chr] = new Tuple <int, bool>(0, false);
}
}
// make sure chr 17 fully completes before 19 starts
if (line.Contains("Processing chromosome 'chr19'"))
{
startedChr19 = true;
}
Assert.False(line.Contains("Processing chromosome 'chr17'") && startedChr19);
}
}
}
private void Write_InFlow(bool threadByChr)
{
var bamFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "SBWriter_Sample_S1.bam");
var vcfFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "SBWriter_Sample_S1.genome.vcf");
var biasFilePath = Path.Combine(UnitTestPaths.TestDataDirectory, "SBWriter_Sample_S1.genome.ReadStrandBias.txt");
if (threadByChr)
{
biasFilePath = biasFilePath + "_chr19"; //Currently when threading by chrom we are outputting one bias file per chromsome. This is not a customer-facing deliverable and is a low-priority feature.
}
var expectedBiasResultsPath = Path.Combine(UnitTestPaths.TestDataDirectory, "Expected_Sample_S1.ReadStrandBias.txt");
var genomeDirectory = Path.Combine(UnitTestPaths.TestGenomesDirectory, "chr19");
var applicationOptions = new ApplicationOptions
{
BAMPaths = new[] { bamFilePath },
IntervalPaths = null,
GenomePaths = new[] { genomeDirectory },
OutputBiasFiles = true,
DebugMode = true,
OutputgVCFFiles = true,
};
// Using GenomeProcessor
//If OutputBiasFiles is true, should output one bias file per vcf
var factory = new MockFactoryWithDefaults(applicationOptions);
var genome = factory.GetReferenceGenome(genomeDirectory);
CreateAndExecuteProcessor(threadByChr, factory, genome);
Assert.True(File.Exists(biasFilePath));
//All variants that are present in VCF where ref!=alt should be included
var biasFileContents = File.ReadAllLines(biasFilePath);
var alleles = VcfReader.GetAllVariantsInFile(vcfFilePath);
var variantCalls = alleles.Where(a => a.VariantAlleles[0] != ".").ToList();
foreach (var variantCall in variantCalls)
{
Console.WriteLine(variantCall);
Assert.True(biasFileContents.Count(l => l.Split('\t')[0] == variantCall.ReferenceName &&
l.Split('\t')[1] == variantCall.ReferencePosition.ToString() &&
l.Split('\t')[2] == variantCall.ReferenceAllele &&
l.Split('\t')[3] == variantCall.VariantAlleles.First()) == 1);
}
foreach (var refCall in alleles.Where(a => a.VariantAlleles[0] == ".").ToList())
{
Assert.False(biasFileContents.Count(l => l.Split('\t')[0] == refCall.ReferenceName &&
l.Split('\t')[1] == refCall.ReferencePosition.ToString() &&
l.Split('\t')[2] == refCall.ReferenceAllele &&
l.Split('\t')[3] == refCall.VariantAlleles.First()) == 1);
}
//Bias files should have expected contents
var expectedBiasFileContents = File.ReadAllLines(expectedBiasResultsPath);
Assert.Equal(expectedBiasFileContents, biasFileContents);
//If OutputBiasFiles is false, should not output any bias files
File.Delete(biasFilePath);
applicationOptions.OutputBiasFiles = false;
factory = new MockFactoryWithDefaults(applicationOptions);
genome = factory.GetReferenceGenome(genomeDirectory);
CreateAndExecuteProcessor(threadByChr, factory, genome);
Assert.False(File.Exists(biasFilePath));
}
