public void GetGeneSection() { var readStream = new BlockGZipStream(ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz")), CompressionMode.Decompress); var indexStream = ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz.jsi")); var outStream = new MemoryStream(); using (var writer = new StreamWriter(outStream, Encoding.UTF8, 512, true)) using (var qp = new QueryProcessor(new StreamReader(readStream), indexStream, writer)) { writer.NewLine = "\r\n"; qp.PrintSection("genes"); } Assert.NotEqual(0, outStream.Length); outStream.Position = 0; using (var reader = new StreamReader(outStream)) { var count = 0; var line = reader.ReadLine(); while (line != null) { count++; line = reader.ReadLine(); } Assert.Equal(127, count); } }
public void EndOfFile() { using (var ms = new MemoryStream()) { var writeBuffer = ByteUtilities.GetRandomBytes(100); var readBuffer = new byte[60]; using (var bgzipStream = new BlockGZipStream(ms, CompressionMode.Compress, true)) { bgzipStream.Write(writeBuffer, 0, writeBuffer.Length); } ms.Position = 0; using (var bgzipStream = new BlockGZipStream(ms, CompressionMode.Decompress)) { int numBytesRead = bgzipStream.Read(readBuffer, 0, 0); Assert.Equal(0, numBytesRead); numBytesRead = bgzipStream.Read(readBuffer, 0, readBuffer.Length); Assert.Equal(readBuffer.Length, numBytesRead); numBytesRead = bgzipStream.Read(readBuffer, 0, readBuffer.Length); Assert.Equal(writeBuffer.Length - readBuffer.Length, numBytesRead); numBytesRead = bgzipStream.Read(readBuffer, 0, readBuffer.Length); Assert.Equal(0, numBytesRead); } } }
public void IndexCreation_multiChromosome() { var jsonStream = new BlockGZipStream(ResourceUtilities.GetReadStream(Resources.TopPath("cosmicv72.indels.json.gz")), CompressionMode.Decompress); var writeStream = new MemoryStream(); using (var indexCreator = new IndexCreator(jsonStream, writeStream)) { indexCreator.CreateIndex(); } JasixIndex readBackIndex; var readStream = new MemoryStream(writeStream.ToArray()); readStream.Seek(0, SeekOrigin.Begin); using (readStream) { readBackIndex = new JasixIndex(readStream); } Assert.Equal(2268, readBackIndex.GetFirstVariantPosition("chr1", 9775924, 9775924)); Assert.Equal(14035925971, readBackIndex.GetFirstVariantPosition("chr2", 16081096, 16081096)); Assert.Equal(433156622693, readBackIndex.GetFirstVariantPosition("chr20", 36026164, 36026164)); Assert.Equal(439602269527, readBackIndex.GetFirstVariantPosition("chrX", 66765044, 66765044)); }
private int ReadGeneLines(Stream jsonStream) { var lineCount = 0; using (var bGzipStream = new BlockGZipStream(jsonStream, CompressionMode.Decompress)) using (var reader = new StreamReader(bGzipStream)) { string line; while ((line = reader.ReadLine()) != null) { if (line == string.Empty) { continue; } if (line == FooterLine) { break; } if (!line.EndsWith(',')) { line += ','; } lineCount++; _geneLines.Add(line); } } return(lineCount); }
public IndexCreator(BlockGZipStream readStream, Stream writeStream) { _reader = new BgzipTextReader(readStream); _writeStream = writeStream; _chromBenchmark = new Benchmark(); _benchmark = new Benchmark(); }
private static (Stream jsonStream, Stream jasixStream) GetNirvanaJsonStreamWithoutGenes(int chromNumber) { var jsonStream = new MemoryStream(); var jasixStream = new MemoryStream(); using (var bgZipStream = new BlockGZipStream(jsonStream, CompressionMode.Compress, true)) using (var writer = new BgzipTextWriter(bgZipStream)) using (var jasixIndex = new JasixIndex()) { writer.Write(NirvanaHeader); writer.Flush(); jasixIndex.BeginSection(JasixCommons.PositionsSectionTag, writer.Position); for (int i = 100 * chromNumber; i < 123 * chromNumber; i++) { writer.WriteLine($"{{\"chromosome\":\"chr{chromNumber}\",\"position\":{i}}},"); if (i % 50 == 0) { writer.Flush(); //creating another block } } writer.WriteLine($"{{\"chromosome\":\"chr{chromNumber}\",\"position\":{100 *chromNumber +25}}}"); writer.Flush(); jasixIndex.EndSection(JasixCommons.PositionsSectionTag, writer.Position); writer.Write(NirvanaFooter); jasixIndex.Write(jasixStream); } jsonStream.Position = 0; jasixStream.Position = 0; return(jsonStream, jasixStream); }
public IndexCreator(BlockGZipStream readStream, Stream writeStream) { _reader = new BgzipTextReader(readStream); _writeStream = writeStream; _processedChromosome = new HashSet <string>(); _chromBenchmark = new Benchmark(); _benchmark = new Benchmark(); }
public void DoubleDispose() { using (var ms = new MemoryStream()) { var bgzipStream = new BlockGZipStream(ms, CompressionMode.Compress); bgzipStream.Dispose(); bgzipStream.Dispose(); } }
public void All_jsons_with_genes() { var jsonStreams = new Stream[3]; var jasixSteams = new Stream[3]; (jsonStreams[0], jasixSteams[0]) = GetJsonStreams(ChromosomeUtilities.Chr1, true); (jsonStreams[1], jasixSteams[1]) = GetJsonStreams(ChromosomeUtilities.Chr2, true); (jsonStreams[2], jasixSteams[2]) = GetJsonStreams(ChromosomeUtilities.Chr3, true); var outStream = new MemoryStream(); using (var stitcher = new JsonStitcher(jsonStreams, jasixSteams, outStream, true)) { stitcher.Stitch(); } outStream.Position = 0; var sb = new StringBuilder(); using (var bgZipStream = new BlockGZipStream(outStream, CompressionMode.Decompress)) using (var reader = new StreamReader(bgZipStream)) { string line; while ((line = reader.ReadLine()) != null) { sb.Append(line + '\n'); } } var fullJson = sb.ToString(); //making sure all the first and last positions are present in the merged JSON Assert.Contains("\"header\":{\"annotator\":\"NirvanaTest\"", fullJson); Assert.Contains("{\"chromosome\":\"chr1\",\"position\":100}", fullJson); Assert.Contains("{\"chromosome\":\"chr1\",\"position\":122}", fullJson); Assert.Contains("{\"chromosome\":\"chr2\",\"position\":200}", fullJson); Assert.Contains("{\"chromosome\":\"chr2\",\"position\":222}", fullJson); Assert.Contains("{\"chromosome\":\"chr3\",\"position\":300}", fullJson); Assert.Contains("{\"chromosome\":\"chr3\",\"position\":322}", fullJson); //checking if all the genes are there Assert.Contains("gene1A", fullJson); Assert.Contains("gene1B", fullJson); Assert.Contains("gene2A", fullJson); Assert.Contains("gene2B", fullJson); Assert.Contains("gene3A", fullJson); Assert.Contains("gene3B", fullJson); //need to check if this is a valid json var jObject = JObject.Parse(fullJson); Assert.NotNull(jObject); }
public void BgzipTextWriter_EndToEnd() { var asterisks = new string('*', BlockGZipStream.BlockGZipFormatCommon.BlockSize); var observedLines = new List <string>(); var observedPositions = new List <long>(); using (var ms = new MemoryStream()) { using (var stream = new BlockGZipStream(ms, CompressionMode.Compress, true)) using (var writer = new BgzipTextWriter(stream)) { writer.Flush(); writer.WriteLine("BOB"); writer.WriteLine(); writer.Flush(); writer.Write("AB"); writer.Write(""); writer.Write("C"); writer.Write(" "); writer.WriteLine("123"); writer.WriteLine(asterisks); writer.WriteLine(asterisks); writer.WriteLine(asterisks); } ms.Position = 0; using (var reader = new BgzipTextReader(new BlockGZipStream(ms, CompressionMode.Decompress))) { while (true) { string line = reader.ReadLine(); observedPositions.Add(reader.Position); if (line == null) { break; } observedLines.Add(line); } } } Assert.Equal(6, observedLines.Count); Assert.Equal("BOB", observedLines[0]); Assert.Equal(0, observedLines[1].Length); Assert.Equal("ABC 123", observedLines[2]); Assert.Equal(asterisks, observedLines[3]); Assert.Equal(4, observedPositions[0]); Assert.Equal(5, observedPositions[1]); Assert.Equal(13, observedPositions[2]); Assert.Equal(43843598, observedPositions[3]); Assert.Equal(87097359, observedPositions[4]); }
private static void WriteGenes(ILogger logger, UgaGene[] genes) { logger.Write($"- writing genes to {Path.GetFileName(ExternalFiles.UniversalGeneFilePath)}... "); using (var stream = new BlockGZipStream(FileUtilities.GetCreateStream(ExternalFiles.UniversalGeneFilePath), CompressionMode.Compress)) using (var writer = new UgaGeneWriter(stream)) { writer.Write(genes); } logger.WriteLine("finished"); }
public static byte[] GetCompressedBlock(string s, int compressionLevel = 1) { using (var stream = new MemoryStream()) { using (var memStream = new BlockGZipStream(stream, CompressionMode.Compress, true)) using (var writer = new StreamWriter(memStream)) { writer.Write(s); } return(stream.ToArray()); } }
public void FileIO() { var observedDecompressedBuffer = new byte[_expectedDecompressedBuffer.Length]; string randomPath = RandomPath.GetRandomPath(); // compress the data long observedPosition; using (var writer = new BlockGZipStream(FileUtilities.GetCreateStream(randomPath), CompressionMode.Compress)) { writer.Write(_expectedDecompressedBuffer, 0, _expectedDecompressedBuffer.Length); observedPosition = writer.Position; var exception = Record.Exception(() => { var buffer = new byte[10]; // ReSharper disable once AccessToDisposedClosure writer.Read(buffer, 0, 1); }); Assert.NotNull(exception); Assert.IsType <CompressionException>(exception); } const long expectedPosition = 979042574; Assert.Equal(expectedPosition, observedPosition); // decompress the data using (var reader = new BlockGZipStream(FileUtilities.GetReadStream(randomPath), CompressionMode.Decompress)) { reader.Read(observedDecompressedBuffer, 0, _expectedDecompressedBuffer.Length); var exception = Record.Exception(() => { var buffer = new byte[10]; // ReSharper disable once AccessToDisposedClosure reader.Write(buffer, 0, 1); }); Assert.NotNull(exception); Assert.IsType <CompressionException>(exception); } Assert.Equal(_expectedDecompressedBuffer, observedDecompressedBuffer); }
public void StreamIO() { byte[] observedCompressedBuffer; var observedDecompressedBuffer = new byte[_expectedDecompressedBuffer.Length]; using (var ms = new MemoryStream()) { // compress the data using (var writer = new BlockGZipStream(ms, CompressionMode.Compress, true, 9)) { Assert.Throws <CompressionException>(delegate { // ReSharper disable once AccessToDisposedClosure writer.Read(observedDecompressedBuffer, 0, 1); }); Assert.True(writer.CanWrite); Assert.False(writer.CanRead); Assert.False(writer.CanSeek); writer.Write(_expectedDecompressedBuffer, 0, _expectedDecompressedBuffer.Length); } observedCompressedBuffer = ms.ToArray(); ms.Seek(0, SeekOrigin.Begin); // decompress the data using (var reader = new BlockGZipStream(ms, CompressionMode.Decompress)) { Assert.Throws <CompressionException>(delegate { // ReSharper disable once AccessToDisposedClosure reader.Write(_expectedDecompressedBuffer, 0, 1); }); Assert.False(reader.CanWrite); Assert.True(reader.CanRead); Assert.True(reader.CanSeek); reader.Read(observedDecompressedBuffer, 0, _expectedDecompressedBuffer.Length); } } Assert.Equal(_expectedDecompressedBuffer, observedDecompressedBuffer); Assert.Equal(9629, observedCompressedBuffer.Length); }
public void WritingTask() { using (FileStream stream = new FileStream(options.ResultFileName, FileMode.Create)) { using (BlockGZipStream compressingStream = new BlockGZipStream(stream, BlockGZipStreamMode.Compressing)) { while (true) { var block = writingQueue.Dequeue(); if (block == stopWorkingTask) { break; } compressingStream.WriteBlock(block); } } } }
private void ReadingTask() { using (Stream fileStream = new FileStream(options.SourceFileName, FileMode.Open, FileAccess.Read)) { using (BlockGZipStream gzipStream = new BlockGZipStream(fileStream, BlockGZipStreamMode.Decompressing)) { for (int i = 0; i < gzipStream.Lenght; i++) { readingQueue.Enqueue(gzipStream.ReadBlock(i)); } } } for (int i = 0; i < options.ThreadsCount; i++) { readingQueue.Enqueue(stopWorkingTask); } }
public void NotImplementedMethods() { using (var ms = new MemoryStream()) { // ReSharper disable AccessToDisposedClosure using (var writer = new BlockGZipStream(ms, CompressionMode.Compress, true)) { Assert.Throws <NotSupportedException>(delegate { // ReSharper disable once UnusedVariable long len = writer.Length; }); Assert.Throws <NotSupportedException>(delegate { writer.SetLength(10); }); Assert.Throws <NotSupportedException>(delegate { writer.Seek(0, SeekOrigin.Begin); }); } // ReSharper restore AccessToDisposedClosure } }
private static AnnotationPosition[] AdjustPartitionGenomicStarts(IReadOnlyList <long> blockBasedOffsets, string vcfUrl, IIntervalForest <IGene> geneIntervalForest, IDictionary <string, IChromosome> refNameToChromosome) { var allAdjustedStarts = new AnnotationPosition[blockBasedOffsets.Count]; for (var index = 0; index < blockBasedOffsets.Count; index++) { long blockBasedOffset = blockBasedOffsets[index]; using (var stream = PersistentStreamUtils.GetReadStream(vcfUrl, blockBasedOffset)) using (var gzipStream = new BlockGZipStream(stream, CompressionMode.Decompress)) { var annotationPosition = GetFirstGenomicPosition(gzipStream, index == 0); allAdjustedStarts[index] = FindProperStartPosition(annotationPosition, geneIntervalForest, refNameToChromosome); } } AnnotationPosition[] adjustedStarts = MergeConsecutiveEqualValues(allAdjustedStarts).ToArray(); return(adjustedStarts); }
public void ReadBlockCorrupted() { using (var ms = new MemoryStream()) using (var truncatedMs = new MemoryStream()) using (var corruptMs = new MemoryStream()) { using (var bgzipStream = new BlockGZipStream(ms, CompressionMode.Compress, true)) using (var writer = new StreamWriter(bgzipStream, Encoding.ASCII, 4096)) { writer.WriteLine("The quick brown fox jumped over the lazy dog."); } var compressedData = ms.ToArray(); truncatedMs.Write(compressedData, 0, compressedData.Length - 10); truncatedMs.Position = 0; corruptMs.Write(compressedData, 0, BlockGZipStream.BlockGZipFormatCommon.BlockHeaderLength); var randomBytes = BlockStreamTests.GetRandomBytes(500); corruptMs.Write(randomBytes, 0, randomBytes.Length); corruptMs.Position = 0; var readBuffer = new byte[60]; Assert.Throws <CompressionException>(delegate { using (var bgzipStream = new BlockGZipStream(truncatedMs, CompressionMode.Decompress)) { bgzipStream.Read(readBuffer, 0, readBuffer.Length); } }); Assert.Throws <CompressionException>(delegate { using (var bgzipStream = new BlockGZipStream(corruptMs, CompressionMode.Decompress)) { bgzipStream.Read(readBuffer, 0, readBuffer.Length); } }); } }
public void GetHeaderOnly() { var readStream = new BlockGZipStream(ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz")), CompressionMode.Decompress); var indexStream = ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz.jsi")); var outStream = new MemoryStream(); using (var writer = new StreamWriter(outStream, Encoding.UTF8, 512, true)) using (var qp = new QueryProcessor(new StreamReader(readStream), indexStream, writer)) { qp.PrintHeaderOnly(); } Assert.NotEqual(0, outStream.Length); outStream.Position = 0; using (var reader = new StreamReader(outStream)) { string headerLine = reader.ReadToEnd(); Assert.Equal("{\"header\":{\"annotator\":\"Nirvana 2.0.9.0\",\"creationTime\":\"2018-04-30 17:17:23\",\"genomeAssembly\":\"GRCh37\",\"schemaVersion\":6,\"dataVersion\":\"91.26.45\",\"dataSources\":[{\"name\":\"VEP\",\"version\":\"91\",\"description\":\"Ensembl\",\"releaseDate\":\"2018-03-05\"},{\"name\":\"ClinVar\",\"version\":\"20180129\",\"description\":\"A freely accessible, public archive of reports of the relationships among human variations and phenotypes, with supporting evidence\",\"releaseDate\":\"2018-01-29\"},{\"name\":\"COSMIC\",\"version\":\"84\",\"description\":\"somatic mutation and related details and information relating to human cancers\",\"releaseDate\":\"2018-02-13\"},{\"name\":\"dbSNP\",\"version\":\"150\",\"description\":\"Identifiers for observed variants\",\"releaseDate\":\"2017-04-03\"},{\"name\":\"gnomAD_exome\",\"version\":\"2.0.2\",\"description\":\"Exome allele frequencies from Genome Aggregation Database (gnomAD)\",\"releaseDate\":\"2017-10-05\"},{\"name\":\"gnomAD\",\"version\":\"2.0.2\",\"description\":\"Whole genome allele frequencies from Genome Aggregation Database (gnomAD)\",\"releaseDate\":\"2017-10-05\"},{\"name\":\"MITOMAP\",\"version\":\"20180228\",\"description\":\"Small variants in the MITOMAP human mitochondrial genome database\",\"releaseDate\":\"2018-02-28\"},{\"name\":\"1000 Genomes Project\",\"version\":\"Phase 3 v5a\",\"description\":\"A public catalogue of human variation and genotype data\",\"releaseDate\":\"2013-05-27\"},{\"name\":\"TOPMed\",\"version\":\"freeze_5\",\"description\":\"Allele frequencies from TOPMed data lifted over using dbSNP ids.\",\"releaseDate\":\"2017-08-28\"},{\"name\":\"ClinGen\",\"version\":\"20160414\",\"releaseDate\":\"2016-04-14\"},{\"name\":\"DGV\",\"version\":\"20160515\",\"description\":\"Provides a comprehensive summary of structural variation in the human genome\",\"releaseDate\":\"2016-05-15\"},{\"name\":\"MITOMAP\",\"version\":\"20180228\",\"description\":\"Large structural variants in the MITOMAP human mitochondrial genome database\",\"releaseDate\":\"2018-02-28\"},{\"name\":\"ExAC\",\"version\":\"0.3.1\",\"description\":\"Gene scores from the ExAC project\",\"releaseDate\":\"2016-03-16\"},{\"name\":\"OMIM\",\"version\":\"20180213\",\"description\":\"An Online Catalog of Human Genes and Genetic Disorders\",\"releaseDate\":\"2018-02-13\"},{\"name\":\"phyloP\",\"version\":\"hg19\",\"description\":\"46 way conservation score between humans and 45 other vertebrates\",\"releaseDate\":\"2009-11-10\"}]}}\r\n", headerLine); } }
public void GetChromosomeList() { var readStream = new BlockGZipStream(ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz")), CompressionMode.Decompress); var indexStream = ResourceUtilities.GetReadStream(Resources.TopPath("Clinvar20150901.json.gz.jsi")); var outStream = new MemoryStream(); using (var writer = new StreamWriter(outStream, Encoding.UTF8, 512, true)) using (var qp = new QueryProcessor(new StreamReader(readStream), indexStream, writer)) { qp.PrintChromosomeList(); } Assert.NotEqual(0, outStream.Length); outStream.Position = 0; using (var reader = new StreamReader(outStream)) { string chromList = reader.ReadToEnd(); Assert.Equal("1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9\r\n10\r\n11\r\n12\r\n13\r\n14\r\n15\r\n16\r\n17\r\n18\r\n19\r\n20\r\n21\r\nX\r\nY\r\n", chromList); } }
private static Stream GetAppropriateStream(Stream stream, CompressionAlgorithm compressionAlgorithm) { Stream newStream; switch (compressionAlgorithm) { case CompressionAlgorithm.BlockGZip: newStream = new BlockGZipStream(stream, CompressionMode.Decompress); break; case CompressionAlgorithm.GZip: newStream = new GZipStream(stream, CompressionMode.Decompress); break; default: newStream = stream; break; } return(newStream); }
private void WriteGeneBlocks(Stream stream) { using (var bGzipStream = new BlockGZipStream(stream, CompressionMode.Compress, _leaveOutStreamOpen)) using (var writer = new StreamWriter(bGzipStream)) { var count = _geneLines.Count; if (count == 0) { writer.WriteLine(FooterLine); return; } writer.WriteLine(GeneHeaderLine); var i = 0; foreach (var geneLine in _geneLines.OrderBy(x => x)) { i++; //the last gene line shouldn't have a comma at the end writer.WriteLine(i == count ? geneLine.TrimEnd(',') : geneLine); } writer.WriteLine(FooterLine); } }
private static Stream GetAppropriateStream(Stream stream, CompressionAlgorithm compressionAlgorithm) { Stream newStream; // ReSharper disable once SwitchStatementMissingSomeCases switch (compressionAlgorithm) { case CompressionAlgorithm.BlockGZip: newStream = new BlockGZipStream(stream, CompressionMode.Decompress); break; case CompressionAlgorithm.GZip: newStream = new GZipStream(stream, CompressionMode.Decompress); break; default: newStream = stream; break; } return(newStream); }
private void ConfigureResourceUtilizationOptions() { availableRamBytes = availableRamBytes / 2; if (CommandName == "compress") { ReadingBlockSizeBytes = availableRamBytes / ThreadsCount; int maxBlockSizeBytes = 1 * 1024 * 1024; ReadingBlockSizeBytes = ReadingBlockSizeBytes > maxBlockSizeBytes ? maxBlockSizeBytes : ReadingBlockSizeBytes; } else { using (Stream fileStream = new FileStream(SourceFileName, FileMode.Open, FileAccess.Read)) { using (BlockGZipStream gzipStream = new BlockGZipStream(fileStream, BlockGZipStreamMode.Decompressing)) { ReadingBlockSizeBytes = gzipStream.GetMaxStoredBlockSizeBytes(); } } } if (availableRamBytes < ReadingBlockSizeBytes) { BadOption("source-filename", "Sorry. You don't have enough ram."); } if (availableRamBytes / ReadingBlockSizeBytes < ThreadsCount) { ThreadsCount = availableRamBytes / ReadingBlockSizeBytes; } var taskByThread = availableRamBytes / ReadingBlockSizeBytes / ThreadsCount; if (taskByThread > TasksByTreadByDefault) { taskByThread = TasksByTreadByDefault; } TaskCount = taskByThread * ThreadsCount; }
public void TestIndexCreation() { var readStream = new BlockGZipStream(ResourceUtilities.GetReadStream(Resources.TopPath("cosmicv72.indels.json.gz")), CompressionMode.Decompress); var tempFile = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); using (var indexCreator = new IndexCreator(readStream, FileUtilities.GetCreateStream(tempFile))) { indexCreator.CreateIndex(); } JasixIndex readBackIndex; using (var stream = FileUtilities.GetReadStream(tempFile)) { readBackIndex = new JasixIndex(stream); } Assert.Equal(1591, readBackIndex.GetFirstVariantPosition("chr1", 9775924, 9775924)); Assert.Equal(11500956299, readBackIndex.GetFirstVariantPosition("chr2", 16081096, 16081096)); Assert.Equal(372100991296, readBackIndex.GetFirstVariantPosition("chr20", 36026164, 36026164)); Assert.Equal(377682846863, readBackIndex.GetFirstVariantPosition("chrX", 66765044, 66765044)); File.Delete(tempFile); }
public void InvalidHeader() { const string dummyString = "The quick brown fox jumped over the lazy dog."; using (var ms = new MemoryStream()) using (var truncatedMs = new MemoryStream()) { using (var writer = new StreamWriter(ms, Encoding.ASCII, 4096, true)) { writer.WriteLine(dummyString); } var observedCompressedBuffer = ms.ToArray(); truncatedMs.Write(ms.ToArray(), 0, 17); ms.Seek(0, SeekOrigin.Begin); truncatedMs.Seek(0, SeekOrigin.Begin); // attempt to decompress the data Assert.Throws <CompressionException>(delegate { using (var reader = new BlockGZipStream(ms, CompressionMode.Decompress, true)) { reader.Read(observedCompressedBuffer, 0, observedCompressedBuffer.Length); } }); Assert.Throws <CompressionException>(delegate { using (var reader = new BlockGZipStream(truncatedMs, CompressionMode.Decompress, true)) { reader.Read(observedCompressedBuffer, 0, observedCompressedBuffer.Length); } }); } }
// ReSharper disable once UnusedMember.Global public AnnotationResult Run(AnnotationConfig config, ILambdaContext context) { var result = new AnnotationResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); string vcfUrl = config.vcfUrl; using (var annotationResources = GetAnnotationResources(config)) { if (annotationResources.InputStartVirtualPosition == -1) { return(GetSuccessOutput(result)); } long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset; using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset)) { annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); } Logger.WriteLine("Scan for positions to preload complete."); using (var aes = new AesCryptoServiceProvider()) { FileMetadata jsonMetadata, jasixMetadata; string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix; string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix; using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress)) using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress)) // using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath)) using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream)) // using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath)) using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream)) { IVcfFilter vcfFilter = config.annotationRange == null ? new NullVcfFilter() as IVcfFilter : new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress)) { StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true); } Logger.WriteLine("Annotation done."); jsonMetadata = jsonMd5Stream.GetFileMetadata(); jasixMetadata = jasixMd5Stream.GetFileMetadata(); } result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix); string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix; var s3Client = config.outputDir.GetS3Client(context.RemainingTime); s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata); Logger.WriteLine("Nirvana result files uploaded."); } } LambdaUtilities.DeleteTempOutput(); if (string.IsNullOrEmpty(result.filePath)) { throw new FileNotFoundException(); } return(GetSuccessOutput(result)); } catch (Exception exception) { LambdaUtilities.DeleteTempOutput(); return(HandleException(runLog, result, exception, snsTopicArn)); } }