public static CustomResult GetSuccessResult(CustomConfig customSaConfig, CustomResult result, List <string> outputFiles) { Logger.WriteLine("All files uploaded."); result.created = new FileList { bucketName = customSaConfig.outputDir.bucketName, outputDir = customSaConfig.outputDir.path, files = outputFiles.ToArray() }; result.status = LambdaUtilities.SuccessMessage; LogUtilities.LogObject("Result", result); LambdaUtilities.DeleteTempOutput(); return(result); }
private static CustomResult HandleException(StringBuilder runLog, CustomResult result, Exception e, string snsTopicArn) { Logger.Log(e); var errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e); result.status = $"{errorCategory}: {e.Message}"; if (errorCategory != ErrorCategory.UserError) { string snsMessage = SNS.CreateMessage(runLog.ToString(), result.status, e.StackTrace); SNS.SendMessage(snsTopicArn, snsMessage); } LogUtilities.LogObject("Result", result); LambdaUtilities.DeleteTempOutput(); return(result); }
// ReSharper disable once UnusedMember.Global public CustomResult Run(CustomConfig config, ILambdaContext context) { var result = new CustomResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); config.CheckRequiredFieldsNotNull(); var s3Client = config.outputDir.GetS3Client(context.RemainingTime); config.CheckResourcesExist(); LambdaUtilities.DeleteTempOutput(); string inputFileName = config.tsvUrl.TrimEndFromFirst("?").TrimStartToLast("/"); Logger.WriteLine($"input file name is: {inputFileName}"); return(IsGeneAnnotationTsv(config.tsvUrl) ? GeneAnnotationCreator.Create(config, inputFileName, result, s3Client) : VariantAnnotationCreator.Create(config, inputFileName, result, s3Client)); } catch (Exception e) { return(HandleException(runLog, result, e, snsTopicArn)); } }
// ReSharper disable once UnusedMember.Global public AnnotationResult Run(AnnotationConfig config, ILambdaContext context) { var result = new AnnotationResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); string vcfUrl = config.vcfUrl; using (var annotationResources = GetAnnotationResources(config)) { if (annotationResources.InputStartVirtualPosition == -1) { return(GetSuccessOutput(result)); } long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset; using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset)) { annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); } Logger.WriteLine("Scan for positions to preload complete."); using (var aes = new AesCryptoServiceProvider()) { FileMetadata jsonMetadata, jasixMetadata; string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix; string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix; using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress)) using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress)) // using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath)) using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream)) // using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath)) using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream)) { IVcfFilter vcfFilter = config.annotationRange == null ? new NullVcfFilter() as IVcfFilter : new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress)) { StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true); } Logger.WriteLine("Annotation done."); jsonMetadata = jsonMd5Stream.GetFileMetadata(); jasixMetadata = jasixMd5Stream.GetFileMetadata(); } result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix); string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix; var s3Client = config.outputDir.GetS3Client(context.RemainingTime); s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata); Logger.WriteLine("Nirvana result files uploaded."); } } LambdaUtilities.DeleteTempOutput(); if (string.IsNullOrEmpty(result.filePath)) { throw new FileNotFoundException(); } return(GetSuccessOutput(result)); } catch (Exception exception) { LambdaUtilities.DeleteTempOutput(); return(HandleException(runLog, result, exception, snsTopicArn)); } }
public static CustomResult Create(CustomConfig config, string inputFileName, CustomResult result, IS3Client s3Client) { string tempPath = Path.GetTempPath(); string inputBaseName = inputFileName.TrimEndFromFirst(".tsv"); string nsaFileName = inputBaseName + SaCommon.SaFileSuffix; string localNsaPath = Path.Combine(tempPath, nsaFileName); string localIndexPath = localNsaPath + SaCommon.IndexSufix; string localSchemaPath = localNsaPath + SaCommon.JsonSchemaSuffix; var outputFiles = new List <string>(); using (var aes = new AesCryptoServiceProvider()) { FileMetadata nsaMetadata, indexMetadata, schemaMetadata; List <CustomInterval> intervals; string jsonTag; SaJsonSchema intervalJsonSchema; DataSourceVersion version; GenomeAssembly genomeAssembly; int nsaItemsCount; ReportFor reportFor; using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl)) using (var parser = GetVariantAnnotationsParserFromCustomTsvStream(customTsvStream)) // using (var nsaStream = FileUtilities.GetCreateStream(localNsaPath)) using (var nsaCryptoStream = new CryptoStream(nsaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsaMd5Stream = new MD5Stream(nsaCryptoStream)) // using (var indexStream = FileUtilities.GetCreateStream(localIndexPath)) using (var indexCryptoStream = new CryptoStream(indexStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var indexMd5Stream = new MD5Stream(indexCryptoStream)) // using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath)) using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream)) { genomeAssembly = parser.Assembly; result.genomeAssembly = genomeAssembly.ToString(); reportFor = parser.ReportFor; using (var nsaWriter = CaUtilities.GetNsaWriter(nsaMd5Stream, indexMd5Stream, parser, inputFileName, parser.SequenceProvider, out version)) using (var schemaWriter = new StreamWriter(schemaMd5Stream)) { (jsonTag, nsaItemsCount, intervalJsonSchema, intervals) = CaUtilities.WriteSmallVariants(parser, nsaWriter, schemaWriter); } nsaMetadata = nsaMd5Stream.GetFileMetadata(); indexMetadata = indexMd5Stream.GetFileMetadata(); schemaMetadata = schemaMd5Stream.GetFileMetadata(); } if (nsaItemsCount > 0) { string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsaFileName); string indexS3Path = nsaS3Path + SaCommon.IndexSufix; string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNsaPath, aes, nsaMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, indexS3Path, localIndexPath, aes, indexMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata); outputFiles.Add(nsaFileName); outputFiles.Add(nsaFileName + SaCommon.IndexSufix); outputFiles.Add(nsaFileName + SaCommon.JsonSchemaSuffix); } if (intervals == null) { return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); } FileMetadata nsiMetadata, nsiSchemaMetadata; string nsiFileName = inputBaseName + SaCommon.SiFileSuffix; string localNsiPath = Path.Combine(tempPath, nsiFileName); string localNsiSchemaPath = localNsiPath + SaCommon.JsonSchemaSuffix; // using (var nsiStream = FileUtilities.GetCreateStream(localNsiPath)) using (var nsiCryptoStream = new CryptoStream(nsiStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsiMd5Stream = new MD5Stream(nsiCryptoStream)) // using (var nsiSchemaSteam = FileUtilities.GetCreateStream(localNsiSchemaPath)) using (var nsiSchemaCryptoStream = new CryptoStream(nsiSchemaSteam, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsiSchemaMd5Stream = new MD5Stream(nsiSchemaCryptoStream)) { using (var nsiWriter = CaUtilities.GetNsiWriter(nsiMd5Stream, version, genomeAssembly, jsonTag, reportFor)) using (var schemaWriter = new StreamWriter(nsiSchemaMd5Stream)) { nsiWriter.Write(intervals); schemaWriter.Write(intervalJsonSchema); } nsiMetadata = nsiMd5Stream.GetFileMetadata(); nsiSchemaMetadata = nsiSchemaMd5Stream.GetFileMetadata(); } string nsiS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsiFileName); string nsiSchemaS3PathFile = nsiS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsiS3Path, localNsiPath, aes, nsiMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, nsiSchemaS3PathFile, localNsiSchemaPath, aes, nsiSchemaMetadata); outputFiles.Add(nsiFileName); outputFiles.Add(nsiFileName + SaCommon.JsonSchemaSuffix); } LambdaUtilities.DeleteTempOutput(); return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); }
public static CustomResult Create(CustomConfig config, string inputBaseName, CustomResult result, IS3Client s3Client) { string ngaFileName = inputBaseName + SaCommon.NgaFileSuffix; string localNgaPath = Path.Combine(Path.GetTempPath(), ngaFileName); string localSchemaPath = localNgaPath + SaCommon.JsonSchemaSuffix; string localLogPath = Path.Combine(Path.GetTempPath(), LogFileName); HttpUtilities.ValidateUrl(LambdaUrlHelper.GetUgaUrl()); var outputFiles = new List <string>(); using (var aes = new AesCryptoServiceProvider()) { FileMetadata ngaMetadata, schemaMetadata, logMetaData; using (var logStream = FileUtilities.GetCreateStream(localLogPath)) using (var logCryptoStream = new CryptoStream(logStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var logMd5Stream = new MD5Stream(logCryptoStream)) // using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl)) using (var parser = GetGeneAnnotationsParserFromCustomTsvStream(customTsvStream)) // using (var ngaStream = FileUtilities.GetCreateStream(localNgaPath)) using (var ngaCryptoStream = new CryptoStream(ngaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var ngaMd5Stream = new MD5Stream(ngaCryptoStream)) // using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath)) using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream)) { using (var ngaWriter = CaUtilities.GetNgaWriter(ngaMd5Stream, parser, config.tsvUrl)) using (var schemaWriter = new StreamWriter(schemaMd5Stream)) using (var logWriter = new StreamWriter(logMd5Stream)) { ngaWriter.Write(parser.GetItems(config.skipGeneIdValidation, logWriter)); schemaWriter.Write(parser.JsonSchema); } //all the writers have to be disposed before GetFileMetaData is called ngaMetadata = ngaMd5Stream.GetFileMetadata(); schemaMetadata = schemaMd5Stream.GetFileMetadata(); logMetaData = logMd5Stream.GetFileMetadata(); } if (config.skipGeneIdValidation) { string logS3Key = string.Join('/', config.outputDir.path.Trim('/'), LogFileName); Logger.WriteLine("uploading log file to " + logS3Key); s3Client.DecryptUpload(config.outputDir.bucketName, logS3Key, localLogPath, aes, logMetaData); } string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), ngaFileName); string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNgaPath, aes, ngaMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata); outputFiles.Add(ngaFileName); outputFiles.Add(ngaFileName + SaCommon.JsonSchemaSuffix); LambdaUtilities.DeleteTempOutput(); return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); } }