public ValidationResult Run(ValidationConfig config, ILambdaContext context) { string snsTopicArn = null; try { LogUtilities.UpdateLogger(context.Logger, null); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); config.Validate(); GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly); string nirvanaS3Ref = LambdaUrlHelper.GetRefUrl(genomeAssembly); var refProvider = ProviderUtilities.GetSequenceProvider(nirvanaS3Ref); using (var stream = PersistentStreamUtils.GetReadStream(config.customStrUrl)) TryLoadStrFile(stream, genomeAssembly, refProvider); } catch (Exception exception) { return(HandleException(config.id, exception, snsTopicArn)); } return(GetSuccessOutput(config.id)); }
// ReSharper disable once UnusedMember.Global public Stream Run(GeneConfig config, ILambdaContext context) { string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); config.Validate(); string result = GetGeneAnnotation(config, _saManifestUrl, _saPathPrefix); return(LambdaResponse.Create(config.id, LambdaUrlHelper.SuccessMessage, result)); } catch (Exception e) { return(HandleException(config.id, snsTopicArn, e)); } }
public static Stream Create(string id, string status, string nirvanaJson) { string statusJson = JsonConvert.SerializeObject(status); string outputStart = $"{{\"id\":\"{id}\",\"status\":{statusJson}"; string output; if (status == LambdaUrlHelper.SuccessMessage) { if (nirvanaJson == null) { throw new NoNullAllowedException("Nirvana annotation cannot be null when the job is successful."); } output = outputStart + OutputBeforeNirvanaJson + nirvanaJson + OutputEnd; } else { output = outputStart + OutputEnd; } LogUtilities.LogObject("Result", output); var outputStream = new MemoryStream(Encoding.UTF8.GetBytes(output)); return(outputStream); }
public static Stream GetStream(string id, string snsTopicArn, Exception e) { Logger.Log(e); GC.Collect(); string snsMessage = SNS.CreateMessage(e.Message, "exception", e.StackTrace); SNS.SendMessage(snsTopicArn, snsMessage); ErrorCategory errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e); string message = GetMessage(errorCategory, e.Message); LogUtilities.LogObject("Result", message); return(SingleResult.Create(id, message, null)); }
private static AnnotationResult HandleException(StringBuilder runLog, AnnotationResult result, Exception e, string snsTopicArn) { Logger.Log(e); result.status = e.Message; result.errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e); Logger.WriteLine($"Error Category: {result.errorCategory}"); if (result.errorCategory != ErrorCategory.UserError) { string snsMessage = SNS.CreateMessage(runLog.ToString(), result.status, e.StackTrace); SNS.SendMessage(snsTopicArn, snsMessage); } LogUtilities.LogObject("Result", result); return(result); }
// ReSharper disable once UnusedMember.Global public Stream Run(SingleConfig config, ILambdaContext context) { string snsTopicArn = null; Stream response; try { LogUtilities.UpdateLogger(context.Logger, null); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); config.Validate(); GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly); var cacheConfiguration = new CacheConfiguration(genomeAssembly, config.supplementaryAnnotations, config.vepVersion); bool preloadRequired = !string.IsNullOrEmpty(config.supplementaryAnnotations); AnnotationResources annotationResources = GetAndCacheAnnotationResources(config, cacheConfiguration); if (genomeAssembly != GenomeAssembly.hg19) { annotationResources.Annotator.EnableMitochondrialAnnotation(); } (IPosition position, string[] sampleNames) = config.GetPositionAndSampleNames(annotationResources.SequenceProvider, annotationResources.RefMinorProvider); if (position.Chromosome.IsEmpty()) { throw new UserErrorException($"An unknown chromosome was specified ({config.variant.chromosome})"); } string annotationResult = GetPositionAnnotation(position, annotationResources, sampleNames, preloadRequired); response = SingleResult.Create(config.id, LambdaUrlHelper.SuccessMessage, annotationResult); } catch (Exception exception) { response = ExceptionHandler.GetStream(config.id, snsTopicArn, exception); } return(response); }
public static CustomResult GetSuccessResult(CustomConfig customSaConfig, CustomResult result, List <string> outputFiles) { Logger.WriteLine("All files uploaded."); result.created = new FileList { bucketName = customSaConfig.outputDir.bucketName, outputDir = customSaConfig.outputDir.path, files = outputFiles.ToArray() }; result.status = LambdaUtilities.SuccessMessage; LogUtilities.LogObject("Result", result); LambdaUtilities.DeleteTempOutput(); return(result); }
private static CustomResult HandleException(StringBuilder runLog, CustomResult result, Exception e, string snsTopicArn) { Logger.Log(e); var errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e); result.status = $"{errorCategory}: {e.Message}"; if (errorCategory != ErrorCategory.UserError) { string snsMessage = SNS.CreateMessage(runLog.ToString(), result.status, e.StackTrace); SNS.SendMessage(snsTopicArn, snsMessage); } LogUtilities.LogObject("Result", result); LambdaUtilities.DeleteTempOutput(); return(result); }
// ReSharper disable once UnusedMember.Global public NirvanaResult Run(NirvanaConfig config, ILambdaContext context) { NirvanaResult result; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey, "annotation_lambda_arn" }); LambdaUtilities.GarbageCollect(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); string annotationLambdaArn = LambdaUtilities.GetEnvironmentVariable(AnnotationLambdaKey); config.Validate(); var genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly); if (!_supportedAssemblies.Contains(genomeAssembly)) { throw new UserErrorException($"Unsupported assembly: {config.genomeAssembly}"); } IEnumerable <AnnotationRange> annotationRanges = GetAnnotationRanges(config, genomeAssembly); result = GetNirvanaResult(annotationRanges, config, annotationLambdaArn, context, runLog, snsTopicArn); } catch (Exception exception) { result = HandleException(runLog, config, exception, snsTopicArn); } LogUtilities.LogObject("Result", result); return(result); }
// ReSharper disable once UnusedMember.Global public CustomResult Run(CustomConfig config, ILambdaContext context) { var result = new CustomResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); config.CheckRequiredFieldsNotNull(); var s3Client = config.outputDir.GetS3Client(context.RemainingTime); config.CheckResourcesExist(); LambdaUtilities.DeleteTempOutput(); string inputFileName = config.tsvUrl.TrimEndFromFirst("?").TrimStartToLast("/"); Logger.WriteLine($"input file name is: {inputFileName}"); return(IsGeneAnnotationTsv(config.tsvUrl) ? GeneAnnotationCreator.Create(config, inputFileName, result, s3Client) : VariantAnnotationCreator.Create(config, inputFileName, result, s3Client)); } catch (Exception e) { return(HandleException(runLog, result, e, snsTopicArn)); } }
// ReSharper disable once UnusedMember.Global public AnnotationResult Run(AnnotationConfig config, ILambdaContext context) { var result = new AnnotationResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); string vcfUrl = config.vcfUrl; using (var annotationResources = GetAnnotationResources(config)) { if (annotationResources.InputStartVirtualPosition == -1) { return(GetSuccessOutput(result)); } long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset; using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset)) { annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); } Logger.WriteLine("Scan for positions to preload complete."); using (var aes = new AesCryptoServiceProvider()) { FileMetadata jsonMetadata, jasixMetadata; string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix; string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix; using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress)) using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress)) // using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath)) using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream)) // using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath)) using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream)) { IVcfFilter vcfFilter = config.annotationRange == null ? new NullVcfFilter() as IVcfFilter : new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress)) { StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true); } Logger.WriteLine("Annotation done."); jsonMetadata = jsonMd5Stream.GetFileMetadata(); jasixMetadata = jasixMd5Stream.GetFileMetadata(); } result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix); string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix; var s3Client = config.outputDir.GetS3Client(context.RemainingTime); s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata); Logger.WriteLine("Nirvana result files uploaded."); } } LambdaUtilities.DeleteTempOutput(); if (string.IsNullOrEmpty(result.filePath)) { throw new FileNotFoundException(); } return(GetSuccessOutput(result)); } catch (Exception exception) { LambdaUtilities.DeleteTempOutput(); return(HandleException(runLog, result, exception, snsTopicArn)); } }
private static AnnotationResult GetSuccessOutput(AnnotationResult result) { result.status = LambdaUtilities.SuccessMessage; LogUtilities.LogObject("Result", result); return(result); }