public ValidationResult Run(ValidationConfig config, ILambdaContext context)
{
string snsTopicArn = null;
try
{
LogUtilities.UpdateLogger(context.Logger, null);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
config.Validate();
GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly);
string nirvanaS3Ref = LambdaUrlHelper.GetRefUrl(genomeAssembly);
var refProvider = ProviderUtilities.GetSequenceProvider(nirvanaS3Ref);
using (var stream = PersistentStreamUtils.GetReadStream(config.customStrUrl))
TryLoadStrFile(stream, genomeAssembly, refProvider);
}
catch (Exception exception)
{
return(HandleException(config.id, exception, snsTopicArn));
}
return(GetSuccessOutput(config.id));
}
// ReSharper disable once UnusedMember.Global
public Stream Run(GeneConfig config, ILambdaContext context)
{
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
config.Validate();
string result = GetGeneAnnotation(config, _saManifestUrl, _saPathPrefix);
return(LambdaResponse.Create(config.id, LambdaUrlHelper.SuccessMessage, result));
}
catch (Exception e)
{
return(HandleException(config.id, snsTopicArn, e));
}
}
public static Stream Create(string id, string status, string nirvanaJson)
{
string statusJson = JsonConvert.SerializeObject(status);
string outputStart = $"{{\"id\":\"{id}\",\"status\":{statusJson}";
string output;
if (status == LambdaUrlHelper.SuccessMessage)
{
if (nirvanaJson == null)
{
throw new NoNullAllowedException("Nirvana annotation cannot be null when the job is successful.");
}
output = outputStart + OutputBeforeNirvanaJson + nirvanaJson + OutputEnd;
}
else
{
output = outputStart + OutputEnd;
}
LogUtilities.LogObject("Result", output);
var outputStream = new MemoryStream(Encoding.UTF8.GetBytes(output));
return(outputStream);
}
public static Stream GetStream(string id, string snsTopicArn, Exception e)
{
Logger.Log(e);
GC.Collect();
string snsMessage = SNS.CreateMessage(e.Message, "exception", e.StackTrace);
SNS.SendMessage(snsTopicArn, snsMessage);
ErrorCategory errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e);
string message = GetMessage(errorCategory, e.Message);
LogUtilities.LogObject("Result", message);
return(SingleResult.Create(id, message, null));
}
private static AnnotationResult HandleException(StringBuilder runLog, AnnotationResult result, Exception e, string snsTopicArn)
{
Logger.Log(e);
result.status = e.Message;
result.errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e);
Logger.WriteLine($"Error Category: {result.errorCategory}");
if (result.errorCategory != ErrorCategory.UserError)
{
string snsMessage = SNS.CreateMessage(runLog.ToString(), result.status, e.StackTrace);
SNS.SendMessage(snsTopicArn, snsMessage);
}
LogUtilities.LogObject("Result", result);
return(result);
}
// ReSharper disable once UnusedMember.Global
public Stream Run(SingleConfig config, ILambdaContext context)
{
string snsTopicArn = null;
Stream response;
try
{
LogUtilities.UpdateLogger(context.Logger, null);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
config.Validate();
GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly);
var cacheConfiguration = new CacheConfiguration(genomeAssembly, config.supplementaryAnnotations, config.vepVersion);
bool preloadRequired = !string.IsNullOrEmpty(config.supplementaryAnnotations);
AnnotationResources annotationResources = GetAndCacheAnnotationResources(config, cacheConfiguration);
if (genomeAssembly != GenomeAssembly.hg19)
{
annotationResources.Annotator.EnableMitochondrialAnnotation();
}
(IPosition position, string[] sampleNames) = config.GetPositionAndSampleNames(annotationResources.SequenceProvider, annotationResources.RefMinorProvider);
if (position.Chromosome.IsEmpty())
{
throw new UserErrorException($"An unknown chromosome was specified ({config.variant.chromosome})");
}
string annotationResult = GetPositionAnnotation(position, annotationResources, sampleNames, preloadRequired);
response = SingleResult.Create(config.id, LambdaUrlHelper.SuccessMessage, annotationResult);
}
catch (Exception exception)
{
response = ExceptionHandler.GetStream(config.id, snsTopicArn, exception);
}
return(response);
}
public static CustomResult GetSuccessResult(CustomConfig customSaConfig, CustomResult result, List <string> outputFiles)
{
Logger.WriteLine("All files uploaded.");
result.created = new FileList
{
bucketName = customSaConfig.outputDir.bucketName,
outputDir = customSaConfig.outputDir.path,
files = outputFiles.ToArray()
};
result.status = LambdaUtilities.SuccessMessage;
LogUtilities.LogObject("Result", result);
LambdaUtilities.DeleteTempOutput();
return(result);
}
private static CustomResult HandleException(StringBuilder runLog, CustomResult result, Exception e, string snsTopicArn)
{
Logger.Log(e);
var errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e);
result.status = $"{errorCategory}: {e.Message}";
if (errorCategory != ErrorCategory.UserError)
{
string snsMessage = SNS.CreateMessage(runLog.ToString(), result.status, e.StackTrace);
SNS.SendMessage(snsTopicArn, snsMessage);
}
LogUtilities.LogObject("Result", result);
LambdaUtilities.DeleteTempOutput();
return(result);
}
// ReSharper disable once UnusedMember.Global
public NirvanaResult Run(NirvanaConfig config, ILambdaContext context)
{
NirvanaResult result;
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey, "annotation_lambda_arn" });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
string annotationLambdaArn = LambdaUtilities.GetEnvironmentVariable(AnnotationLambdaKey);
config.Validate();
var genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly);
if (!_supportedAssemblies.Contains(genomeAssembly))
{
throw new UserErrorException($"Unsupported assembly: {config.genomeAssembly}");
}
IEnumerable <AnnotationRange> annotationRanges = GetAnnotationRanges(config, genomeAssembly);
result = GetNirvanaResult(annotationRanges, config, annotationLambdaArn, context, runLog, snsTopicArn);
}
catch (Exception exception)
{
result = HandleException(runLog, config, exception, snsTopicArn);
}
LogUtilities.LogObject("Result", result);
return(result);
}
// ReSharper disable once UnusedMember.Global
public CustomResult Run(CustomConfig config, ILambdaContext context)
{
var result = new CustomResult {
id = config.id
};
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
LambdaUtilities.DeleteTempOutput();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
config.CheckRequiredFieldsNotNull();
var s3Client = config.outputDir.GetS3Client(context.RemainingTime);
config.CheckResourcesExist();
LambdaUtilities.DeleteTempOutput();
string inputFileName = config.tsvUrl.TrimEndFromFirst("?").TrimStartToLast("/");
Logger.WriteLine($"input file name is: {inputFileName}");
return(IsGeneAnnotationTsv(config.tsvUrl)
? GeneAnnotationCreator.Create(config, inputFileName, result, s3Client)
: VariantAnnotationCreator.Create(config, inputFileName, result, s3Client));
}
catch (Exception e)
{
return(HandleException(runLog, result, e, snsTopicArn));
}
}
// ReSharper disable once UnusedMember.Global
public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
{
var result = new AnnotationResult {
id = config.id
};
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
LambdaUtilities.DeleteTempOutput();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
string vcfUrl = config.vcfUrl;
using (var annotationResources = GetAnnotationResources(config))
{
if (annotationResources.InputStartVirtualPosition == -1)
{
return(GetSuccessOutput(result));
}
long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset;
using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset))
{
annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome));
}
Logger.WriteLine("Scan for positions to preload complete.");
using (var aes = new AesCryptoServiceProvider())
{
FileMetadata jsonMetadata, jasixMetadata;
string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix;
string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix;
using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress))
using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress))
//
using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath))
using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream))
//
using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath))
using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream))
{
IVcfFilter vcfFilter = config.annotationRange == null
? new NullVcfFilter() as IVcfFilter
: new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome));
using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress))
{
StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true);
}
Logger.WriteLine("Annotation done.");
jsonMetadata = jsonMd5Stream.GetFileMetadata();
jasixMetadata = jasixMd5Stream.GetFileMetadata();
}
result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix);
string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix;
var s3Client = config.outputDir.GetS3Client(context.RemainingTime);
s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata);
Logger.WriteLine("Nirvana result files uploaded.");
}
}
LambdaUtilities.DeleteTempOutput();
if (string.IsNullOrEmpty(result.filePath))
{
throw new FileNotFoundException();
}
return(GetSuccessOutput(result));
}
catch (Exception exception)
{
LambdaUtilities.DeleteTempOutput();
return(HandleException(runLog, result, exception, snsTopicArn));
}
}
private static AnnotationResult GetSuccessOutput(AnnotationResult result)
{
result.status = LambdaUtilities.SuccessMessage;
LogUtilities.LogObject("Result", result);
return(result);
}
