private static IEnumerable <AnnotationRange> GetAnnotationRanges(NirvanaConfig config, GenomeAssembly genomeAssembly)
{
string cachePathPrefix = LambdaUtilities.GetCachePathPrefix(genomeAssembly);
IntervalForest <IGene> geneIntervalForest;
IDictionary <string, IChromosome> refNameToChromosome;
List <long> blockOffsets;
using (var tabixStream = PersistentStreamUtils.GetReadStream(config.tabixUrl))
using (var tabixReader = new BinaryReader(new BlockGZipStream(tabixStream, CompressionMode.Decompress)))
using (var referenceStream = PersistentStreamUtils.GetReadStream(LambdaUrlHelper.GetRefUrl(genomeAssembly)))
using (var sequenceProvider = new ReferenceSequenceProvider(referenceStream))
using (var taProvider = new TranscriptAnnotationProvider(cachePathPrefix, sequenceProvider, null))
{
long vcfSize = HttpUtilities.GetLength(config.vcfUrl);
int numPartitions = Math.Max(Math.Min((int)((vcfSize - 1) / MinPartitionSize + 1), MaxNumPartitions), MinNumPartitions);
var tabixIndex = Reader.Read(tabixReader, sequenceProvider.RefNameToChromosome);
blockOffsets = PartitionUtilities.GetFileOffsets(config.vcfUrl, numPartitions, tabixIndex);
IntervalArray <ITranscript>[] transcriptIntervalArrays = taProvider.TranscriptIntervalArrays;
geneIntervalForest = GeneForestGenerator.GetGeneForest(transcriptIntervalArrays);
refNameToChromosome = sequenceProvider.RefNameToChromosome;
}
IEnumerable <AnnotationRange> annotationRanges = PartitionUtilities.GenerateAnnotationRanges(blockOffsets, config.vcfUrl, geneIntervalForest, refNameToChromosome);
return(annotationRanges);
}
private static NirvanaResult HandleException(StringBuilder runLog, NirvanaConfig config, Exception e, string snsTopicArn)
{
Logger.Log(e);
var errorCategory = ExceptionUtilities.ExceptionToErrorCategory(e);
return(GetNirvanaFailResult(runLog, config, errorCategory, e.Message, e.StackTrace, snsTopicArn));
}
private static Task <AnnotationResultSummary> RunAnnotationJob(NirvanaConfig config, string annotationLambdaArn, ILambdaContext context, AnnotationRange range, int jobIndex)
{
var annotationConfig = GetAnnotationConfig(config, range, jobIndex);
Logger.WriteLine($"Job: {jobIndex}, Annotation region: {DescribeAnnotationRegion(range)}");
string configString = JsonUtilities.Stringify(annotationConfig);
var annotationJob = new AnnotationJob(context, jobIndex);
return(Task.Run(() => annotationJob.Invoke(annotationLambdaArn, configString)));
}
private static AnnotationConfig GetAnnotationConfig(NirvanaConfig config, AnnotationRange annotationRange, int jobIndex) => new AnnotationConfig
{
id = config.id + $"_job{jobIndex}",
genomeAssembly = config.genomeAssembly,
vcfUrl = config.vcfUrl,
tabixUrl = config.tabixUrl,
outputDir = config.outputDir,
outputPrefix = GetIndexedPrefix(config.vcfUrl, jobIndex),
supplementaryAnnotations = config.supplementaryAnnotations,
customAnnotations = config.customAnnotations,
customStrUrl = config.customStrUrl,
annotationRange = annotationRange
};
private static NirvanaResult GetNirvanaFailResult(StringBuilder runLog, NirvanaConfig config, ErrorCategory errorCategory, string errorMessage, string stackTrace, string snsTopicArn)
{
string status = GetFailedRunStatus(errorCategory, errorMessage);
if (errorCategory != ErrorCategory.UserError)
{
string snsMessage = SNS.CreateMessage(runLog.ToString(), status, stackTrace);
SNS.SendMessage(snsTopicArn, snsMessage);
}
return(new NirvanaResult
{
id = config.id,
status = status
});
}
// ReSharper disable once UnusedMember.Global
public NirvanaResult Run(NirvanaConfig config, ILambdaContext context)
{
NirvanaResult result;
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey, "annotation_lambda_arn" });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
string annotationLambdaArn = LambdaUtilities.GetEnvironmentVariable(AnnotationLambdaKey);
config.Validate();
var genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly);
if (!_supportedAssemblies.Contains(genomeAssembly))
{
throw new UserErrorException($"Unsupported assembly: {config.genomeAssembly}");
}
IEnumerable <AnnotationRange> annotationRanges = GetAnnotationRanges(config, genomeAssembly);
result = GetNirvanaResult(annotationRanges, config, annotationLambdaArn, context, runLog, snsTopicArn);
}
catch (Exception exception)
{
result = HandleException(runLog, config, exception, snsTopicArn);
}
LogUtilities.LogObject("Result", result);
return(result);
}
private static Task <AnnotationResultSummary>[] CallAnnotationLambdas(NirvanaConfig config, string annotationLambdaArn, ILambdaContext context, IEnumerable <AnnotationRange> annotationRanges) =>
annotationRanges?.Select((x, i) => RunAnnotationJob(config, annotationLambdaArn, context, x, i + 1)).ToArray()
?? new[] { RunAnnotationJob(config, annotationLambdaArn, context, null, 1) };
private NirvanaResult GetNirvanaResult(IEnumerable <AnnotationRange> annotationRanges, NirvanaConfig config, string annotationLambdaArn, ILambdaContext context, StringBuilder runLog, string snsTopicArn)
{
Task <AnnotationResultSummary>[] annotationTasks = CallAnnotationLambdas(config, annotationLambdaArn, context, annotationRanges);
AnnotationResultSummary[] processedAnnotationResults = Task.WhenAll(annotationTasks).Result;
(ErrorCategory? errorCategory, string errorMessage) = GetMostSevereErrorCategoryAndMessage(processedAnnotationResults);
if (errorCategory != null)
{
return(GetNirvanaFailResult(runLog, config, errorCategory.Value, errorMessage, null, snsTopicArn));
}
string[] fileNames = processedAnnotationResults.Select(x => x.FileName).ToArray();
return(new NirvanaResult
{
id = config.id,
status = LambdaUrlHelper.SuccessMessage,
created = new FileList
{
bucketName = config.outputDir.bucketName,
outputDir = config.outputDir.path,
files = fileNames
}
});
}
