public void ProcessEvent(SNSEvent snsEvent, ILambdaContext context) { context.Logger.LogLine($"Beginning to process {snsEvent.Records.Count} records..."); string bearerToken = $"Bearer {this.oauthClient.GetBearerTokenAsync().Result}"; QueueProcessor processor = new QueueProcessor(queue, notifier, productAnalyticsAPIClient, persister, udrData, httpClientFactory.CreateClient(), bearerToken, slack, headToHeadAPIClient); foreach (SNSEvent.SNSRecord record in snsEvent.Records) { GermPlasmSNSRequest body = JsonConvert.DeserializeObject <GermPlasmSNSRequest>(record.Sns.Message); processor.Process(context, body, record.Sns.TopicArn); } context.Logger.LogLine($"SNS event processing is complete."); }
public async Task SendSelfProcessStartAsync(ILambdaContext context, GermPlasmSNSRequest message, string topicArn) { context.Logger.Log($"starting another instance of this lambda by sending message to: {topicArn}."); var client = new AmazonSimpleNotificationServiceClient(); var request = new PublishRequest(topicArn, JsonConvert.SerializeObject(message)); PublishResponse response = await client.PublishAsync(request); if (response.HttpStatusCode != System.Net.HttpStatusCode.OK) { throw new Exception($"failed sending a message to source SNS: {response.HttpStatusCode}"); } else { context.Logger.LogLine($"SNS response: {response.HttpStatusCode}; ID: {response.MessageId}"); } }
public void Initialize(GermPlasmSNSRequest request) { this.gpQueueUrl = request.GPQueueUrl; this.request = new ReceiveMessageRequest(gpQueueUrl); this.request.MaxNumberOfMessages = 1; this.request.WaitTimeSeconds = 0; this.request.VisibilityTimeout = 10; this.deleteRequest = new DeleteMessageRequest(); this.deleteRequest.QueueUrl = gpQueueUrl; if (this.client != null) { this.client.Dispose(); } this.client = new AmazonSQSClient(); }
public void Process(ILambdaContext context, GermPlasmSNSRequest request, string topicArn) { string keyReportName = request.ReportName.Replace(' ', '`'); int reportId = request.ReportId; ulong count = this.persister.InitializeCounter(keyReportName, request.MyCount); if (count == (ulong)request.MyCount) { //Removed the CacheClean Up we can put it back //this.persister.CleanupCache(context, keyReportName, reportId); count = this.persister.InitializeCounter(keyReportName, request.MyCount); } string queueUrl = request.GPQueueUrl; context.Logger.LogLine($"starting to process against SQS: {queueUrl} and count: {count}"); // give lambda 20 secods to create a new notification to kick off a new lambda processor TimeSpan limit = new TimeSpan(0, 0, 20); bool remaining = true; // initialize Analytics API bool initialized = false; queue.Initialize(request); QueueGermPlasmEvent gpEvent; while (remaining && context.RemainingTime > limit) { context.Logger.LogLine($"getting from SQS: {queueUrl} with {context.RemainingTime} remaining"); try { // Load Q data gpEvent = this.queue.GetNext(context); if (gpEvent == null) { remaining = false; } else { if (!initialized) { this.persister.InitializeObservations(keyReportName, gpEvent.Observations); productAnalyticsAPIClient.Initialize(this.httpClient, bearerToken, gpEvent.UserId); this.headToHeadAPIClient.Initialize(this.httpClient, bearerToken, gpEvent.UserId); List <string> udrNames = gpEvent.Bands.Where(b => b.BandingGroup == "UDR").Select(b => b.BandName).ToList(); udrNames.AddRange(gpEvent.DataFilters.Where(df => df.StartsWith("UserDefinedRegions")).SelectMany(df => df.Split('=')[1].Split('&'))); this.udrList = new UDRList(this.udrData.GetUDRsForCrop(this.httpClient, gpEvent.Crop, udrNames).Result); initialized = true; } remaining = ProcessGermPlasmEvent(context, request, gpEvent); } } catch (AggregateException exc) { string error; foreach (Exception iExc in exc.InnerExceptions) { error = $"ERROR: queue: {queueUrl}: error: {iExc.ToString()}"; string slackResponse = slack.SlackIntegrationPostAsync(this.httpClient, error).Result; headToHeadAPIClient.ReportFailure(context, this.httpClient, reportId, "AggregateException = " + error); context.Logger.Log(error); context.Logger.Log(slackResponse); } remaining = false; } catch (Exception exc) { string error = $"ERROR: queue: {queueUrl}: error: {exc.ToString()}"; context.Logger.Log(error); string slackResponse = slack.SlackIntegrationPostAsync(this.httpClient, error).Result; headToHeadAPIClient.ReportFailure(context, this.httpClient, reportId, "Exception= " + error); context.Logger.Log(slackResponse); remaining = false; } } // need to queue a new processor to the SNS if (remaining) { // use notifier to send to first entrypoint topic notifier.SendSelfProcessStartAsync(context, request, topicArn).Wait(); } }
private bool ProcessGermPlasmEvent(ILambdaContext context, GermPlasmSNSRequest request, QueueGermPlasmEvent gpEvent) { long germPlasmId = gpEvent.Germplasm.GermplasmId; context.Logger.LogLine($"processing data keys to cache for germ Plasm Id: {germPlasmId}; key:{gpEvent.Germplasm.GetProduct()}; isMetric: {request.IsMetric}"); if (filter == null) { filter = new FilterApplicator(gpEvent.DataFilters, this.udrList); } string cacheReportName = gpEvent.ReportName.Replace(' ', '`'); int reportId = gpEvent.ReportId; TimeSpan ts = context.RemainingTime.Subtract(TimeSpan.FromSeconds(20)); if (ts.Ticks <= 0) { context.Logger.LogLine($"CreateMessageAsync for germ Plasm Id: {germPlasmId}"); queue.CreateMessageAsync(gpEvent); return(true); } Dictionary <string, BaseBand> bands = new Dictionary <string, BaseBand>(); foreach (BandDefinition bandDefintion in gpEvent.Bands) { BandFactory.Create(bands, context, this.httpClient, bandDefintion, this.productAnalyticsAPIClient, this.udrList); } CancellationTokenSource cancellationToken = new CancellationTokenSource(ts); if (!this.persister.GermPlasmProcessed(cacheReportName, reportId, germPlasmId)) { try { context.Logger.Log($"processing the germplasm {germPlasmId}"); List <string> observations = gpEvent.Observations.Select(o => o.ObsRefCd).ToList(); // load data, supplying API supported filters List <Dictionary <string, dynamic> > data = this.productAnalyticsAPIClient.GetEntryMeansAsynch( context, this.httpClient, cancellationToken.Token, gpEvent.UserId, germPlasmId, filter.ApiFilters, observations, gpEvent.Crop, gpEvent.RegionName ).Result; // S<IP for now: then load any missing banding/observational data // output to summarize valid records List <ReducedEntryMeans> reportOutputs = ReportGrouperFactory.GetStandardReducedEntryMeans( bands, gpEvent.AnalysisType, observations); object cEvent; foreach (Dictionary <string, dynamic> row in data) { if (gpEvent.Germplasm.CEvent == null && row.TryGetValue("cEvent", out cEvent) && cEvent != null && !string.IsNullOrEmpty(cEvent.ToString())) { gpEvent.Germplasm.CEvent = cEvent.ToString().Replace(',', '|'); } if (filter.IsApplicable(row)) // first, see if the row should be filtered { context.Logger.Log($"first, see if the row should be filtered {JsonConvert.SerializeObject(row)}"); foreach (ReducedEntryMeans output in reportOutputs) { output.ProcessRecord(cancellationToken.Token, request.IsMetric, row); // apply bands and gather observations } } } context.Logger.LogLine($"persisting data to cache for germ Plasm Id: {germPlasmId}, cEvent: {gpEvent.Germplasm.CEvent}"); persister.PersistGermPlasmOutputAsync(context, cacheReportName, reportId, gpEvent.Germplasm, gpEvent.Category, reportOutputs).Wait(); } catch (AggregateException exc) { context.Logger.LogLine($"ProcessGermPlasmEvent : {exc.StackTrace}-ExceptionMessage:{exc.Message}-reportName:{cacheReportName}- germplasmId: {germPlasmId}"); //headToHeadAPIClient.ReportFailure(context, this.httpClient, request.ReportId, "AggregateException= " +exc.Message); queue.CreateMessageAsync(gpEvent); bool cancelled = cancellationToken.IsCancellationRequested || exc.InnerExceptions.Select(e => e is TaskCanceledException).Count() > 0; if (!cancelled) { throw; } context.Logger.LogLine($"cancelled: {cancellationToken.IsCancellationRequested}"); return(true); } } else { context.Logger.Log($"skipping processing for report: {cacheReportName} and germ plasm id: {germPlasmId}"); } if (AreWeFinished(context, cacheReportName, gpEvent.UserId, germPlasmId)) //when we are finished { context.Logger.Log($"starting up band generation"); //put single SNS message to initiiate next process. this.notifier.SendBandProcessStartAsync(new BandNotificationMessage( userId: request.UserId, reportName: request.ReportName, reportId: request.ReportId, crop: gpEvent.Crop, region: gpEvent.Region, year: gpEvent.Year, compareHeads: request.CompareHeads, reportIdentifier: request.ReportIdentifier)) .Wait(); this.queue.DeleteGPQueueAsync().Wait(); context.Logger.Log($"Delete Queue DeleteGPQueueAsync End"); return(false); } return(true); }