public static async Task BatchProcessor( [QueueTrigger(_QUEUENAME)] string message, [Table(_INDEXERTABLENAME)] CloudTable summaryTable, [EventHub(_EVENTHUBEPATH, Connection = "EventHubConnection")] IAsyncCollector <string> eventHub, ILogger logger) { var run = DateTime.Now; logger.LogInformation($"Processing batch {run}"); var analytics = LogAnalyticQuery.GetInstance(logger); if (_local) { await analytics.AuthenticateWithClient(tenantId, _clientId, _clientSecret); } else { await analytics.AuthenticateWithMSI(); } var timer = Stopwatch.StartNew(); var summary = JsonConvert.DeserializeObject <Summary>(message); var events = await analytics.FetchEvents($"datetime({summary.LastCursor})", $"datetime({summary.NextCursor})"); await eventHub.SendEvents(events, logger); summary = new Summary(summary.LastCursor, summary.NextCursor, events.Count, "OK", timer.ElapsedMilliseconds, run); summary.ETag = "*"; //Etag required for replace await summaryTable.ExecuteAsync(TableOperation.Replace(summary)); }
public static async Task DatabaseCursorSlicer( [TimerTrigger("0 * * * * *")] TimerInfo myTimer, [Queue(_QUEUENAME)] IAsyncCollector <Summary> queuesCollector, [Table(_INDEXERTABLENAME)] IAsyncCollector <Summary> summaryCollector, [Table(_INDEXERTABLENAME)] CloudTable summaryTable, ILogger logger ) { var run = DateTime.Now; logger.LogInformation($"C# Timer trigger function executed at: {run}"); //Need to retrieve the latest cursor value var initialCursor = DateTime.Now.AddMinutes(-31).ToUniversalTime().ToString(_CURSORFORMAT); Summary summary = null; var lastSummaryQuery = new TableQuery <Summary>().Take(1); try { summary = (await summaryTable.ExecuteQuerySegmentedAsync(lastSummaryQuery, null)).FirstOrDefault(); } catch (Exception e) { logger.LogError(e, "Impossible to find the latest cursor from the Table"); } var previousCursor = summary?.NextCursor ?? initialCursor; //Get the log analytic instance var analytics = LogAnalyticQuery.GetInstance(logger); //Authenticate to thje log analytic REST API if (_local) { await analytics.AuthenticateWithClient(tenantId, _clientId, _clientSecret); } else { await analytics.AuthenticateWithMSI(); } bool exit = true; do { exit = true; var timer = Stopwatch.StartNew(); //get next cursor ranges within the safety boundaries var ranges = analytics.GetCursorRanges($"{previousCursor}", _SAFETYLAG, _TAKE); await foreach (var r in ranges) { logger.LogDebug($"Summary: {JsonConvert.SerializeObject(r)}"); await queuesCollector.AddAsync(r); await summaryCollector.AddAsync(r); //continue the loop if some items were found exit = false; previousCursor = r.LastCursor; } await queuesCollector.FlushAsync(); await summaryCollector.FlushAsync(); } while (exit); logger.LogInformation("Complete"); }