public static async IAsyncEnumerable <Summary> GetCursorRanges(this LogAnalyticQuery query, string lastCursor, string safetyLag, int limit)
{
//Making 1 second grouped ranges for better performances
var q = $@"union withsource=SourceTable *
| extend ItemId = _ItemId
| extend _ingestionTime = ingestion_time()
| extend cursor = format_datetime(_ingestionTime,'yyyy-MM-dd HH:mm:ss.fffffff')
| where _ingestionTime > datetime({lastCursor})
| where _ingestionTime <= {safetyLag}
| order by _ingestionTime asc
| limit 10000
| summarize max(cursor), count() by bin(_ingestionTime, 1s)";
var cursors = await query.GetNextCursors(workspaceId, q, limit);
foreach (var c in cursors)
{
yield return(new Summary(lastCursor, c.Item1)
{
EventsCount = c.Item2
});
lastCursor = c.Item1;
}
}
public static async Task BatchProcessor(
[QueueTrigger(_QUEUENAME)] string message,
[Table(_INDEXERTABLENAME)] CloudTable summaryTable,
[EventHub(_EVENTHUBEPATH, Connection = "EventHubConnection")] IAsyncCollector <string> eventHub,
ILogger logger)
{
var run = DateTime.Now;
logger.LogInformation($"Processing batch {run}");
var analytics = LogAnalyticQuery.GetInstance(logger);
if (_local)
{
await analytics.AuthenticateWithClient(tenantId, _clientId, _clientSecret);
}
else
{
await analytics.AuthenticateWithMSI();
}
var timer = Stopwatch.StartNew();
var summary = JsonConvert.DeserializeObject <Summary>(message);
var events = await analytics.FetchEvents($"datetime({summary.LastCursor})", $"datetime({summary.NextCursor})");
await eventHub.SendEvents(events, logger);
summary = new Summary(summary.LastCursor, summary.NextCursor, events.Count, "OK", timer.ElapsedMilliseconds, run);
summary.ETag = "*"; //Etag required for replace
await summaryTable.ExecuteAsync(TableOperation.Replace(summary));
}
public static async Task <IList <Dictionary <string, object> > > FetchEvents(this LogAnalyticQuery query, string after, string to)
{
var q = $@"union withsource=SourceTable *
| extend ItemId = _ItemId
| extend _ingestionTime = ingestion_time()
| extend cursor = format_datetime(_ingestionTime,'yyyy-MM-dd HH:mm:ss.fffffff')
| where _ingestionTime > {after}
| where _ingestionTime <= {to}
| order by _ingestionTime asc";
return(await query.ExecuteQuery(workspaceId, q));
}
public static LogAnalyticQuery GetInstance(ILogger log)
{
if (_instance == null)
{
lock (_lock)
{
if (_instance == null)
{
_instance = new LogAnalyticQuery(log);
}
}
}
return(_instance);
}
public static async Task DatabaseCursorSlicer(
[TimerTrigger("0 * * * * *")] TimerInfo myTimer,
[Queue(_QUEUENAME)] IAsyncCollector <Summary> queuesCollector,
[Table(_INDEXERTABLENAME)] IAsyncCollector <Summary> summaryCollector,
[Table(_INDEXERTABLENAME)] CloudTable summaryTable,
ILogger logger
)
{
var run = DateTime.Now;
logger.LogInformation($"C# Timer trigger function executed at: {run}");
//Need to retrieve the latest cursor value
var initialCursor = DateTime.Now.AddMinutes(-31).ToUniversalTime().ToString(_CURSORFORMAT);
Summary summary = null;
var lastSummaryQuery = new TableQuery <Summary>().Take(1);
try
{
summary = (await summaryTable.ExecuteQuerySegmentedAsync(lastSummaryQuery, null)).FirstOrDefault();
}
catch (Exception e)
{
logger.LogError(e, "Impossible to find the latest cursor from the Table");
}
var previousCursor = summary?.NextCursor ?? initialCursor;
//Get the log analytic instance
var analytics = LogAnalyticQuery.GetInstance(logger);
//Authenticate to thje log analytic REST API
if (_local)
{
await analytics.AuthenticateWithClient(tenantId, _clientId, _clientSecret);
}
else
{
await analytics.AuthenticateWithMSI();
}
bool exit = true;
do
{
exit = true;
var timer = Stopwatch.StartNew();
//get next cursor ranges within the safety boundaries
var ranges = analytics.GetCursorRanges($"{previousCursor}", _SAFETYLAG, _TAKE);
await foreach (var r in ranges)
{
logger.LogDebug($"Summary: {JsonConvert.SerializeObject(r)}");
await queuesCollector.AddAsync(r);
await summaryCollector.AddAsync(r);
//continue the loop if some items were found
exit = false;
previousCursor = r.LastCursor;
}
await queuesCollector.FlushAsync();
await summaryCollector.FlushAsync();
} while (exit);
logger.LogInformation("Complete");
}