public async Task BatchPoolQueryExecutionAsync_WhenSucceeds()
{
var queryExecutions = new List <QueryExecution>
{
new QueryExecution {
QueryExecutionId = "1", Status = new QueryExecutionStatus {
State = QueryExecutionState.SUCCEEDED
}
}
};
var queryExecutionId = queryExecutions.Select(s => s.QueryExecutionId).ToList();
A.CallTo(() => _amazonAthena.BatchGetQueryExecutionAsync(
A <BatchGetQueryExecutionRequest> .That.Matches(m => m.QueryExecutionIds.SequenceEqual(queryExecutionId)), CancellationToken.None))
.Returns(Task.FromResult(new BatchGetQueryExecutionResponse {
HttpStatusCode = HttpStatusCode.OK, QueryExecutions = queryExecutions
}));
await _athenaClientService.BatchPoolQueryExecutionAsync(queryExecutionId, 1);
A.CallTo(() => _amazonAthena.BatchGetQueryExecutionAsync(
A <BatchGetQueryExecutionRequest> .Ignored, CancellationToken.None))
.MustHaveHappenedOnceExactly();
}
private Amazon.Athena.Model.BatchGetQueryExecutionResponse CallAWSServiceOperation(IAmazonAthena client, Amazon.Athena.Model.BatchGetQueryExecutionRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Athena", "BatchGetQueryExecution");
try
{
#if DESKTOP
return(client.BatchGetQueryExecution(request));
#elif CORECLR
return(client.BatchGetQueryExecutionAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
/// <summary>
/// Processes the execution Ids that need be retried because they weren't finished or cancelled
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task RetryAsync(CloudWatchScheduledEvent request, ILambdaContext context)
{
context.LogInfo($"Received scheduled event for retries:\n{JsonConvert.SerializeObject(request)}");
List <string> RetryIds = await GetRetryFileAsync(Environment.GetEnvironmentVariable(RETRY_BUCKET), Environment.GetEnvironmentVariable(RETRY_KEY), context);
context.LogInfo($"Found {RetryIds.Count} ids to retry.");
List <string> RemainingIds = new List <string>();
if (RetryIds != null && RetryIds.Any() && !RetryIds.All(x => String.IsNullOrEmpty(x)))
{
int Counter = 0;
foreach (List <string> Chunk in ChunkList <string>(RetryIds, 50))
{
BatchGetQueryExecutionRequest BatchRequest = new BatchGetQueryExecutionRequest()
{
QueryExecutionIds = Chunk
};
BatchGetQueryExecutionResponse BatchResponse = await _AthenaClient.BatchGetQueryExecutionAsync(BatchRequest);
if (BatchResponse == null)
{
string Message = $"The batch response was null, this shouldn't happen.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// Make sure we received a good status code
if (BatchResponse.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"The batch request did not return a success status code: {(int)BatchResponse.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// Make sure we actually received data back
if (BatchResponse.QueryExecutions == null || !BatchResponse.QueryExecutions.Any())
{
string Message = $"The batch response did not contain any query executions.";
context.LogError(Message);
await SNSNotify(Message, context);
}
else
{
// These are all the transformed records
IEnumerable <AthenaQueryMetric> Records = BatchResponse.QueryExecutions.Select(x => AthenaQueryMetric.Build(x));
// These are the queries that either succeeded or were cancelled and are done
List <AthenaQueryMetric> FinishedQueries = Records.Where(x => x.Status == QueryExecutionState.SUCCEEDED.Value || x.Status == QueryExecutionState.CANCELLED.Value).ToList();
// These are the queries that are still running or are queued
List <string> NotFinishedQueries = Records.Where(x => x.Status == QueryExecutionState.RUNNING.Value || x.Status == QueryExecutionState.QUEUED.Value).Select(x => x.QueryExecutionId).ToList();
if (NotFinishedQueries.Any())
{
RemainingIds.AddRange(NotFinishedQueries);
}
// Nothing to write, so skip to next iteration
if (!FinishedQueries.Any())
{
context.LogInfo("No successful queries found in this list.");
continue;
}
else
{
Counter += FinishedQueries.Count;
await WriteDataAsync(
FinishedQueries,
Environment.GetEnvironmentVariable(RESULT_BUCKET),
Environment.GetEnvironmentVariable(OUTPUT_FORMAT),
context
);
}
}
}
context.LogInfo($"Finished pulling query execution data and writing to S3. Wrote {Counter} records.");
if (RemainingIds.Count < RetryIds.Count)
{
context.LogInfo("Updating retry file.");
await SetRetryFileAsync(Environment.GetEnvironmentVariable(RETRY_BUCKET), Environment.GetEnvironmentVariable(RETRY_KEY), RemainingIds, context);
context.LogInfo("Finished updating retry file.");
}
else
{
context.LogInfo("No updates need to made to the retry file.");
}
}
else
{
context.LogInfo("No ids in the retry file.");
}
}
