/// <summary> /// Unmarshaller the response from the service to the response class. /// </summary> /// <param name="context"></param> /// <returns></returns> public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context) { ListQueryExecutionsResponse response = new ListQueryExecutionsResponse(); context.Read(); int targetDepth = context.CurrentDepth; while (context.ReadAtDepth(targetDepth)) { if (context.TestExpression("NextToken", targetDepth)) { var unmarshaller = StringUnmarshaller.Instance; response.NextToken = unmarshaller.Unmarshall(context); continue; } if (context.TestExpression("QueryExecutionIds", targetDepth)) { var unmarshaller = new ListUnmarshaller <string, StringUnmarshaller>(StringUnmarshaller.Instance); response.QueryExecutionIds = unmarshaller.Unmarshall(context); continue; } } return(response); }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonAthenaConfig config = new AmazonAthenaConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonAthenaClient client = new AmazonAthenaClient(creds, config); ListQueryExecutionsResponse resp = new ListQueryExecutionsResponse(); do { ListQueryExecutionsRequest req = new ListQueryExecutionsRequest { NextToken = resp.NextToken , MaxResults = maxItems }; resp = client.ListQueryExecutions(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.QueryExecutionIds) { AddObject(obj); } }while (!string.IsNullOrEmpty(resp.NextToken)); }
/// <summary> /// A Lambda function to respond to HTTP Get methods from API Gateway /// </summary> /// <param name="request"></param> /// <returns>The list of blogs</returns> public async Task ExecAsync(CloudWatchScheduledEvent request, ILambdaContext context) { context.LogInfo($"Received scheduled event:\n{JsonConvert.SerializeObject(request)}"); // The list request for the query execution Ids ListQueryExecutionsRequest ListRequest = new ListQueryExecutionsRequest(); // Retrieve the last query execution id that was processed, i.e. the most recent one // the last time it ran string LastReadQueryExecutionId = await GetLastQueryExecutionIdAsync( Environment.GetEnvironmentVariable(MARKER_BUCKET), Environment.GetEnvironmentVariable(MARKER_KEY), context ); context.LogInfo($"Previous run last processed query execution id: {LastReadQueryExecutionId}."); // Track whether we're done in the do/while loop bool Finished = false; // Track whether this is the first time through the loop so we // can grab the first execution id bool FirstLoop = true; // This will be considered the most recent query, grab it here // and we'll write it at the end when everything's done and we're sure this all succeeded string NewLastQueryExecutionId = String.Empty; // This will count the number of successful queries written to S3 in total int Counter = 0; do { // Get the same list we got above again ListQueryExecutionsResponse ListResponse = await _AthenaClient.ListQueryExecutionsAsync(ListRequest); if (ListResponse.HttpStatusCode != HttpStatusCode.OK) { string Message = $"The list request did not return a success status code: {(int)ListResponse.HttpStatusCode}."; context.LogError(Message); await SNSNotify(Message, context); return; } // If the list response is null of doesn't have query execution ids, stop processing if (ListResponse == null || ListResponse.QueryExecutionIds == null || !ListResponse.QueryExecutionIds.Any()) { context.LogWarning("The list response was null or the query execution Ids were null or empty."); break; } // If it's the first loop if (FirstLoop) { NewLastQueryExecutionId = ListResponse.QueryExecutionIds.First(); context.LogInfo($"The new last processed query execution id will be: {NewLastQueryExecutionId}."); FirstLoop = false; if (LastReadQueryExecutionId == NewLastQueryExecutionId) { context.LogInfo("No new query execution ids."); break; } } // Batch get the query executions based on ids BatchGetQueryExecutionRequest BatchRequest = new BatchGetQueryExecutionRequest() { QueryExecutionIds = ListResponse.QueryExecutionIds }; // If any of the ids match the last read id, then we're done listing ids since // we've gotten back to the start of the last run if (ListResponse.QueryExecutionIds.Any(x => x.Equals(LastReadQueryExecutionId))) { // Take everything until we reach the last read id BatchRequest.QueryExecutionIds = BatchRequest.QueryExecutionIds.TakeWhile(x => !x.Equals(LastReadQueryExecutionId)).ToList(); Finished = true; } // Make sure there were ids in the request if (BatchRequest.QueryExecutionIds.Any()) { // Get query execution details BatchGetQueryExecutionResponse BatchResponse = await _AthenaClient.BatchGetQueryExecutionAsync(BatchRequest); if (BatchResponse == null) { string Message = $"The batch response was null, this shouldn't happen."; context.LogError(Message); await SNSNotify(Message, context); return; } // Make sure we received a good status code if (BatchResponse.HttpStatusCode != HttpStatusCode.OK) { string Message = $"The batch request did not return a success status code: {(int)BatchResponse.HttpStatusCode}."; context.LogError(Message); await SNSNotify(Message, context); return; } // Make sure we actually received data back if (BatchResponse.QueryExecutions == null || !BatchResponse.QueryExecutions.Any()) { string Message = $"The batch response did not contain any query executions."; context.LogError(Message); await SNSNotify(Message, context); } else { // These are all the transformed records IEnumerable <AthenaQueryMetric> Records = BatchResponse.QueryExecutions.Select(x => AthenaQueryMetric.Build(x)); // These are the queries that either succeeded or were cancelled and are done List <AthenaQueryMetric> FinishedQueries = Records.Where(x => x.Status == QueryExecutionState.SUCCEEDED.Value || x.Status == QueryExecutionState.CANCELLED.Value).ToList(); // These are the queries that are still running or are queued List <string> NotFinishedQueries = Records.Where(x => x.Status == QueryExecutionState.RUNNING.Value || x.Status == QueryExecutionState.QUEUED.Value).Select(x => x.QueryExecutionId).ToList(); // This block updates the retry list stored in S3 if (NotFinishedQueries.Any()) { context.LogInfo("Adding to the not finished queries list."); PutObjectResponse Response = await UpdateRetryFileAsync( Environment.GetEnvironmentVariable(RETRY_BUCKET), Environment.GetEnvironmentVariable(RETRY_KEY), NotFinishedQueries, context ); if (Response.HttpStatusCode != HttpStatusCode.OK) { string Message = $"Failed to upload retry file with status code: {(int)Response.HttpStatusCode}. Request Id: {Response.ResponseMetadata.RequestId}."; context.LogError(Message); await SNSNotify(Message, context); } } // Nothing to write, so skip to next iteration if (!FinishedQueries.Any()) { context.LogInfo("No successful queries found in this list."); continue; } // Add the finished queries to the total count Counter += FinishedQueries.Count; // Write the finished query data to S3 await WriteDataAsync(FinishedQueries, Environment.GetEnvironmentVariable(RESULT_BUCKET), Environment.GetEnvironmentVariable(OUTPUT_FORMAT), context); } } if (!String.IsNullOrEmpty(ListResponse.NextToken)) { ListRequest.NextToken = ListResponse.NextToken; } else { ListRequest.NextToken = String.Empty; } } while (!String.IsNullOrEmpty(ListRequest.NextToken) && !Finished); context.LogInfo($"Finished pulling query execution data and writing to S3. Wrote {Counter} records."); // Only update the new last query id if it's not empty, which might happen if there are no Ids in the first list // response, of if the new is the same as the old, meaning we didn't process any new queries if (!String.IsNullOrEmpty(NewLastQueryExecutionId) && NewLastQueryExecutionId != LastReadQueryExecutionId) { await SetLastQueryExecutionIdAsync( Environment.GetEnvironmentVariable(MARKER_BUCKET), Environment.GetEnvironmentVariable(MARKER_KEY), NewLastQueryExecutionId, context ) ; context.LogInfo($"Completed updating marker to {NewLastQueryExecutionId}."); } else { context.LogInfo($"No new query executions, not updating marker."); } context.LogInfo("Function complete."); }