/// <summary>
/// Called when the Kinesis Stream Awaiter is created in the CF script, it will wait on the specified stream to enter
/// ACTIVE status
/// </summary>
/// <param name="request"></param>
/// <param name="context"></param>
/// <returns></returns>
public override async Task <CustomResourceResponse> CreateAsync(CustomResourceRequest request, ILambdaContext context)
{
if (request.ResourceProperties.ContainsKey("StreamName"))
{
context.LogInfo($"Beginning await for Kinesis stream {request.ResourceProperties["StreamName"]}.");
DescribeStreamRequest Request = new DescribeStreamRequest()
{
StreamName = request.ResourceProperties["StreamName"].ToString()
};
while (true)
{
if (context.RemainingTime.TotalMilliseconds < 1500)
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "Timeout waiting for stream to become active.", request));
}
DescribeStreamResponse Response = await this._KinesisClient.DescribeStreamAsync(Request);
if ((int)Response.HttpStatusCode < 300)
{
if (Response.StreamDescription.StreamStatus == StreamStatus.ACTIVE)
{
break;
}
}
else
{
context.LogWarning($"Received an unsuccessful response to the describe stream request: {(int)Response.HttpStatusCode}.");
}
Thread.Sleep(_WaitTimeInMillis);
}
context.LogInfo($"Successfully created Kinesis stream {Request.StreamName}.");
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.SUCCESS, "Created", Request.StreamName, request.StackId, request.RequestId, request.LogicalResourceId));
}
else
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, "The StreamName property was not provided.", "stream", request.StackId, request.RequestId, request.LogicalResourceId));
}
}
/// <summary>
/// Converts the price list data from json into our formatted csv
/// </summary>
/// <param name="json"></param>
/// <param name="writer"></param>
private async Task GetFromJson(Stream json, CsvWriter writer)
{
json.Position = 0;
ProductOffer offer = ProductOffer.FromJsonStream(json);
/*
* Old implementation, don't need such a complex grouping, just iterate
* every sku in the reserved terms and only process the ones that have a matching
* product and on demand term
*
* Hashset<string> ApplicableProductSkus = new Hashset<string>(Offer.Terms[Term.RESERVED].Keys)
*
* foreach (IGrouping<string, PricingTerm> CommonSkus in Offer.Terms
* .SelectMany(x => x.Value) // Get all of the product item dictionaries from on demand and reserved
* .Where(x => ApplicableProductSkus.Contains(x.Key)) // Only get the pricing terms for products we care about
* .SelectMany(x => x.Value) // Get all of the pricing term key value pairs
* .Select(x => x.Value) // Get just the pricing terms
* .GroupBy(x => x.Sku)) // Put all of the same skus together
* {
* try
* {
* IEnumerable<ReservedInstancePricingTerm> Terms = ReservedInstancePricingTerm.Build2(CommonSkus, Offer.Products[CommonSkus.Key]);
* writer.WriteRecords<ReservedInstancePricingTerm>(Terms);
* }
* catch (Exception e)
* {
* this._Context.LogError(e);
* }
* }
*/
foreach (string Sku in offer.Terms[Term.RESERVED].Keys)
{
if (!offer.Products.ContainsKey(Sku))
{
_context.LogWarning($"There is no product that matches the Sku {Sku}.");
continue;
}
if (!offer.Terms[Term.ON_DEMAND].ContainsKey(Sku))
{
_context.LogWarning($"There is no on-demand pricing term for sku {Sku}.");
continue;
}
try
{
IEnumerable <ReservedInstancePricingTerm> terms = ReservedInstancePricingTerm.Build(
offer.Products[Sku], // The product
offer.Terms[Term.ON_DEMAND][Sku].FirstOrDefault().Value, // OnDemand PricingTerm
offer.Terms[Term.RESERVED][Sku].Select(x => x.Value) // IEnumerable<PricingTerm> Reserved Terms
);
writer.WriteRecords <ReservedInstancePricingTerm>(terms);
}
catch (Exception e)
{
_context.LogError(e);
await SNSNotify(e, _context);
throw e;
}
}
}
public override async Task <CustomResourceResponse> UpdateAsync(CustomResourceRequest request, ILambdaContext context)
{
try
{
context.LogInfo("Initiating update for pipeline.");
UpdatePipelineRequest PipelineRequest = JsonConvert.DeserializeObject <UpdatePipelineRequest>(JsonConvert.SerializeObject(request.ResourceProperties));
ListPipelinesRequest Listing = new ListPipelinesRequest();
List <Pipeline> Pipelines = new List <Pipeline>();
ListPipelinesResponse Pipes;
do
{
Pipes = await this._ETClient.ListPipelinesAsync(Listing);
Pipelines.AddRange(Pipes.Pipelines.Where(x => x.Name.Equals(request.ResourceProperties["Name"] as string) &&
x.InputBucket.Equals(request.ResourceProperties["InputBucket"]) &&
x.Role.Equals(request.ResourceProperties["Role"])
));
} while (Pipes.NextPageToken != null);
if (Pipelines.Count > 1)
{
context.LogWarning($"{Pipelines.Count} pipelines were found matching the Name, InputBucket, and Role specified.");
}
if (Pipelines.Count > 0)
{
PipelineRequest.Id = Pipelines.First().Id;
UpdatePipelineResponse UpdateResponse = await this._ETClient.UpdatePipelineAsync(PipelineRequest);
if ((int)UpdateResponse.HttpStatusCode < 200 || (int)UpdateResponse.HttpStatusCode > 299)
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, $"Received HTTP status code {(int)UpdateResponse.HttpStatusCode}.", request));
}
else
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.SUCCESS,
$"See the details in CloudWatch Log Stream: {context.LogStreamName}.",
request,
false,
new Dictionary <string, object>()
{
{ "Name", UpdateResponse.Pipeline.Name },
{ "Arn", UpdateResponse.Pipeline.Arn },
{ "Id", UpdateResponse.Pipeline.Id }
}
));
}
}
else
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.FAILED,
"No pipelines could be found with the matching characteristics.",
request
));
}
}
catch (AmazonElasticTranscoderException e)
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.FAILED,
e.Message,
request
));
}
catch (Exception e)
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.FAILED,
e.Message,
request
));
}
}
public override async Task <CustomResourceResponse> DeleteAsync(CustomResourceRequest request, ILambdaContext context)
{
try
{
context.LogInfo("Attempting to delete a pipeline.");
ListPipelinesRequest Listing = new ListPipelinesRequest();
List <Pipeline> Pipelines = new List <Pipeline>();
ListPipelinesResponse Pipes;
do
{
Pipes = await this._ETClient.ListPipelinesAsync(Listing);
Pipelines.AddRange(Pipes.Pipelines.Where(x => x.Name.Equals(request.ResourceProperties["Name"] as string) &&
x.InputBucket.Equals(request.ResourceProperties["InputBucket"]) &&
x.Role.Equals(request.ResourceProperties["Role"])
));
} while (Pipes.NextPageToken != null);
if (Pipelines.Count > 1)
{
context.LogWarning($"{Pipelines.Count} pipelines were found matching the Name, InputBucket, and Role specified.");
}
if (Pipelines.Count > 0)
{
DeletePipelineRequest PipelineRequest = new DeletePipelineRequest()
{
Id = Pipelines.First().Id
};
DeletePipelineResponse DeleteResponse = await this._ETClient.DeletePipelineAsync(PipelineRequest);
if ((int)DeleteResponse.HttpStatusCode < 200 || (int)DeleteResponse.HttpStatusCode > 299)
{
return(new CustomResourceResponse(CustomResourceResponse.RequestStatus.FAILED, $"Received HTTP status code {(int)DeleteResponse.HttpStatusCode}.", request));
}
else
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.SUCCESS,
$"See the details in CloudWatch Log Stream: {context.LogStreamName}.",
request,
false
));
}
}
else
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.SUCCESS,
"No pipelines could be found with the matching characteristics.",
request
));
}
}
catch (AmazonElasticTranscoderException e)
{
// If the pipeline doesn't exist, consider it deleted
if (e.StatusCode == HttpStatusCode.NotFound)
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.SUCCESS,
$"See the details in CloudWatch Log Stream: {context.LogStreamName}.",
request
));
}
else
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.FAILED,
e.Message,
request
));
}
}
catch (Exception e)
{
return(new CustomResourceResponse(
CustomResourceResponse.RequestStatus.FAILED,
e.Message,
request
));
}
}
/// <summary>
/// Processes a single manifest file and all of the report keys it contains
/// </summary>
/// <param name="item"></param>
/// <param name="context"></param>
/// <returns></returns>
private static async Task <Manifest> ProcessItemAsync(S3EventNotificationRecord item, string destinationBucket, ILambdaContext context)
{
context.LogInfo(JsonConvert.SerializeObject(item));
// Make sure the event was when a new object was created
if (item.EventName != EventType.ObjectCreatedPut && item.EventName != EventType.ObjectCreatedPost)
{
string Message = $"This Lambda function was triggered by a non ObjectCreated Put or Post event, {item.EventName}, for object {item.S3.Object.Key}; check the CloudFormation template configuration and S3 Event setup.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Get the manifest file contents
GetObjectRequest Request = new GetObjectRequest()
{
BucketName = item.S3.Bucket.Name,
Key = item.S3.Object.Key
};
string Body = "";
using (GetObjectResponse Response = await _S3Client.GetObjectAsync(Request))
{
using (Stream ResponseStream = Response.ResponseStream)
{
using (StreamReader Reader = new StreamReader(ResponseStream))
{
Body = await Reader.ReadToEndAsync();
}
}
}
Manifest ManifestFile = Manifest.Build(Body);
string Prefix = GetDestinationPrefix(ManifestFile);
// Build the destination key map to link source key to destination key
Dictionary <string, string> DestinationKeyMap = ManifestFile.ReportKeys.ToDictionary(x => x, x => $"{Prefix}/{Path.GetFileName(x)}");
// If there are no destination keys
// then there is nothing to do, return
if (!DestinationKeyMap.Any())
{
string Message = $"No destination keys produced for s3://{Request.BucketName}/{Request.Key}";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Copy all of the files over first to replace existing files, this way there
// is no period of time where a file may not exist and break an active query
List <Task <CopyResponse> > CopyTasks = new List <Task <CopyResponse> >();
// Initiate a copy object task for each key
foreach (KeyValuePair <string, string> KeySet in DestinationKeyMap)
{
try
{
context.LogInfo($"Copying CUR from s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}");
CopyTasks.Add(CopyObjectAsync(KeySet.Key, KeySet.Value, item.S3.Bucket.Name, _DestinationBucket));
}
catch (Exception e)
{
string Message = $"Failed to add a copy object task to the queue for s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Process the copy object results
foreach (Task <CopyResponse> Response in CopyTasks.Interleaved())
{
try
{
CopyResponse Result = await Response;
if (Result.IsError)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.";
context.LogError(Message, Result.Exception);
await SNSNotify(Message, context);
return(null);
}
else
{
if (Result.Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey} with http code {(int)Result.Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully copied CUR from s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.");
}
}
}
catch (Exception e)
{
string Message = $"Internal error processing the copy async task.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Delete all of the keys in the that are not the files we just copied over
List <KeyVersion> KeysToDelete;
try
{
// Find all keys under the same prefix, and that aren't one of the keys of the files that have been copied
KeysToDelete = await ListAllObjectsAsync(destinationBucket, Prefix, x => x.Where(y => !DestinationKeyMap.Values.Contains(y.Key)));
}
catch (Exception e)
{
context.LogError(e);
await SNSNotify($"{e.Message}\n{e.StackTrace}", context);
return(null);
}
// Delete the old CUR files in the destination bucket
try
{
if (KeysToDelete != null && KeysToDelete.Any())
{
int DeletedCount = await DeleteObjectsAsync(KeysToDelete, destinationBucket);
if (DeletedCount != KeysToDelete.Count)
{
string Message = $"Unable to delete all objects, expected to delete {KeysToDelete.Count} but only deleted {DeletedCount}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully deleted {DeletedCount} objects.");
}
}
}
catch (Exception e)
{
string Message = "Unable to delete all old CUR files.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
return(ManifestFile);
}
/// <summary>
/// If provided, runs a Glue job after the files have been copied
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
private static async Task RunGlueJob(string table, ILambdaContext context)
{
if (String.IsNullOrEmpty(table))
{
throw new ArgumentNullException("table");
}
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "The Glue database name was provided. Not running job.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return;
}
if (String.IsNullOrEmpty(_GlueJobName))
{
string Message = "The Glue job name for the job was not provided as an environment variable. Not running job.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return;
}
context.LogInfo($"Running glue job on table {table} in database {_GlueDatabaseName}.");
try
{
StartJobRunRequest Request = new StartJobRunRequest()
{
JobName = _GlueJobName,
Timeout = 1440, // 24 Hours
Arguments = new Dictionary <string, string>()
{
{ "--table", table },
{ "--database", _GlueDatabaseName }
}
};
if (!String.IsNullOrEmpty(_GlueDestinationBucket))
{
Request.Arguments.Add("--destination_bucket", _GlueDestinationBucket);
}
StartJobRunResponse Response = await _GlueClient.StartJobRunAsync(Request);
if (Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to start job with status code ${(int)Response.HttpStatusCode}";
context.LogError(Message);
await SNSNotify(Message, context);
}
else
{
context.LogInfo($"Successfully started job {Response.JobRunId}");
}
}
catch (Exception e)
{
string Message = "Failed to start Glue job.";
context.LogError(Message, e);
await SNSNotify(Message + $" {e.Message}", context);
}
}
/// <summary>
/// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
/// and applied to the table. This will end up creating a new table for each billing period.
/// </summary>
/// <param name="manifest"></param>
/// <param name="context"></param>
/// <returns>The table name</returns>
private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
{
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "No Glue database name defined, cannot create a table.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");
string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);
Dictionary <string, string> Parameters;
StorageDescriptor Descriptor;
switch (Format)
{
case "csv":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "skip.header.line.count", "1" },
{ "columnsOrdered", "true" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = $"{x.Category}/{x.Name}", Type = "string"
}).ToList(),
InputFormat = "org.apache.hadoop.mapred.TextInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "OpenCSVSerde",
SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
Parameters = new Dictionary <string, string>()
{
{ "escapeChar", "\\" },
{ "quoteChar", "\"" },
{ "separatorChar", "," }
}
}
};
break;
}
case "parquet":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
}).ToList(),
InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "ParquetHiveSerDe",
SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
Parameters = new Dictionary <string, string>()
{
{ "serialization.format", "1" }
}
}
};
break;
}
default:
{
string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
await SNSNotify(Message, context);
return(String.Empty);
}
}
// The updated table input for this particular CUR
TableInput TblInput = new TableInput()
{
Description = Date,
Name = Date,
TableType = "EXTERNAL_TABLE",
Parameters = Parameters,
StorageDescriptor = Descriptor
};
// Make sure the database exists
GetDatabaseRequest GetDb = new GetDatabaseRequest()
{
Name = _GlueDatabaseName
};
try
{
await _GlueClient.GetDatabaseAsync(GetDb);
context.LogInfo($"Database {_GlueDatabaseName} already exists.");
}
catch (EntityNotFoundException)
{
try
{
CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
{
DatabaseInput = new DatabaseInput()
{
Name = _GlueDatabaseName
}
};
CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
}
else
{
context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
}
}
catch (Exception ex)
{
string Message = $"Failed to create the database {_GlueDatabaseName}.";
context.LogError(Message, ex);
await SNSNotify(Message + $" {ex.Message}", context);
return(String.Empty);
}
}
// Make sure the table exists
GetTableRequest GetTable = new GetTableRequest()
{
DatabaseName = _GlueDatabaseName,
Name = Date
};
try
{
GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);
UpdateTableRequest UpdateReq = new UpdateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
catch (EntityNotFoundException) // This means the table does not exist
{
CreateTableRequest CreateReq = new CreateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
}
/// <summary>
/// Entrypoint for the lambda function, processes each manifest file
/// </summary>
/// <param name="s3Event"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task Exec(S3Event s3Event, ILambdaContext context)
{
context.LogInfo($"Recevied S3 Event : {JsonConvert.SerializeObject(s3Event)}");
// Validate the only required env variable has been set
if (String.IsNullOrEmpty(_DestinationBucket))
{
string Message = "The environment variable DESTINATION_S3_BUCKET was not set.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// Keep track of each copy task in this list
List <Task <Manifest> > Tasks = new List <Task <Manifest> >();
// Process each event record
foreach (S3EventNotificationRecord Item in s3Event.Records)
{
if (ValidManifestFile(Item.S3.Object.Key))
{
Tasks.Add(ProcessItemAsync(Item, _DestinationBucket, context));
}
else
{
context.LogInfo($"The object s3://{Item.S3.Bucket.Name}/{Item.S3.Object.Key} is not a top level manifest file");
}
}
// Process each copy task as it finishes
foreach (Task <Manifest> Task in Tasks.Interleaved())
{
try
{
Manifest Result = await Task;
if (Result == null)
{
string Message = "A task did not return successfully";
context.LogWarning(Message);
await SNSNotify(Message, context);
}
else
{
// Create or update the glue data catalog table
// for this CUR
string TableName = await CreateOrUpdateGlueTable(Result, context);
if (!String.IsNullOrEmpty(TableName))
{
// If provided, run a glue job
await RunGlueJob(TableName, context);
}
else
{
string Message = "The CreateOrUpdateGlueTable method returned an empty string for the table name, indicating either the DB or Table could not be created.";
context.LogWarning(Message);
await SNSNotify(Message, context);
}
}
}
catch (Exception e)
{
string Message = "A process item async task failed with an exception.";
context.LogError(Message, e);
await SNSNotify(Message + $" {e.Message}", context);
}
}
context.LogInfo("Function completed.");
}
/// <summary>
/// A Lambda function to respond to HTTP Get methods from API Gateway
/// </summary>
/// <param name="request"></param>
/// <returns>The list of blogs</returns>
public async Task ExecAsync(CloudWatchScheduledEvent request, ILambdaContext context)
{
context.LogInfo($"Received scheduled event:\n{JsonConvert.SerializeObject(request)}");
// The list request for the query execution Ids
ListQueryExecutionsRequest ListRequest = new ListQueryExecutionsRequest();
// Retrieve the last query execution id that was processed, i.e. the most recent one
// the last time it ran
string LastReadQueryExecutionId = await GetLastQueryExecutionIdAsync(
Environment.GetEnvironmentVariable(MARKER_BUCKET),
Environment.GetEnvironmentVariable(MARKER_KEY),
context
);
context.LogInfo($"Previous run last processed query execution id: {LastReadQueryExecutionId}.");
// Track whether we're done in the do/while loop
bool Finished = false;
// Track whether this is the first time through the loop so we
// can grab the first execution id
bool FirstLoop = true;
// This will be considered the most recent query, grab it here
// and we'll write it at the end when everything's done and we're sure this all succeeded
string NewLastQueryExecutionId = String.Empty;
// This will count the number of successful queries written to S3 in total
int Counter = 0;
do
{
// Get the same list we got above again
ListQueryExecutionsResponse ListResponse = await _AthenaClient.ListQueryExecutionsAsync(ListRequest);
if (ListResponse.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"The list request did not return a success status code: {(int)ListResponse.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// If the list response is null of doesn't have query execution ids, stop processing
if (ListResponse == null || ListResponse.QueryExecutionIds == null || !ListResponse.QueryExecutionIds.Any())
{
context.LogWarning("The list response was null or the query execution Ids were null or empty.");
break;
}
// If it's the first loop
if (FirstLoop)
{
NewLastQueryExecutionId = ListResponse.QueryExecutionIds.First();
context.LogInfo($"The new last processed query execution id will be: {NewLastQueryExecutionId}.");
FirstLoop = false;
if (LastReadQueryExecutionId == NewLastQueryExecutionId)
{
context.LogInfo("No new query execution ids.");
break;
}
}
// Batch get the query executions based on ids
BatchGetQueryExecutionRequest BatchRequest = new BatchGetQueryExecutionRequest()
{
QueryExecutionIds = ListResponse.QueryExecutionIds
};
// If any of the ids match the last read id, then we're done listing ids since
// we've gotten back to the start of the last run
if (ListResponse.QueryExecutionIds.Any(x => x.Equals(LastReadQueryExecutionId)))
{
// Take everything until we reach the last read id
BatchRequest.QueryExecutionIds = BatchRequest.QueryExecutionIds.TakeWhile(x => !x.Equals(LastReadQueryExecutionId)).ToList();
Finished = true;
}
// Make sure there were ids in the request
if (BatchRequest.QueryExecutionIds.Any())
{
// Get query execution details
BatchGetQueryExecutionResponse BatchResponse = await _AthenaClient.BatchGetQueryExecutionAsync(BatchRequest);
if (BatchResponse == null)
{
string Message = $"The batch response was null, this shouldn't happen.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// Make sure we received a good status code
if (BatchResponse.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"The batch request did not return a success status code: {(int)BatchResponse.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return;
}
// Make sure we actually received data back
if (BatchResponse.QueryExecutions == null || !BatchResponse.QueryExecutions.Any())
{
string Message = $"The batch response did not contain any query executions.";
context.LogError(Message);
await SNSNotify(Message, context);
}
else
{
// These are all the transformed records
IEnumerable <AthenaQueryMetric> Records = BatchResponse.QueryExecutions.Select(x => AthenaQueryMetric.Build(x));
// These are the queries that either succeeded or were cancelled and are done
List <AthenaQueryMetric> FinishedQueries = Records.Where(x => x.Status == QueryExecutionState.SUCCEEDED.Value || x.Status == QueryExecutionState.CANCELLED.Value).ToList();
// These are the queries that are still running or are queued
List <string> NotFinishedQueries = Records.Where(x => x.Status == QueryExecutionState.RUNNING.Value || x.Status == QueryExecutionState.QUEUED.Value).Select(x => x.QueryExecutionId).ToList();
// This block updates the retry list stored in S3
if (NotFinishedQueries.Any())
{
context.LogInfo("Adding to the not finished queries list.");
PutObjectResponse Response = await UpdateRetryFileAsync(
Environment.GetEnvironmentVariable(RETRY_BUCKET),
Environment.GetEnvironmentVariable(RETRY_KEY),
NotFinishedQueries,
context
);
if (Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to upload retry file with status code: {(int)Response.HttpStatusCode}. Request Id: {Response.ResponseMetadata.RequestId}.";
context.LogError(Message);
await SNSNotify(Message, context);
}
}
// Nothing to write, so skip to next iteration
if (!FinishedQueries.Any())
{
context.LogInfo("No successful queries found in this list.");
continue;
}
// Add the finished queries to the total count
Counter += FinishedQueries.Count;
// Write the finished query data to S3
await WriteDataAsync(FinishedQueries, Environment.GetEnvironmentVariable(RESULT_BUCKET), Environment.GetEnvironmentVariable(OUTPUT_FORMAT), context);
}
}
if (!String.IsNullOrEmpty(ListResponse.NextToken))
{
ListRequest.NextToken = ListResponse.NextToken;
}
else
{
ListRequest.NextToken = String.Empty;
}
} while (!String.IsNullOrEmpty(ListRequest.NextToken) && !Finished);
context.LogInfo($"Finished pulling query execution data and writing to S3. Wrote {Counter} records.");
// Only update the new last query id if it's not empty, which might happen if there are no Ids in the first list
// response, of if the new is the same as the old, meaning we didn't process any new queries
if (!String.IsNullOrEmpty(NewLastQueryExecutionId) && NewLastQueryExecutionId != LastReadQueryExecutionId)
{
await SetLastQueryExecutionIdAsync(
Environment.GetEnvironmentVariable(MARKER_BUCKET),
Environment.GetEnvironmentVariable(MARKER_KEY),
NewLastQueryExecutionId,
context
)
;
context.LogInfo($"Completed updating marker to {NewLastQueryExecutionId}.");
}
else
{
context.LogInfo($"No new query executions, not updating marker.");
}
context.LogInfo("Function complete.");
}