public async Task <string> LoadPartitionIfNotExists(string tableName, string keyFieldAssignment, string s3Location)
{
if (awsAthenaOptions.SQSOptions != null)
{
AWSSQSAPI awsSQSAPI = new AWSSQSAPI(awsAthenaOptions.SQSOptions);
var request = new StartQueryExecutionRequest()
{
QueryString = $@"ALTER TABLE {tableName} ADD IF NOT EXISTS PARTITION ({keyFieldAssignment}) LOCATION '{s3Location}'",
ResultConfiguration = new ResultConfiguration()
{
OutputLocation = awsAthenaOptions.DefaultOutputLocation //"s3://aws-athena-query-results-855250023996-ap-southeast-2/"
}
};
await awsSQSAPI.SendMessage(JsonConvert.SerializeObject(request));
return($"SQS-{awsAthenaOptions.SQSOptions.Url}");
}
else
{
var queryInfo = await amazonAthenaClient.StartQueryExecutionAsync(new StartQueryExecutionRequest()
{
QueryString = $@"ALTER TABLE {tableName} ADD IF NOT EXISTS PARTITION ({keyFieldAssignment}) LOCATION '{s3Location}'",
ResultConfiguration = new ResultConfiguration()
{
OutputLocation = awsAthenaOptions.DefaultOutputLocation //"s3://aws-athena-query-results-855250023996-ap-southeast-2/"
}
});
return(queryInfo.QueryExecutionId);
}
}
public async Task <string> LoadPartition(string tableName, string keyFieldAssignment, string s3Location)
{
/*
* ALTER TABLE twilio_log.twilogs ADD PARTITION (logdate=20181203)
* LOCATION 's3://datascience-twilio-sms-logs/twilio-sms-log-2018-12-03/'
*/
if (awsAthenaOptions.SQSOptions != null)
{
AWSSQSAPI awsSQSAPI = new AWSSQSAPI(awsAthenaOptions.SQSOptions);
var request = new StartQueryExecutionRequest()
{
QueryString = $@"ALTER TABLE {tableName} ADD PARTITION ({keyFieldAssignment}) LOCATION '{s3Location}'",
ResultConfiguration = new ResultConfiguration()
{
OutputLocation = awsAthenaOptions.DefaultOutputLocation //"s3://aws-athena-query-results-855250023996-ap-southeast-2/"
}
};
await awsSQSAPI.SendMessage(JsonConvert.SerializeObject(request));
return($"SQS-{awsAthenaOptions.SQSOptions.Url}");
}
else
{
var queryInfo = await amazonAthenaClient.StartQueryExecutionAsync(new StartQueryExecutionRequest()
{
QueryString = $@"ALTER TABLE {tableName} ADD PARTITION ({keyFieldAssignment}) LOCATION '{s3Location}'",
ResultConfiguration = new ResultConfiguration()
{
OutputLocation = awsAthenaOptions.DefaultOutputLocation //"s3://aws-athena-query-results-855250023996-ap-southeast-2/"
}
});
return(queryInfo.QueryExecutionId);
}
//var queryStatus = amazonAthenaClient.GetQueryExecutionAsync(new GetQueryExecutionRequest() { QueryExecutionId = queryInfo.QueryExecutionId });
//while(!queryStatus.IsCanceled && !queryStatus.IsCompleted && !queryStatus.IsFaulted)
//{
// Thread.Sleep(100);
// queryStatus = amazonAthenaClient.GetQueryExecutionAsync(new GetQueryExecutionRequest() { QueryExecutionId = queryInfo.QueryExecutionId });
//}
//if (queryStatus.IsCanceled) return "canceled";
//if (queryStatus.IsCompleted) return "completed";
//if (queryStatus.IsFaulted) return "faulted";
//return "unknown";
}