public static async Task DropAthenaTable(this AWSAthenaAPI athenaApi, string tableName)
{
string query = $"DROP TABLE IF EXISTS {tableName}";
Console.WriteLine(query);
var executionId = await athenaApi.StartQuery(query);
while (!await athenaApi.IsExecutionCompleted(executionId))
{
Thread.Sleep(2000);
}
}
public static async Task LoadAthenaParition(this AWSAthenaAPI athenaApi, string tableName, string key, string location)
{
string dropQuery = $"ALTER TABLE {tableName} DROP IF EXISTS PARTITION ({key})";
Console.WriteLine(dropQuery);
var dropExecutionId = await athenaApi.StartQuery(dropQuery);
while (!await athenaApi.IsExecutionCompleted(dropExecutionId))
{
Thread.Sleep(500);
}
string addQuery = $"ALTER TABLE {tableName} ADD IF NOT EXISTS PARTITION ({key}) LOCATION '{location}'";
Console.WriteLine(addQuery);
var addExecutionId = await athenaApi.StartQuery(addQuery);
while (!await athenaApi.IsExecutionCompleted(addExecutionId))
{
Thread.Sleep(500);
}
}
public static async Task <string> StartSampleDataBySQL(this AWSAthenaAPI athenaApi, string sql)
{
var result = new DataSampleWithSchema()
{
FieldMappings = new List <FieldMapping>(),
};
var sample = new DataSample()
{
Rows = new List <DataRow>()
};
result.DataSample = sample;
// var response = await athenaApi.ExecuteQuery(sql);
return(await athenaApi.StartQuery(sql));
}
public static async Task ClearAthenaTable(this AWSAthenaAPI athenaApi, AWSS3API awsS3Api, string tableName, string s3Path)
{
Console.WriteLine($"DROP TABLE IF EXISTS {tableName}");
var executionId = await athenaApi.StartQuery($"DROP TABLE IF EXISTS {tableName}");
while (!await athenaApi.IsExecutionCompleted(executionId))
{
Thread.Sleep(2000);
}
var s3Object = s3Path.ParseS3URI();
if (s3Object is S3Object)
{
Console.WriteLine($"Delete S3: {s3Path}");
var files = await awsS3Api.ListFiles(s3Object.Key, "/", s3Object.BucketName);
if (files.Any())
{
await awsS3Api.Delete(files.Select(key => $"{s3Object.Key}{key}"), s3Object.BucketName);
}
Console.WriteLine($"{s3Path}: {files.Count} S3 Files Deleted");
}
}
public async Task <AsyncJobState> CreateAsyncJobState(AsyncJob job)
{
AsyncJobState state = new AsyncJobState()
{
success = false,
wait = job.wait
};
if (state.wait <= 0)
{
state.wait = 5;
}
switch (job.type.ToLower())
{
case "athena":
{
state.type = "Athena";
string sql = job.query;
state.id = await awsAthenaAPI.StartQuery(sql);
Console.WriteLine($"Athena Query:\n" + sql);
}
break;
case "batch":
{
state.type = "Batch";
string json = job.query;
BatchPayload payload = JsonConvert.DeserializeObject <BatchPayload>(json);
state.id = await awsBatchAPI.SubmitJobAndGetID(payload.name, payload.job, payload.queue, payload.parameters);
Console.WriteLine($"Batch Job(Name -> {payload.name}, Job -> {payload.job}, Queue -> {payload.queue}\n" + JsonConvert.SerializeObject(payload.parameters));
}
break;
}
return(state);
}