public static async Task DropAthenaTable(this AWSAthenaAPI athenaApi, string tableName)
{
string query = $"DROP TABLE IF EXISTS {tableName}";
Console.WriteLine(query);
var executionId = await athenaApi.StartQuery(query);
while (!await athenaApi.IsExecutionCompleted(executionId))
{
Thread.Sleep(2000);
}
}
public static async Task LoadAthenaParition(this AWSAthenaAPI athenaApi, string tableName, string key, string location)
{
string dropQuery = $"ALTER TABLE {tableName} DROP IF EXISTS PARTITION ({key})";
Console.WriteLine(dropQuery);
var dropExecutionId = await athenaApi.StartQuery(dropQuery);
while (!await athenaApi.IsExecutionCompleted(dropExecutionId))
{
Thread.Sleep(500);
}
string addQuery = $"ALTER TABLE {tableName} ADD IF NOT EXISTS PARTITION ({key}) LOCATION '{location}'";
Console.WriteLine(addQuery);
var addExecutionId = await athenaApi.StartQuery(addQuery);
while (!await athenaApi.IsExecutionCompleted(addExecutionId))
{
Thread.Sleep(500);
}
}
public static async Task ClearAthenaTable(this AWSAthenaAPI athenaApi, AWSS3API awsS3Api, string tableName, string s3Path)
{
Console.WriteLine($"DROP TABLE IF EXISTS {tableName}");
var executionId = await athenaApi.StartQuery($"DROP TABLE IF EXISTS {tableName}");
while (!await athenaApi.IsExecutionCompleted(executionId))
{
Thread.Sleep(2000);
}
var s3Object = s3Path.ParseS3URI();
if (s3Object is S3Object)
{
Console.WriteLine($"Delete S3: {s3Path}");
var files = await awsS3Api.ListFiles(s3Object.Key, "/", s3Object.BucketName);
if (files.Any())
{
await awsS3Api.Delete(files.Select(key => $"{s3Object.Key}{key}"), s3Object.BucketName);
}
Console.WriteLine($"{s3Path}: {files.Count} S3 Files Deleted");
}
}
public static async Task <DataSampleWithSchema> TryObtainSampleDataResult(this AWSAthenaAPI athenaApi, string executionId)
{
if (await athenaApi.IsExecutionCompleted(executionId))
{
var result = new DataSampleWithSchema()
{
FieldMappings = new List <FieldMapping>(),
};
var sample = new DataSample()
{
Rows = new List <DataRow>()
};
result.DataSample = sample;
var response = await athenaApi.ReadOneResult(new GetQueryResultsRequest()
{
QueryExecutionId = executionId
});
var data = response.ReadData();
result.FieldMappings = response.ToFieldMapping();
foreach (var row in data)
{
var dataRow = new DataRow()
{
Items = row.Select(item => item.ToString()).ToList()
};
sample.Rows.Add(dataRow);
}
return(result);
}
else
{
return(null);
}
}