public static async Task ClearAthenaTable(this EtlSettings etlSettings, string tableName, string s3Path)
{
if (etlSettings.SourceType != EtlSourceEnum.AmazonAthenaPipes)
{
return;
}
var pipesSource = etlSettings.AthenaQueryPipesSource;
var athenaApi = etlSettings.CreatePipesSourceAthenaAPI();
Console.WriteLine($"DROP TABLE IF EXISTS {tableName}");
var executionId = await athenaApi.StartQuery($"DROP TABLE IF EXISTS {tableName}");
while (!await athenaApi.IsExecutionCompleted(executionId))
{
Thread.Sleep(2000);
}
var s3Api = etlSettings.CreatePipesSourceS3API();
var s3Object = s3Path.ParseS3URI();
if (s3Object is S3Object)
{
Console.WriteLine($"Delete S3: {s3Path}");
var files = await s3Api.ListFiles(s3Object.Key, "/", s3Object.BucketName);
await s3Api.Delete(files.Select(key => $"{s3Object.Key}{key}"), s3Object.BucketName);
Console.WriteLine($"{s3Path}: {files.Count} S3 Files Deleted");
}
}