/// <summary>
/// run the athena query pipes
/// </summary>
/// <param name="etlSettings"></param>
/// <param name="useDate"></param>
/// <returns></returns>
public static async Task RunAthenaQueryPipes(this EtlSettings etlSettings, DateTime?useDate = null)
{
if (etlSettings.SourceType != EtlSourceEnum.AmazonAthenaPipes)
{
return;
}
var pipesSource = etlSettings.AthenaQueryPipesSource;
AthenaParserSetting parserSetting = new AthenaParserSetting();
parserSetting.DefaultExportPath = $"s3://{etlSettings.TargetS3BucketName}/{etlSettings.TargetS3Prefix}".FixPathEnd();
parserSetting.DefaultTableName = $"`{etlSettings.AthenaDatabaseName}`.`{etlSettings.AthenaTableName}`";
parserSetting.Date = useDate == null?DateTime.UtcNow.AddDays(-pipesSource.DaysAgo) : useDate.Value.AddDays(-pipesSource.DaysAgo);
parserSetting.DateFormat = pipesSource.DateFormat;
parserSetting.TempDatabase = pipesSource.TempDatabase;
parserSetting.TempTablePath = pipesSource.TempDataPath.FixPathEnd();
var caches = JsonConvert.DeserializeObject <List <CacheSetting> >(etlSettings.AthenaQueryPipesSource.Caches);
foreach (var cache in caches)
{
if (!cache.S3Path.EndsWith("/"))
{
cache.S3Path += "/";
}
parserSetting.Caches.Add(cache.Key, cache);
}
var parsed = pipesSource.AthenaSQL.ParseAthenaPipes(parserSetting);
await etlSettings.ExecuteControlFlow(parsed, parserSetting);
var athenaApi = etlSettings.CreatePipesSourceAthenaAPI();
foreach (var kvp in parserSetting.Partitions)
{
await athenaApi.LoadPartitionIfNotExists(parserSetting.DefaultTableName, kvp.Key, kvp.Value);
}
}