public static async Task DropAthenaTables(this StateMachineQueryContext context, AWSAthenaAPI athenaApi)
{
foreach (var dropping in context.settings.DroppingTables)
{
await DropAthenaTable(athenaApi, dropping);
}
}
public static async Task LoadPartitions(this StateMachineQueryContext context, AWSAthenaAPI athenaApi)
{
//Console.WriteLine("LoadPartitions:");
foreach (var patition in context.settings.Partitions)
{
await LoadAthenaParition(athenaApi, context.settings.DefaultTableName, patition.Key, patition.Value);
}
}
public static async Task ClearTempTables(this StateMachineQueryContext context, AWSAthenaAPI athenaApi, AWSS3API awsS3Api)
{
foreach (var clearing in context.settings.Clearings)
{
await athenaApi.ClearAthenaTable(awsS3Api, clearing.Key, clearing.Value);
}
context.settings.Clearings.Clear();
}
public static void Print(this StateMachineQueryContext context)
{
context.state.Print();
Debug.WriteLine($"SQL Query: {context.query}");
if (!context.state.Any())
{
Debug.WriteLine($"*** End of Pipes ***");
}
}
public static async Task ClearAthenaTables(this StateMachineQueryContext context, AWSAthenaAPI athenaApi, AWSS3API awsS3Api)
{
var parserSetting = context.BuildParserSetting();
var pipes = context.raw.ParseAthenaPipes(parserSetting);
foreach (var clearing in parserSetting.Clearings)
{
await athenaApi.ClearAthenaTable(awsS3Api, clearing.Key, clearing.Value);
}
}
public static AthenaParserSetting BuildParserSetting(this StateMachineQueryContext context)
{
StateMachineSettings settings = context.settings;
AthenaParserSetting parserSetting = new AthenaParserSetting();
parserSetting.DefaultExportPath = settings.DefaultExportPath;
parserSetting.DefaultTableName = settings.DefaultTableName;
parserSetting.Date = settings.Date;
parserSetting.DateFormat = settings.DateFormat;
parserSetting.TempDatabase = settings.TempDatabase;
parserSetting.TempTablePath = settings.TempTablePath;
return(parserSetting);
}
public static void RunWithResult(this StateMachineQueryContext context, StateMachineQueryResult result)
{
context.result = result;
context.ExecuteStateMachineQueryContext();
if (context.settings.Variables != null && context.settings.Variables.Any())
{
Debug.WriteLine("Context Variables:");
foreach (var kv in context.settings.Variables)
{
Debug.WriteLine($" Variable: {kv.Key} -> {kv.Value}");
}
}
context.Print();
}
public async void StateQueryTests()
{
int i = 0;
StateMachineExecutionResult result;
StateMachineQueryContext context;
{
i = 1;
AthenaParserSetting athenaParserLogger = new AthenaParserSetting();
Debug.WriteLine($"****** Begin File {i} ******");
var filename = $"{AppContext.BaseDirectory}/query{i}.sql";
Debug.WriteLine($"File {i}: {filename}");
var query = File.ReadAllText(filename);
try
{
var pipes = query.ParseAthenaPipes(athenaParserLogger);
Debug.WriteLine($"****** End File {i} ******");
Debug.WriteLine($"****** Json File {i} ******");
var tree = JsonConvert.SerializeObject(pipes, Formatting.Indented);
Debug.WriteLine($"****** Parsed File {i} ******");
bool seeking = false;
context = new StateMachineQueryContext();
// context.ExecuteStateMachineQueryContext()
result = pipes.LoadNextStateMachineQuery(new AthenaParserSetting(), new LinkedList <int>(), context, ref seeking);
seeking = true;
result = pipes.LoadNextStateMachineQuery(new AthenaParserSetting(), new LinkedList <int>(), context, ref seeking);
Debug.WriteLine(pipes.ToQueryString().StripEmptyLines());
}
catch (Exception ex)
{
Debug.WriteLine(athenaParserLogger.ToString());
Debug.Write(ex.Message);
}
}
}
public static StateMachineQueryContext BuildStateMachineQueryContext(this EtlSettings etlSettings, DateTime?useDate = null)
{
if (etlSettings.SourceType != EtlSourceEnum.AmazonAthenaPipes)
{
return(null);
}
var pipesSource = etlSettings.AthenaQueryPipesSource;
AthenaParserSetting parserSetting = new AthenaParserSetting();
parserSetting.DefaultExportPath = $"s3://{etlSettings.TargetS3BucketName}/{etlSettings.TargetS3Prefix}".FixPathEnd();
parserSetting.DefaultTableName = $"`{etlSettings.AthenaDatabaseName}`.`{etlSettings.AthenaTableName}`";
parserSetting.Date = useDate == null?DateTime.UtcNow.AddDays(-pipesSource.DaysAgo) : useDate.Value.AddDays(-pipesSource.DaysAgo);
parserSetting.DateFormat = pipesSource.DateFormat;
parserSetting.TempDatabase = pipesSource.TempDatabase;
parserSetting.TempTablePath = pipesSource.TempDataPath.FixPathEnd();
var caches = JsonConvert.DeserializeObject <List <CacheSetting> >(etlSettings.AthenaQueryPipesSource.Caches);
foreach (var cache in caches)
{
if (!cache.S3Path.EndsWith("/"))
{
cache.S3Path += "/";
}
parserSetting.Caches.Add(cache.Key, cache);
}
StateMachineQueryContext context = new StateMachineQueryContext();
context.raw = pipesSource.AthenaSQL;
context.settings = new StateMachineSettings()
{
DefaultExportPath = parserSetting.DefaultExportPath,
DefaultTableName = parserSetting.DefaultTableName,
Date = parserSetting.Date,
DateFormat = parserSetting.DateFormat,
TempDatabase = parserSetting.TempDatabase,
TempTablePath = parserSetting.TempTablePath,
Caches = parserSetting.Caches.Values.ToList(),
Clearings = parserSetting.Clearings.Select(kvp => new KeyValueEntry()
{
Key = kvp.Key, Value = kvp.Value
}).ToList(),
Commands = parserSetting.Commands,
DroppingTables = parserSetting.DroppingTables,
Partitions = parserSetting.Partitions.Select(kvp => new KeyValueEntry()
{
Key = kvp.Key, Value = kvp.Value
}).ToList(),
Variables = parserSetting.Variables.Select(kvp => new KeyValueEntry()
{
Key = kvp.Key, Value = kvp.Value
}).ToList()
};
//var parsed = pipesSource.AthenaSQL.ParseAthenaPipes(parserSetting);
return(context);
//var athenaApi = etlSettings.CreatePipesSourceAthenaAPI();
//foreach (var kvp in parserSetting.Partitions)
//{
// await athenaApi.LoadPartitionIfNotExists(parserSetting.DefaultTableName, kvp.Key, kvp.Value);
//}
}
