public static List <Dictionary <String, String> > runQuery(String StrQuery)
{
var client = new AmazonAthenaClient("AKIAWR5GGX5MW7QKSAWV", "jEdL+v3zQa8oTg/3Pkx0nd9+y3j5ZUP9AULNRW/H", Amazon.RegionEndpoint.APSouth1);
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
List <Dictionary <String, String> > items = new List <Dictionary <String, String> >();
try
{
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = StrQuery,
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
/* Executes the query in an async manner */
StartQueryExecutionResponse qRes = client.StartQueryExecution(qReq); // await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
items = getQueryExecution(client, qRes.QueryExecutionId); //await getQueryExecution(client, qRes.QueryExecutionId);
}
catch (InvalidRequestException e)
{
Console.WriteLine("Run Error: {0}", e.Message);
}
return(items);
}
/**
* Submits a sample query to Athena and returns the execution ID of the query.
*/
private static String submitAthenaQuery(AmazonAthenaClient athenaClient)
{
// The QueryExecutionContext allows us to set the Database.
var queryExecutionContext = new QueryExecutionContext()
{
Database = ExampleConstants.ATHENA_DEFAULT_DATABASE
};
// The result configuration specifies where the results of the query should go in S3 and encryption options
var resultConfiguration = new ResultConfiguration()
{
// You can provide encryption options for the output that is written.
// EncryptionConfiguration = encryptionConfiguration
OutputLocation = ExampleConstants.ATHENA_OUTPUT_BUCKET
};
// Create the StartQueryExecutionRequest to send to Athena which will start the query.
var startQueryExecutionRequest = new StartQueryExecutionRequest()
{
QueryString = ExampleConstants.ATHENA_SAMPLE_QUERY,
QueryExecutionContext = queryExecutionContext,
ResultConfiguration = resultConfiguration
};
var startQueryExecutionResponse = athenaClient.StartQueryExecution(startQueryExecutionRequest);
return(startQueryExecutionResponse.QueryExecutionId);
}
private async Task <string> SubmitAthenaQuery(AmazonAthenaClient client, string theQuery, ParsedAthenaResponse niceAthenaResult)
{
// The QueryExecutionContext allows us to set the Database.
QueryExecutionContext queryExecutionContext = new QueryExecutionContext();
queryExecutionContext.Database = Functions.DatabaseName;
// The result configuration specifies where the results of the query should go in S3 and encryption options
ResultConfiguration resultConfiguration = new ResultConfiguration();
resultConfiguration.OutputLocation = Functions.QueryOutputLocation;
// Create the StartQueryExecutionRequest to send to Athena which will start the query.
StartQueryExecutionRequest startQueryExecutionRequest = new StartQueryExecutionRequest();
startQueryExecutionRequest.QueryString = theQuery;
//Now reading this dynamically
niceAthenaResult.columnOrder = new List <string>();
startQueryExecutionRequest.QueryExecutionContext = queryExecutionContext;
startQueryExecutionRequest.ResultConfiguration = resultConfiguration;
var startQueryExecutionResponse = await client.StartQueryExecutionAsync(startQueryExecutionRequest);
return(startQueryExecutionResponse.QueryExecutionId);
}
async static Task run(IAmazonAthena client, QueryExecutionContext qContext, ResultConfiguration resConf, string searchkey)
{
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = "SELECT * FROM book where bookid='" + searchkey + "'",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
try
{
/* Executes the query in an async manner */
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
List <Dictionary <String, String> > items = await getQueryExecution(client, qRes.QueryExecutionId);
foreach (var item in items)
{
foreach (KeyValuePair <String, String> pair in item)
{
Console.WriteLine("Col: {0}", pair.Key);
Console.WriteLine("Val: {0}", pair.Value);
}
}
}
catch (InvalidRequestException e)
{
Console.WriteLine("Run Error: {0}", e.Message);
}
}
public void ExecuteQuery(string sql)
{
var awsCredentials = new Amazon.Runtime.BasicAWSCredentials("A", "B");
var c = new AmazonAthenaClient(awsCredentials);
QueryExecutionContext queryExecutionContext = new QueryExecutionContext();
queryExecutionContext.Database = ATHENA_DEFAULT_DATABASE;
// The result configuration specifies where the results of the query should go in S3 and encryption options
ResultConfiguration resultConfiguration = new ResultConfiguration();
// You can provide encryption options for the output that is written.
// .withEncryptionConfiguration(encryptionConfiguration)
resultConfiguration.OutputLocation = ATHENA_OUTPUT_BUCKET;
// Create the StartQueryExecutionRequest to send to Athena which will start the query.
StartQueryExecutionRequest startQueryExecutionRequest = new StartQueryExecutionRequest();
startQueryExecutionRequest.QueryString = sql;
startQueryExecutionRequest.QueryExecutionContext = queryExecutionContext;
startQueryExecutionRequest.ResultConfiguration = resultConfiguration;
var startQueryExecutionResponse = c.StartQueryExecutionAsync(startQueryExecutionRequest);
//Console.WriteLine($"Query ID {startQueryExecutionResponse.QueryExecutionId}");
// return startQueryExecutionResponse.QueryExecutionId();
}
public void GivenNoPollingFrequencyInConfiguration_WhenAskingForValue_ThenItShouldThrow()
{
// arrange
IResultConfiguration resultConfiguration = new ResultConfiguration();
// act
Action action = () => resultConfiguration.PollingFrequency();
// assert
action.Should().Throw <ConfigurationItemNotFoundException>();
}
public void GivenNoResultTimeoutInConfiguration_WhenAskingForValue_ThenItShouldThrow()
{
// arrange
IResultConfiguration resultConfiguration = new ResultConfiguration();
// act
Action action = () => resultConfiguration.UniqueResultTimeout();
// assert
action.Should().Throw <ConfigurationItemNotFoundException>();
}
private async Task <List <Dictionary <String, String> > > queryAthena(String date)
{
using (var athenaClient = new AmazonAthenaClient(Amazon.RegionEndpoint.USEast2)) {
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
List <Dictionary <String, String> > items = await runQuery(athenaClient, qContext, resConf, date);
return(items);
}
}
public void GivenResultTimeoutNotParseable_WhenAskingForValue_ThenItShouldThrow()
{
// arrange
Environment.SetEnvironmentVariable(new UniqueResultTimeoutMillisecondsKey(), "Dammit, Bobby!");
IResultConfiguration resultConfiguration = new ResultConfiguration();
// act
Action action = () => resultConfiguration.UniqueResultTimeout();
// assert
action.Should().Throw <ConfigurationItemParsingException>();
}
public void GivenResultTimeout_WhenAskingForValue_ThenItShouldReturnCorrectValue()
{
// arrange
Environment.SetEnvironmentVariable(new UniqueResultTimeoutMillisecondsKey(), "123");
IResultConfiguration resultConfiguration = new ResultConfiguration();
// act
TimeSpan actual = resultConfiguration.UniqueResultTimeout();
// assert
actual.TotalMilliseconds.Should().Be(123);
}
public void GivenPollingFrequency_WhenAskingForValue_ThenItShouldReturnCorrectValue()
{
// arrange
Environment.SetEnvironmentVariable(new UniqueResultPollingFrequencyKey(), "234");
IResultConfiguration resultConfiguration = new ResultConfiguration();
// act
TimeSpan actual = resultConfiguration.PollingFrequency();
// assert
actual.TotalMilliseconds.Should().Be(234);
}
static void Main(string[] args)
{
using (var client = new AmazonAthenaClient(Amazon.RegionEndpoint.USEast1)) {
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
Console.WriteLine("Created Athena Client");
run(client, qContext, resConf).Wait();
}
}
public static async Task <JRaw> QueryAthenaAndSend(string datestring)
{
using (var client = new AmazonAthenaClient(Amazon.RegionEndpoint.USEast2))
{
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
Console.WriteLine("Created Athena Client");
List <Dictionary <String, String> > items = await Run(client, qContext, resConf, datestring);
if (items.Count == 1 && items[0].ContainsKey("error"))
{
items[0].TryGetValue("error", out string errorinfo);
return(new JRaw(errorinfo));
}
// return items.ToString();
JObject obj = new JObject();
if (items.Count == 0)
{
obj.Add("count", "zero");
return(obj.ToObject <JRaw>());
}
obj.Add("count", items.Count);
for (int i = 1; i < items.Count; i++)
{
JProperty nameProp = null;
JProperty idProp = null;
foreach (KeyValuePair <String, String> pair in items[i])
{
if (pair.Key == "emp_name")
{
nameProp = new JProperty("name", pair.Value);
}
if (pair.Key == "emp_id")
{
idProp = new JProperty("id", pair.Value);
}
}
if (nameProp != null && idProp != null)
{
obj.Add("emp " + i, new JObject(nameProp, idProp));
}
else
{
return(null);
}
}
return(obj.ToObject <JRaw>());
}
}
public IResultConfiguration Clone()
{
var resultConfiguration = new ResultConfiguration();
resultConfiguration.Folder = this.Folder;
if (this.FileName != null)
{
resultConfiguration.FileName = this.FileName.Clone();
}
return(resultConfiguration);
}
public JMResult HandleException(Exception e)
{
DefaultLogger.Error(e);
string code = "System";
if (e is JMException)
{
code = (e as JMException).Code;
}
JMResult result = null;
ResultConfiguration resultConfiguration = resultConfigurationsStaticDataProvider.GetResultConfiguration(code);
if (resultConfiguration != null)
{
result = new JMResult
{
RedirectionInfo = new JMRedirectionInfo
{
RedirectionParameter = resultConfiguration.RedirectionParameter,
RedirectionType = resultConfiguration.RedirectionType
},
Errors = new JMResultItem[]
{
new JMResultItem
{
Code = code,
Message = resultConfiguration.ResultMessages.FirstOrDefault(e => e.Culture == CultureInfo.CurrentCulture.Name)?.Content
}
}
};
}
else
{
result = new JMResult
{
Errors = new JMResultItem[]
{
new JMResultItem
{
Code = code,
Message = e.Message
}
}
};
}
return(result);
}
public async Task <string> FunctionHandler(ILambdaContext context)
{
string strResult = "";
using (var client = new AmazonAthenaClient(Amazon.RegionEndpoint.APSouth1))
{
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
strResult = await run(client, qContext, resConf);
}
return(strResult);
}
public static async Task AWSAthenaSearchByDescription(string searchkey)
{
using (var client = new AmazonAthenaClient(AWS_ACCESS_KEY, AWS_SECRET_KEY, Amazon.RegionEndpoint.USEast2))
{
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = "bookdb";
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = "s3://test-bucket-kalam/";
Console.WriteLine("Created Athena Client");
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = "SELECT * FROM book where bookdescription like '%" + searchkey + "%'",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
try
{
/* Executes the query in an async manner */
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
List <Dictionary <String, String> > items = await getQueryExecution(client, qRes.QueryExecutionId);
foreach (var item in items)
{
foreach (KeyValuePair <String, String> pair in item)
{
Console.WriteLine("Col: {0}", pair.Key);
Console.WriteLine("Val: {0}", pair.Value);
}
}
}
catch (InvalidRequestException e)
{
Console.WriteLine("Run Error: {0}", e.Message);
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Starting Owin Normalizing Service");
if (_deploymentSettings.IsFirstDeploy)
{
_logger.LogInformation("Fetching AWS Security Credentials");
var securityCredentials = await _securityProvider.GetTemporaryCredentialsAsync();
_logger.LogInformation($"Security Credientials fetched correctly - Temp Token = {securityCredentials.GetCredentials().Token} ");
using (var athenaClient = new AmazonAthenaClient(securityCredentials,
Amazon.RegionEndpoint.EUWest1))
{
var queryContext = new QueryExecutionContext
{
Catalog = AthenaConstants.ATHENA_CATALOG,
Database = AthenaConstants.ATHENA_DEFAULT_DATABASE
};
var ressultConfig = new ResultConfiguration();
ressultConfig.OutputLocation = AthenaConstants.ATHENA_OUTPUT_BUCKET;
_logger.LogInformation("Created Athena Client");
var listOfCountryData = await _athenaDataService.Run(athenaClient, queryContext, ressultConfig);
foreach (var countryEnrichedData in listOfCountryData)
{
_logger.LogInformation($"CountryCode: {countryEnrichedData.IsoCode}, Population: {countryEnrichedData.Population}, Median Age: {countryEnrichedData.MedianAge}, GPD Per Capita: {countryEnrichedData.GdpPerCapita}");
}
List <CountryEnrichedData> list = new List <CountryEnrichedData>();
await _athenaUpdateService.MapAndUpdate(listOfCountryData);
}
}
_logger.LogInformation($"Data is currently up to date, shutting down Owin Normalizer");
}
async static Task <List <Dictionary <String, String> > > runQuery(IAmazonAthena client, QueryExecutionContext qContext, ResultConfiguration resConf, String dateToQuery)
{
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = $"SELECT * FROM employee where day(employee.emp_dob) = {dateToQuery}",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
try {
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
List <Dictionary <String, String> > items = await getQueryExecution(client, qRes.QueryExecutionId);
return(items);
}
catch (InvalidRequestException e) {
LambdaLogger.Log($"Run Error: {e.Message}");
return(new List <Dictionary <String, String> >());
}
}
public APIGatewayProxyResponse Get(APIGatewayProxyRequest request, ILambdaContext context)
{
context.Logger.LogLine("Get Request\n");
using (var client = new AmazonAthenaClient(Amazon.RegionEndpoint.USEast1))
{
String date = JsonConvert.DeserializeObject <String>(request?.Body);
QueryExecutionContext qContext = new QueryExecutionContext();
qContext.Database = ATHENA_DB;
ResultConfiguration resConf = new ResultConfiguration();
resConf.OutputLocation = ATHENA_TEMP_PATH;
Console.WriteLine("Created Athena Client");
run(client, qContext, resConf, date).Wait();
}
async Task run(IAmazonAthena client, QueryExecutionContext qContext, ResultConfiguration resConf, String date)
{
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = "SELECT emp_name FROM emp_table WHERE date = " + date + "limit 10;",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
try
{
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
List <Dictionary <String, String> > items = await getQueryExecution(client, qRes.QueryExecutionId);
foreach (var item in items)
{
foreach (KeyValuePair <String, String> pair in item)
{
Console.WriteLine("Col: {0}", pair.Key);
Console.WriteLine("Val: {0}", pair.Value);
}
}
}
catch (InvalidRequestException e)
{
Console.WriteLine("Run Error: {0}", e.Message);
}
}
async Task <List <Dictionary <String, String> > > getQueryExecution(IAmazonAthena client, String id)
{
List <Dictionary <String, String> > items = new List <Dictionary <String, String> >();
GetQueryExecutionResponse results = null;
QueryExecution q = null;
GetQueryExecutionRequest qReq = new GetQueryExecutionRequest()
{
QueryExecutionId = id
};
do
{
try
{
results = await client.GetQueryExecutionAsync(qReq);
q = results.QueryExecution;
Console.WriteLine("Status: {0}... {1}", q.Status.State, q.Status.StateChangeReason);
await Task.Delay(5000);
}
catch (InvalidRequestException e)
{
Console.WriteLine("GetQueryExec Error: {0}", e.Message);
}
} while (q.Status.State == "RUNNING" || q.Status.State == "QUEUED");
Console.WriteLine("Data Scanned for {0}: {1} Bytes", id, q.Statistics.DataScannedInBytes);
GetQueryResultsRequest resReq = new GetQueryResultsRequest()
{
QueryExecutionId = id,
MaxResults = 10
};
GetQueryResultsResponse resResp = null;
do
{
resResp = await client.GetQueryResultsAsync(resReq);
foreach (Row row in resResp.ResultSet.Rows)
{
Dictionary <String, String> dict = new Dictionary <String, String>();
for (var i = 0; i < resResp.ResultSet.ResultSetMetadata.ColumnInfo.Count; i++)
{
dict.Add(resResp.ResultSet.ResultSetMetadata.ColumnInfo[i].Name, row.Data[i].VarCharValue);
}
items.Add(dict);
}
if (resResp.NextToken != null)
{
resReq.NextToken = resResp.NextToken;
}
} while (resResp.NextToken != null);
return(items);
}
var response = new APIGatewayProxyResponse
{
StatusCode = (int)HttpStatusCode.OK,
Body = "GET Method Executed",
Headers = new Dictionary <string,
string> {
{ "Content-Type", "application/json" }
}
};
return(response);
}
async static Task <List <Dictionary <String, String> > > Run(IAmazonAthena client, QueryExecutionContext qContext, ResultConfiguration resConf, string datestring)
{
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = $@"SELECT * FROM emptable where emp_dob = '{datestring}'",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
List <Dictionary <String, String> > items = null;
try
{
/* Executes the query in an async manner */
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
items = await getQueryExecution(client, qRes.QueryExecutionId);
if (items == null)
{
Dictionary <string, string> dic1 = new Dictionary <string, string>();
dic1.Add("error", "items from query execution is empty");
items.Add(dic1);
}
}
catch (InvalidRequestException e)
{
Dictionary <string, string> dic1 = new Dictionary <string, string>();
dic1.Add("error", "exception at " + " (Run method) " + " : " + e.Message);
if (items == null)
{
items = new List <Dictionary <string, string> >();
}
items.Add(dic1);
Console.WriteLine("Run Error: {0}", e.Message);
}
return(items);
}
async static Task <string> run(IAmazonAthena client, QueryExecutionContext qContext, ResultConfiguration resConf)
{
/* Execute a simple query on a table */
StartQueryExecutionRequest qReq = new StartQueryExecutionRequest()
{
QueryString = "SELECT * FROM tbl_logger7;",
QueryExecutionContext = qContext,
ResultConfiguration = resConf
};
try
{
/* Executes the query in an async manner */
StartQueryExecutionResponse qRes = await client.StartQueryExecutionAsync(qReq);
/* Call internal method to parse the results and return a list of key/value dictionaries */
List <Dictionary <String, String> > items = await getQueryExecution(client, qRes.QueryExecutionId);
string json = JsonConvert.SerializeObject(items);
return(json);
}
catch (InvalidRequestException e)
{
return("{\"Error\":\"" + e.Message + "\"}");
}
}
public string ExecuteQuery(string query, ref QueryStageStatistics queryStats)
{
#region Setup
QueryExecutionContext queryExecContext = new QueryExecutionContext()
{
Database = this.DataBase
};
ResultConfiguration resultConfig = new ResultConfiguration()
{
OutputLocation = this.OutputLocation
};
StartQueryExecutionRequest startQueryExecRequest = new StartQueryExecutionRequest()
{
QueryExecutionContext = queryExecContext,
QueryString = query,
ResultConfiguration = resultConfig
};
#endregion
var watch = new Stopwatch();
#region Query Execution
//Start Query execution and wait till start command has completed
var startQueryExecResult = _athenaClient.StartQueryExecutionAsync(startQueryExecRequest);
//Start measurement
watch.Start();
while (!startQueryExecResult.IsCompleted)
{
Thread.Sleep(100);
Console.Write($"\r START {watch.Elapsed}");
}
if (startQueryExecResult.Exception != null)
{
throw new Exception("Query Execution", startQueryExecResult.Exception);
}
if (startQueryExecResult.IsCanceled)
{
return("- Cancelled -");
}
//Query if query execution has finished
GetQueryExecutionRequest getQueryExecRequest = new GetQueryExecutionRequest()
{
QueryExecutionId = startQueryExecResult.Result.QueryExecutionId
};
Task <Amazon.Athena.Model.GetQueryExecutionResponse> getQueryExecResult = null;
bool isQueryRunning = true;
while (isQueryRunning)
{
getQueryExecResult = _athenaClient.GetQueryExecutionAsync(getQueryExecRequest);
var state = getQueryExecResult.Result.QueryExecution.Status.State;
if (state == Amazon.Athena.QueryExecutionState.FAILED)
{
throw new Exception("Query Execution", getQueryExecResult.Exception);
}
else if (state == Amazon.Athena.QueryExecutionState.CANCELLED)
{
return("- Canceled -");
}
else if (state == Amazon.Athena.QueryExecutionState.SUCCEEDED)
{
isQueryRunning = false;
queryStats.QueryExecutionTime = watch.Elapsed;
}
else
{
Thread.Sleep(100);
Console.Write($"\r EXEC {watch.Elapsed}");
}
}
queryStats.EngineExecutionTimeInMillis = getQueryExecResult.Result.QueryExecution.Statistics.EngineExecutionTimeInMillis;
queryStats.DataScannedInBytes = getQueryExecResult.Result.QueryExecution.Statistics.DataScannedInBytes;
#endregion
watch.Restart();
#region Get Query Result
GetQueryResultsRequest getQueryResultRequest = new GetQueryResultsRequest()
{
QueryExecutionId = getQueryExecResult.Result.QueryExecution.QueryExecutionId
};
var getQueryResultsResult = _athenaClient.GetQueryResultsAsync(getQueryResultRequest);
//No data process is taken account. Only take meassurements.
long contentSize = 0, totalRows = 0;
while (true)
{
while (!getQueryResultsResult.IsCompleted)
{
Thread.Sleep(100);
Console.Write($"\r FETCH {watch.Elapsed}");
}
if (getQueryResultsResult.Exception != null)
{
throw new Exception("Retrieving Query Rows", getQueryResultsResult.Exception);
}
if (getQueryResultsResult.IsCanceled)
{
return("- Canceled -");
}
//while(getQueryResultsResult.Status == System.Threading.Tasks.TaskStatus.Running)
//{
// Thread.Sleep(100);
// Console.Write(".");
//}
totalRows += getQueryResultsResult.Result.ResultSet.Rows.Count;
if (getQueryResultsResult.Result.NextToken == null || totalRows >= SampleContext.MAX_FETCHED_RECORDS)
{
queryStats.DataFetchingTime = watch.Elapsed;
contentSize += getQueryResultsResult.Result.ContentLength;
break;
}
getQueryResultsResult = _athenaClient.GetQueryResultsAsync(new GetQueryResultsRequest()
{
QueryExecutionId = getQueryResultRequest.QueryExecutionId,
NextToken = getQueryResultRequest.NextToken
}
);
}
#endregion
return($"{contentSize} bytes took {queryStats.QueryExecutionTime} executing query and {queryStats.DataFetchingTime} fetching first {SampleContext.MAX_FETCHED_RECORDS} records.");
}
public async Task <List <CountryEnrichedData> > Run(IAmazonAthena client, QueryExecutionContext queryContext, ResultConfiguration resultConfig)
{
/* Execute a simple query on a table */
var queryRequest = new StartQueryExecutionRequest()
{
QueryString = AthenaConstants.ATHENA_COUNTRY_DATA_QUERY,
QueryExecutionContext = queryContext,
ResultConfiguration = resultConfig
};
try
{
var ListOfCountryEnrichedData = new List <CountryEnrichedData>();
/* Executes the query in an async manner */
var queryResult = await client.StartQueryExecutionAsync(queryRequest);
/* Call internal method to parse the results and return a list of key/value dictionaries */
List <Dictionary <String, String> > items = await GetQueryExecution(client, queryResult.QueryExecutionId);
foreach (var item in items)
{
var countryUpdate = new CountryEnrichedData();
foreach (KeyValuePair <String, String> pair in item)
{
switch (pair.Key)
{
case "iso_code":
if (pair.Value == "iso_code")
{
break;
}
else
{
countryUpdate.IsoCode = pair.Value;
}
break;
case "population":
if (pair.Value == "population")
{
break;
}
else
{
countryUpdate.Population = Convert.ToDouble(pair.Value);
}
break;
case "population_density":
if (pair.Value == "population_density")
{
break;
}
else
{
countryUpdate.PopulationDensity = Convert.ToDouble(pair.Value);
}
break;
case "median_age":
if (pair.Value == "median_age")
{
break;
}
else
{
countryUpdate.MedianAge = Convert.ToDouble(pair.Value);
}
break;
case "aged_65_older":
if (pair.Value == "aged_65_older")
{
break;
}
else
{
countryUpdate.Aged65Older = Convert.ToDouble(pair.Value);
}
break;
case "aged_70_older":
if (pair.Value == "aged_70_older")
{
break;
}
else
{
countryUpdate.Aged70Older = Convert.ToDouble(pair.Value);
}
break;
case "gdp_per_capita":
if (pair.Value == "gdp_per_capita")
{
break;
}
else
{
countryUpdate.GdpPerCapita = Convert.ToDouble(pair.Value);
}
break;
case "diabetes_prevalence":
if (pair.Value == "diabetes_prevalence")
{
break;
}
else
{
countryUpdate.DiabetesPrevalence = Convert.ToDouble(pair.Value);
}
break;
case "handwashing_facilities":
if (pair.Value == "handwashing_facilities")
{
break;
}
else
{
countryUpdate.HandwashingFacilities = Convert.ToDouble(pair.Value);
}
break;
case "hospital_beds_per_thousand":
if (pair.Value == "hospital_beds_per_thousand")
{
break;
}
else
{
countryUpdate.HospitalBedsPerThousand = Convert.ToDouble(pair.Value);
}
break;
case "life_expectancy":
if (pair.Value == "life_expectancy")
{
break;
}
else
{
countryUpdate.LifeExpectancy = Convert.ToDouble(pair.Value);
}
break;
}
}
ListOfCountryEnrichedData.Add(countryUpdate);
}
return(ListOfCountryEnrichedData);
}
catch (InvalidRequestException e)
{
_logger.LogInformation("Run Error: {0}", e.Message);
}
return(new List <CountryEnrichedData>());
}
