private void ReadHeadersFromColumnInfo(ResultSet resultSet, ParsedAthenaResponse niceAthenaResult) { niceAthenaResult.columnOrder = new List <string>(); //Get the headers ordered foreach (var header in resultSet.ResultSetMetadata.ColumnInfo) { niceAthenaResult.columnOrder.Add(header.Name); } }
private void ProcessRow_NameCounts(ResultSet resultSet, ParsedAthenaResponse niceAthenaResult) { List <Row> rows = resultSet.Rows; //foreach (var row in resultSet.Rows) //{ for (int r = 0; r < resultSet.Rows.Count; r++) { if (r == 0) { continue; //Row 0 has headers. Ignore me } Dictionary <string, string> newRow = new Dictionary <string, string>(); for (int i = 0; i < niceAthenaResult.columnOrder.Count; i++) { newRow.Add(niceAthenaResult.columnOrder[i], resultSet.Rows[r].Data[i].VarCharValue); } niceAthenaResult.Events.Add(newRow); } }
internal async Task <ParsedAthenaResponse> PerformAthenaQuery(string theQuery) { ParsedAthenaResponse retVal = new ParsedAthenaResponse(); retVal.SqlQuery = theQuery; AmazonAthenaClient athenaClient = CreateAthenaClient(); Console.WriteLine("Submitting Query"); string queryId = await SubmitAthenaQuery(athenaClient, theQuery, retVal); Console.WriteLine(string.Format("QueryId: {0}", queryId)); Console.WriteLine("Waiting for Query to complete"); await WaitForQueryToComplete(athenaClient, queryId); Console.WriteLine("Query complete. Lets read the response!"); await ProcessResultRows(athenaClient, queryId, retVal); Console.WriteLine("Query complete. Response extracted!"); return(retVal); }
private async Task <string> SubmitAthenaQuery(AmazonAthenaClient client, string theQuery, ParsedAthenaResponse niceAthenaResult) { // The QueryExecutionContext allows us to set the Database. QueryExecutionContext queryExecutionContext = new QueryExecutionContext(); queryExecutionContext.Database = Functions.DatabaseName; // The result configuration specifies where the results of the query should go in S3 and encryption options ResultConfiguration resultConfiguration = new ResultConfiguration(); resultConfiguration.OutputLocation = Functions.QueryOutputLocation; // Create the StartQueryExecutionRequest to send to Athena which will start the query. StartQueryExecutionRequest startQueryExecutionRequest = new StartQueryExecutionRequest(); startQueryExecutionRequest.QueryString = theQuery; //Now reading this dynamically niceAthenaResult.columnOrder = new List <string>(); startQueryExecutionRequest.QueryExecutionContext = queryExecutionContext; startQueryExecutionRequest.ResultConfiguration = resultConfiguration; var startQueryExecutionResponse = await client.StartQueryExecutionAsync(startQueryExecutionRequest); return(startQueryExecutionResponse.QueryExecutionId); }
private async Task ProcessResultRows(AmazonAthenaClient client, string queryExecutionId, ParsedAthenaResponse niceAthenaResult) { GetQueryResultsRequest getQueryResultsRequest = new GetQueryResultsRequest(); // Max Results can be set but if its not set, // it will choose the maximum page size // As of the writing of this code, the maximum value is 1000 // .withMaxResults(1000) getQueryResultsRequest.QueryExecutionId = queryExecutionId; var getQueryResultsResponse = await client.GetQueryResultsAsync(getQueryResultsRequest); List <ColumnInfo> columnInfoList = getQueryResultsResponse.ResultSet.ResultSetMetadata.ColumnInfo; ResultSet responseData = getQueryResultsResponse.ResultSet; ReadHeadersFromColumnInfo(getQueryResultsResponse.ResultSet, niceAthenaResult); while (true) { //Convert the returned response to a serialisable JSON object ProcessRow_NameCounts(responseData, niceAthenaResult); // If the nextToken is null, there are no more pages to read. Break out of the loop. if (getQueryResultsResponse.NextToken == null) { break; } getQueryResultsResponse = await client.GetQueryResultsAsync( new GetQueryResultsRequest { NextToken = getQueryResultsResponse.NextToken }); Console.WriteLine("getting more data from response..."); } }