// Implementation of ReadPageAsync for an already-complete job.
private async Task <BigQueryPage> ReadPageAsyncImpl(int pageSize, CancellationToken cancellationToken)
{
GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions();
List <BigQueryRow> rows = new List <BigQueryRow>(pageSize);
if ((clonedOptions.PageSize == null || clonedOptions.PageSize > pageSize) && _response.Rows?.Count > pageSize)
{
// Oops. Do it again from scratch, with a useful page size.
// Still, we know it's already completed, so we can skip straight to the Impl method again.
clonedOptions.PageSize = pageSize;
var newQueryJob = await _client.GetQueryResultsAsync(JobReference, clonedOptions, cancellationToken).ConfigureAwait(false);
return(await newQueryJob.ReadPageAsyncImpl(pageSize, cancellationToken).ConfigureAwait(false));
}
// First add the rows from the first response which is part of the state
// of the object.
rows.AddRange(ResponseRows);
string pageToken = _response.PageToken;
clonedOptions.StartIndex = null;
// Now keep going until we've filled the result set or know there's no more data.
while (rows.Count > pageSize && pageToken != null)
{
clonedOptions.PageToken = pageToken;
clonedOptions.PageSize = pageSize - rows.Count;
var job = await _client.GetQueryResultsAsync(JobReference, clonedOptions, cancellationToken).ConfigureAwait(false);
rows.AddRange(job.ResponseRows);
pageToken = job._response.PageToken;
}
return(new BigQueryPage(rows, Schema, JobReference, pageToken));
}
// Implementation of ReadPage for an already-complete job.
private BigQueryPage ReadPageImpl(int pageSize)
{
if (!Completed)
{
throw new InvalidOperationException($"Cannot call {nameof(ReadPage)} on an incomplete job");
}
GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions();
List <BigQueryRow> rows = new List <BigQueryRow>(pageSize);
if ((clonedOptions.PageSize == null || clonedOptions.PageSize > pageSize) && _response.Rows?.Count > pageSize)
{
// Oops. Do it again from scratch, with a useful page size.
// Still, we know it's already completed, so we can skip straight to the Impl method again.
clonedOptions.PageSize = pageSize;
return(_client.GetQueryResults(JobReference, clonedOptions).ReadPageImpl(pageSize));
}
// First add the rows from the first response which is part of the state
// of the object.
rows.AddRange(ResponseRows);
string pageToken = _response.PageToken;
clonedOptions.StartIndex = null;
// Now keep going until we've filled the result set or know there's no more data.
while (rows.Count > pageSize && pageToken != null)
{
clonedOptions.PageToken = pageToken;
clonedOptions.PageSize = pageSize - rows.Count;
var job = _client.GetQueryResults(JobReference, clonedOptions);
rows.AddRange(job.ResponseRows);
pageToken = job._response.PageToken;
}
return(new BigQueryPage(rows, Schema, JobReference, pageToken));
}
/// <summary>
/// Asynchronously but eagerly fetches a set of rows, up to the specified count, providing a page of results with a next page token
/// if more results are available. This is typically used within web applications, where the next page token
/// is propagated to the client along with the results, so that the next page can be retrieved in another request.
/// </summary>
/// <param name="pageSize">The maximum number of rows to retrieve. Must be positive.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous operation. When complete, the result is
/// an in-memory result set of at most the given number of rows.</returns>
public async Task <BigQueryPage> ReadPageAsync(int pageSize, CancellationToken cancellationToken = default)
{
GaxPreconditions.CheckArgumentRange(pageSize, nameof(pageSize), 1, int.MaxValue);
GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions();
List <BigQueryRow> rows = new List <BigQueryRow>(pageSize);
// Work out whether to use the response we've already got, or create a new one.
GetQueryResultsResponse response = _response;
if (response.Rows?.Count > pageSize)
{
// Oops. Do it again from scratch, with a useful page size.
clonedOptions.PageSize = pageSize;
response = await _client.GetRawQueryResultsAsync(JobReference, clonedOptions, timeoutBase : null, cancellationToken).ConfigureAwait(false);
}
// First add the rows from the existing response.
rows.AddRange(ConvertResponseRows(response));
string pageToken = response.PageToken;
clonedOptions.StartIndex = null;
// Now keep going until we've filled the result set or know there's no more data.
while (rows.Count < pageSize && pageToken != null)
{
clonedOptions.PageToken = pageToken;
clonedOptions.PageSize = pageSize - rows.Count;
var nextResponse = await _client.GetRawQueryResultsAsync(JobReference, clonedOptions, timeoutBase : null, cancellationToken).ConfigureAwait(false);
rows.AddRange(ConvertResponseRows(nextResponse));
pageToken = nextResponse.PageToken;
}
return(new BigQueryPage(rows, Schema, JobReference, TableReference, pageToken));
}
/// <inheritdoc />
public override BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var job = GetJob(jobReference);
return(job.GetQueryResults(options));
}
/// <summary>
/// Constructs a new set of results.
/// </summary>
/// <remarks>
/// This is public to allow tests to construct instances for production code to consume;
/// production code should not normally construct instances itself.
/// </remarks>
/// <param name="client">The client to use for further operations. Must not be null.</param>
/// <param name="response">The response to a GetQueryResults API call. Must not be null.</param>
/// <param name="tableReference">A reference to the table containing the results.
/// May be null. (For example, script queries don't store results in tables.)</param>
/// <param name="options">Options to use when fetching results. May be null.</param>
public BigQueryResults(BigQueryClient client, GetQueryResultsResponse response, TableReference tableReference, GetQueryResultsOptions options)
{
_client = GaxPreconditions.CheckNotNull(client, nameof(client));
_response = GaxPreconditions.CheckNotNull(response, nameof(response));
TableReference = tableReference;
_options = options;
_fieldNames = response.Schema?.IndexFieldNames();
}
public AsyncRowEnumerator(BigQueryResults parent)
{
_parent = parent;
_options = parent._options?.Clone() ?? new GetQueryResultsOptions();
_options.StartIndex = null;
_options.PageToken = parent._response.PageToken;
_underlyingEnumerator = parent.ResponseRows.GetEnumerator();
}
public AsyncRowEnumerator(BigQueryResults job, PollSettings pollSettings)
{
_job = job;
_pollSettings = pollSettings;
_options = _job._options?.Clone() ?? new GetQueryResultsOptions();
_options.StartIndex = null;
_options.PageToken = _job._response.PageToken;
}
/// <inheritdoc />
public override BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
options?.ModifyRequest(request);
var firstResponse = request.Execute();
return(new BigQueryResults(this, firstResponse, options));
}
// Implementation of GetRows for a completed job.
private IEnumerable <BigQueryRow> GetRowsImpl()
{
foreach (var row in ResponseRows)
{
yield return(row);
}
GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions();
clonedOptions.StartIndex = null;
clonedOptions.PageToken = _response.PageToken;
while (clonedOptions.PageToken != null)
{
var job = _client.GetQueryResults(JobReference, clonedOptions);
foreach (var row in job.ResponseRows)
{
yield return(row);
}
clonedOptions.PageToken = job._response.PageToken;
}
}
/// <summary>
/// Returns an iterator over the query results.
/// </summary>
/// <returns>An iterator over the query results.</returns>
public IEnumerator <BigQueryRow> GetEnumerator()
{
foreach (var row in ResponseRows)
{
yield return(row);
}
GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions();
clonedOptions.StartIndex = null;
clonedOptions.PageToken = _response.PageToken;
while (clonedOptions.PageToken != null)
{
var response = _client.GetRawQueryResults(JobReference, clonedOptions, timeoutBase: null);
foreach (var row in ConvertResponseRows(response))
{
yield return(row);
}
clonedOptions.PageToken = response.PageToken;
}
}
internal override Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken)
{
var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc());
return(request.ExecuteAsync(cancellationToken));
}
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
DateTime start = Clock.GetCurrentDateTimeUtc();
while (true)
{
// This will throw if the query has timed out. Otherwise, the RPC will have a timeout,
// and the server won't return an incomplete job before that timeout. So although this
// looks like a tight loop without any delay, the delay is on the server side.
var response = GetRawQueryResults(jobReference, options, start);
if (response.JobComplete == true)
{
return(new BigQueryResults(this, response, tableReference, options));
}
}
}
/// <summary>
/// Asynchronously executes a query.
/// </summary>
/// <remarks>
/// <para>
/// The task returned by this method will only complete when the query has completed.
/// This method simply delegates to <see cref="CreateQueryJobAsync(string, IEnumerable{BigQueryParameter}, QueryOptions, CancellationToken)"/>
/// and then <see cref="BigQueryJob.GetQueryResultsAsync(GetQueryResultsOptions, CancellationToken)"/>.
/// </para>
/// </remarks>
/// <param name="sql">The SQL query. Must not be null.</param>
/// <param name="parameters">The parameters for the query. May be null, which is equivalent to specifying an empty list of parameters. Must not contain null elements.</param>
/// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param>
/// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous operation. When complete, the result is
/// the <see cref="BigQueryResults"/> representing the query.</returns>
public virtual async Task <BigQueryResults> ExecuteQueryAsync(string sql, IEnumerable <BigQueryParameter> parameters, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null, CancellationToken cancellationToken = default)
{
var job = await CreateQueryJobAsync(sql, parameters, queryOptions, cancellationToken).ConfigureAwait(false);
return(await job.GetQueryResultsAsync(resultsOptions, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Executes a query.
/// </summary>
/// <remarks>
/// <para>
/// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(string, IEnumerable{BigQueryParameter}, QueryOptions)"/>
/// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>.
/// </para>
/// </remarks>
/// <param name="sql">The SQL query. Must not be null.</param>
/// <param name="parameters">The parameters for the query. May be null, which is equivalent to specifying an empty list of parameters. Must not contain null elements.</param>
/// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param>
/// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param>
/// <returns>The result of the query.</returns>
public virtual BigQueryResults ExecuteQuery(string sql, IEnumerable <BigQueryParameter> parameters, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) =>
CreateQueryJob(sql, parameters, queryOptions).GetQueryResults(resultsOptions);
internal virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary>
/// Asynchronously executes a command. This overload allows query parameterization, and is preferred over
/// <see cref="ExecuteQueryAsync(string, QueryOptions, GetQueryResultsOptions, CancellationToken)"/> when values need to be passed in.
/// </summary>
/// <remarks>
/// <para>
/// The task returned by this method will only complete when the query has completed.
/// This method simply delegates to <see cref="CreateQueryJobAsync(BigQueryCommand, QueryOptions, CancellationToken)"/>
/// and then <see cref="BigQueryJob.GetQueryResultsAsync(GetQueryResultsOptions, CancellationToken)"/>.
/// </para>
/// </remarks>
/// <param name="command">The command to execute. Must not be null.</param>
/// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param>
/// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous operation. When complete, the result is
/// the <see cref="BigQueryResults"/> representing the query.</returns>
public virtual async Task <BigQueryResults> ExecuteQueryAsync(BigQueryCommand command, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null, CancellationToken cancellationToken = default(CancellationToken))
{
var job = await CreateQueryJobAsync(command, queryOptions, cancellationToken).ConfigureAwait(false);
return(await job.GetQueryResultsAsync(resultsOptions, cancellationToken).ConfigureAwait(false));
}
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
{
var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart;
var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;
if (timeRemainingMs < 1)
{
// TODO: Check this is correct
throw new TimeoutException("Query timed out");
}
var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
request.Location = jobReference.Location;
request.TimeoutMs = requestTimeoutMs;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
/// <summary> /// "Raw" version of GetQueryResults, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <returns>The results of the query.</returns> internal virtual GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase) => throw new NotImplementedException();
internal override GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase)
{
var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc());
return(request.Execute());
}
/// <summary> /// Retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The results of the query.</returns> public virtual BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) => throw new NotImplementedException();
/// <summary> /// Retrieves the results of the specified job, which must be a query job. /// This method just creates a <see cref="JobReference"/> and delegates to <see cref="GetQueryResults(JobReference, GetQueryResultsOptions)"/>. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="projectId">The project ID. Must not be null.</param> /// <param name="jobId">The job ID. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The results of the query.</returns> public virtual BigQueryResults GetQueryResults(string projectId, string jobId, GetQueryResultsOptions options = null) => GetQueryResults(GetJobReference(projectId, jobId), options);
/// <summary> /// Executes a command. This overload allows query parameterization, and is preferred over /// <see cref="ExecuteQuery(string, QueryOptions, GetQueryResultsOptions)"/> when values need to be passed in. /// </summary> /// <remarks> /// <para> /// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(BigQueryCommand, QueryOptions)"/> /// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>. /// </para> /// </remarks> /// <param name="command">The command to execute. Must not be null.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <returns>The result of the query.</returns> public virtual BigQueryResults ExecuteQuery(BigQueryCommand command, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) => CreateQueryJob(command, queryOptions).GetQueryResults(resultsOptions);
/// <summary> /// Executes a query. /// </summary> /// <remarks> /// <para> /// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(string, QueryOptions)"/> /// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>. /// </para> /// </remarks> /// <param name="sql">The SQL query. Must not be null.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <returns>The result of the query.</returns> public virtual BigQueryResults ExecuteQuery(string sql, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) => CreateQueryJob(sql, queryOptions).GetQueryResults(resultsOptions);
/// <inheritdoc />
public override async Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var job = await GetJobAsync(jobReference, cancellationToken : cancellationToken).ConfigureAwait(false);
return(await job.GetQueryResultsAsync(options, cancellationToken).ConfigureAwait(false));
}
internal override async Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
DateTime start = Clock.GetCurrentDateTimeUtc();
while (true)
{
// This will throw if the query has timed out.
var response = await GetRawQueryResultsAsync(jobReference, options, start, cancellationToken).ConfigureAwait(false);
if (response.JobComplete == true)
{
return(new BigQueryResults(this, response, tableReference, options));
}
}
}
/// <summary> /// "Raw" version of GetQueryResultsAsync, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>The results of the query.</returns> internal virtual Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary> /// Asynchronously retrieves the results of the specified job, which must be a query job. /// This method just creates a <see cref="JobReference"/> and delegates to <see cref="GetQueryResultsAsync(JobReference, GetQueryResultsOptions, CancellationToken)"/>. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="projectId">The project ID. Must not be null.</param> /// <param name="jobId">The job ID. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the results of the query.</returns> public virtual Task <BigQueryResults> GetQueryResultsAsync(string projectId, string jobId, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) => GetQueryResultsAsync(GetJobReference(projectId, jobId), options, cancellationToken);
// Note - these methods are not part of the regular "pattern", so are not in the GetQueryResults region above. // We want to remove them, if the underlying GetQueryResultsResponse starts including the table reference. // These methods allow us to call GetQueryResultsAsync from BigQueryJob without fetching the job again. internal virtual BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) => throw new NotImplementedException();
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
DateTime start = Clock.GetCurrentDateTimeUtc();
while (true)
{
// This will throw if the query has timed out.
var response = GetRawQueryResults(jobReference, options, start);
if (response.JobComplete == true)
{
return(new BigQueryResults(this, response, tableReference, options));
}
}
}
/// <summary> /// Asynchronously retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the results of the query.</returns> public virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();