// Implementation of ReadPageAsync for an already-complete job. private async Task <BigQueryPage> ReadPageAsyncImpl(int pageSize, CancellationToken cancellationToken) { GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions(); List <BigQueryRow> rows = new List <BigQueryRow>(pageSize); if ((clonedOptions.PageSize == null || clonedOptions.PageSize > pageSize) && _response.Rows?.Count > pageSize) { // Oops. Do it again from scratch, with a useful page size. // Still, we know it's already completed, so we can skip straight to the Impl method again. clonedOptions.PageSize = pageSize; var newQueryJob = await _client.GetQueryResultsAsync(JobReference, clonedOptions, cancellationToken).ConfigureAwait(false); return(await newQueryJob.ReadPageAsyncImpl(pageSize, cancellationToken).ConfigureAwait(false)); } // First add the rows from the first response which is part of the state // of the object. rows.AddRange(ResponseRows); string pageToken = _response.PageToken; clonedOptions.StartIndex = null; // Now keep going until we've filled the result set or know there's no more data. while (rows.Count > pageSize && pageToken != null) { clonedOptions.PageToken = pageToken; clonedOptions.PageSize = pageSize - rows.Count; var job = await _client.GetQueryResultsAsync(JobReference, clonedOptions, cancellationToken).ConfigureAwait(false); rows.AddRange(job.ResponseRows); pageToken = job._response.PageToken; } return(new BigQueryPage(rows, Schema, JobReference, pageToken)); }
// Implementation of ReadPage for an already-complete job. private BigQueryPage ReadPageImpl(int pageSize) { if (!Completed) { throw new InvalidOperationException($"Cannot call {nameof(ReadPage)} on an incomplete job"); } GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions(); List <BigQueryRow> rows = new List <BigQueryRow>(pageSize); if ((clonedOptions.PageSize == null || clonedOptions.PageSize > pageSize) && _response.Rows?.Count > pageSize) { // Oops. Do it again from scratch, with a useful page size. // Still, we know it's already completed, so we can skip straight to the Impl method again. clonedOptions.PageSize = pageSize; return(_client.GetQueryResults(JobReference, clonedOptions).ReadPageImpl(pageSize)); } // First add the rows from the first response which is part of the state // of the object. rows.AddRange(ResponseRows); string pageToken = _response.PageToken; clonedOptions.StartIndex = null; // Now keep going until we've filled the result set or know there's no more data. while (rows.Count > pageSize && pageToken != null) { clonedOptions.PageToken = pageToken; clonedOptions.PageSize = pageSize - rows.Count; var job = _client.GetQueryResults(JobReference, clonedOptions); rows.AddRange(job.ResponseRows); pageToken = job._response.PageToken; } return(new BigQueryPage(rows, Schema, JobReference, pageToken)); }
/// <summary> /// Asynchronously but eagerly fetches a set of rows, up to the specified count, providing a page of results with a next page token /// if more results are available. This is typically used within web applications, where the next page token /// is propagated to the client along with the results, so that the next page can be retrieved in another request. /// </summary> /// <param name="pageSize">The maximum number of rows to retrieve. Must be positive.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// an in-memory result set of at most the given number of rows.</returns> public async Task <BigQueryPage> ReadPageAsync(int pageSize, CancellationToken cancellationToken = default) { GaxPreconditions.CheckArgumentRange(pageSize, nameof(pageSize), 1, int.MaxValue); GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions(); List <BigQueryRow> rows = new List <BigQueryRow>(pageSize); // Work out whether to use the response we've already got, or create a new one. GetQueryResultsResponse response = _response; if (response.Rows?.Count > pageSize) { // Oops. Do it again from scratch, with a useful page size. clonedOptions.PageSize = pageSize; response = await _client.GetRawQueryResultsAsync(JobReference, clonedOptions, timeoutBase : null, cancellationToken).ConfigureAwait(false); } // First add the rows from the existing response. rows.AddRange(ConvertResponseRows(response)); string pageToken = response.PageToken; clonedOptions.StartIndex = null; // Now keep going until we've filled the result set or know there's no more data. while (rows.Count < pageSize && pageToken != null) { clonedOptions.PageToken = pageToken; clonedOptions.PageSize = pageSize - rows.Count; var nextResponse = await _client.GetRawQueryResultsAsync(JobReference, clonedOptions, timeoutBase : null, cancellationToken).ConfigureAwait(false); rows.AddRange(ConvertResponseRows(nextResponse)); pageToken = nextResponse.PageToken; } return(new BigQueryPage(rows, Schema, JobReference, TableReference, pageToken)); }
/// <inheritdoc /> public override BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); var job = GetJob(jobReference); return(job.GetQueryResults(options)); }
/// <summary> /// Constructs a new set of results. /// </summary> /// <remarks> /// This is public to allow tests to construct instances for production code to consume; /// production code should not normally construct instances itself. /// </remarks> /// <param name="client">The client to use for further operations. Must not be null.</param> /// <param name="response">The response to a GetQueryResults API call. Must not be null.</param> /// <param name="tableReference">A reference to the table containing the results. /// May be null. (For example, script queries don't store results in tables.)</param> /// <param name="options">Options to use when fetching results. May be null.</param> public BigQueryResults(BigQueryClient client, GetQueryResultsResponse response, TableReference tableReference, GetQueryResultsOptions options) { _client = GaxPreconditions.CheckNotNull(client, nameof(client)); _response = GaxPreconditions.CheckNotNull(response, nameof(response)); TableReference = tableReference; _options = options; _fieldNames = response.Schema?.IndexFieldNames(); }
public AsyncRowEnumerator(BigQueryResults parent) { _parent = parent; _options = parent._options?.Clone() ?? new GetQueryResultsOptions(); _options.StartIndex = null; _options.PageToken = parent._response.PageToken; _underlyingEnumerator = parent.ResponseRows.GetEnumerator(); }
public AsyncRowEnumerator(BigQueryResults job, PollSettings pollSettings) { _job = job; _pollSettings = pollSettings; _options = _job._options?.Clone() ?? new GetQueryResultsOptions(); _options.StartIndex = null; _options.PageToken = _job._response.PageToken; }
/// <inheritdoc /> public override BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId); options?.ModifyRequest(request); var firstResponse = request.Execute(); return(new BigQueryResults(this, firstResponse, options)); }
// Implementation of GetRows for a completed job. private IEnumerable <BigQueryRow> GetRowsImpl() { foreach (var row in ResponseRows) { yield return(row); } GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions(); clonedOptions.StartIndex = null; clonedOptions.PageToken = _response.PageToken; while (clonedOptions.PageToken != null) { var job = _client.GetQueryResults(JobReference, clonedOptions); foreach (var row in job.ResponseRows) { yield return(row); } clonedOptions.PageToken = job._response.PageToken; } }
/// <summary> /// Returns an iterator over the query results. /// </summary> /// <returns>An iterator over the query results.</returns> public IEnumerator <BigQueryRow> GetEnumerator() { foreach (var row in ResponseRows) { yield return(row); } GetQueryResultsOptions clonedOptions = _options?.Clone() ?? new GetQueryResultsOptions(); clonedOptions.StartIndex = null; clonedOptions.PageToken = _response.PageToken; while (clonedOptions.PageToken != null) { var response = _client.GetRawQueryResults(JobReference, clonedOptions, timeoutBase: null); foreach (var row in ConvertResponseRows(response)) { yield return(row); } clonedOptions.PageToken = response.PageToken; } }
internal override Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken) { var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc()); return(request.ExecuteAsync(cancellationToken)); }
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. Otherwise, the RPC will have a timeout, // and the server won't return an incomplete job before that timeout. So although this // looks like a tight loop without any delay, the delay is on the server side. var response = GetRawQueryResults(jobReference, options, start); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
/// <summary> /// Asynchronously executes a query. /// </summary> /// <remarks> /// <para> /// The task returned by this method will only complete when the query has completed. /// This method simply delegates to <see cref="CreateQueryJobAsync(string, IEnumerable{BigQueryParameter}, QueryOptions, CancellationToken)"/> /// and then <see cref="BigQueryJob.GetQueryResultsAsync(GetQueryResultsOptions, CancellationToken)"/>. /// </para> /// </remarks> /// <param name="sql">The SQL query. Must not be null.</param> /// <param name="parameters">The parameters for the query. May be null, which is equivalent to specifying an empty list of parameters. Must not contain null elements.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the <see cref="BigQueryResults"/> representing the query.</returns> public virtual async Task <BigQueryResults> ExecuteQueryAsync(string sql, IEnumerable <BigQueryParameter> parameters, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null, CancellationToken cancellationToken = default) { var job = await CreateQueryJobAsync(sql, parameters, queryOptions, cancellationToken).ConfigureAwait(false); return(await job.GetQueryResultsAsync(resultsOptions, cancellationToken).ConfigureAwait(false)); }
/// <summary> /// Executes a query. /// </summary> /// <remarks> /// <para> /// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(string, IEnumerable{BigQueryParameter}, QueryOptions)"/> /// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>. /// </para> /// </remarks> /// <param name="sql">The SQL query. Must not be null.</param> /// <param name="parameters">The parameters for the query. May be null, which is equivalent to specifying an empty list of parameters. Must not contain null elements.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <returns>The result of the query.</returns> public virtual BigQueryResults ExecuteQuery(string sql, IEnumerable <BigQueryParameter> parameters, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) => CreateQueryJob(sql, parameters, queryOptions).GetQueryResults(resultsOptions);
internal virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary> /// Asynchronously executes a command. This overload allows query parameterization, and is preferred over /// <see cref="ExecuteQueryAsync(string, QueryOptions, GetQueryResultsOptions, CancellationToken)"/> when values need to be passed in. /// </summary> /// <remarks> /// <para> /// The task returned by this method will only complete when the query has completed. /// This method simply delegates to <see cref="CreateQueryJobAsync(BigQueryCommand, QueryOptions, CancellationToken)"/> /// and then <see cref="BigQueryJob.GetQueryResultsAsync(GetQueryResultsOptions, CancellationToken)"/>. /// </para> /// </remarks> /// <param name="command">The command to execute. Must not be null.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the <see cref="BigQueryResults"/> representing the query.</returns> public virtual async Task <BigQueryResults> ExecuteQueryAsync(BigQueryCommand command, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null, CancellationToken cancellationToken = default(CancellationToken)) { var job = await CreateQueryJobAsync(command, queryOptions, cancellationToken).ConfigureAwait(false); return(await job.GetQueryResultsAsync(resultsOptions, cancellationToken).ConfigureAwait(false)); }
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart) { var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart; var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout; var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds; if (timeRemainingMs < 1) { // TODO: Check this is correct throw new TimeoutException("Query timed out"); } var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout); var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId); request.Location = jobReference.Location; request.TimeoutMs = requestTimeoutMs; options?.ModifyRequest(request); RetryHandler.MarkAsRetriable(request); return(request); }
/// <summary> /// "Raw" version of GetQueryResults, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <returns>The results of the query.</returns> internal virtual GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase) => throw new NotImplementedException();
internal override GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase) { var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc()); return(request.Execute()); }
/// <summary> /// Retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The results of the query.</returns> public virtual BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) => throw new NotImplementedException();
/// <summary> /// Retrieves the results of the specified job, which must be a query job. /// This method just creates a <see cref="JobReference"/> and delegates to <see cref="GetQueryResults(JobReference, GetQueryResultsOptions)"/>. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="projectId">The project ID. Must not be null.</param> /// <param name="jobId">The job ID. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The results of the query.</returns> public virtual BigQueryResults GetQueryResults(string projectId, string jobId, GetQueryResultsOptions options = null) => GetQueryResults(GetJobReference(projectId, jobId), options);
/// <summary> /// Executes a command. This overload allows query parameterization, and is preferred over /// <see cref="ExecuteQuery(string, QueryOptions, GetQueryResultsOptions)"/> when values need to be passed in. /// </summary> /// <remarks> /// <para> /// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(BigQueryCommand, QueryOptions)"/> /// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>. /// </para> /// </remarks> /// <param name="command">The command to execute. Must not be null.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <returns>The result of the query.</returns> public virtual BigQueryResults ExecuteQuery(BigQueryCommand command, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) => CreateQueryJob(command, queryOptions).GetQueryResults(resultsOptions);
/// <summary> /// Executes a query. /// </summary> /// <remarks> /// <para> /// This method will only return when the query has completed. It simply delegates to <see cref="CreateQueryJob(string, QueryOptions)"/> /// and then <see cref="BigQueryJob.GetQueryResults(GetQueryResultsOptions)"/>. /// </para> /// </remarks> /// <param name="sql">The SQL query. Must not be null.</param> /// <param name="queryOptions">The options for the query. May be null, in which case defaults will be supplied.</param> /// <param name="resultsOptions">The options for retrieving query results. May be null, in which case defaults will be supplied.</param> /// <returns>The result of the query.</returns> public virtual BigQueryResults ExecuteQuery(string sql, QueryOptions queryOptions = null, GetQueryResultsOptions resultsOptions = null) => CreateQueryJob(sql, queryOptions).GetQueryResults(resultsOptions);
/// <inheritdoc /> public override async Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); var job = await GetJobAsync(jobReference, cancellationToken : cancellationToken).ConfigureAwait(false); return(await job.GetQueryResultsAsync(options, cancellationToken).ConfigureAwait(false)); }
internal override async Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. var response = await GetRawQueryResultsAsync(jobReference, options, start, cancellationToken).ConfigureAwait(false); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
/// <summary> /// "Raw" version of GetQueryResultsAsync, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>The results of the query.</returns> internal virtual Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary> /// Asynchronously retrieves the results of the specified job, which must be a query job. /// This method just creates a <see cref="JobReference"/> and delegates to <see cref="GetQueryResultsAsync(JobReference, GetQueryResultsOptions, CancellationToken)"/>. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="projectId">The project ID. Must not be null.</param> /// <param name="jobId">The job ID. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the results of the query.</returns> public virtual Task <BigQueryResults> GetQueryResultsAsync(string projectId, string jobId, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) => GetQueryResultsAsync(GetJobReference(projectId, jobId), options, cancellationToken);
// Note - these methods are not part of the regular "pattern", so are not in the GetQueryResults region above. // We want to remove them, if the underlying GetQueryResultsResponse starts including the table reference. // These methods allow us to call GetQueryResultsAsync from BigQueryJob without fetching the job again. internal virtual BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) => throw new NotImplementedException();
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. var response = GetRawQueryResults(jobReference, options, start); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
/// <summary> /// Asynchronously retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the results of the query.</returns> public virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();