private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out IReadOnlyList <BigQueryInsertRow> validatedRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
validatedRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row);
}).ToList().AsReadOnly();
var body = new TableDataInsertAllRequest
{
Rows = new RawRowList(validatedRows, options?.AllowEmptyInsertIds ?? false)
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData(options?.AllowEmptyInsertIds ?? false));
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
if (insertRows.All(ir => ir.InsertId != null))
{
RetryHandler.MarkAsRetriable(request);
}
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
// We ensure that every row has an insert ID, so we can always retry.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options)
{
var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListDatasetsRequest(ProjectReference projectReference, ListDatasetsOptions options)
{
var request = Service.Datasets.List(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListModelsRequest(DatasetReference datasetReference, ListModelsOptions options)
{
var request = Service.Models.List(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
/// <inheritdoc />
private ListRequest CreateListProjectsRequest(ListProjectsOptions options)
{
var request = Service.Projects.List();
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
static internal void RetryIfETagPresent <TResource, TResponse>(BigqueryBaseServiceRequest <TResponse> request, TResource resource)
where TResource : IDirectResponseSchema
{
if (resource.ETag != null)
{
RetryHandler.MarkAsRetriable(request);
}
}
private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
{
var request = Service.Jobs.List(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
// Request creation
private GetRequest CreateGetDatasetRequest(DatasetReference datasetReference, GetDatasetOptions options)
{
GaxPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var request = Service.Datasets.Get(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertRequest CreateInsertJobRequest(JobConfiguration configuration, JobCreationOptions options)
{
var job = CreateJob(configuration, options);
var request = Service.Jobs.Insert(job, job.JobReference.ProjectId);
// It's safe to retry job inserts when they include a job ID, which ours always do.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
{
var request = Service.Jobs.List(projectReference.ProjectId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
/// <inheritdoc />
private ListRequest CreateListTablesRequest(DatasetReference datasetReference, ListTablesOptions options)
{
var request = Service.Tables.List(datasetReference.ProjectId, datasetReference.DatasetId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListRoutinesRequest(DatasetReference datasetReference, ListRoutinesOptions options)
{
var request = Service.Routines.List(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
private GetServiceAccountRequest CreateGetServiceAccountEmailRequest(ProjectReference projectReference, GetBigQueryServiceAccountEmailOptions options)
{
GaxPreconditions.CheckNotNull(projectReference, nameof(projectReference));
var request = Service.Projects.GetServiceAccount(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private GetRequest CreateGetRoutineRequest(RoutineReference routineReference, GetRoutineOptions options)
{
GaxPreconditions.CheckNotNull(routineReference, nameof(routineReference));
var request = Service.Routines.Get(routineReference.ProjectId, routineReference.DatasetId, routineReference.RoutineId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private CancelRequest CreateCancelJobRequest(JobReference jobReference, CancelJobOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.Cancel(jobReference.ProjectId, jobReference.JobId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options, TableSchema schema)
{
var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
options?.ModifyRequest(request);
// null and empty schemas are handled by BuildSelectedFields,
// but both values mean the same, and that is to return whole rows.
request.SelectedFields = schema.BuildSelectedFields();
RetryHandler.MarkAsRetriable(request);
return(request);
}
// Request creation
private GetRequest CreateGetJobRequest(JobReference jobReference, GetJobOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.Get(jobReference.ProjectId, jobReference.JobId);
request.Location = jobReference.Location;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
// Request creation
private GetRequest CreateGetTableRequest(TableReference tableReference, GetTableOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
var request = Service.Tables.Get(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private GetRequest CreateGetModelRequest(ModelReference modelReference, GetModelOptions options)
{
GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference));
var request = Service.Models.Get(modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
{
var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart;
var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;
if (timeRemainingMs < 1)
{
// TODO: Check this is correct
throw new TimeoutException("Query timed out");
}
var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
request.Location = jobReference.Location;
request.TimeoutMs = requestTimeoutMs;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
{
var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart;
var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;
if (timeRemainingMs < 1)
{
// TODO: Check this is correct
throw new TimeoutException("Query timed out");
}
var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
// We never use the results within the first response; instead, we're just checking that the job has
// completed and using the statistics and schema from it.
request.MaxResults = 0;
request.ModifyRequest += _versionHeaderAction;
request.TimeoutMs = requestTimeoutMs;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
/// <summary>
/// Constructs a new client wrapping the given <see cref="BigqueryService"/>.
/// </summary>
/// <remarks>
/// Care should be taken when constructing the service: if the default serializer settings are used,
/// result values which can be parsed as date/time values can cause problems. Where possible, either use
/// <see cref="BigQueryClient.Create(string, Apis.Auth.OAuth2.GoogleCredential)"/> or construct a service
/// using serializer settings from <see cref="BigQueryClient.CreateJsonSerializersSettings"/>.
/// </remarks>
/// <param name="projectId">The ID of the project to work with. Must not be null.</param>
/// <param name="service">The service to wrap. Must not be null.</param>
public BigQueryClientImpl(string projectId, BigqueryService service)
{
ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
Service = GaxPreconditions.CheckNotNull(service, nameof(service));
RetryHandler.Install(Service);
}