private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
if (insertRows.All(ir => ir.InsertId != null))
{
RetryHandler.MarkAsRetriable(request);
}
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData(options?.AllowEmptyInsertIds ?? false));
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
// We ensure that every row has an insert ID, so we can always retry.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out IReadOnlyList <BigQueryInsertRow> validatedRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
validatedRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row);
}).ToList().AsReadOnly();
var body = new TableDataInsertAllRequest
{
Rows = new RawRowList(validatedRows, options?.AllowEmptyInsertIds ?? false)
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
static internal void RetryIfETagPresent <TResource, TResponse>(BigqueryBaseServiceRequest <TResponse> request, TResource resource)
where TResource : IDirectResponseSchema
{
if (resource.ETag != null)
{
RetryHandler.MarkAsRetriable(request);
}
}
private ListRequest CreateListModelsRequest(DatasetReference datasetReference, ListModelsOptions options)
{
var request = Service.Models.List(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
/// <inheritdoc />
private ListRequest CreateListProjectsRequest(ListProjectsOptions options)
{
var request = Service.Projects.List();
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options)
{
var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListDatasetsRequest(ProjectReference projectReference, ListDatasetsOptions options)
{
var request = Service.Datasets.List(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListRoutinesRequest(DatasetReference datasetReference, ListRoutinesOptions options)
{
var request = Service.Routines.List(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
/// <inheritdoc />
private ListRequest CreateListTablesRequest(DatasetReference datasetReference, ListTablesOptions options)
{
var request = Service.Tables.List(datasetReference.ProjectId, datasetReference.DatasetId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertRequest CreateInsertJobRequest(JobConfiguration configuration, JobCreationOptions options)
{
var job = CreateJob(configuration, options);
var request = Service.Jobs.Insert(job, job.JobReference.ProjectId);
// It's safe to retry job inserts when they include a job ID, which ours always do.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
{
var request = Service.Jobs.List(projectReference.ProjectId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
// Request creation
private GetRequest CreateGetDatasetRequest(DatasetReference datasetReference, GetDatasetOptions options)
{
GaxPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var request = Service.Datasets.Get(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
{
var request = Service.Jobs.List(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
private GetRequest CreateGetRoutineRequest(RoutineReference routineReference, GetRoutineOptions options)
{
GaxPreconditions.CheckNotNull(routineReference, nameof(routineReference));
var request = Service.Routines.Get(routineReference.ProjectId, routineReference.DatasetId, routineReference.RoutineId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private CancelRequest CreateCancelJobRequest(JobReference jobReference, CancelJobOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.Cancel(jobReference.ProjectId, jobReference.JobId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private GetServiceAccountRequest CreateGetServiceAccountEmailRequest(ProjectReference projectReference, GetBigQueryServiceAccountEmailOptions options)
{
GaxPreconditions.CheckNotNull(projectReference, nameof(projectReference));
var request = Service.Projects.GetServiceAccount(projectReference.ProjectId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options, TableSchema schema)
{
var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
options?.ModifyRequest(request);
// null and empty schemas are handled by BuildSelectedFields,
// but both values mean the same, and that is to return whole rows.
request.SelectedFields = schema.BuildSelectedFields();
RetryHandler.MarkAsRetriable(request);
return(request);
}
// Request creation
private GetRequest CreateGetJobRequest(JobReference jobReference, GetJobOptions options)
{
GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.Get(jobReference.ProjectId, jobReference.JobId);
request.Location = jobReference.Location;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private GetRequest CreateGetModelRequest(ModelReference modelReference, GetModelOptions options)
{
GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference));
var request = Service.Models.Get(modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId);
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
// Request creation
private GetRequest CreateGetTableRequest(TableReference tableReference, GetTableOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
var request = Service.Tables.Get(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
{
var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart;
var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;
if (timeRemainingMs < 1)
{
// TODO: Check this is correct
throw new TimeoutException("Query timed out");
}
var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
request.Location = jobReference.Location;
request.TimeoutMs = requestTimeoutMs;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
{
var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart;
var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;
if (timeRemainingMs < 1)
{
// TODO: Check this is correct
throw new TimeoutException("Query timed out");
}
var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
// We never use the results within the first response; instead, we're just checking that the job has
// completed and using the statistics and schema from it.
request.MaxResults = 0;
request.ModifyRequest += _versionHeaderAction;
request.TimeoutMs = requestTimeoutMs;
options?.ModifyRequest(request);
RetryHandler.MarkAsRetriable(request);
return(request);
}
