private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out IReadOnlyList <BigQueryInsertRow> validatedRows)
        {
            GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
            GaxPreconditions.CheckNotNull(rows, nameof(rows));

            validatedRows = rows.Select(row =>
            {
                GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
                return(row);
            }).ToList().AsReadOnly();
            var body = new TableDataInsertAllRequest
            {
                Rows = new RawRowList(validatedRows, options?.AllowEmptyInsertIds ?? false)
            };

            options?.ModifyRequest(body);
            var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            // Even though empty InsertIds might be allowed, this can be retried as per guidance from
            // the API team. Previous de-duplicating was on a best effort basis anyways and client code
            // needs to explicitly allow for empty InsertId and should be aware that doing so will be at
            // the expense of de-duplication efforts.
            RetryHandler.MarkAsRetriable(request);
            request.PrettyPrint = PrettyPrint;
            return(request);
        }
Exemplo n.º 2
0
        private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
        {
            GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
            GaxPreconditions.CheckNotNull(rows, nameof(rows));

            var insertRows = rows.Select(row =>
            {
                GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
                return(row.ToRowsData(options?.AllowEmptyInsertIds ?? false));
            }).ToList();
            var body = new TableDataInsertAllRequest
            {
                Rows = insertRows
            };

            // It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
            hasRows = body.Rows.Any();
            options?.ModifyRequest(body);
            var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            // Even though empty InsertIds might be allowed, this can be retried as per guidance from
            // the API team. Previous de-duplicating was on a best effort basis anyways and client code
            // needs to explicitly allow for empty InsertId and should be aware that doing so will be at
            // the expense of de-duplication efforts.
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 3
0
        private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options)
        {
            GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
            GaxPreconditions.CheckNotNull(rows, nameof(rows));

            var insertRows = rows.Select(row =>
            {
                GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
                return(row.ToRowsData());
            }).ToList();
            var body = new TableDataInsertAllRequest
            {
                Rows = insertRows
            };

            options?.ModifyRequest(body);
            var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            request.ModifyRequest += _versionHeaderAction;
            if (insertRows.All(ir => ir.InsertId != null))
            {
                RetryHandler.MarkAsRetriable(request);
            }
            return(request);
        }
        private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
        {
            GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
            GaxPreconditions.CheckNotNull(rows, nameof(rows));

            var insertRows = rows.Select(row =>
            {
                GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
                return(row.ToRowsData());
            }).ToList();
            var body = new TableDataInsertAllRequest
            {
                Rows = insertRows
            };

            // It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
            hasRows = body.Rows.Any();
            options?.ModifyRequest(body);
            var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            request.ModifyRequest += _versionHeaderAction;
            // We ensure that every row has an insert ID, so we can always retry.
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 5
0
        private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options)
        {
            var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        private ListRequest CreateListDatasetsRequest(ProjectReference projectReference, ListDatasetsOptions options)
        {
            var request = Service.Datasets.List(projectReference.ProjectId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        private ListRequest CreateListModelsRequest(DatasetReference datasetReference, ListModelsOptions options)
        {
            var request = Service.Models.List(datasetReference.ProjectId, datasetReference.DatasetId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        /// <inheritdoc />
        private ListRequest CreateListProjectsRequest(ListProjectsOptions options)
        {
            var request = Service.Projects.List();

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
 static internal void RetryIfETagPresent <TResource, TResponse>(BigqueryBaseServiceRequest <TResponse> request, TResource resource)
     where TResource : IDirectResponseSchema
 {
     if (resource.ETag != null)
     {
         RetryHandler.MarkAsRetriable(request);
     }
 }
        private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
        {
            var request = Service.Jobs.List(projectReference.ProjectId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            request.PrettyPrint = PrettyPrint;
            return(request);
        }
        // Request creation
        private GetRequest CreateGetDatasetRequest(DatasetReference datasetReference, GetDatasetOptions options)
        {
            GaxPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
            var request = Service.Datasets.Get(datasetReference.ProjectId, datasetReference.DatasetId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        private InsertRequest CreateInsertJobRequest(JobConfiguration configuration, JobCreationOptions options)
        {
            var job     = CreateJob(configuration, options);
            var request = Service.Jobs.Insert(job, job.JobReference.ProjectId);

            // It's safe to retry job inserts when they include a job ID, which ours always do.
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 13
0
        private ListRequest CreateListJobsRequest(ProjectReference projectReference, ListJobsOptions options)
        {
            var request = Service.Jobs.List(projectReference.ProjectId);

            request.ModifyRequest += _versionHeaderAction;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        /// <inheritdoc />
        private ListRequest CreateListTablesRequest(DatasetReference datasetReference, ListTablesOptions options)
        {
            var request = Service.Tables.List(datasetReference.ProjectId, datasetReference.DatasetId);

            request.ModifyRequest += _versionHeaderAction;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        private ListRequest CreateListRoutinesRequest(DatasetReference datasetReference, ListRoutinesOptions options)
        {
            var request = Service.Routines.List(datasetReference.ProjectId, datasetReference.DatasetId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            request.PrettyPrint = PrettyPrint;
            return(request);
        }
Exemplo n.º 16
0
        private GetServiceAccountRequest CreateGetServiceAccountEmailRequest(ProjectReference projectReference, GetBigQueryServiceAccountEmailOptions options)
        {
            GaxPreconditions.CheckNotNull(projectReference, nameof(projectReference));

            var request = Service.Projects.GetServiceAccount(projectReference.ProjectId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 17
0
        private GetRequest CreateGetRoutineRequest(RoutineReference routineReference, GetRoutineOptions options)
        {
            GaxPreconditions.CheckNotNull(routineReference, nameof(routineReference));

            var request = Service.Routines.Get(routineReference.ProjectId, routineReference.DatasetId, routineReference.RoutineId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 18
0
        private CancelRequest CreateCancelJobRequest(JobReference jobReference, CancelJobOptions options)
        {
            GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));
            var request = Service.Jobs.Cancel(jobReference.ProjectId, jobReference.JobId);

            request.ModifyRequest += _versionHeaderAction;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 19
0
        private TabledataResource.ListRequest CreateListRequest(TableReference tableReference, ListRowsOptions options, TableSchema schema)
        {
            var request = Service.Tabledata.List(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            options?.ModifyRequest(request);
            // null and empty schemas are handled by BuildSelectedFields,
            // but both values mean the same, and that is to return whole rows.
            request.SelectedFields = schema.BuildSelectedFields();
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        // Request creation
        private GetRequest CreateGetJobRequest(JobReference jobReference, GetJobOptions options)
        {
            GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference));

            var request = Service.Jobs.Get(jobReference.ProjectId, jobReference.JobId);

            request.Location = jobReference.Location;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        // Request creation
        private GetRequest CreateGetTableRequest(TableReference tableReference, GetTableOptions options)
        {
            GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));

            var request = Service.Tables.Get(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);

            request.ModifyRequest += _versionHeaderAction;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
        private GetRequest CreateGetModelRequest(ModelReference modelReference, GetModelOptions options)
        {
            GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference));

            var request = Service.Models.Get(modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId);

            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            request.PrettyPrint = PrettyPrint;
            return(request);
        }
Exemplo n.º 23
0
        private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
        {
            var timeSoFar       = Clock.GetCurrentDateTimeUtc() - loopStart;
            var timeout         = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
            var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;

            if (timeRemainingMs < 1)
            {
                // TODO: Check this is correct
                throw new TimeoutException("Query timed out");
            }
            var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
            var request          = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);

            request.Location  = jobReference.Location;
            request.TimeoutMs = requestTimeoutMs;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
Exemplo n.º 24
0
        private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart)
        {
            var timeSoFar       = Clock.GetCurrentDateTimeUtc() - loopStart;
            var timeout         = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout;
            var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds;

            if (timeRemainingMs < 1)
            {
                // TODO: Check this is correct
                throw new TimeoutException("Query timed out");
            }
            var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout);
            var request          = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);

            // We never use the results within the first response; instead, we're just checking that the job has
            // completed and using the statistics and schema from it.
            request.MaxResults     = 0;
            request.ModifyRequest += _versionHeaderAction;
            request.TimeoutMs      = requestTimeoutMs;
            options?.ModifyRequest(request);
            RetryHandler.MarkAsRetriable(request);
            return(request);
        }
 /// <summary>
 /// Constructs a new client wrapping the given <see cref="BigqueryService"/>.
 /// </summary>
 /// <remarks>
 /// Care should be taken when constructing the service: if the default serializer settings are used,
 /// result values which can be parsed as date/time values can cause problems. Where possible, either use
 /// <see cref="BigQueryClient.Create(string, Apis.Auth.OAuth2.GoogleCredential)"/> or construct a service
 /// using serializer settings from <see cref="BigQueryClient.CreateJsonSerializersSettings"/>.
 /// </remarks>
 /// <param name="projectId">The ID of the project to work with. Must not be null.</param>
 /// <param name="service">The service to wrap. Must not be null.</param>
 public BigQueryClientImpl(string projectId, BigqueryService service)
 {
     ProjectId = GaxPreconditions.CheckNotNull(projectId, nameof(projectId));
     Service   = GaxPreconditions.CheckNotNull(service, nameof(service));
     RetryHandler.Install(Service);
 }