private void DownloadObjectImpl(
string baseUri,
Stream destination,
DownloadObjectOptions options,
IProgress <IDownloadProgress> progress)
{
// URI will definitely not be null; that's constructed internally.
GaxRestPreconditions.CheckNotNull(destination, nameof(destination));
var downloader = new MediaDownloader(Service);
options?.ModifyDownloader(downloader);
string uri = options == null ? baseUri : options.GetUri(baseUri);
if (progress != null)
{
downloader.ProgressChanged += progress.Report;
progress.Report(InitialDownloadProgress.Instance);
}
var result = downloader.Download(uri, destination);
if (result.Status == DownloadStatus.Failed)
{
throw result.Exception;
}
}
internal void ModifyRequest(UpdateRequest request)
{
if (IfMetagenerationMatch != null && IfMetagenerationNotMatch != null)
{
throw new ArgumentException($"Cannot specify {nameof(IfMetagenerationMatch)} and {nameof(IfMetagenerationNotMatch)} in the same options", "options");
}
if (IfMetagenerationMatch != null)
{
request.IfMetagenerationMatch = IfMetagenerationMatch;
}
if (IfMetagenerationNotMatch != null)
{
request.IfMetagenerationNotMatch = IfMetagenerationNotMatch;
}
if (Projection != null)
{
request.Projection = GaxRestPreconditions.CheckEnumValue((ProjectionEnum)Projection, nameof(Projection));
}
if (PredefinedAcl != null)
{
request.PredefinedAcl =
GaxRestPreconditions.CheckEnumValue((PredefinedAclEnum)PredefinedAcl, nameof(PredefinedAcl));
}
if (PredefinedDefaultObjectAcl != null)
{
request.PredefinedDefaultObjectAcl =
GaxRestPreconditions.CheckEnumValue((PredefinedDefaultObjectAclEnum)PredefinedDefaultObjectAcl, nameof(PredefinedDefaultObjectAcl));
}
}
/// <summary>
/// Synchronously creates a <see cref="BigqueryClient"/>, using application default credentials if
/// no credentials are specified.
/// </summary>
/// <remarks>
/// The credentials are scoped as necessary.
/// </remarks>
/// <param name="credential">Optional <see cref="GoogleCredential"/>.</param>
/// <returns>The created <see cref="BigqueryClient"/>.</returns>
public static BigqueryClient Create(string projectId, GoogleCredential credential = null)
{
GaxRestPreconditions.CheckNotNull(projectId, nameof(projectId));
try
{
credential = credential ?? Task.Run(() => GoogleCredential.GetApplicationDefaultAsync()).Result;
}
catch (AggregateException e)
{
// Unwrap the first exception, a bit like await would.
// It's very unlikely that we'd ever see an AggregateException without an inner exceptions,
// but let's handle it relatively gracefully.
throw e.InnerExceptions.FirstOrDefault() ?? e;
}
if (credential.IsCreateScopedRequired)
{
credential = credential.CreateScoped(
// FIXME: Work out which of these we actually need.
BigqueryService.Scope.Bigquery,
BigqueryService.Scope.BigqueryInsertdata,
BigqueryService.Scope.DevstorageFullControl,
BigqueryService.Scope.CloudPlatform);
}
var service = new BigqueryService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
ApplicationName = BigqueryClientImpl.ApplicationName,
});
return(new BigqueryClientImpl(projectId, service));
}
private async Task DownloadObjectAsyncImpl(
string baseUri,
Stream destination,
DownloadObjectOptions options,
CancellationToken cancellationToken,
IProgress <IDownloadProgress> progress)
{
GaxRestPreconditions.CheckNotNull(destination, nameof(destination));
var downloader = new MediaDownloader(Service);
options?.ModifyDownloader(downloader);
string uri = options == null ? baseUri : options.GetUri(baseUri);
if (progress != null)
{
downloader.ProgressChanged += progress.Report;
// Avoid reporting progress synchronously in the original call.
await Task.Yield();
progress.Report(InitialDownloadProgress.Instance);
}
var result = await downloader.DownloadAsync(uri, destination, cancellationToken).ConfigureAwait(false);
if (result.Status == DownloadStatus.Failed)
{
throw result.Exception;
}
}
/// <inheritdoc />
public override IPagedAsyncEnumerable <Buckets, Bucket> ListBucketsAsync(
string projectId, ListBucketsOptions options = null)
{
GaxRestPreconditions.CheckNotNull(projectId, nameof(projectId));
return(new PagedAsyncEnumerable <BucketsResource.ListRequest, Buckets, Bucket>(
() => CreateListBucketsRequest(projectId, options), BucketPageManager.Instance));
}
/// <summary>
/// Creates a <see cref="TableReference"/> from the given project ID, dataset ID and table ID.
/// </summary>
/// <param name="projectId">The project ID. Must not be null.</param>
/// <param name="datasetId">The dataset ID. Must not be null.</param>
/// <param name="tableId">The table ID. Must not be null.</param>
/// <returns>A <see cref="TableReference"/> representing the requested table.</returns>
public TableReference GetTableReference(string projectId, string datasetId, string tableId) =>
new TableReference
{
ProjectId = GaxRestPreconditions.CheckNotNull(projectId, nameof(projectId)),
DatasetId = GaxRestPreconditions.CheckNotNull(datasetId, nameof(datasetId)),
TableId = GaxRestPreconditions.CheckNotNull(tableId, nameof(tableId)),
};
/// <summary>
/// Validates that a bucket only contains valid characters, and is not too long. This is far from
/// complete validation, but is all that's required to ensure that it's safe to include in a URL.
/// This method also checks for nullity, so callers don't need to do that first.
/// This method is internal rather than private for testing purposes.
/// </summary>
internal static void ValidateBucketName(string bucket)
{
GaxRestPreconditions.CheckNotNull(bucket, nameof(bucket));
if (!ValidBucketName.IsMatch(bucket))
{
throw new ArgumentException($"Invalid bucket name '{bucket}' - see https://cloud.google.com/storage/docs/bucket-naming", nameof(bucket));
}
}
/// <summary>
/// Adds all the values in the specified dictionary to the row.
/// </summary>
/// <remarks>
/// This being named <c>Add</c> rather than <c>AddRange</c> allows
/// it to be specified in a collection initializer, which can be useful
/// to provide a set of common fields and then some extra values.
/// </remarks>
/// <param name="fields">The fields to add to the row. Must not be null.</param>
public void Add(IDictionary <string, object> fields)
{
GaxRestPreconditions.CheckNotNull(fields, nameof(fields));
foreach (var entry in fields)
{
Add(entry.Key, entry.Value);
}
}
/// <inheritdoc />
public override BigqueryDataset GetDataset(DatasetReference datasetReference, GetDatasetOptions options = null)
{
GaxRestPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var request = Service.Datasets.Get(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
return(new BigqueryDataset(this, request.Execute()));
}
/// <inheritdoc />
public override void DeleteTable(TableReference tableReference, DeleteTableOptions options = null)
{
GaxRestPreconditions.CheckNotNull(tableReference, nameof(tableReference));
var request = Service.Tables.Delete(tableReference.ProjectId, tableReference.TableId, tableReference.TableId);
options?.ModifyRequest(request);
request.Execute();
}
/// <summary>
/// Validates that the given Bucket has a "somewhat valid" (no URI encoding required) bucket name.
/// </summary>
/// <param name="bucket">Bucket to validate</param>
/// <param name="paramName">The parameter name in the calling method</param>
private void ValidateBucket(Bucket bucket, string paramName)
{
GaxRestPreconditions.CheckNotNull(bucket, paramName);
GaxRestPreconditions.CheckArgument(bucket.Name != null, paramName, "Bucket must have a name");
GaxRestPreconditions.CheckArgument(ValidBucketName.IsMatch(bucket.Name),
paramName,
"Invalid bucket name '{0}' - see https://cloud.google.com/storage/docs/bucket-naming", bucket.Name);
}
/// <inheritdoc />
public override void DeleteDataset(DatasetReference datasetReference, DeleteDatasetOptions options = null)
{
GaxRestPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var request = Service.Datasets.Delete(datasetReference.ProjectId, datasetReference.DatasetId);
options?.ModifyRequest(request);
request.Execute();
}
private BucketsResource.UpdateRequest CreateUpdateBucketRequest(Bucket bucket, UpdateBucketOptions options)
{
ValidateBucket(bucket, nameof(bucket));
GaxRestPreconditions.CheckArgument(bucket.Acl != null, nameof(bucket), "The Acl property of the bucket to update is null");
var request = Service.Buckets.Update(bucket, bucket.Name);
options?.ModifyRequest(request);
return(request);
}
private BucketsResource.InsertRequest CreateCreateBucketRequest(string projectId, Bucket bucket, CreateBucketOptions options)
{
GaxRestPreconditions.CheckNotNull(projectId, nameof(projectId));
ValidateBucket(bucket, nameof(bucket));
var request = Service.Buckets.Insert(bucket, projectId);
options?.ModifyRequest(request);
return(request);
}
private ObjectsResource.DeleteRequest CreateDeleteObjectRequest(string bucket, string name, DeleteObjectOptions options)
{
ValidateBucketName(bucket);
GaxRestPreconditions.CheckNotNull(name, nameof(name));
var request = Service.Objects.Delete(bucket, name);
options?.ModifyRequest(request);
return(request);
}
/// <inheritdoc />
public override IPagedEnumerable <DatasetList, BigqueryDataset> ListDatasets(ProjectReference projectReference, ListDatasetsOptions options = null)
{
GaxRestPreconditions.CheckNotNull(projectReference, nameof(projectReference));
var pageManager = new DatasetPageManager(this);
return(new PagedEnumerable <ListRequest, DatasetList, BigqueryDataset>(
() => CreateListDatasetsRequest(projectReference, options),
pageManager));
}
/// <inheritdoc />
public override BigqueryJob CancelJob(JobReference jobReference, CancelJobOptions options = null)
{
GaxRestPreconditions.CheckNotNull(jobReference, nameof(jobReference));
var request = Service.Jobs.Cancel(jobReference.ProjectId, jobReference.JobId);
options?.ModifyRequest(request);
var job = request.Execute().Job;
return(new BigqueryJob(this, job));
}
/// <inheritdoc />
public override IPagedEnumerable <TableList, BigqueryTable> ListTables(DatasetReference datasetReference, ListTablesOptions options = null)
{
GaxRestPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var pageManager = new TablePageManager(this);
return(new PagedEnumerable <ListRequest, TableList, BigqueryTable>(
() => CreateListTablesRequest(datasetReference, options),
pageManager));
}
/// <inheritdoc />
public override Task <Object> GetObjectAsync(
string bucket,
string objectName,
GetObjectOptions options = null,
CancellationToken cancellationToken = default(CancellationToken))
{
ValidateBucketName(bucket);
GaxRestPreconditions.CheckNotNull(objectName, nameof(objectName));
return(CreateGetObjectRequest(bucket, objectName, options).ExecuteAsync(cancellationToken));
}
/// <inheritdoc />
public override IPagedEnumerable <JobList, BigqueryJob> ListJobs(ProjectReference projectReference, ListJobsOptions options = null)
{
GaxRestPreconditions.CheckNotNull(projectReference, nameof(projectReference));
var pageManager = new JobPageManager(this);
return(new PagedEnumerable <ListRequest, JobList, BigqueryJob>(
() => CreateListJobsRequest(projectReference, options),
pageManager));
}
private ObjectsResource.PatchRequest CreatePatchObjectRequest(Object obj, PatchObjectOptions options)
{
GaxRestPreconditions.CheckNotNull(obj, nameof(obj));
GaxRestPreconditions.CheckArgument(obj.Bucket != null, nameof(obj), "The Bucket property of the object to update is null");
GaxRestPreconditions.CheckArgument(obj.Name != null, nameof(obj), "The Name property of the object to update is null");
var request = Service.Objects.Patch(obj, obj.Bucket, obj.Name);
options?.ModifyRequest(request);
return(request);
}
/// <inheritdoc />
public override BigqueryDataset CreateDataset(DatasetReference datasetReference, CreateDatasetOptions options = null)
{
GaxRestPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
var dataset = new Dataset {
DatasetReference = datasetReference
};
var request = Service.Datasets.Insert(dataset, datasetReference.ProjectId);
options?.ModifyRequest(dataset, request);
return(new BigqueryDataset(this, request.Execute()));
}
/// <inheritdoc />
public override BigqueryTable GetTable(TableReference tableReference, GetTableOptions options = null)
{
GaxRestPreconditions.CheckNotNull(tableReference, nameof(tableReference));
var request = Service.Tables.Get(tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
options?.ModifyRequest(request);
var resource = request.Execute();
return(new BigqueryTable(this, resource));
}
internal static T ToValue(string apiValue, string paramName = "name")
{
GaxRestPreconditions.CheckNotNull(apiValue, paramName);
T value;
if (s_stringToValue.TryGetValue(apiValue, out value))
{
return(value);
}
throw new ArgumentException($"Value {apiValue} is undefined in {typeof(T).Name}", paramName);
}
/// <summary>
/// Validates that the given Object has a "somewhat valid" (no URI encoding required) bucket name and an object name.
/// </summary>
/// <param name="obj">Object to validate</param>
/// <param name="paramName">The parameter name in the calling method</param>
private void ValidateObject(Object obj, string paramName)
{
GaxRestPreconditions.CheckNotNull(obj, paramName);
GaxRestPreconditions.CheckArgument(
obj.Name != null && obj.Bucket != null,
paramName,
"Object must have a name and bucket");
GaxRestPreconditions.CheckArgument(ValidBucketName.IsMatch(obj.Bucket),
paramName,
"Object bucket '{0}' is invalid", obj.Bucket);
}
/// <inheritdoc />
public override BigqueryDataset GetOrCreateDataset(DatasetReference datasetReference, GetDatasetOptions getOptions = null, CreateDatasetOptions createOptions = null)
{
GaxRestPreconditions.CheckNotNull(datasetReference, nameof(datasetReference));
try
{
return(GetDataset(datasetReference, getOptions));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
return(CreateDataset(datasetReference, createOptions));
}
}
/// <summary>
/// Creates a field with the specified details, and adds it to the schema being built.
/// </summary>
/// <param name="name">The name of the field. Must be a valid field name.</param>
/// <param name="type">The type of the field. Must be a defined member within <see cref="BigqueryDbType"/>, other than <c>Record</c>.</param>
/// <param name="mode">The mode of the field. Must be a defined member within <see cref="FieldMode"/>.</param>
/// <param name="description">The description of the field. May be null.</param>
public void Add(string name, BigqueryDbType type, FieldMode mode = FieldMode.Nullable, string description = null)
{
ValidateFieldName(name, nameof(name));
GaxRestPreconditions.CheckArgument(type != BigqueryDbType.Record, nameof(type), "Record fields must be specified with their schema");
Add(new TableFieldSchema
{
Name = name,
Type = EnumMap <BigqueryDbType> .ToApiValue(type, nameof(type)),
Mode = EnumMap <FieldMode> .ToApiValue(mode, nameof(mode)),
Description = description,
});
}
/// <inheritdoc />
public override BigqueryResult ExecuteQuery(string sql, ExecuteQueryOptions options = null)
{
GaxRestPreconditions.CheckNotNull(sql, nameof(sql));
var queryRequest = new Apis.Bigquery.v2.Data.QueryRequest {
Query = sql
};
options?.ModifyRequest(queryRequest);
var request = Service.Jobs.Query(queryRequest, ProjectId);
var queryResponse = request.Execute();
return(new BigqueryResult(this, queryResponse));
}
/// <inheritdoc />
public override BigqueryResult GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null)
{
GaxRestPreconditions.CheckNotNull(jobReference, nameof(jobReference));
Func <GetQueryResultsRequest> requestProvider = () =>
{
var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId);
options?.ModifyRequest(request);
return(request);
};
var firstResponse = requestProvider().Execute();
return(new BigqueryResult(this, firstResponse, requestProvider));
}
/// <summary>
/// Creates a nested field with the specified schema, and adds it to the schema being built.
/// </summary>
/// <param name="name">The name of the field. Must be a valid field name.</param>
/// <param name="nestedSchema">The schema for the nested field. Must not be null.</param>
/// <param name="mode">The mode of the field. Must be a defined member within <see cref="FieldMode"/>.</param>
/// <param name="description">The description of the field. May be null.</param>
public void Add(string name, TableSchema nestedSchema, FieldMode mode = FieldMode.Nullable, string description = null)
{
ValidateFieldName(name, nameof(name));
GaxRestPreconditions.CheckNotNull(nestedSchema, nameof(nestedSchema));
Add(new TableFieldSchema
{
Name = name,
Fields = nestedSchema.Fields,
Type = EnumMap <BigqueryDbType> .ToApiValue(BigqueryDbType.Record),
Mode = EnumMap <FieldMode> .ToApiValue(mode, nameof(mode)),
Description = description,
});
}
