public void ModifyRequest()
{
EncryptionConfiguration config = new EncryptionConfiguration {
KmsKeyName = "dataset_default_key"
};
var options = new CreateDatasetOptions
{
DefaultTableExpiration = TimeSpan.FromSeconds(10),
Description = "A description",
FriendlyName = "A friendly name",
Location = "EU",
DefaultEncryptionConfiguration = config,
DefaultPartitionExpiration = TimeSpan.FromSeconds(20)
};
Dataset dataset = new Dataset {
Location = "US"
};
InsertRequest request = new InsertRequest(new BigqueryService(), dataset, "project");
options.ModifyRequest(dataset, request);
Assert.Equal(10 * 1000, dataset.DefaultTableExpirationMs);
Assert.Equal("A description", dataset.Description);
Assert.Equal("A friendly name", dataset.FriendlyName);
Assert.Equal("EU", dataset.Location);
Assert.Equal(20 * 1000, dataset.DefaultPartitionExpirationMs);
Assert.Equal(config, dataset.DefaultEncryptionConfiguration);
}
public async Task <IBigQueryTableClient> GetTableClient(
string datasetId,
string tableId,
TableSchema schema,
CreateTableOptions createTableOptions = null,
CreateDatasetOptions createDatasetOptions = null,
CancellationToken ct = default(CancellationToken)
)
{
try
{
var dataset = await InnerClient.GetOrCreateDatasetAsync(
datasetId,
createOptions : createDatasetOptions,
cancellationToken : ct);
var table = await dataset.GetOrCreateTableAsync(
tableId,
schema,
createOptions : createTableOptions,
cancellationToken : ct);
return(new BigQueryTableClient(table));
}
catch (Exception ex)
{
throw new BigQuerierException($"Failed to create dataset {datasetId} or table {tableId}", ex);
}
}
public void ModifyRequest()
{
var options = new CreateDatasetOptions();
InsertRequest request = new InsertRequest(new BigqueryService(), new Dataset(), "project");
options.ModifyRequest(request);
}
public void CreateDatasetEquivalents()
{
var datasetId = "dataset";
var reference = GetDatasetReference(datasetId);
var options = new CreateDatasetOptions();
VerifyEquivalent(new BigQueryDataset(new DerivedBigQueryClient(), GetDataset(reference)),
client => client.CreateDataset(MatchesWhenSerialized(reference), options),
client => client.CreateDataset(datasetId, options),
client => client.CreateDataset(ProjectId, datasetId, options));
}
public void CreateDatasetAsyncEquivalents()
{
var datasetId = "dataset";
var reference = GetDatasetReference(datasetId);
var options = new CreateDatasetOptions();
var token = new CancellationTokenSource().Token;
VerifyEquivalentAsync(new BigQueryDataset(new DerivedBigQueryClient(), GetDataset(reference)),
client => client.CreateDatasetAsync(MatchesWhenSerialized(reference), options, token),
client => client.CreateDatasetAsync(datasetId, options, token),
client => client.CreateDatasetAsync(ProjectId, datasetId, options, token));
}
public BigQueryDataset CreateDataset(string projectId,
string location, string datasetId
)
{
BigQueryClient client = BigQueryClient.Create(projectId);
var createDatasetOptions = new CreateDatasetOptions()
{
// Specify the geographic location where the dataset should reside.
Location = location
};
// Create the dataset
return(client.CreateDataset(
datasetId: datasetId, options: createDatasetOptions));
}
public void ModifyRequest()
{
var options = new CreateDatasetOptions
{
DefaultTableExpiration = TimeSpan.FromSeconds(10),
Description = "A description",
FriendlyName = "A friendly name"
};
Dataset dataset = new Dataset();
InsertRequest request = new InsertRequest(new BigqueryService(), dataset, "project");
options.ModifyRequest(dataset, request);
Assert.Equal(10 * 1000, dataset.DefaultTableExpirationMs);
Assert.Equal("A description", dataset.Description);
Assert.Equal("A friendly name", dataset.FriendlyName);
}
public void ModifyRequest_NoLocation()
{
// When the options don't specify a location, we should keep the one in the original Dataset.
var options = new CreateDatasetOptions
{
DefaultTableExpiration = TimeSpan.FromSeconds(10),
Description = "A description",
FriendlyName = "A friendly name",
};
Dataset dataset = new Dataset {
Location = "US"
};
InsertRequest request = new InsertRequest(new BigqueryService(), dataset, "project");
options.ModifyRequest(dataset, request);
Assert.Equal("US", dataset.Location);
}
public DispatcherService(
[NotNull] IBigQueryClient client,
[NotNull] TableSchema schema,
[NotNull] string datasetId,
[NotNull] Func <DateTime, string> tableNameFun,
CreateTableOptions createTableOptions = null,
CreateDatasetOptions createDatasetOptions = null,
IDispatchLogger logger = null)
{
_client = client;
_schema = schema;
_datasetId = datasetId;
_tableNameFun = tableNameFun;
_createTableOptions = createTableOptions;
_createDatasetOptions = createDatasetOptions;
_logger = logger;
_tokenSource = new CancellationTokenSource();
_consumeTask = Task.Factory.StartNew(() => RunConsume(_tokenSource.Token), TaskCreationOptions.LongRunning);
_storageTask = Task.Factory.StartNew(() => RunStorage(_tokenSource.Token),
TaskCreationOptions.LongRunning);
}
