public void PropertiesSetOnRequest()
{
var options = new CreateTableOptions
{
Expiration = new DateTimeOffset(1970, 1, 1, 0, 0, 5, 0, TimeSpan.Zero),
Description = "A description",
FriendlyName = "A friendly name",
TimePartitioning = TimePartition.CreateDailyPartitioning(TimeSpan.FromDays(10)),
ExternalDataConfiguration = new ExternalDataConfiguration(),
EncryptionConfiguration = new EncryptionConfiguration {
KmsKeyName = "projects/1/locations/us/keyRings/1/cryptoKeys/1"
},
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
options.ModifyRequest(table, request);
Assert.Equal(5 * 1000, table.ExpirationTime);
Assert.Equal("A description", table.Description);
Assert.Equal("A friendly name", table.FriendlyName);
Assert.Equal("DAY", table.TimePartitioning.Type);
Assert.Equal(10 * 24 * 60 * 60 * 1000L, table.TimePartitioning.ExpirationMs);
Assert.Same(options.ExternalDataConfiguration, table.ExternalDataConfiguration);
Assert.Equal("projects/1/locations/us/keyRings/1/cryptoKeys/1", table.EncryptionConfiguration.KmsKeyName);
}
public async Task <IBigQueryTableClient> GetTableClient(
string datasetId,
string tableId,
TableSchema schema,
CreateTableOptions createTableOptions = null,
CreateDatasetOptions createDatasetOptions = null,
CancellationToken ct = default(CancellationToken)
)
{
try
{
var dataset = await InnerClient.GetOrCreateDatasetAsync(
datasetId,
createOptions : createDatasetOptions,
cancellationToken : ct);
var table = await dataset.GetOrCreateTableAsync(
tableId,
schema,
createOptions : createTableOptions,
cancellationToken : ct);
return(new BigQueryTableClient(table));
}
catch (Exception ex)
{
throw new BigQuerierException($"Failed to create dataset {datasetId} or table {tableId}", ex);
}
}
public void PropertiesSetOnRequest()
{
var options = new CreateTableOptions();
InsertRequest request = new InsertRequest(new BigqueryService(), new Table(), "project", "dataset");
options.ModifyRequest(request);
}
public void TimePartitioningExpirationWithoutTimePartitioningType()
{
var options = new CreateTableOptions
{
TimePartitionExpiration = TimeSpan.FromDays(1)
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
Assert.Throws <ArgumentException>(() => options.ModifyRequest(table, request));
}
public void InvalidTimePartitioningType()
{
var options = new CreateTableOptions
{
TimePartitionType = (TimePartitionType)2
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
Assert.Throws <ArgumentException>(() => options.ModifyRequest(table, request));
}
public void ExternalConfigurationAndViewInvalid()
{
var options = new CreateTableOptions
{
ExternalDataConfiguration = new ExternalDataConfiguration(),
View = new ViewDefinition()
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
Assert.Throws <ArgumentException>(() => options.ModifyRequest(table, request));
}
public void init()
{
var dataset = this.client.GetOrCreateDataset(datasetName);
var file = File.OpenText("tableSchema.json");
var serializer = new JsonSerializer();
var schema = (TableSchema)serializer.Deserialize(file, typeof(TableSchema));
var options = new CreateTableOptions
{
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null)
};
var table = dataset.GetOrCreateTable(tableName, schema, null, options);
}
public void CreateTableEquivalents()
{
var datasetId = "dataset";
var tableId = "table";
var schema = new TableSchemaBuilder().Build();
var reference = GetTableReference(datasetId, tableId);
var options = new CreateTableOptions();
VerifyEquivalent(new BigQueryTable(new DerivedBigQueryClient(), GetTable(reference)),
client => client.CreateTable(MatchesWhenSerialized(reference), schema, options),
client => client.CreateTable(datasetId, tableId, schema, options),
client => client.CreateTable(ProjectId, datasetId, tableId, schema, options),
client => new BigQueryDataset(client, GetDataset(datasetId)).CreateTable(tableId, schema, options));
}
public void ObsoleteTimePartitioning()
{
var options = new CreateTableOptions
{
TimePartitionType = TimePartitionType.Day,
TimePartitionExpiration = TimeSpan.FromDays(10),
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
options.ModifyRequest(table, request);
Assert.Equal("DAY", table.TimePartitioning.Type);
Assert.Equal(10 * 24 * 60 * 60 * 1000L, table.TimePartitioning.ExpirationMs);
}
public void CreateTableAsyncEquivalents()
{
var datasetId = "dataset";
var tableId = "table";
var schema = new TableSchemaBuilder().Build();
var reference = GetTableReference(datasetId, tableId);
var options = new CreateTableOptions();
var token = new CancellationTokenSource().Token;
VerifyEquivalentAsync(new BigqueryTable(new DerivedBigqueryClient(), GetTable(reference)),
client => client.CreateTableAsync(MatchesWhenSerialized(reference), schema, options, token),
client => client.CreateTableAsync(datasetId, tableId, schema, options, token),
client => client.CreateTableAsync(ProjectId, datasetId, tableId, schema, options, token),
client => new BigqueryDataset(client, GetDataset(datasetId)).CreateTableAsync(tableId, schema, options, token));
}
public void PropertiesSetOnRequest()
{
var options = new CreateTableOptions
{
Expiration = new DateTimeOffset(1970, 1, 1, 0, 0, 5, 0, TimeSpan.Zero),
Description = "A description",
FriendlyName = "A friendly name"
};
Table table = new Table();
InsertRequest request = new InsertRequest(new BigqueryService(), table, "project", "dataset");
options.ModifyRequest(table, request);
Assert.Equal(5 * 1000, table.ExpirationTime);
Assert.Equal("A description", table.Description);
Assert.Equal("A friendly name", table.FriendlyName);
}
public async Task <BigQueryTable> GetOrCreateTableAsync <T>(string optionsName = "")
{
var currentOptions = _options.Get(optionsName);
var client = _bigQueryClientFactory.GetOrCreateClient(currentOptions.ProjectId);
var type = typeof(T);
_logger.LogInformation($"Building schema from Type: {type.Name}");
var schemaBuilder = BigQuerySchemaBuilder.GetSchemaBuilder(type, "> ");
var createTableOptions = new CreateTableOptions()
{
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null)
};
_logger.LogInformation($"Get or Create Dataset: {currentOptions.DatasetName}");
_ = await client.GetOrCreateDatasetAsync(currentOptions.DatasetName);
_logger.LogInformation($"Get or Create Table: {currentOptions.GetTableName(type.Name)}");
return(await client.GetOrCreateTableAsync(currentOptions.DatasetName, currentOptions.GetTableName(type.Name), schemaBuilder.Build(), createOptions : createTableOptions));
}
public DispatcherService(
[NotNull] IBigQueryClient client,
[NotNull] TableSchema schema,
[NotNull] string datasetId,
[NotNull] Func <DateTime, string> tableNameFun,
CreateTableOptions createTableOptions = null,
CreateDatasetOptions createDatasetOptions = null,
IDispatchLogger logger = null)
{
_client = client;
_schema = schema;
_datasetId = datasetId;
_tableNameFun = tableNameFun;
_createTableOptions = createTableOptions;
_createDatasetOptions = createDatasetOptions;
_logger = logger;
_tokenSource = new CancellationTokenSource();
_consumeTask = Task.Factory.StartNew(() => RunConsume(_tokenSource.Token), TaskCreationOptions.LongRunning);
_storageTask = Task.Factory.StartNew(() => RunStorage(_tokenSource.Token),
TaskCreationOptions.LongRunning);
}