public void ModifyRequest()
{
var options = new UploadAvroOptions
{
AllowUnknownFields = true,
CreateDisposition = CreateDisposition.CreateIfNeeded,
MaxBadRecords = 10,
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null),
WriteDisposition = WriteDisposition.WriteAppend,
DestinationEncryptionConfiguration = new EncryptionConfiguration {
KmsKeyName = "projects/1/locations/us/keyRings/1/cryptoKeys/1"
},
DestinationSchemaUpdateOptions = SchemaUpdateOption.AllowFieldAddition | SchemaUpdateOption.AllowFieldRelaxation
};
JobConfigurationLoad config = new JobConfigurationLoad();
options.ModifyConfiguration(config);
Assert.Equal(true, config.IgnoreUnknownValues);
Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition);
Assert.Equal(10, config.MaxBadRecords);
Assert.Equal("WRITE_APPEND", config.WriteDisposition);
Assert.Equal("DAY", config.TimePartitioning.Type);
Assert.Null(config.TimePartitioning.ExpirationMs);
Assert.Equal("projects/1/locations/us/keyRings/1/cryptoKeys/1", config.DestinationEncryptionConfiguration.KmsKeyName);
Assert.Equal(2, config.SchemaUpdateOptions.Count);
Assert.Contains("ALLOW_FIELD_ADDITION", config.SchemaUpdateOptions);
Assert.Contains("ALLOW_FIELD_RELAXATION", config.SchemaUpdateOptions);
}
protected override void ProcessRecord()
{
Project = InputObject.ProjectId;
try
{
using (Stream fileInput = File.OpenRead(Filename))
{
BigQueryJob bqj;
switch (Type)
{
case DataFormats.AVRO:
UploadAvroOptions AvroOptions = new UploadAvroOptions();
AvroOptions.WriteDisposition = WriteMode;
AvroOptions.AllowUnknownFields = AllowUnknownFields;
bqj = Client.UploadAvro(InputObject, null, fileInput, AvroOptions);
break;
case DataFormats.JSON:
UploadJsonOptions JsonOptions = new UploadJsonOptions();
JsonOptions.WriteDisposition = WriteMode;
JsonOptions.AllowUnknownFields = AllowUnknownFields;
bqj = Client.UploadJson(InputObject, null, fileInput, JsonOptions);
break;
case DataFormats.CSV:
UploadCsvOptions CsvOptions = new UploadCsvOptions();
CsvOptions.WriteDisposition = WriteMode;
CsvOptions.AllowJaggedRows = AllowJaggedRows;
CsvOptions.AllowQuotedNewlines = AllowQuotedNewlines;
CsvOptions.AllowTrailingColumns = AllowUnknownFields;
CsvOptions.FieldDelimiter = FieldDelimiter;
CsvOptions.Quote = Quote;
CsvOptions.SkipLeadingRows = SkipLeadingRows;
bqj = Client.UploadCsv(InputObject, null, fileInput, CsvOptions);
break;
default:
throw UnknownParameterSetException;
}
bqj.PollUntilCompleted().ThrowOnAnyError();
}
}
catch (IOException ex)
{
ThrowTerminatingError(new ErrorRecord(ex,
$"Error while reading file '{Filename}'.",
ErrorCategory.ReadError, Filename));
return;
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(ex,
$"Error while uploading file '{Filename}' to table '{InputObject.TableId}'.",
ErrorCategory.WriteError, Filename));
}
}
public void ModifyRequest()
{
var options = new UploadAvroOptions
{
AllowUnknownFields = true,
CreateDisposition = CreateDisposition.CreateIfNeeded,
MaxBadRecords = 10,
WriteDisposition = WriteDisposition.WriteAppend
};
JobConfigurationLoad config = new JobConfigurationLoad();
options.ModifyConfiguration(config);
Assert.Equal(true, config.IgnoreUnknownValues);
Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition);
Assert.Equal(10, config.MaxBadRecords);
Assert.Equal("WRITE_APPEND", config.WriteDisposition);
}
