internal void ModifyConfiguration(JobConfigurationLoad loadRequest)
{
if (SkipLeadingRows != null)
{
loadRequest.SkipLeadingRows = SkipLeadingRows;
}
if (Quote != null)
{
loadRequest.Quote = Quote;
}
if (AllowQuotedNewlines != null)
{
loadRequest.AllowQuotedNewlines = AllowQuotedNewlines;
}
if (AllowJaggedRows != null)
{
loadRequest.AllowJaggedRows = AllowJaggedRows;
}
if (FieldDelimiter != null)
{
loadRequest.FieldDelimiter = FieldDelimiter;
}
if (AllowTrailingColumns != null)
{
loadRequest.IgnoreUnknownValues = AllowTrailingColumns;
}
if (MaxBadRecords != null)
{
loadRequest.MaxBadRecords = MaxBadRecords;
}
if (CreateDisposition != null)
{
loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value);
}
if (WriteDisposition != null)
{
loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value);
}
// TODO: Encoding? Only UTF-8 and ISO-8859-1 are supported... unsure what to do with this.
}
public void PropertiesSetOnRequest()
{
var options = new CreateLoadJobOptions
{
AllowJaggedRows = true,
AllowQuotedNewlines = true,
Autodetect = false,
CreateDisposition = CreateDisposition.CreateIfNeeded,
FieldDelimiter = "/",
IgnoreUnknownValues = true,
MaxBadRecords = 100,
NullMarker = "null",
ProjectionFields = new[] { "foo", "bar" },
Quote = "q",
SkipLeadingRows = 10,
SourceFormat = FileFormat.DatastoreBackup,
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null),
WriteDisposition = WriteDisposition.WriteAppend
};
JobConfigurationLoad load = new JobConfigurationLoad();
options.ModifyRequest(load);
Assert.Equal(true, load.AllowJaggedRows);
Assert.Equal(true, load.AllowQuotedNewlines);
Assert.Equal(false, load.Autodetect);
Assert.Equal("CREATE_IF_NEEDED", load.CreateDisposition);
Assert.Equal("/", load.FieldDelimiter);
Assert.Equal(true, load.IgnoreUnknownValues);
Assert.Equal(100, load.MaxBadRecords);
Assert.Equal("null", load.NullMarker);
Assert.Equal(new[] { "foo", "bar" }, load.ProjectionFields);
Assert.Equal("q", load.Quote);
Assert.Equal(10, load.SkipLeadingRows);
Assert.Equal("DATASTORE_BACKUP", load.SourceFormat);
Assert.Equal("WRITE_APPEND", load.WriteDisposition);
Assert.Equal("DAY", load.TimePartitioning.Type);
Assert.Null(load.TimePartitioning.ExpirationMs);
}
public void ModifyRequest()
{
var options = new UploadCsvOptions
{
AllowJaggedRows = true,
AllowQuotedNewlines = true,
AllowTrailingColumns = true,
CreateDisposition = CreateDisposition.CreateIfNeeded,
FieldDelimiter = "!",
MaxBadRecords = 10,
Quote = "'",
SkipLeadingRows = 5,
WriteDisposition = WriteDisposition.WriteAppend,
Autodetect = true,
NullMarker = "custom-null",
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null),
DestinationEncryptionConfiguration = new EncryptionConfiguration {
KmsKeyName = "projects/1/locations/us/keyRings/1/cryptoKeys/1"
},
};
JobConfigurationLoad config = new JobConfigurationLoad();
options.ModifyConfiguration(config);
Assert.Equal(true, config.AllowJaggedRows);
Assert.Equal(true, config.AllowQuotedNewlines);
Assert.Equal(true, config.IgnoreUnknownValues);
Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition);
Assert.Equal("!", config.FieldDelimiter);
Assert.Equal(10, config.MaxBadRecords);
Assert.Equal("'", config.Quote);
Assert.Equal(5, config.SkipLeadingRows);
Assert.Equal("WRITE_APPEND", config.WriteDisposition);
Assert.True(config.Autodetect);
Assert.Equal("custom-null", config.NullMarker);
Assert.Equal("DAY", config.TimePartitioning.Type);
Assert.Null(config.TimePartitioning.ExpirationMs);
Assert.Equal("projects/1/locations/us/keyRings/1/cryptoKeys/1", config.DestinationEncryptionConfiguration.KmsKeyName);
}
internal void ModifyConfiguration(JobConfigurationLoad loadRequest)
{
if (AllowUnknownFields != null)
{
loadRequest.IgnoreUnknownValues = AllowUnknownFields;
}
if (MaxBadRecords != null)
{
loadRequest.MaxBadRecords = MaxBadRecords;
}
if (CreateDisposition != null)
{
loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value);
}
if (WriteDisposition != null)
{
loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value);
}
if (Autodetect != null)
{
loadRequest.Autodetect = Autodetect;
}
}
public void ModifyRequest()
{
var options = new UploadJsonOptions
{
AllowUnknownFields = true,
CreateDisposition = CreateDisposition.CreateIfNeeded,
MaxBadRecords = 10,
WriteDisposition = WriteDisposition.WriteAppend,
Autodetect = true,
TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null)
};
JobConfigurationLoad config = new JobConfigurationLoad();
options.ModifyConfiguration(config);
Assert.Equal(true, config.IgnoreUnknownValues);
Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition);
Assert.Equal(10, config.MaxBadRecords);
Assert.Equal("WRITE_APPEND", config.WriteDisposition);
Assert.True(config.Autodetect);
Assert.Equal("DAY", config.TimePartitioning.Type);
Assert.Null(config.TimePartitioning.ExpirationMs);
}
private InsertRequest CreateLoadJobRequest(IEnumerable <string> sourceUris, TableReference destination, TableSchema schema, CreateLoadJobOptions options)
{
GaxPreconditions.CheckNotNull(sourceUris, nameof(sourceUris));
GaxPreconditions.CheckNotNull(destination, nameof(destination));
List <string> sourceList = sourceUris.ToList();
GaxPreconditions.CheckArgument(sourceList.Count != 0, nameof(sourceUris), "Source URIs collection cannot be empty");
var load = new JobConfigurationLoad {
SourceUris = sourceList, DestinationTable = destination, Schema = schema
};
options?.ModifyRequest(load);
var request = Service.Jobs.Insert(new Job
{
Configuration = new JobConfiguration
{
Load = load
}
}, ProjectId);
request.ModifyRequest += _versionHeaderAction;
return(request);
}
internal void ModifyRequest(JobConfigurationLoad load)
{
if (SkipLeadingRows != null)
{
load.SkipLeadingRows = SkipLeadingRows;
}
if (Quote != null)
{
load.Quote = Quote;
}
if (AllowQuotedNewlines != null)
{
load.AllowQuotedNewlines = AllowQuotedNewlines;
}
if (AllowJaggedRows != null)
{
load.AllowJaggedRows = AllowJaggedRows;
}
if (FieldDelimiter != null)
{
load.FieldDelimiter = FieldDelimiter;
}
if (IgnoreUnknownValues != null)
{
load.IgnoreUnknownValues = IgnoreUnknownValues;
}
if (MaxBadRecords != null)
{
load.MaxBadRecords = MaxBadRecords;
}
if (CreateDisposition != null)
{
load.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value);
}
if (WriteDisposition != null)
{
load.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value);
}
if (Autodetect != null)
{
load.Autodetect = Autodetect;
}
if (ProjectionFields != null)
{
load.ProjectionFields = ProjectionFields;
}
if (NullMarker != null)
{
load.NullMarker = NullMarker;
}
if (SourceFormat != null)
{
load.SourceFormat = EnumMap.ToApiValue(SourceFormat.Value);
}
if (TimePartitioning != null)
{
load.TimePartitioning = TimePartitioning;
}
if (DestinationEncryptionConfiguration != null)
{
load.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration;
}
if (DestinationSchemaUpdateOptions != null)
{
load.SchemaUpdateOptions = new List <string>(EnumMap.ToApiValues(DestinationSchemaUpdateOptions.Value));
}
if (UseAvroLogicalTypes != null)
{
load.UseAvroLogicalTypes = UseAvroLogicalTypes;
}
}
void UploadData(Table table)
{
Job job = new Job();
var config = new JobConfiguration();
var configLoad = new JobConfigurationLoad {
Schema = table.Schema,
DestinationTable = table.TableReference,
Encoding = "ISO-8859-1",
CreateDisposition = "CREATE_IF_NEEDED",
WriteDisposition = "",
FieldDelimiter = ",",
AllowJaggedRows = true,
SourceFormat = "CSV"
};
config.Load = configLoad;
job.Configuration = config;
var jobId = "---" + Environment.TickCount;
var jobRef = new JobReference {
JobId = jobId,
ProjectId = connection.ProjectId
};
job.JobReference = jobRef;
using (
Stream stream =
Assembly.GetExecutingAssembly()
.GetManifestResourceStream(string.Format("DevExpress.DataAccess.BigQuery.Tests.{0}.csv",
table.TableReference.TableId))) {
var insertMediaUpload = new JobsResource.InsertMediaUpload(connection.Service,
job, job.JobReference.ProjectId, stream, "application/octet-stream");
insertMediaUpload.Upload();
}
while (true) {
Job job1 = connection.Service.Jobs.Get(connection.ProjectId, jobId).Execute();
if (job1.Status.State.Equals("DONE")) {
break;
}
Thread.Sleep(5000);
}
}
