internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (SkipLeadingRows != null) { loadRequest.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { loadRequest.Quote = Quote; } if (AllowQuotedNewlines != null) { loadRequest.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { loadRequest.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { loadRequest.FieldDelimiter = FieldDelimiter; } if (AllowTrailingColumns != null) { loadRequest.IgnoreUnknownValues = AllowTrailingColumns; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } // TODO: Encoding? Only UTF-8 and ISO-8859-1 are supported... unsure what to do with this. }
public void PropertiesSetOnRequest() { var options = new CreateLoadJobOptions { AllowJaggedRows = true, AllowQuotedNewlines = true, Autodetect = false, CreateDisposition = CreateDisposition.CreateIfNeeded, FieldDelimiter = "/", IgnoreUnknownValues = true, MaxBadRecords = 100, NullMarker = "null", ProjectionFields = new[] { "foo", "bar" }, Quote = "q", SkipLeadingRows = 10, SourceFormat = FileFormat.DatastoreBackup, TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null), WriteDisposition = WriteDisposition.WriteAppend }; JobConfigurationLoad load = new JobConfigurationLoad(); options.ModifyRequest(load); Assert.Equal(true, load.AllowJaggedRows); Assert.Equal(true, load.AllowQuotedNewlines); Assert.Equal(false, load.Autodetect); Assert.Equal("CREATE_IF_NEEDED", load.CreateDisposition); Assert.Equal("/", load.FieldDelimiter); Assert.Equal(true, load.IgnoreUnknownValues); Assert.Equal(100, load.MaxBadRecords); Assert.Equal("null", load.NullMarker); Assert.Equal(new[] { "foo", "bar" }, load.ProjectionFields); Assert.Equal("q", load.Quote); Assert.Equal(10, load.SkipLeadingRows); Assert.Equal("DATASTORE_BACKUP", load.SourceFormat); Assert.Equal("WRITE_APPEND", load.WriteDisposition); Assert.Equal("DAY", load.TimePartitioning.Type); Assert.Null(load.TimePartitioning.ExpirationMs); }
public void ModifyRequest() { var options = new UploadCsvOptions { AllowJaggedRows = true, AllowQuotedNewlines = true, AllowTrailingColumns = true, CreateDisposition = CreateDisposition.CreateIfNeeded, FieldDelimiter = "!", MaxBadRecords = 10, Quote = "'", SkipLeadingRows = 5, WriteDisposition = WriteDisposition.WriteAppend, Autodetect = true, NullMarker = "custom-null", TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null), DestinationEncryptionConfiguration = new EncryptionConfiguration { KmsKeyName = "projects/1/locations/us/keyRings/1/cryptoKeys/1" }, }; JobConfigurationLoad config = new JobConfigurationLoad(); options.ModifyConfiguration(config); Assert.Equal(true, config.AllowJaggedRows); Assert.Equal(true, config.AllowQuotedNewlines); Assert.Equal(true, config.IgnoreUnknownValues); Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition); Assert.Equal("!", config.FieldDelimiter); Assert.Equal(10, config.MaxBadRecords); Assert.Equal("'", config.Quote); Assert.Equal(5, config.SkipLeadingRows); Assert.Equal("WRITE_APPEND", config.WriteDisposition); Assert.True(config.Autodetect); Assert.Equal("custom-null", config.NullMarker); Assert.Equal("DAY", config.TimePartitioning.Type); Assert.Null(config.TimePartitioning.ExpirationMs); Assert.Equal("projects/1/locations/us/keyRings/1/cryptoKeys/1", config.DestinationEncryptionConfiguration.KmsKeyName); }
internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (AllowUnknownFields != null) { loadRequest.IgnoreUnknownValues = AllowUnknownFields; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { loadRequest.Autodetect = Autodetect; } }
public void ModifyRequest() { var options = new UploadJsonOptions { AllowUnknownFields = true, CreateDisposition = CreateDisposition.CreateIfNeeded, MaxBadRecords = 10, WriteDisposition = WriteDisposition.WriteAppend, Autodetect = true, TimePartitioning = TimePartition.CreateDailyPartitioning(expiration: null) }; JobConfigurationLoad config = new JobConfigurationLoad(); options.ModifyConfiguration(config); Assert.Equal(true, config.IgnoreUnknownValues); Assert.Equal("CREATE_IF_NEEDED", config.CreateDisposition); Assert.Equal(10, config.MaxBadRecords); Assert.Equal("WRITE_APPEND", config.WriteDisposition); Assert.True(config.Autodetect); Assert.Equal("DAY", config.TimePartitioning.Type); Assert.Null(config.TimePartitioning.ExpirationMs); }
private InsertRequest CreateLoadJobRequest(IEnumerable <string> sourceUris, TableReference destination, TableSchema schema, CreateLoadJobOptions options) { GaxPreconditions.CheckNotNull(sourceUris, nameof(sourceUris)); GaxPreconditions.CheckNotNull(destination, nameof(destination)); List <string> sourceList = sourceUris.ToList(); GaxPreconditions.CheckArgument(sourceList.Count != 0, nameof(sourceUris), "Source URIs collection cannot be empty"); var load = new JobConfigurationLoad { SourceUris = sourceList, DestinationTable = destination, Schema = schema }; options?.ModifyRequest(load); var request = Service.Jobs.Insert(new Job { Configuration = new JobConfiguration { Load = load } }, ProjectId); request.ModifyRequest += _versionHeaderAction; return(request); }
internal void ModifyRequest(JobConfigurationLoad load) { if (SkipLeadingRows != null) { load.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { load.Quote = Quote; } if (AllowQuotedNewlines != null) { load.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { load.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { load.FieldDelimiter = FieldDelimiter; } if (IgnoreUnknownValues != null) { load.IgnoreUnknownValues = IgnoreUnknownValues; } if (MaxBadRecords != null) { load.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { load.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { load.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { load.Autodetect = Autodetect; } if (ProjectionFields != null) { load.ProjectionFields = ProjectionFields; } if (NullMarker != null) { load.NullMarker = NullMarker; } if (SourceFormat != null) { load.SourceFormat = EnumMap.ToApiValue(SourceFormat.Value); } if (TimePartitioning != null) { load.TimePartitioning = TimePartitioning; } if (DestinationEncryptionConfiguration != null) { load.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } if (DestinationSchemaUpdateOptions != null) { load.SchemaUpdateOptions = new List <string>(EnumMap.ToApiValues(DestinationSchemaUpdateOptions.Value)); } if (UseAvroLogicalTypes != null) { load.UseAvroLogicalTypes = UseAvroLogicalTypes; } }
void UploadData(Table table) { Job job = new Job(); var config = new JobConfiguration(); var configLoad = new JobConfigurationLoad { Schema = table.Schema, DestinationTable = table.TableReference, Encoding = "ISO-8859-1", CreateDisposition = "CREATE_IF_NEEDED", WriteDisposition = "", FieldDelimiter = ",", AllowJaggedRows = true, SourceFormat = "CSV" }; config.Load = configLoad; job.Configuration = config; var jobId = "---" + Environment.TickCount; var jobRef = new JobReference { JobId = jobId, ProjectId = connection.ProjectId }; job.JobReference = jobRef; using ( Stream stream = Assembly.GetExecutingAssembly() .GetManifestResourceStream(string.Format("DevExpress.DataAccess.BigQuery.Tests.{0}.csv", table.TableReference.TableId))) { var insertMediaUpload = new JobsResource.InsertMediaUpload(connection.Service, job, job.JobReference.ProjectId, stream, "application/octet-stream"); insertMediaUpload.Upload(); } while (true) { Job job1 = connection.Service.Jobs.Get(connection.ProjectId, jobId).Execute(); if (job1.Status.State.Equals("DONE")) { break; } Thread.Sleep(5000); } }