internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (SkipLeadingRows != null) { loadRequest.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { loadRequest.Quote = Quote; } if (AllowQuotedNewlines != null) { loadRequest.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { loadRequest.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { loadRequest.FieldDelimiter = FieldDelimiter; } if (AllowTrailingColumns != null) { loadRequest.IgnoreUnknownValues = AllowTrailingColumns; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { loadRequest.Autodetect = Autodetect; } if (NullMarker != null) { loadRequest.NullMarker = NullMarker; } if (TimePartitioning != null) { loadRequest.TimePartitioning = TimePartitioning; } if (DestinationEncryptionConfiguration != null) { loadRequest.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } if (DestinationSchemaUpdateOptions != null) { loadRequest.SchemaUpdateOptions = new List <string>(EnumMap.ToApiValues(DestinationSchemaUpdateOptions.Value)); } // TODO: Encoding? Only UTF-8 and ISO-8859-1 are supported... unsure what to do with this. }
internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (AllowUnknownFields != null) { loadRequest.IgnoreUnknownValues = AllowUnknownFields; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { loadRequest.Autodetect = Autodetect; } if (TimePartitioning != null) { loadRequest.TimePartitioning = TimePartitioning; } if (DestinationEncryptionConfiguration != null) { loadRequest.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } }
internal void ModifyRequest(Table table, InsertRequest request) { if (Description != null) { table.Description = Description; } if (FriendlyName != null) { table.FriendlyName = FriendlyName; } if (Expiration != null) { table.ExpirationTime = (long)(Expiration.Value - UnixEpoch).TotalMilliseconds; } #pragma warning disable CS0618 // Type or member is obsolete if (TimePartitionType != null) { if (TimePartitionType.Value == V2.TimePartitionType.None) { table.TimePartitioning = null; } else { table.TimePartitioning = new TimePartitioning { Type = EnumMap.ToApiValue(TimePartitionType.Value) }; } } if (TimePartitionExpiration != null) { if (table.TimePartitioning == null) { throw new ArgumentException($"Cannot specify {nameof(TimePartitionExpiration)} when not using time partitioning."); } table.TimePartitioning.ExpirationMs = (long)TimePartitionExpiration.Value.TotalMilliseconds; } // TODO: There's currently no way of forcing table.TimePartitioning to be set to null. Not a problem at the moment, // but we may want a sentinel value at some point. if (TimePartitioning != null) { if (TimePartitionType != null) { throw new ArgumentException($"Cannot specify {nameof(TimePartitioning)} and {nameof(TimePartitionType)} together."); } table.TimePartitioning = TimePartitioning; } #pragma warning restore CS0618 // Type or member is obsolete if (ExternalDataConfiguration != null && View != null) { throw new ArgumentException($"Cannot specify both {nameof(ExternalDataConfiguration)} and {nameof(View)}"); } if (ExternalDataConfiguration != null) { table.ExternalDataConfiguration = ExternalDataConfiguration; } if (View != null) { table.View = View; } }
internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (AllowUnknownFields != null) { loadRequest.IgnoreUnknownValues = AllowUnknownFields; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (TimePartitioning != null) { loadRequest.TimePartitioning = TimePartitioning; } if (DestinationEncryptionConfiguration != null) { loadRequest.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } if (DestinationSchemaUpdateOptions != null) { loadRequest.SchemaUpdateOptions = new List <string>(EnumMap.ToApiValues(DestinationSchemaUpdateOptions.Value)); } }
internal void ModifyRequest(JobConfigurationExtract extract) { if (DestinationFormat != null) { if (DestinationFormat == FileFormat.DatastoreBackup) { throw new ArgumentException($"{nameof(FileFormat.DatastoreBackup)} is not supported for extract operations", "options"); } extract.DestinationFormat = EnumMap.ToApiValue(DestinationFormat.Value); } if (Compression != null) { extract.Compression = EnumMap.ToApiValue(Compression.Value); } if (FieldDelimiter != null) { extract.FieldDelimiter = FieldDelimiter; } if (PrintHeader != null) { extract.PrintHeader = PrintHeader; } if (UseAvroLogicalTypes != null) { extract.UseAvroLogicalTypes = UseAvroLogicalTypes; } }
private static QueryParameter PopulateArrayParameter(QueryParameter parameter, object value, BigQueryDbType?arrayType) { if (value == null) { throw new InvalidOperationException("The value of an array parameter cannot be null"); } if (!IsArrayValue(value)) { throw new InvalidOperationException($"Invalid value for array parameter: {value.GetType()}"); } List <object> values = ((IEnumerable)value).Cast <object>().ToList(); if (values.Any(v => v == null)) { throw new InvalidOperationException("Array parameter values cannot contain null elements"); } BigQueryDbType actualArrayType = arrayType ?? s_typeMapping[GetArrayElementType(value)]; parameter.ParameterType = new QueryParameterType { Type = EnumMap.ToApiValue(BigQueryDbType.Array), ArrayType = new QueryParameterType { Type = EnumMap.ToApiValue(actualArrayType) } }; var parameterValues = values .Select(p => new BigQueryParameter(actualArrayType, p).ToQueryParameter(BigQueryParameterMode.Positional).ParameterValue) .ToList(); parameter.ParameterValue = new QueryParameterValue { ArrayValues = parameterValues }; return(parameter); }
internal void ModifyRequest(JobConfigurationExtract extract) { if (DestinationFormat != null) { extract.DestinationFormat = EnumMap.ToApiValue(DestinationFormat.Value); } }
internal void ModifyRequest(JobConfigurationLoad load) { if (SkipLeadingRows != null) { load.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { load.Quote = Quote; } if (AllowQuotedNewlines != null) { load.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { load.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { load.FieldDelimiter = FieldDelimiter; } if (IgnoreUnknownValues != null) { load.IgnoreUnknownValues = IgnoreUnknownValues; } if (MaxBadRecords != null) { load.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { load.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { load.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { load.Autodetect = Autodetect; } if (ProjectionFields != null) { load.ProjectionFields = ProjectionFields; } if (NullMarker != null) { load.NullMarker = NullMarker; } if (SourceFormat != null) { load.SourceFormat = EnumMap.ToApiValue(SourceFormat.Value); } if (TimePartitioning != null) { load.TimePartitioning = TimePartitioning; } }
internal void ModifyRequest(JobConfigurationQuery query) { // Note: no validation of combinations (flatten results etc). Leave this to the server, // to avoid restrictions getting out of date. if (AllowLargeResults != null) { query.AllowLargeResults = AllowLargeResults; } if (CreateDisposition != null) { query.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (DefaultDataset != null) { query.DefaultDataset = DefaultDataset; } if (DestinationTable != null) { query.DestinationTable = DestinationTable; } if (FlattenResults != null) { query.FlattenResults = FlattenResults; } if (MaximumBillingTier != null) { query.MaximumBillingTier = MaximumBillingTier; } if (MaximumBytesBilled != null) { query.MaximumBytesBilled = MaximumBytesBilled; } if (Priority != null) { query.Priority = EnumMap.ToApiValue(Priority.Value); } if (UseQueryCache != null) { query.UseQueryCache = UseQueryCache; } if (WriteDisposition != null) { query.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (UseLegacySql != null) { query.UseLegacySql = UseLegacySql; } if (ParameterMode != null) { // Safe for now; we only have "named" or "positional". This is unlikely to change. query.ParameterMode = ParameterMode.ToString().ToLowerInvariant(); } if (DestinationEncryptionConfiguration != null) { query.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } }
/// <summary> /// If the value is null, populate the parameter with an QueryParameterValue. Otherwise, /// throw an exception - this is expected to be the last call in a chain, so at this point we know /// we can't handle a value of this type. /// </summary> internal static QueryParameter UseNullScalarOrThrow(this QueryParameter parameter, object value) { if (value == null) { parameter.ParameterValue = new QueryParameterValue(); return parameter; } var clrEnum = EnumMap<BigQueryDbType>.ToValue(parameter.ParameterType.Type); throw new InvalidOperationException($"Value of type {value.GetType()} cannot be used for a parameter of type {clrEnum}"); }
internal void ModifyRequest(JobConfigurationTableCopy copy) { if (CreateDisposition != null) { copy.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { copy.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } }
/// <summary> /// Creates a nested field with the specified schema, and adds it to the schema being built. /// </summary> /// <param name="name">The name of the field. Must be a valid field name.</param> /// <param name="nestedSchema">The schema for the nested field. Must not be null.</param> /// <param name="mode">The mode of the field. Must be a defined member within <see cref="BigQueryFieldMode"/>.</param> /// <param name="description">The description of the field. May be null.</param> public void Add(string name, TableSchema nestedSchema, BigQueryFieldMode mode = BigQueryFieldMode.Nullable, string description = null) { ValidateFieldName(name, nameof(name)); GaxPreconditions.CheckNotNull(nestedSchema, nameof(nestedSchema)); Add(new TableFieldSchema { Name = name, Fields = nestedSchema.Fields, Type = EnumMap.ToApiValue(BigQueryDbType.Struct), Mode = EnumMap.ToApiValue(mode, nameof(mode)), Description = description, }); }
internal void ModifyRequest(JobConfigurationQuery query) { // Note: no validation of combinations (flatten results etc). Leave this to the server, // to avoid restrictions getting out of date. if (AllowLargeResults != null) { query.AllowLargeResults = AllowLargeResults; } if (CreateDisposition != null) { query.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (DefaultDataset != null) { query.DefaultDataset = DefaultDataset; } if (DestinationTable != null) { query.DestinationTable = DestinationTable; } if (FlattenResults != null) { query.FlattenResults = FlattenResults; } if (MaximumBillingTier != null) { query.MaximumBillingTier = MaximumBillingTier; } if (MaximumBytesBilled != null) { query.MaximumBytesBilled = MaximumBytesBilled; } if (Priority != null) { query.Priority = EnumMap.ToApiValue(Priority.Value); } if (UseQueryCache != null) { query.UseQueryCache = UseQueryCache; } if (WriteDisposition != null) { query.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (UseLegacySql != null) { query.UseLegacySql = UseLegacySql; } }
internal void ModifyRequest(Table table, InsertRequest request) { if (Description != null) { table.Description = Description; } if (FriendlyName != null) { table.FriendlyName = FriendlyName; } if (Expiration != null) { table.ExpirationTime = (long)(Expiration.Value - UnixEpoch).TotalMilliseconds; } if (TimePartitionType != null) { if (TimePartitionType.Value == V2.TimePartitionType.None) { table.TimePartitioning = null; } else { table.TimePartitioning = new TimePartitioning { Type = EnumMap.ToApiValue(TimePartitionType.Value) }; } } if (TimePartitionExpiration != null) { if (table.TimePartitioning == null) { throw new ArgumentException($"Cannot specify {nameof(TimePartitionExpiration)} when not using time partitioning."); } table.TimePartitioning.ExpirationMs = (long)TimePartitionExpiration.Value.TotalMilliseconds; } if (ExternalDataConfiguration != null && View != null) { throw new ArgumentException($"Cannot specify both {nameof(ExternalDataConfiguration)} and {nameof(View)}"); } if (ExternalDataConfiguration != null) { table.ExternalDataConfiguration = ExternalDataConfiguration; } if (View != null) { table.View = View; } }
/// <summary> /// Creates a field with the specified details, and adds it to the schema being built. /// </summary> /// <param name="name">The name of the field. Must be a valid field name.</param> /// <param name="type">The type of the field. Must be a defined member within <see cref="BigQueryDbType"/>, other than <c>Struct</c> or <c>Array</c>.</param> /// <param name="mode">The mode of the field. Must be a defined member within <see cref="BigQueryFieldMode"/>.</param> /// <param name="description">The description of the field. May be null.</param> public void Add(string name, BigQueryDbType type, BigQueryFieldMode mode = BigQueryFieldMode.Nullable, string description = null) { ValidateFieldName(name, nameof(name)); GaxPreconditions.CheckArgument(type != BigQueryDbType.Struct, nameof(type), $"{nameof(BigQueryDbType.Struct)} fields must be specified with their schema"); GaxPreconditions.CheckArgument(type != BigQueryDbType.Array, nameof(type), $"{nameof(BigQueryDbType.Array)} fields must be specified with by element type with a {nameof(BigQueryFieldMode)} of {nameof(BigQueryFieldMode.Repeated)}"); Add(new TableFieldSchema { Name = name, Type = EnumMap.ToApiValue(type, nameof(type)), Mode = EnumMap.ToApiValue(mode, nameof(mode)), Description = description, }); }
internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (AllowUnknownFields != null) { loadRequest.IgnoreUnknownValues = AllowUnknownFields; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } }
internal void ModifyRequest(ListRequest request) { if (PageSize != null) { request.MaxResults = PageSize; } if (PageToken != null) { request.PageToken = PageToken; } if (ReadMask != null) { request.ReadMask = ReadMask; } if (RoutineTypeFilter.HasValue) { request.Filter = $"{RoutineTypeFilterName}:{EnumMap.ToApiValue(RoutineTypeFilter.Value, nameof(RoutineTypeFilter))}"; } }
internal void ModifyConfiguration(JobConfigurationLoad loadRequest) { if (SkipLeadingRows != null) { loadRequest.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { loadRequest.Quote = Quote; } if (AllowQuotedNewlines != null) { loadRequest.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { loadRequest.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { loadRequest.FieldDelimiter = FieldDelimiter; } if (AllowTrailingColumns != null) { loadRequest.IgnoreUnknownValues = AllowTrailingColumns; } if (MaxBadRecords != null) { loadRequest.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { loadRequest.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { loadRequest.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } // TODO: Encoding? Only UTF-8 and ISO-8859-1 are supported... unsure what to do with this. }
/// <summary> /// Returns the mode of a field as a <see cref="BigQueryFieldMode"/>. /// </summary> internal static BigQueryFieldMode GetFieldMode(this TableFieldSchema field) => EnumMap <BigQueryFieldMode> .ToValue(field.Mode);
internal static string ToApiValue <T>(T value, string paramName = "value") where T : struct => EnumMap <T> .ToApiValue(value, paramName);
/// <summary> /// Returns the mode of a field as a <see cref="BigQueryFieldMode"/>. /// If the mode isn't specified, it defaults to Nullable. /// </summary> internal static BigQueryFieldMode GetFieldMode(this TableFieldSchema field) => field.Mode == null ? BigQueryFieldMode.Nullable : EnumMap <BigQueryFieldMode> .ToValue(field.Mode);
/// <summary> /// Returns the type of a field as a <see cref="BigQueryDbType"/>. /// </summary> internal static BigQueryDbType GetFieldType(this TableFieldSchema field) => EnumMap <BigQueryDbType> .ToValue(field.Type);
internal void ModifyRequest(JobConfigurationLoad load) { if (SkipLeadingRows != null) { load.SkipLeadingRows = SkipLeadingRows; } if (Quote != null) { load.Quote = Quote; } if (AllowQuotedNewlines != null) { load.AllowQuotedNewlines = AllowQuotedNewlines; } if (AllowJaggedRows != null) { load.AllowJaggedRows = AllowJaggedRows; } if (FieldDelimiter != null) { load.FieldDelimiter = FieldDelimiter; } if (IgnoreUnknownValues != null) { load.IgnoreUnknownValues = IgnoreUnknownValues; } if (MaxBadRecords != null) { load.MaxBadRecords = MaxBadRecords; } if (CreateDisposition != null) { load.CreateDisposition = EnumMap.ToApiValue(CreateDisposition.Value); } if (WriteDisposition != null) { load.WriteDisposition = EnumMap.ToApiValue(WriteDisposition.Value); } if (Autodetect != null) { load.Autodetect = Autodetect; } if (ProjectionFields != null) { load.ProjectionFields = ProjectionFields; } if (NullMarker != null) { load.NullMarker = NullMarker; } if (SourceFormat != null) { load.SourceFormat = EnumMap.ToApiValue(SourceFormat.Value); } if (TimePartitioning != null) { load.TimePartitioning = TimePartitioning; } if (DestinationEncryptionConfiguration != null) { load.DestinationEncryptionConfiguration = DestinationEncryptionConfiguration; } if (DestinationSchemaUpdateOptions != null) { load.SchemaUpdateOptions = new List <string>(EnumMap.ToApiValues(DestinationSchemaUpdateOptions.Value)); } if (UseAvroLogicalTypes != null) { load.UseAvroLogicalTypes = UseAvroLogicalTypes; } }
internal QueryParameter ToQueryParameter(BigQueryParameterMode parameterMode) { if (parameterMode == BigQueryParameterMode.Named && string.IsNullOrEmpty(Name)) { throw new InvalidOperationException("Unnamed parameters cannot be used in command using BigQueryParameterMode.Named"); } var value = Value; if (Type == null && value == null) { throw new InvalidOperationException("A null-valued parameter must have an explicitly specified type"); } var type = Type ?? InferParameterType(value); var parameter = new QueryParameter { Name = Name, ParameterType = new QueryParameterType { Type = EnumMap.ToApiValue(type) }, }; switch (type) { case BigQueryDbType.Array: return(PopulateArrayParameter(parameter, value, ArrayType)); case BigQueryDbType.Bool: return(parameter.PopulateScalar <bool>(value, x => x ? "TRUE" : "FALSE") ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Bytes: return(parameter.PopulateScalar <byte[]>(value, x => Convert.ToBase64String(x)) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Date: return(parameter.PopulateScalar <DateTime>(value, x => x.ToString("yyyy-MM-dd", InvariantCulture)) ?? parameter.PopulateScalar <DateTimeOffset>(value, x => x.ToString("yyyy-MM-dd", InvariantCulture)) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.DateTime: return(parameter.PopulateScalar <DateTime>(value, x => x.ToString("yyyy-MM-dd HH:mm:ss.FFFFFF", InvariantCulture)) ?? parameter.PopulateScalar <DateTimeOffset>(value, x => x.ToString("yyyy-MM-dd HH:mm:ss.FFFFFF", InvariantCulture)) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Float64: return(parameter.PopulateInteger(value) ?? parameter.PopulateFloatingPoint(value) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Int64: return(parameter.PopulateInteger(value) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.String: return(parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Struct: throw new NotImplementedException("Struct parameters are not yet implemented"); case BigQueryDbType.Time: return(parameter.PopulateScalar <TimeSpan>(value, FormatTimeSpan) ?? parameter.PopulateScalar <DateTimeOffset>(value, x => x.ToString("HH:mm:ss.FFFFFF", InvariantCulture)) ?? parameter.PopulateScalar <DateTime>(value, x => x.ToString("HH:mm:ss.FFFFFF", InvariantCulture)) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); case BigQueryDbType.Timestamp: return(parameter.PopulateScalar <DateTime>(value, x => { if (x.Kind != DateTimeKind.Utc) { throw new InvalidOperationException($"A DateTime with a Kind of {x.Kind} cannot be used for a Timestamp parameter"); } return x.ToString("yyyy-MM-dd HH:mm:ss.FFFFFF"); }) ?? parameter.PopulateScalar <DateTimeOffset>(value, x => x.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFzzz", InvariantCulture)) ?? parameter.PopulateScalar <string>(value, x => x) ?? parameter.UseNullScalarOrThrow(value)); default: throw new InvalidOperationException($"No conversion available for parameter type {type}"); } }