/// <summary> /// Load Job main processing function. /// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not /// support Load opertations at the time of writing. /// </summary> public Apis.Bigquery.v2.Data.Job DoLoad() { if (ShouldProcess($"{Destination.TableId} (LOAD)")) { try { var loadjob = new Apis.Bigquery.v2.Data.Job { Configuration = new JobConfiguration { Load = new JobConfigurationLoad { DestinationTable = Destination, SourceFormat = (Type == DataFormats.JSON) ? JSON_TEXT : Type.ToString(), SourceUris = SourceUris, Encoding = Encoding, FieldDelimiter = FieldDelimiter, Quote = Quote, SkipLeadingRows = SkipLeadingRows, IgnoreUnknownValues = AllowUnknownFields, AllowJaggedRows = AllowJaggedRows, AllowQuotedNewlines = AllowQuotedNewlines, WriteDisposition = WriteMode.ToString(), MaxBadRecords = MaxBadRecords ?? 0 } } }; return(Service.Jobs.Insert(loadjob, Project).Execute()); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden) { ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Access denied", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound) { ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Resource not found", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict) { ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Duplicate resource", ErrorCategory.InvalidOperation, this)); } } return(null); }
/// <summary> /// This function waits for a job to reach the "DONE" status and then returns. /// </summary> /// <param name="job">Job to poll for completion.</param> public Apis.Bigquery.v2.Data.Job PollForCompletion(Apis.Bigquery.v2.Data.Job job) { while (!job.Status.State.Equals(STATUS_DONE)) { // Poll every 250 ms, or 4 times/sec. System.Threading.Thread.Sleep(250); try { job = Service.Jobs.Get(job.JobReference.ProjectId, job.JobReference.JobId).Execute(); } catch (Exception ex) { ThrowTerminatingError(new ErrorRecord(ex, "Polling for status was interrupted.", ErrorCategory.InvalidOperation, this)); return(null); } } return(job); }
/// <summary> /// Extract Job main processing function. /// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not /// support Extract opertations at the time of writing. /// </summary> public Apis.Bigquery.v2.Data.Job DoExtract() { if (ShouldProcess($"{Source.TableId} (EXTRACT)")) { try { var extractjob = new Apis.Bigquery.v2.Data.Job { Configuration = new JobConfiguration { Extract = new JobConfigurationExtract { SourceTable = Source, DestinationUris = DestinationUris, DestinationFormat = (Type == DataFormats.JSON) ? JSON_TEXT : Type.ToString(), Compression = (Compress) ? COMPRESSION_GZIP : COMPRESSION_NONE, PrintHeader = !NoHeader, FieldDelimiter = FieldDelimiter } } }; return(Service.Jobs.Insert(extractjob, Project).Execute()); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden) { ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Access denied", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound) { ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Resource not found", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict) { ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Duplicate resource", ErrorCategory.InvalidOperation, this)); } } return(null); }
/// <summary> /// Copy Job main processing function. /// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not /// support Copy opertations at the time of writing. /// </summary> public Apis.Bigquery.v2.Data.Job DoCopy() { if (ShouldProcess($"Copying {Source.TableId} to {Destination.TableId}")) { try { var copyjob = new Apis.Bigquery.v2.Data.Job { Configuration = new JobConfiguration { Copy = new JobConfigurationTableCopy { DestinationTable = Destination, SourceTable = Source, WriteDisposition = WriteMode?.ToString() ?? WriteDisposition.WriteIfEmpty.ToString() } } }; return(Service.Jobs.Insert(copyjob, Project).Execute()); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden) { ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Access denied", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound) { ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Resource not found", ErrorCategory.InvalidOperation, this)); } catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict) { ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Duplicate resource", ErrorCategory.InvalidOperation, this)); } } return(null); }