/// <summary>
/// Load Job main processing function.
/// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not
/// support Load opertations at the time of writing.
/// </summary>
public Apis.Bigquery.v2.Data.Job DoLoad()
{
if (ShouldProcess($"{Destination.TableId} (LOAD)"))
{
try
{
var loadjob = new Apis.Bigquery.v2.Data.Job
{
Configuration = new JobConfiguration
{
Load = new JobConfigurationLoad
{
DestinationTable = Destination,
SourceFormat = (Type == DataFormats.JSON) ? JSON_TEXT : Type.ToString(),
SourceUris = SourceUris,
Encoding = Encoding,
FieldDelimiter = FieldDelimiter,
Quote = Quote,
SkipLeadingRows = SkipLeadingRows,
IgnoreUnknownValues = AllowUnknownFields,
AllowJaggedRows = AllowJaggedRows,
AllowQuotedNewlines = AllowQuotedNewlines,
WriteDisposition = WriteMode.ToString(),
MaxBadRecords = MaxBadRecords ?? 0
}
}
};
return(Service.Jobs.Insert(loadjob, Project).Execute());
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden)
{
ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Access denied",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Resource not found",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict)
{
ThrowTerminatingError(new ErrorRecord(ex, "Load failed: Duplicate resource",
ErrorCategory.InvalidOperation, this));
}
}
return(null);
}
/// <summary>
/// This function waits for a job to reach the "DONE" status and then returns.
/// </summary>
/// <param name="job">Job to poll for completion.</param>
public Apis.Bigquery.v2.Data.Job PollForCompletion(Apis.Bigquery.v2.Data.Job job)
{
while (!job.Status.State.Equals(STATUS_DONE))
{
// Poll every 250 ms, or 4 times/sec.
System.Threading.Thread.Sleep(250);
try
{
job = Service.Jobs.Get(job.JobReference.ProjectId, job.JobReference.JobId).Execute();
}
catch (Exception ex)
{
ThrowTerminatingError(new ErrorRecord(ex,
"Polling for status was interrupted.",
ErrorCategory.InvalidOperation, this));
return(null);
}
}
return(job);
}
/// <summary>
/// Extract Job main processing function.
/// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not
/// support Extract opertations at the time of writing.
/// </summary>
public Apis.Bigquery.v2.Data.Job DoExtract()
{
if (ShouldProcess($"{Source.TableId} (EXTRACT)"))
{
try
{
var extractjob = new Apis.Bigquery.v2.Data.Job
{
Configuration = new JobConfiguration
{
Extract = new JobConfigurationExtract
{
SourceTable = Source,
DestinationUris = DestinationUris,
DestinationFormat = (Type == DataFormats.JSON) ? JSON_TEXT : Type.ToString(),
Compression = (Compress) ? COMPRESSION_GZIP : COMPRESSION_NONE,
PrintHeader = !NoHeader,
FieldDelimiter = FieldDelimiter
}
}
};
return(Service.Jobs.Insert(extractjob, Project).Execute());
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden)
{
ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Access denied",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Resource not found",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict)
{
ThrowTerminatingError(new ErrorRecord(ex, "Extract failed: Duplicate resource",
ErrorCategory.InvalidOperation, this));
}
}
return(null);
}
/// <summary>
/// Copy Job main processing function.
/// *This is written using Apis.BigQuery.v2 becuase Cloud.BigQuery did not
/// support Copy opertations at the time of writing.
/// </summary>
public Apis.Bigquery.v2.Data.Job DoCopy()
{
if (ShouldProcess($"Copying {Source.TableId} to {Destination.TableId}"))
{
try
{
var copyjob = new Apis.Bigquery.v2.Data.Job
{
Configuration = new JobConfiguration
{
Copy = new JobConfigurationTableCopy
{
DestinationTable = Destination,
SourceTable = Source,
WriteDisposition = WriteMode?.ToString() ?? WriteDisposition.WriteIfEmpty.ToString()
}
}
};
return(Service.Jobs.Insert(copyjob, Project).Execute());
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Forbidden)
{
ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Access denied",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Resource not found",
ErrorCategory.InvalidOperation, this));
}
catch (GoogleApiException ex) when(ex.HttpStatusCode == HttpStatusCode.Conflict)
{
ThrowTerminatingError(new ErrorRecord(ex, "Copy failed: Duplicate resource",
ErrorCategory.InvalidOperation, this));
}
}
return(null);
}