private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
if (insertRows.All(ir => ir.InsertId != null))
{
RetryHandler.MarkAsRetriable(request);
}
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
// We ensure that every row has an insert ID, so we can always retry.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData(options?.AllowEmptyInsertIds ?? false));
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out IReadOnlyList <BigQueryInsertRow> validatedRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
validatedRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row);
}).ToList().AsReadOnly();
var body = new TableDataInsertAllRequest
{
Rows = new RawRowList(validatedRows, options?.AllowEmptyInsertIds ?? false)
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
internal void ModifyRequest(TableDataInsertAllRequest body)
{
if (AllowUnknownFields != null)
{
body.IgnoreUnknownValues = AllowUnknownFields;
}
}
public void ModifyRequest()
{
var options = new InsertOptions
{
AllowUnknownFields = true
};
TableDataInsertAllRequest request = new TableDataInsertAllRequest();
options.ModifyRequest(request);
Assert.Equal(true, request.IgnoreUnknownValues);
}
internal void ModifyRequest(TableDataInsertAllRequest body)
{
if (AllowUnknownFields != null)
{
body.IgnoreUnknownValues = AllowUnknownFields;
}
if (SkipInvalidRows != null)
{
body.SkipInvalidRows = SkipInvalidRows;
}
if (TemplateSuffix != null)
{
body.TemplateSuffix = TemplateSuffix;
}
}
public void ModifyRequest()
{
var options = new InsertOptions
{
AllowUnknownFields = true,
SkipInvalidRows = true,
TemplateSuffix = "fromTemplate"
};
TableDataInsertAllRequest request = new TableDataInsertAllRequest();
options.ModifyRequest(request);
Assert.Equal(true, request.IgnoreUnknownValues);
Assert.Equal(true, request.SkipInvalidRows);
Assert.Equal("fromTemplate", request.TemplateSuffix);
}
private string AddNewPhotoBigQuery(Photo photo)
{
try
{
GoogleCredential credential;
using (Stream stream = new FileStream(bigqueryFileKey, FileMode.Open, FileAccess.Read, FileShare.Read))
{
credential = GoogleCredential.FromStream(stream);
}
string[] scopes = new string[] {
BigqueryService.Scope.Bigquery,
BigqueryService.Scope.CloudPlatform,
};
credential = credential.CreateScoped(scopes);
BaseClientService.Initializer initializer = new BaseClientService.Initializer()
{
HttpClientInitializer = (IConfigurableHttpClientInitializer)credential,
ApplicationName = bigqueryApplicationName,
GZipEnabled = true,
};
BigqueryService service = new BigqueryService(initializer);
var rowList = new List <TableDataInsertAllRequest.RowsData>();
// Check @ https://developers.google.com/bigquery/streaming-data-into-bigquery for InsertId usage
var row = new TableDataInsertAllRequest.RowsData();
row.Json = new Dictionary <string, object>();
row.Json.Add("Id", photo.Id);
row.Json.Add("Title", photo.Title);
row.Json.Add("Url", photo.Url);
rowList.Add(row);
var content = new TableDataInsertAllRequest();
content.Rows = rowList;
content.Kind = "bigquery#tableDataInsertAllRequest";
content.IgnoreUnknownValues = true;
content.SkipInvalidRows = true;
var requestResponse = service.Tabledata.InsertAll(content, bigqueryProjectId, "dsbigquery", "Photo").Execute();
return("true");
}
catch (Exception ex)
{
return(ex.Message);
}
}
/// <inheritdoc />
public override void Insert(TableReference tableReference, IEnumerable <InsertRow> rows, InsertOptions options = null)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var body = new TableDataInsertAllRequest
{
Rows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList()
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
var response = request.Execute();
HandleInsertAllResponse(response);
}
/// <inheritdoc />
public override async Task InsertAsync(TableReference tableReference, IEnumerable <InsertRow> rows,
InsertOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var body = new TableDataInsertAllRequest
{
Rows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList()
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
var response = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
HandleInsertAllResponse(response);
}
/// <summary>
/// Streams data into BigQuery one record at a time without needing to run a load job. Requires the WRITER dataset role.
/// Documentation https://developers.google.com/bigquery/v2/reference/tabledata/insertAll
/// Generation Note: This does not always build corectly. Google needs to standardise things I need to figuer out which ones are wrong.
/// </summary>
/// <param name="service">Authenticated bigquery service.</param>
/// <param name="projectId">Project ID of the destination table.</param>
/// <param name="datasetId">Dataset ID of the destination table.</param>
/// <param name="tableId">Table ID of the destination table.</param>
/// <param name="body">A valid bigquery v2 body.</param>
/// <returns>TableDataInsertAllResponseResponse</returns>
public static TableDataInsertAllResponse InsertAll(bigqueryService service, string projectId, string datasetId, string tableId, TableDataInsertAllRequest body)
{
try
{
// Initial validation.
if (service == null)
{
throw new ArgumentNullException("service");
}
if (body == null)
{
throw new ArgumentNullException("body");
}
if (projectId == null)
{
throw new ArgumentNullException(projectId);
}
if (datasetId == null)
{
throw new ArgumentNullException(datasetId);
}
if (tableId == null)
{
throw new ArgumentNullException(tableId);
}
// Make the request.
return(service.Tabledata.InsertAll(body, projectId, datasetId, tableId).Execute());
}
catch (Exception ex)
{
throw new Exception("Request Tabledata.InsertAll failed.", ex);
}
}
public async Task InsertDataAsync(IList <TableDataInsertAllRequest.RowsData> rows,
CancellationToken cancellationToken)
{
ExpandTableIdIfNecessary();
await EnsureTableExistsAsync(cancellationToken);
var req = new TableDataInsertAllRequest
{
Rows = rows
};
var rowsCount = req.Rows.Count;
var retry = 1;
while (retry < _BackOff.MaxNumOfRetries)
{
try
{
var response = await _BQSvc.Tabledata.InsertAll(req,
Config.ProjectId, Config.DatasetId, TableIdExpanded)
.ExecuteAsync(cancellationToken);
if (response.InsertErrors == null || !response.InsertErrors.Any())
{
return;
}
var messages = response.InsertErrors
.Zip(req.Rows, (x, r) => x.Errors.Select(e => new { x, r, e }).ToArray())
.SelectMany(xs => xs)
.Where(x => x.e.Reason != "stopped")
.Select(x =>
{
return(string.Format(@"Index:{0}
DebugInfo:{1}
ETag:{2}
Location:{3}
Message:{4}
Reason:{5}
PostRawJSON:{6}",
x.x.Index, x.e.DebugInfo, x.e.ETag, x.e.Location,
x.e.Message, x.e.Reason,
JsonConvert.SerializeObject(x.r.Json, Formatting.None)));
});
this.healthReporter.ReportWarning(String.Join("\n", messages), EventFlowContextIdentifiers.Output);
}
catch (OperationCanceledException)
{
throw;
}
catch (GoogleApiException ex)
{
string errorMessage = nameof(BigQueryOutput) + ": insert has failed." + Environment.NewLine + ex.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
if (ex.HttpStatusCode == System.Net.HttpStatusCode.Unauthorized)
{
return; // something wrong in authentication. no retry
}
}
catch (Exception ex)
{
string errorMessage = nameof(BigQueryOutput) + ": insert has failed." + Environment.NewLine + ex.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
return;
}
retry++;
await Task.Delay(_BackOff.GetNextBackOff(retry));
}
this.healthReporter.ReportWarning(nameof(BigQueryOutput) + ": Retry over.", EventFlowContextIdentifiers.Output);
}
public async Task InsertDataAsync(IList <TableDataInsertAllRequest.RowsData> rows)
{
ExpandTableIdIfNecessary();
await EnsureTableExistsAsync();
var req = new TableDataInsertAllRequest
{
Rows = rows
};
var rowsCount = req.Rows.Count;
var retry = 1;
while (retry < _BackOff.MaxNumOfRetries)
{
try
{
BigQuerySinkEventSource.Log.BigQueryInsertBegan(rowsCount);
var response = await _BQSvc.Tabledata.InsertAll(req,
ProjectId, DatasetId, TableIdExpanded).ExecuteAsync();
if (response.InsertErrors == null || !response.InsertErrors.Any())
{
BigQuerySinkEventSource.Log.BigQueryInserted(rowsCount);
return;
}
var messages = response.InsertErrors
.Zip(req.Rows, (x, r) => x.Errors.Select(e => new { x, r, e }).ToArray())
.SelectMany(xs => xs)
.Where(x => x.e.Reason != "stopped")
.Select(x =>
{
return(string.Format(@"Index:{0}
DebugInfo:{1}
ETag:{2}
Location:{3}
Message:{4}
Reason:{5}
PostRawJSON:{6}",
x.x.Index, x.e.DebugInfo, x.e.ETag, x.e.Location,
x.e.Message, x.e.Reason,
JsonConvert.SerializeObject(x.r.Json, Formatting.None)));
});
BigQuerySinkEventSource.Log.BigQueryInsertFault(String.Join("\n", messages), retry);
}
catch (OperationCanceledException)
{
throw;
}
catch (GoogleApiException ex)
{
BigQuerySinkEventSource.Log.BigQueryInsertFault(ex.ToString(), retry);
if (ex.HttpStatusCode == System.Net.HttpStatusCode.Unauthorized)
{
return; // something wrong in authentication. no retry
}
}
catch (Exception ex)
{
BigQuerySinkEventSource.Log.BigQueryInsertFault(ex.ToString(), retry);
return;
}
retry++;
await Task.Delay(_BackOff.GetNextBackOff(retry));
}
BigQuerySinkEventSource.Log.BigQueryRetryOver(req.ToString());
}
