/// <inheritdoc />
public override BigQueryInsertResults InsertRows(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options = null)
{
var request = CreateInsertAllRequest(tableReference, rows, options, out IReadOnlyList <BigQueryInsertRow> validatedRows);
if (validatedRows.Count == 0)
{
return(new BigQueryInsertResults(this, options, validatedRows, new TableDataInsertAllResponse()));
}
TableDataInsertAllResponse response = request.Execute();
BigQueryInsertResults results = new BigQueryInsertResults(this, options, validatedRows, response);
return(results.ThrowIfNotSuppressing(options?.SuppressInsertErrors));
}
public void AllRowsInserted(TableDataInsertAllResponse response)
{
BigQueryInsertResults results = new BigQueryInsertResults(
_client,
new InsertOptions(),
new List <BigQueryInsertRow> {
new BigQueryInsertRow()
},
response);
Assert.Equal(BigQueryInsertStatus.AllRowsInserted, results.Status);
Assert.Equal(0, results.OriginalRowsWithErrors);
Assert.Equal(1, results.InsertAttemptRowCount);
Assert.Empty(results.Errors);
Assert.Same(results, results.ThrowOnNoneInserted());
Assert.Same(results, results.ThrowOnNotAllInserted());
Assert.Same(results, results.ThrowOnAnyError());
}
private void HandleInsertAllResponse(TableDataInsertAllResponse response)
{
if (response.InsertErrors != null)
{
var exception = new GoogleApiException(Service.Name, "Error inserting data")
{
Error = new RequestError
{
Errors = response.InsertErrors
.SelectMany(errors => (errors.Errors ?? Enumerable.Empty <ErrorProto>()).Select(error => new SingleError
{
Location = error.Location,
Reason = error.Reason,
Message = $"Row {errors.Index}: {error.Message}"
}))
.ToList()
}
};
throw exception;
}
}
private void HandleInsertAllResponse(TableDataInsertAllResponse response, InsertOptions options)
{
var errors = response.InsertErrors;
bool shouldThrow = options == null || !options.SuppressInsertErrors;
if (errors?.Count > 0 && shouldThrow)
{
var exception = new GoogleApiException(Service.Name, "Error inserting data")
{
Error = new RequestError
{
Errors = response.InsertErrors
.SelectMany(rowErrors => (rowErrors.Errors ?? Enumerable.Empty<ErrorProto>()).Select(error => new SingleError
{
Location = error.Location,
Reason = error.Reason,
Message = $"Row {rowErrors.Index}: {error.Message}"
}))
.ToList()
}
};
throw exception;
}
}
/// <summary>
/// Constructs a new set of insert results.
/// </summary>
/// <param name="client">The client used for the insert request. Must not be null.</param>
/// <param name="options">The options used for the insert request. May be null.</param>
/// <param name="originalRows">The rows whose insert was attempted. Must not be null.</param>
/// <param name="insertResponse">The response obtained after attempting the insert. Must not be null.</param>
public BigQueryInsertResults(BigQueryClient client, InsertOptions options, IReadOnlyList <BigQueryInsertRow> originalRows, TableDataInsertAllResponse insertResponse)
{
GaxPreconditions.CheckNotNull(insertResponse, nameof(insertResponse));
_client = GaxPreconditions.CheckNotNull(client, nameof(client));
GaxPreconditions.CheckNotNull(originalRows, nameof(originalRows));
InsertAttemptRowCount = originalRows.Count;
_options = options;
var errorsByRow = insertResponse.InsertErrors
?.Where(error => error != null)
.GroupBy(error => error.Index) ?? Enumerable.Empty <IGrouping <long?, InsertErrorsData> >();
int originalRowsWithErrors = 0;
_errors = errorsByRow
.Select(rowErrors => new BigQueryInsertRowErrors(GetRow(rowErrors.Key), rowErrors.ToList().AsReadOnly()))
.ToList().AsReadOnly();
OriginalRowsWithErrors = originalRowsWithErrors;
BigQueryInsertRow GetRow(long?index)
{
if (index.HasValue && index.Value >= 0 && index.Value < InsertAttemptRowCount)
{
originalRowsWithErrors++;
return(originalRows[(int)index.Value]);
}
return(null);
}
}
public void SomeRowsInserted()
{
ErrorProto row1Error1 = new ErrorProto {
Location = "field_1", Reason = "reason_1", Message = "message_1"
};
ErrorProto row1Error2 = new ErrorProto {
Location = "field_2", Message = "message_2"
};
InsertErrorsData row1List1 = new InsertErrorsData {
Index = 1, Errors = new List <ErrorProto> {
row1Error1, row1Error2
}
};
ErrorProto row5Error1 = new ErrorProto {
Location = "field_3", Reason = "reason_3"
};
InsertErrorsData row5List1 = new InsertErrorsData {
Index = 5, Errors = new List <ErrorProto> {
row5Error1
}
};
InsertErrorsData row6List1 = new InsertErrorsData {
Index = 6
};
ErrorProto row1Error3 = new ErrorProto {
Location = "field_4", Reason = "reason_4", Message = "message_4"
};
InsertErrorsData row1List2 = new InsertErrorsData {
Index = 1, Errors = new List <ErrorProto> {
row1Error3
}
};
TableDataInsertAllResponse response = new TableDataInsertAllResponse
{
InsertErrors = new List <InsertErrorsData>
{
row1List1,
row5List1,
row6List1,
row1List2
}
};
IReadOnlyList <BigQueryInsertRow> rows = Enumerable.Range(0, 8).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
List <BigQueryInsertRowErrors> expectedInsertRowErrors = new List <BigQueryInsertRowErrors>
{
new BigQueryInsertRowErrors(rows[1], new List <InsertErrorsData> {
row1List1, row1List2
}),
new BigQueryInsertRowErrors(rows[5], new List <InsertErrorsData> {
row5List1
}),
new BigQueryInsertRowErrors(rows[6], new List <InsertErrorsData> {
row6List1
}),
};
List <SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
BigQueryInsertResults results = new BigQueryInsertResults(
_client,
new InsertOptions {
SkipInvalidRows = true
},
rows,
response);
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, results.Status);
Assert.Equal(3, results.OriginalRowsWithErrors);
Assert.Equal(8, results.InsertAttemptRowCount);
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
Assert.Same(results, results.ThrowOnNoneInserted());
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
}
public void Constructor_NullParameters_Fail(
BigQueryClient client, InsertOptions options, IReadOnlyList <BigQueryInsertRow> rows, TableDataInsertAllResponse response) =>
Assert.Throws <ArgumentNullException>(() => new BigQueryInsertResults(client, options, rows, response));
public void NoRowsInserted_NoSkip()
{
ErrorProto row1Error1 = new ErrorProto {
Location = "field_1", Reason = "reason_1", Message = "message_1"
};
ErrorProto row1Error2 = new ErrorProto {
Location = "field_2", Message = "message_2"
};
InsertErrorsData row1List1 = new InsertErrorsData {
Index = 1, Errors = new List <ErrorProto> {
row1Error1, row1Error2
}
};
ErrorProto row5Error1 = new ErrorProto {
Location = "field_3", Reason = "reason_3"
};
InsertErrorsData row5List1 = new InsertErrorsData {
Index = 5, Errors = new List <ErrorProto> {
row5Error1
}
};
InsertErrorsData row6List1 = new InsertErrorsData {
Index = 6
};
ErrorProto row1Error3 = new ErrorProto {
Location = "field_4", Reason = "reason_4", Message = "message_4"
};
InsertErrorsData row1List2 = new InsertErrorsData {
Index = 1, Errors = new List <ErrorProto> {
row1Error3
}
};
TableDataInsertAllResponse response = new TableDataInsertAllResponse
{
InsertErrors = new List <InsertErrorsData>
{
row1List1,
row5List1,
row6List1,
row1List2
}
};
IReadOnlyList <BigQueryInsertRow> rows = Enumerable.Range(0, 8).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
// We don't need to have errors for all rows to know that no rows were inserted.
// If SkipInvalidRows is not set to true, then even if only one row has errors, no rows are inserted.
List <BigQueryInsertRowErrors> expectedInsertRowErrors = new List <BigQueryInsertRowErrors>
{
new BigQueryInsertRowErrors(rows[1], new List <InsertErrorsData> {
row1List1, row1List2
}),
new BigQueryInsertRowErrors(rows[5], new List <InsertErrorsData> {
row5List1
}),
new BigQueryInsertRowErrors(rows[6], new List <InsertErrorsData> {
row6List1
}),
};
List <SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
BigQueryInsertResults results = new BigQueryInsertResults(
_client,
new InsertOptions(),
rows,
response);
Assert.Equal(BigQueryInsertStatus.NoRowsInserted, results.Status);
Assert.Equal(3, results.OriginalRowsWithErrors);
Assert.Equal(8, results.InsertAttemptRowCount);
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
AssertException(results.ThrowOnNoneInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
}
/// <param name="retryStrategy">If not null, try retry.</param>
public async Task InsertAllAsync <T>(BigqueryService service, IEnumerable <T> data, IBackOff retryStrategy = null, Func <T, string> insertIdSelector = null, JsonSerializerSettings serializerSettings = null)
{
if (insertIdSelector == null)
{
insertIdSelector = _ => Guid.NewGuid().ToString();
}
var rows = data
.Select(x => new Google.Apis.Bigquery.v2.Data.TableDataInsertAllRequest.RowsData
{
InsertId = insertIdSelector(x),
Json = JsonConvert.DeserializeObject <Dictionary <string, object> >(JsonConvert.SerializeObject(x, serializerSettings))
})
.Where(x => x.Json != null)
.ToArray();
if (!rows.Any())
{
return;
}
var request = service.Tabledata.InsertAll(new TableDataInsertAllRequest
{
Rows = rows
}, this.project_id, this.dataset_id, this.table_id);
var retry = 0;
TableDataInsertAllResponse response = null;
Exception lastError;
do
{
try
{
lastError = null;
response = await request.ExecuteAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
lastError = ex;
}
if (retryStrategy == null)
{
break;
}
if (response != null && response.InsertErrors == null)
{
break;
}
retry++;
var nextDelay = retryStrategy.GetNextBackOff(retry);
if (nextDelay == TimeSpan.MinValue)
{
break;
}
await Task.Delay(nextDelay).ConfigureAwait(false);
} while (true);
if (lastError != null)
{
var exception = new InsertAllFailedException("", lastError)
{
RetryCount = retry,
InternalErrorInfos = new InsertAllFailedException.ErrorInfo[0],
};
throw exception;
}
if (response.InsertErrors != null && response.InsertErrors.Any())
{
var errorMessages = response.InsertErrors.Zip(rows, (x, r) =>
{
return(x.Errors.Select(e =>
{
return new InsertAllFailedException.ErrorInfo
{
Index = x.Index,
DebugInfo = e.DebugInfo,
ETag = e.ETag,
Location = e.Location,
Message = e.Message,
Reason = e.Reason,
PostRawJSON = JsonConvert.SerializeObject(r.Json, Formatting.None)
};
}));
}).SelectMany(xs => xs);
var exception = new InsertAllFailedException
{
RetryCount = retry,
InternalErrorInfos = errorMessages.ToArray()
};
throw exception;
}
}
