/// <summary>
/// Constructs a new set of insert results.
/// </summary>
/// <param name="client">The client used for the insert request. Must not be null.</param>
/// <param name="options">The options used for the insert request. May be null.</param>
/// <param name="originalRows">The rows whose insert was attempted. Must not be null.</param>
/// <param name="insertResponse">The response obtained after attempting the insert. Must not be null.</param>
public BigQueryInsertResults(BigQueryClient client, InsertOptions options, IReadOnlyList <BigQueryInsertRow> originalRows, TableDataInsertAllResponse insertResponse)
{
GaxPreconditions.CheckNotNull(insertResponse, nameof(insertResponse));
_client = GaxPreconditions.CheckNotNull(client, nameof(client));
GaxPreconditions.CheckNotNull(originalRows, nameof(originalRows));
InsertAttemptRowCount = originalRows.Count;
_options = options;
var errorsByRow = insertResponse.InsertErrors
?.Where(error => error != null)
.GroupBy(error => error.Index) ?? Enumerable.Empty <IGrouping <long?, InsertErrorsData> >();
int originalRowsWithErrors = 0;
_errors = errorsByRow
.Select(rowErrors => new BigQueryInsertRowErrors(GetRow(rowErrors.Key), rowErrors.ToList().AsReadOnly()))
.ToList().AsReadOnly();
OriginalRowsWithErrors = originalRowsWithErrors;
BigQueryInsertRow GetRow(long?index)
{
if (index.HasValue && index.Value >= 0 && index.Value < InsertAttemptRowCount)
{
originalRowsWithErrors++;
return(originalRows[(int)index.Value]);
}
return(null);
}
}
/// <inheritdoc />
public override async Task InsertRowsAsync(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows,
InsertOptions options = null, CancellationToken cancellationToken = default)
{
var request = CreateInsertAllRequest(tableReference, rows, options);
var response = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
HandleInsertAllResponse(response);
}
/// <inheritdoc />
public override async Task <BigQueryInsertResults> InsertRowsAsync(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows,
InsertOptions options = null, CancellationToken cancellationToken = default)
{
var request = CreateInsertAllRequest(tableReference, rows, options, out IReadOnlyList <BigQueryInsertRow> validatedRows);
if (validatedRows.Count == 0)
{
return(new BigQueryInsertResults(this, options, validatedRows, new TableDataInsertAllResponse()));
}
var response = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
BigQueryInsertResults results = new BigQueryInsertResults(this, options, validatedRows, response);
return(results.ThrowIfNotSuppressing(options?.SuppressInsertErrors));
}
/// <inheritdoc />
public override async Task InsertAsync(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows,
InsertOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var body = new TableDataInsertAllRequest
{
Rows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList()
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
var response = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
HandleInsertAllResponse(response);
}
private void HandleInsertAllResponse(TableDataInsertAllResponse response, InsertOptions options)
{
var errors = response.InsertErrors;
bool shouldThrow = options == null || !options.SuppressInsertErrors;
if (errors?.Count > 0 && shouldThrow)
{
var exception = new GoogleApiException(Service.Name, "Error inserting data")
{
Error = new RequestError
{
Errors = response.InsertErrors
.SelectMany(rowErrors => (rowErrors.Errors ?? Enumerable.Empty<ErrorProto>()).Select(error => new SingleError
{
Location = error.Location,
Reason = error.Reason,
Message = $"Row {rowErrors.Index}: {error.Message}"
}))
.ToList()
}
};
throw exception;
}
}
/// <summary>
/// Asynchronously inserts all the given rows of data into this table.
/// This method just creates a <see cref="TableReference"/> and delegates to <see cref="BigQueryClient.InsertRowsAsync(TableReference, IEnumerable{BigQueryInsertRow}, InsertOptions, CancellationToken)"/>.
/// </summary>
/// <param name="rows">The rows to insert. Must not be null or contain null entries.</param>
/// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>A task representing the asynchronous operation.</returns>
public Task InsertRowsAsync(IEnumerable <BigQueryInsertRow> rows, InsertOptions options = null, CancellationToken cancellationToken = default) =>
_client.InsertRowsAsync(Reference, rows, options, cancellationToken);
/// <summary> /// Asynchronously inserts a single row of data into this table. /// This method just creates a <see cref="TableReference"/> and delegates to <see cref="BigQueryClient.InsertRowAsync(TableReference, BigQueryInsertRow, InsertOptions, CancellationToken)"/>. /// </summary> /// <param name="row">The data to insert. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation.</returns> public Task InsertRowAsync(BigQueryInsertRow row, InsertOptions options = null, CancellationToken cancellationToken = default) => _client.InsertRowAsync(Reference, row, options, cancellationToken);
/// <summary>
/// Inserts all the given rows of data into this table.
/// This method just creates a <see cref="TableReference"/> and delegates to <see cref="BigQueryClient.InsertRows(TableReference, IEnumerable{BigQueryInsertRow}, InsertOptions)"/>.
/// </summary>
/// <param name="rows">The rows to insert. Must not be null or contain null entries.</param>
/// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param>
public void InsertRows(IEnumerable <BigQueryInsertRow> rows, InsertOptions options = null) =>
_client.InsertRows(Reference, rows, options);
/// <summary> /// Inserts a single row of data into this table. /// This method just creates a <see cref="TableReference"/> and delegates to <see cref="BigQueryClient.InsertRow(TableReference, BigQueryInsertRow, InsertOptions)"/>. /// </summary> /// <param name="row">The data to insert. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> public void InsertRow(BigQueryInsertRow row, InsertOptions options = null) => _client.InsertRow(Reference, row, options);
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable<BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return row.ToRowsData();
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// We ensure that every row has an insert ID, so we can always retry.
RetryHandler.MarkAsRetriable(request);
return request;
}
/// <inheritdoc />
public override void InsertRows(TableReference tableReference, IEnumerable<BigQueryInsertRow> rows, InsertOptions options = null)
{
var request = CreateInsertAllRequest(tableReference, rows, options, out bool hasRows);
if (!hasRows)
{
return;
}
var response = request.Execute();
HandleInsertAllResponse(response, options);
}
/// <inheritdoc />
public override void Insert(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options = null)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var body = new TableDataInsertAllRequest
{
Rows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList()
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
var response = request.Execute();
HandleInsertAllResponse(response);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData());
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
request.ModifyRequest += _versionHeaderAction;
if (insertRows.All(ir => ir.InsertId != null))
{
RetryHandler.MarkAsRetriable(request);
}
return(request);
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out IReadOnlyList <BigQueryInsertRow> validatedRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
validatedRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row);
}).ToList().AsReadOnly();
var body = new TableDataInsertAllRequest
{
Rows = new RawRowList(validatedRows, options?.AllowEmptyInsertIds ?? false)
};
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
request.PrettyPrint = PrettyPrint;
return(request);
}
/// <inheritdoc />
public override BigQueryInsertResults InsertRows(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options = null)
{
var request = CreateInsertAllRequest(tableReference, rows, options, out IReadOnlyList <BigQueryInsertRow> validatedRows);
if (validatedRows.Count == 0)
{
return(new BigQueryInsertResults(this, options, validatedRows, new TableDataInsertAllResponse()));
}
TableDataInsertAllResponse response = request.Execute();
BigQueryInsertResults results = new BigQueryInsertResults(this, options, validatedRows, response);
return(results.ThrowIfNotSuppressing(options?.SuppressInsertErrors));
}
private InsertAllRequest CreateInsertAllRequest(TableReference tableReference, IEnumerable <BigQueryInsertRow> rows, InsertOptions options, out bool hasRows)
{
GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference));
GaxPreconditions.CheckNotNull(rows, nameof(rows));
var insertRows = rows.Select(row =>
{
GaxPreconditions.CheckArgument(row != null, nameof(rows), "Entries must not be null");
return(row.ToRowsData(options?.AllowEmptyInsertIds ?? false));
}).ToList();
var body = new TableDataInsertAllRequest
{
Rows = insertRows
};
// It's annoying to use an out parameter for this, but InsertAllRequest doesn't allow access to the body.
hasRows = body.Rows.Any();
options?.ModifyRequest(body);
var request = Service.Tabledata.InsertAll(body, tableReference.ProjectId, tableReference.DatasetId, tableReference.TableId);
// Even though empty InsertIds might be allowed, this can be retried as per guidance from
// the API team. Previous de-duplicating was on a best effort basis anyways and client code
// needs to explicitly allow for empty InsertId and should be aware that doing so will be at
// the expense of de-duplication efforts.
RetryHandler.MarkAsRetriable(request);
return(request);
}
