/// <summary>
/// Handler for back-off, if retry is not possible it returns <c>false</c>. Otherwise it blocks for some time (miliseconds)
/// and returns <c>true</c>, so call is retried.
/// </summary>
/// <param name="args">The arguments object to handler call.</param>
/// <returns>If request could be retried.</returns>
public async Task <bool> HandleResponseAsync(HandleUnsuccessfulResponseArgs args)
{
if (!args.SupportsRetry || _backoff.MaxNumOfRetries < args.CurrentFailedTry)
{
return(false);
}
if (IsTransientError(args.Response.StatusCode, _service.DeserializeError(args.Response).Result))
{
var delay = _backoff.GetNextBackOff(args.CurrentFailedTry);
if (delay > _maxTimeSpan || delay < TimeSpan.Zero)
{
return(false);
}
await Task.Delay(delay, args.CancellationToken);
Logger.Log("Back-Off waited " + delay.TotalMilliseconds + "ms before next retry...", EventType.Debug);
return(true);
}
return(false);
}
public async Task InsertDataAsync(IList <TableDataInsertAllRequest.RowsData> rows,
CancellationToken cancellationToken)
{
ExpandTableIdIfNecessary();
await EnsureTableExistsAsync(cancellationToken);
var req = new TableDataInsertAllRequest
{
Rows = rows
};
var rowsCount = req.Rows.Count;
var retry = 1;
while (retry < _BackOff.MaxNumOfRetries)
{
try
{
var response = await _BQSvc.Tabledata.InsertAll(req,
Config.ProjectId, Config.DatasetId, TableIdExpanded)
.ExecuteAsync(cancellationToken);
if (response.InsertErrors == null || !response.InsertErrors.Any())
{
return;
}
var messages = response.InsertErrors
.Zip(req.Rows, (x, r) => x.Errors.Select(e => new { x, r, e }).ToArray())
.SelectMany(xs => xs)
.Where(x => x.e.Reason != "stopped")
.Select(x =>
{
return(string.Format(@"Index:{0}
DebugInfo:{1}
ETag:{2}
Location:{3}
Message:{4}
Reason:{5}
PostRawJSON:{6}",
x.x.Index, x.e.DebugInfo, x.e.ETag, x.e.Location,
x.e.Message, x.e.Reason,
JsonConvert.SerializeObject(x.r.Json, Formatting.None)));
});
this.healthReporter.ReportWarning(String.Join("\n", messages), EventFlowContextIdentifiers.Output);
}
catch (OperationCanceledException)
{
throw;
}
catch (GoogleApiException ex)
{
string errorMessage = nameof(BigQueryOutput) + ": insert has failed." + Environment.NewLine + ex.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
if (ex.HttpStatusCode == System.Net.HttpStatusCode.Unauthorized)
{
return; // something wrong in authentication. no retry
}
}
catch (Exception ex)
{
string errorMessage = nameof(BigQueryOutput) + ": insert has failed." + Environment.NewLine + ex.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
return;
}
retry++;
await Task.Delay(_BackOff.GetNextBackOff(retry));
}
this.healthReporter.ReportWarning(nameof(BigQueryOutput) + ": Retry over.", EventFlowContextIdentifiers.Output);
}
/// <param name="retryStrategy">If not null, try retry.</param>
public async Task InsertAllAsync <T>(BigqueryService service, IEnumerable <T> data, IBackOff retryStrategy = null, Func <T, string> insertIdSelector = null, JsonSerializerSettings serializerSettings = null)
{
if (insertIdSelector == null)
{
insertIdSelector = _ => Guid.NewGuid().ToString();
}
var rows = data
.Select(x => new Google.Apis.Bigquery.v2.Data.TableDataInsertAllRequest.RowsData
{
InsertId = insertIdSelector(x),
Json = JsonConvert.DeserializeObject <Dictionary <string, object> >(JsonConvert.SerializeObject(x, serializerSettings))
})
.Where(x => x.Json != null)
.ToArray();
if (!rows.Any())
{
return;
}
var request = service.Tabledata.InsertAll(new TableDataInsertAllRequest
{
Rows = rows
}, this.project_id, this.dataset_id, this.table_id);
var retry = 0;
TableDataInsertAllResponse response = null;
Exception lastError;
do
{
try
{
lastError = null;
response = await request.ExecuteAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
lastError = ex;
}
if (retryStrategy == null)
{
break;
}
if (response != null && response.InsertErrors == null)
{
break;
}
retry++;
var nextDelay = retryStrategy.GetNextBackOff(retry);
if (nextDelay == TimeSpan.MinValue)
{
break;
}
await Task.Delay(nextDelay).ConfigureAwait(false);
} while (true);
if (lastError != null)
{
var exception = new InsertAllFailedException("", lastError)
{
RetryCount = retry,
InternalErrorInfos = new InsertAllFailedException.ErrorInfo[0],
};
throw exception;
}
if (response.InsertErrors != null && response.InsertErrors.Any())
{
var errorMessages = response.InsertErrors.Zip(rows, (x, r) =>
{
return(x.Errors.Select(e =>
{
return new InsertAllFailedException.ErrorInfo
{
Index = x.Index,
DebugInfo = e.DebugInfo,
ETag = e.ETag,
Location = e.Location,
Message = e.Message,
Reason = e.Reason,
PostRawJSON = JsonConvert.SerializeObject(r.Json, Formatting.None)
};
}));
}).SelectMany(xs => xs);
var exception = new InsertAllFailedException
{
RetryCount = retry,
InternalErrorInfos = errorMessages.ToArray()
};
throw exception;
}
}
public async Task InsertDataAsync(IList <TableDataInsertAllRequest.RowsData> rows)
{
ExpandTableIdIfNecessary();
await EnsureTableExistsAsync();
var req = new TableDataInsertAllRequest
{
Rows = rows
};
var rowsCount = req.Rows.Count;
var retry = 1;
while (retry < _BackOff.MaxNumOfRetries)
{
try
{
BigQuerySinkEventSource.Log.BigQueryInsertBegan(rowsCount);
var response = await _BQSvc.Tabledata.InsertAll(req,
ProjectId, DatasetId, TableIdExpanded).ExecuteAsync();
if (response.InsertErrors == null || !response.InsertErrors.Any())
{
BigQuerySinkEventSource.Log.BigQueryInserted(rowsCount);
return;
}
var messages = response.InsertErrors
.Zip(req.Rows, (x, r) => x.Errors.Select(e => new { x, r, e }).ToArray())
.SelectMany(xs => xs)
.Where(x => x.e.Reason != "stopped")
.Select(x =>
{
return(string.Format(@"Index:{0}
DebugInfo:{1}
ETag:{2}
Location:{3}
Message:{4}
Reason:{5}
PostRawJSON:{6}",
x.x.Index, x.e.DebugInfo, x.e.ETag, x.e.Location,
x.e.Message, x.e.Reason,
JsonConvert.SerializeObject(x.r.Json, Formatting.None)));
});
BigQuerySinkEventSource.Log.BigQueryInsertFault(String.Join("\n", messages), retry);
}
catch (OperationCanceledException)
{
throw;
}
catch (GoogleApiException ex)
{
BigQuerySinkEventSource.Log.BigQueryInsertFault(ex.ToString(), retry);
if (ex.HttpStatusCode == System.Net.HttpStatusCode.Unauthorized)
{
return; // something wrong in authentication. no retry
}
}
catch (Exception ex)
{
BigQuerySinkEventSource.Log.BigQueryInsertFault(ex.ToString(), retry);
return;
}
retry++;
await Task.Delay(_BackOff.GetNextBackOff(retry));
}
BigQuerySinkEventSource.Log.BigQueryRetryOver(req.ToString());
}