internal static async Task <object[]> RunItemHooks <TEntity>(this IBulkRequest request,
ICrudRequestConfig config,
object[] items,
CancellationToken ct)
where TEntity : class
{
var hooks = config.GetItemHooksFor <TEntity>();
foreach (var hook in hooks)
{
for (var i = 0; i < items.Length; ++i)
{
try
{
items[i] = await hook.Run(request, items[i], ct).Configure();
ct.ThrowIfCancellationRequested();
}
catch (Exception e) when(IsNonCancellationFailure(e))
{
throw new CrudHookFailedException(GenericHookError("item"), e)
{
HookProperty = hook
};
}
}
}
return(items);
}
internal static async Task <TEntity[]> UpdateEntities <TEntity>(this IBulkRequest request,
ICrudRequestConfig config,
IEnumerable <Tuple <object, TEntity> > items,
CancellationToken token)
where TEntity : class
{
var updator = config.GetUpdatorFor <TEntity>();
try
{
var entities = await Task.WhenAll(items.Select(x => updator(request, x.Item1, x.Item2, token))).Configure();
token.ThrowIfCancellationRequested();
return(entities);
}
catch (Exception e) when(IsNonCancellationFailure(e))
{
throw new CrudUpdateEntityFailedException(GenericUpdateEntityError, e)
{
ItemProperty = items.Select(x => x.Item1).ToArray(),
EntityProperty = items.Select(x => x.Item2).ToArray()
};
}
}
private Task <BulkResponse> BulkData(IBulkRequest request)
{
var response = _client.BulkAsync(request);
if (!response.Result.IsValid)
{
if ((response.Result.OriginalException != null && response.Result.OriginalException.Message.Contains("The underlying connection was closed: A connection that was expected to be kept alive was closed by the server")) ||
(response.Result.ItemsWithErrors != null && response.Result.ItemsWithErrors.First().Status == 503))
{
_logger.Warn("Elasticsearch overload, retrying");
throw new TooManyRequestsException();
}
if (response.Exception != null)
{
foreach (var ex in response.Exception.InnerExceptions)
{
_logger.Error(ex, ex.Message);
}
}
throw new HttpException(response.Result.ItemsWithErrors.First().Status, "Something failed trying to insert data into the bulk service", response.Exception);
}
return(response);
}
public virtual Task <IBulkResponse> BulkAsync(IBulkRequest request, string callerName = "")
{
var timer = Stopwatch.StartNew();
var result = _client.BulkAsync(request);
SendLog(null, null, timer.ElapsedMilliseconds, $"Bulk Async {callerName}");
return(result);
}
/// <inheritdoc />
public Task<IBulkResponse> BulkAsync(IBulkRequest bulkRequest)
{
return this.DispatchAsync<IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>(
bulkRequest,
(p, d) =>
{
var json = Serializer.SerializeBulkDescriptor(d);
return this.RawDispatch.BulkDispatchAsync<BulkResponse>(p, json);
}
);
}
/// <inheritdoc />
public Task <IBulkResponse> BulkAsync(IBulkRequest bulkRequest)
{
return(this.DispatchAsync <IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>(
bulkRequest,
(p, d) =>
{
var json = Serializer.SerializeBulkDescriptor(d);
return this.RawDispatch.BulkDispatchAsync <BulkResponse>(p, json);
}
));
}
public virtual async Task <BulkResponse> BulkAsync(IBulkRequest request, string callerName = "")
{
var timer = Stopwatch.StartNew();
var policy = Policy
.Handle <TooManyRequestsException>()
.WaitAndRetry(5, retryAttempt =>
TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) * 10));
var result = await policy.Execute(() => BulkData(request));
SendLog(null, null, timer.ElapsedMilliseconds, $"Elasticsearch.BulkAsync.{callerName}");
return(result);
}
public string SerializeBulkDescriptor(IBulkRequest bulkRequest)
{
bulkRequest.ThrowIfNull("bulkRequest");
bulkRequest.Operations.ThrowIfEmpty("Bulk request does not define any operations");
var sb = new StringBuilder();
var inferrer = new ElasticInferrer(this._settings);
foreach (var operation in bulkRequest.Operations)
{
var command = operation.Operation;
var index = operation.Index
?? inferrer.IndexName(bulkRequest.Index)
?? inferrer.IndexName(operation.ClrType);
var typeName = operation.Type
?? inferrer.TypeName(bulkRequest.Type)
?? inferrer.TypeName(operation.ClrType);
var id = operation.GetIdForOperation(inferrer);
if (index.EqualsMarker(bulkRequest.Index))
{
operation.Index = null;
}
else
{
operation.Index = index;
}
operation.Type = typeName;
operation.Id = id;
var opJson = this.Serialize(operation, SerializationFormatting.None).Utf8String();
var action = "{{ \"{0}\" : {1} }}\n".F(command, opJson);
sb.Append(action);
var body = operation.GetBody();
if (body == null)
{
continue;
}
var jsonCommand = this.Serialize(body, SerializationFormatting.None).Utf8String();
sb.Append(jsonCommand + "\n");
}
var json = sb.ToString();
return(json);
}
/// <inheritdoc/> public IBulkResponse Bulk(IBulkRequest request) => this.Dispatcher.Dispatch<IBulkRequest, BulkRequestParameters, BulkResponse>( request, this.LowLevelDispatch.BulkDispatch<BulkResponse> );
/// <inheritdoc/> public Task<IBulkResponse> BulkAsync(IBulkRequest request) => this.Dispatcher.DispatchAsync<IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>( request, this.LowLevelDispatch.BulkDispatchAsync<BulkResponse> );
public static void Update(ElasticsearchPathInfo <BulkRequestParameters> pathInfo, IBulkRequest request)
{
pathInfo.HttpMethod = PathInfoHttpMethod.POST;
}
/// <inheritdoc />
public Task <IBulkResponse> BulkAsync(IBulkRequest request, CancellationToken cancellationToken = default(CancellationToken)) =>
Dispatcher.DispatchAsync <IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>(
request, cancellationToken, LowLevelDispatch.BulkDispatchAsync <BulkResponse>
);
/// <inheritdoc /> public Task <BulkResponse> BulkAsync(IBulkRequest request, CancellationToken ct = default) => DoRequestAsync <IBulkRequest, BulkResponse>(request, request.RequestParameters, ct);
public Task <BulkResponse> BulkAsync(IBulkRequest request, CancellationToken ct = new CancellationToken())
{
throw new NotImplementedException();
}
/// <inheritdoc/> public Task<IBulkResponse> BulkAsync(IBulkRequest request, CancellationToken cancellationToken = default(CancellationToken)) => this.Dispatcher.DispatchAsync<IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>( request, cancellationToken, this.LowLevelDispatch.BulkDispatchAsync<BulkResponse> );
public BulkWithBulkRequestAsyncAction(IBulkRequest request)
{
_request = request;
}
public IBulkResponse Bulk(IBulkRequest request)
{
var client = new ElasticClient(connctionSettings);
return(client.Bulk(request));
}
public BulkResponse Bulk(IBulkRequest request)
{
throw new NotImplementedException();
}
/// <inheritdoc/>
public Task <IBulkResponse> BulkAsync(IBulkRequest request) =>
this.Dispatcher.DispatchAsync <IBulkRequest, BulkRequestParameters, BulkResponse, IBulkResponse>(
request, this.LowLevelDispatch.BulkDispatchAsync <BulkResponse>
);
/// <inheritdoc />
public IBulkResponse Bulk(IBulkRequest request) =>
Dispatcher.Dispatch <IBulkRequest, BulkRequestParameters, BulkResponse>(
request, LowLevelDispatch.BulkDispatch <BulkResponse>
);
/// <inheritdoc /> public BulkResponse Bulk(IBulkRequest request) => DoRequest <IBulkRequest, BulkResponse>(request, request.RequestParameters);
