public async Task Invoke(HttpContext context)
{
try
{
await _next.Invoke(context);
}
catch (Exception exception)
{
ErrorResponseDto response;
if (exception is ErrorCodeException errorCodeException)
{
foreach (var(key, value) in errorCodeException.Headers)
{
context.Response.Headers.Add(key, value);
}
response = new ErrorResponseDto(errorCodeException);
}
else
{
var addStackTrace = _environment.IsDevelopment();
var serverErrorException = new ServerErrorException(exception?.Message, exception);
response = new ErrorResponseDto(serverErrorException, addStackTrace);
}
context.Response.StatusCode = response.GetStatusCode();
context.Response.Headers.Add("Content-Type", "application/json");
await context.Response.WriteAsync(JsonSerializer.Serialize(response, new JsonSerializerOptions
{
PropertyNamingPolicy = SnakeCaseNamingPolicy.Instance,
IgnoreNullValues = true
}));
}
}
private void App_DispatcherUnhandledException(object sender, DispatcherUnhandledExceptionEventArgs e)
{
ServerErrorException ex = e.Exception as ServerErrorException;
if (ex != null)
{
MessageBox.Show(ex.Message, "Server Error", MessageBoxButton.OK, MessageBoxImage.Error);
e.Handled = true;
}
}
public void ReturnsFailedTransitionWhenHttpFailsWithServerErrorException(ServerErrorException exception)
{
var state = CreateState(repository, sinceParameterRepository);
var observables = createFetchObservablesWhichThrow(exception);
var transition = state.Start(observables).SingleAsync().Wait();
var reason = ((Transition <Exception>)transition).Parameter;
transition.Result.Should().Be(state.Failed);
reason.Should().BeAssignableTo <ServerErrorException>();
}
public void SerializableTest()
{
var request = new HttpRequest(new Uri("http://localhost"), new Dictionary <string, string> {
{ "hello", "tyria" }
});
var response = new HttpResponse <ErrorObject>(new ErrorObject {
Text = "Error"
}, HttpStatusCode.InternalServerError, null, null);
var exception = new ServerErrorException(request, response);
exception.Should().BeBinarySerializable();
}
public void ReturnsTheServerErrorTransitionWhenHttpFailsWithServerError(ServerErrorException exception)
{
var state = (UpdateEntityState <ITestModel, IThreadSafeTestModel>)CreateState();
var entity = new TestModel(1, SyncStatus.InSync);
api.Update(Arg.Any <ITestModel>()).ReturnsThrowingTaskOf(exception);
var transition = state.Start(entity).SingleAsync().Wait();
var parameter = ((Transition <ServerErrorException>)transition).Parameter;
transition.Result.Should().Be(state.ServerError);
parameter.Should().BeAssignableTo <ServerErrorException>();
}
public void ReturnsTheServerErrorTransitionWhenHttpFailsWithServerError(ServerErrorException exception)
{
var state = createUpdateState(api, repository);
var entity = CreateDirtyEntity(1);
GetUpdateFunction(api)(Arg.Any <TModel>())
.Returns(_ => Observable.Throw <TApiModel>(exception));
var transition = state.Start(entity).SingleAsync().Wait();
var parameter = ((Transition <(Exception Reason, TModel)>)transition).Parameter;
transition.Result.Should().Be(state.ServerError);
parameter.Reason.Should().BeAssignableTo <ServerErrorException>();
}
private async Task DisplayError(string message, Exception exception)
{
Exception e = exception;
while (e.InnerException != null)
{
e = e.InnerException;
}
if (e is ServerErrorException)
{
ServerErrorException serv = (ServerErrorException)e;
await this.ShowMessageAsync("NuxeoClient error", message + Environment.NewLine + "Status code:" + serv.StatusCode + Environment.NewLine + e.Message, MessageDialogStyle.Affirmative);
}
else
{
await this.ShowMessageAsync(message, e.Message, MessageDialogStyle.Affirmative);
}
}
public ActionResult ServerError()
{
var exception = new ServerErrorException();
// Need to populate errors.
var errorHandler = (IErrorHandler) new StandardErrorHandler();
var message = errorHandler.FormatErrorMessage(exception);
var errorCode = errorHandler.GetErrorCode(exception);
return(Json(new JsonResponseModel
{
Success = false,
Errors = new List <JsonError> {
new JsonError {
Code = errorCode, Message = message
}
}
}, JsonRequestBehavior.AllowGet));
}
protected virtual IActionResult InternalServerError(Exception ex, string message, params object[] args)
{
if (ex == null)
{
Logger.LogError(message, args);
var error = new ServerErrorException(string.Format(message, args));
return(Content(new AjaxResult {
state = ResultType.error.ToString(), message = _exceptionMapper.Resolve(error).ToString()
}.ToJson()));
}
else
{
var errorMessage = String.Format(message, args);
Logger.LogError("{0} : {1}", errorMessage, ExceptionHelper.GetAllToStrings(ex));
var error = new ServerErrorException(string.Format("{0} : {1}", errorMessage, ex.Message));
return(Content(new AjaxResult {
state = ResultType.error.ToString(), message = _exceptionMapper.Resolve(error).ToString()
}.ToJson()));
}
}
public void ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(ServerErrorException exception)
{
var state = CreateState();
var entity = new TestModel(-1, SyncStatus.SyncNeeded);
PrepareApiCallFunctionToThrow(exception);
var transition = state.Start(entity).SingleAsync().Wait();
var parameter = ((Transition <ServerErrorException>)transition).Parameter;
transition.Result.Should().Be(state.ServerError);
parameter.Should().BeAssignableTo <ServerErrorException>();
}
public void DelaysTheTransitionAtLeastByTheNextFastDelayTimeFromTheRetryDelayServiceWhenAServerErrorOtherThanInternalServerErrorOccurs(ServerErrorException exception)
{
api.Status.IsAvailable().Returns(Observable.Throw <Unit>(exception));
statusDelay.NextFastDelay().Returns(TimeSpan.FromSeconds(10));
statusDelay.NextSlowDelay().Returns(TimeSpan.FromSeconds(1));
var hasCompleted = false;
var transition = state.Start();
var subscription = transition.Subscribe(_ => hasCompleted = true);
scheduler.AdvanceBy(TimeSpan.FromSeconds(10).Ticks - 1);
subscription.Dispose();
hasCompleted.Should().BeFalse();
}
// to allow proper rescheduling of the first request from a bucket
private async Task ExecuteRequestAsync(BaseRestRequest request, RateLimitBucket bucket, TaskCompletionSource <bool> ratelimitTcs)
{
if (this._disposed)
{
return;
}
HttpResponseMessage res = default;
try
{
await this.GlobalRateLimitEvent.WaitAsync().ConfigureAwait(false);
if (bucket == null)
{
bucket = request.RateLimitBucket;
}
if (ratelimitTcs == null)
{
ratelimitTcs = await this.WaitForInitialRateLimit(bucket).ConfigureAwait(false);
}
if (ratelimitTcs == null) // ckeck rate limit only if we are not the probe request
{
var now = DateTimeOffset.UtcNow;
await bucket.TryResetLimitAsync(now).ConfigureAwait(false);
// Decrement the remaining number of requests as there can be other concurrent requests before this one finishes and has a chance to update the bucket
if (Interlocked.Decrement(ref bucket._remaining) < 0)
{
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is blocked", bucket.ToString());
var delay = bucket.Reset - now;
var resetDate = bucket.Reset;
if (this.UseResetAfter)
{
delay = bucket.ResetAfter.Value;
resetDate = bucket.ResetAfterOffset;
}
if (delay < new TimeSpan(-TimeSpan.TicksPerMinute))
{
this.Logger.LogError(LoggerEvents.RatelimitDiag, "Failed to retrieve ratelimits - giving up and allowing next request for bucket");
bucket._remaining = 1;
}
if (delay < TimeSpan.Zero)
{
delay = TimeSpan.FromMilliseconds(100);
}
this.Logger.LogWarning(LoggerEvents.RatelimitPreemptive, "Pre-emptive ratelimit triggered - waiting until {0:yyyy-MM-dd HH:mm:ss zzz} ({1:c}).", resetDate, delay);
Task.Delay(delay)
.ContinueWith(_ => this.ExecuteRequestAsync(request, null, null))
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while executing request");
return;
}
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is allowed", bucket.ToString());
}
else
{
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Initial request for {Bucket} is allowed", bucket.ToString());
}
var req = this.BuildRequest(request);
var response = new RestResponse();
try
{
if (this._disposed)
{
return;
}
res = await this.HttpClient.SendAsync(req, HttpCompletionOption.ResponseContentRead, CancellationToken.None).ConfigureAwait(false);
var bts = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
var txt = Utilities.UTF8.GetString(bts, 0, bts.Length);
this.Logger.LogTrace(LoggerEvents.RestRx, txt);
response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value), StringComparer.OrdinalIgnoreCase);
response.Response = txt;
response.ResponseCode = (int)res.StatusCode;
}
catch (HttpRequestException httpex)
{
this.Logger.LogError(LoggerEvents.RestError, httpex, "Request to {Url} triggered an HttpException", request.Url);
request.SetFaulted(httpex);
this.FailInitialRateLimitTest(request, ratelimitTcs);
return;
}
this.UpdateBucket(request, response, ratelimitTcs);
Exception ex = null;
switch (response.ResponseCode)
{
case 400:
case 405:
ex = new BadRequestException(request, response);
break;
case 401:
case 403:
ex = new UnauthorizedException(request, response);
break;
case 404:
ex = new NotFoundException(request, response);
break;
case 413:
ex = new RequestSizeException(request, response);
break;
case 429:
ex = new RateLimitException(request, response);
// check the limit info and requeue
this.Handle429(response, out var wait, out var global);
if (wait != null)
{
if (global)
{
this.Logger.LogError(LoggerEvents.RatelimitHit, "Global ratelimit hit, cooling down");
try
{
this.GlobalRateLimitEvent.Reset();
await wait.ConfigureAwait(false);
}
finally
{
// we don't want to wait here until all the blocked requests have been run, additionally Set can never throw an exception that could be suppressed here
_ = this.GlobalRateLimitEvent.SetAsync();
}
this.ExecuteRequestAsync(request, bucket, ratelimitTcs)
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request");
}
else
{
this.Logger.LogError(LoggerEvents.RatelimitHit, "Ratelimit hit, requeueing request to {Url}", request.Url);
await wait.ConfigureAwait(false);
this.ExecuteRequestAsync(request, bucket, ratelimitTcs)
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request");
}
return;
}
break;
case 500:
case 502:
case 503:
case 504:
ex = new ServerErrorException(request, response);
break;
}
if (ex != null)
{
request.SetFaulted(ex);
}
else
{
request.SetCompleted(response);
}
}
catch (Exception ex)
{
this.Logger.LogError(LoggerEvents.RestError, ex, "Request to {Url} triggered an exception", request.Url);
// if something went wrong and we couldn't get rate limits for the first request here, allow the next request to run
if (bucket != null && ratelimitTcs != null && bucket._limitTesting != 0)
{
this.FailInitialRateLimitTest(request, ratelimitTcs);
}
if (!request.TrySetFaulted(ex))
{
throw;
}
}
finally
{
res?.Dispose();
// Get and decrement active requests in this bucket by 1.
_ = this.RequestQueue.TryGetValue(bucket.BucketId, out var count);
this.RequestQueue[bucket.BucketId] = Interlocked.Decrement(ref count);
// If it's 0 or less, we can remove the bucket from the active request queue,
// along with any of its past routes.
if (count <= 0)
{
foreach (var r in bucket.RouteHashes)
{
if (this.RequestQueue.ContainsKey(r))
{
_ = this.RequestQueue.TryRemove(r, out _);
}
}
}
}
}
public void ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(ServerErrorException exception)
{
var state = CreateState(api, repository);
var entity = CreateDirtyEntityWithNegativeId();
PrepareApiCallFunctionToThrow(exception);
var transition = state.Start(entity).SingleAsync().Wait();
var parameter = ((Transition <(Exception Reason, TModel)>)transition).Parameter;
transition.Result.Should().Be(state.ServerError);
parameter.Reason.Should().BeAssignableTo <ServerErrorException>();
}
public void ReturnsTheServerErrorTransitionWhenHttpFailsWithServerError(ServerErrorException exception) => helper.ReturnsTheServerErrorTransitionWhenHttpFailsWithServerError(exception);
public void ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(ServerErrorException reason) => helper.ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(reason);
public void ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(ServerErrorException exception)
{
var transition = state.Start(exception).SingleAsync().Wait();
var reason = ((Transition <ApiException>)transition).Parameter;
transition.Result.Should().Be(state.Retry);
reason.Should().BeAssignableTo <ServerErrorException>();
}
protected virtual void OnServerError(ServerErrorException serverError)
{
Log.Error(serverError, "An server error occurred when receiving an event from EventStore");
}
public void ReturnsServerErrorTransitionWhenHttpFailsWithServerErrorException(ServerErrorException exception)
{
internalState.Start(Arg.Any <IFetchObservables>()).Returns(Observable.Throw <ITransition>(exception));
var transition = state.Start(fetchObservables).SingleAsync().Wait();
var reason = ((Transition <Exception>)transition).Parameter;
transition.Result.Should().Be(state.Failed);
reason.Should().BeAssignableTo <ServerErrorException>();
}
