public void SetState(LeakyBucketState bucketInfo)
{
//Shopify Plus customers have a bucket that is twice the size (80) so we resize the bucket capacity accordingly
//It is apparently possible to request the bucket size to be even larger
//https://ecommerce.shopify.com/c/shopify-apis-and-technology/t/what-is-the-default-api-call-limit-on-shopify-stores-407292
//Note that when the capacity doubles, the leak rate also doubles. So, not only can request bursts be larger, it is also possible to sustain a faster rate over the long term.
if (bucketInfo.Capacity > this._bucketCapacity)
{
lock (_semaphore)
{
if (bucketInfo.Capacity > this._bucketCapacity)
{
_semaphore.Release(bucketInfo.Capacity - this._bucketCapacity);
_bucketCapacity = bucketInfo.Capacity;
_leakRate = bucketInfo.Capacity / DEFAULT_BUCKET_CAPACITY;
}
}
}
//Corrects the grant capacity of the bucket based on the size returned by Shopify.
//Shopify may know that the remaining capacity is less than we think it is (for example if multiple programs are using that same token)
//Shopify may also think that the remaining capacity is more than we know, but we do not ever empty the bucket because Shopify is not
//considering requests that we know are already in flight.
int grantCapacity = this._bucketCapacity - bucketInfo.CurrentFillLevel;
while (_semaphore.CurrentCount > grantCapacity)
{
//We refill the virtual bucket accordingly.
_semaphore.Wait();
}
}
public ShopifyRateLimitException(HttpResponseMessage response,
HttpStatusCode httpStatusCode,
IEnumerable <string> errors,
string message,
string jsonError,
string requestId,
LeakyBucketState leakyBucket)
: base(response, httpStatusCode, errors, message, jsonError, requestId)
{
LeakyBucket = leakyBucket;
ExtractRetryAfterSeconds(response);
}
public async Task <RequestResult <T> > Run <T>(CloneableRequestMessage baseRequest, ExecuteRequestAsync <T> executeRequestAsync, CancellationToken cancellationToken)
{
var accessToken = GetAccessToken(baseRequest);
LeakyBucket bucket = null;
if (accessToken != null)
{
bucket = _shopAccessTokenToLeakyBucket.GetOrAdd(accessToken, _ => new LeakyBucket());
}
while (true)
{
var request = baseRequest.Clone();
if (accessToken != null)
{
await bucket.GrantAsync();
}
try
{
var fullResult = await executeRequestAsync(request);
var bucketState = LeakyBucketState.Get(fullResult.Response);
if (bucketState != null)
{
bucket?.SetState(bucketState);
}
return(fullResult);
}
catch (ShopifyRateLimitException ex) when(ex.Reason == ShopifyRateLimitReason.BucketFull || !_retryOnlyIfLeakyBucketFull)
{
//Only retry if breach caused by full bucket
//Other limits will bubble the exception because it's not clear how long the program should wait
//Even if there is a Retry-After header, we probably don't want the thread to sleep for potentially many hours
//
//An exception may still occur:
//-Shopify may have a slightly different algorithm
//-Shopify may change to a different algorithm in the future
//-There may be timing and latency delays
//-Multiple programs may use the same access token
//-Multiple instances of the same program may use the same access token
await Task.Delay(THROTTLE_DELAY, cancellationToken);
}
}
}
/// <summary>
/// Checks a response for exceptions or invalid status codes. Throws an exception when necessary.
/// </summary>
/// <param name="response">The response.</param>
/// <<param name="rawResponse">The response body returned by Shopify.</param>
public static void CheckResponseExceptions(HttpResponseMessage response, string rawResponse)
{
var statusCode = (int)response.StatusCode;
// No error if response was between 200 and 300.
if (statusCode >= 200 && statusCode < 300)
{
return;
}
var requestIdHeader = response.Headers.FirstOrDefault(h => h.Key.Equals("X-Request-Id", StringComparison.OrdinalIgnoreCase));
var requestId = requestIdHeader.Value?.FirstOrDefault();
var code = response.StatusCode;
var statusMessage = $"{(int)code} {response.ReasonPhrase}";
// If the error was caused by reaching the API rate limit, throw a rate limit exception.
if ((int)code == 429 /* Too many requests */)
{
string rateExceptionMessage;
IEnumerable <string> errors;
if (TryParseErrorJson(rawResponse, out var rateLimitErrors))
{
rateExceptionMessage = $"({statusMessage}) {rateLimitErrors.First()}";
errors = rateLimitErrors;
}
else
{
var baseMessage = "Exceeded the rate limit for api client. Reduce request rates to resume uninterrupted service.";
rateExceptionMessage = $"({statusMessage}) {baseMessage}";
errors = new List <string> {
baseMessage
};
}
throw new ShopifyRateLimitException(response, code, errors, rateExceptionMessage, rawResponse, requestId, LeakyBucketState.Get(response));
}
var contentType = response.Content.Headers.GetValues("Content-Type").FirstOrDefault();
if (contentType.StartsWithIgnoreCase("application/json") || contentType.StartsWithIgnoreCase("text/json"))
{
IEnumerable <string> errors;
string exceptionMessage;
if (TryParseErrorJson(rawResponse, out var parsedErrors))
{
var firstError = parsedErrors.First();
var totalErrors = parsedErrors.Count();
var baseErrorMessage = $"({statusMessage}) {firstError}";
switch (totalErrors)
{
case 1:
exceptionMessage = baseErrorMessage;
break;
case 2:
exceptionMessage = $"{baseErrorMessage} (and one other error)";
break;
default:
exceptionMessage = $"{baseErrorMessage} (and {totalErrors} other errors)";
break;
}
errors = parsedErrors;
}
else
{
exceptionMessage = $"({statusMessage}) Shopify returned {statusMessage}, but ShopifySharp was unable to parse the response JSON.";
errors = new List <string>
{
exceptionMessage
};
}
throw new ShopifyException(response, code, errors, exceptionMessage, rawResponse, requestId);
}
var message = $"({statusMessage}) Shopify returned {statusMessage}, but there was no JSON to parse into an error message.";
var customErrors = new List <string>
{
message
};
throw new ShopifyException(response, code, customErrors, message, rawResponse, requestId);
}