public static RateLimitInfo ParseRateLimitHeaders(IList <Parameter> headers)
{
var result = new RateLimitInfo();
foreach (var header in headers)
{
switch (header.Name.ToLower())
{
case Constants.HEADER_RATELIMIT_LIMIT:
result.Limit = int.Parse((string)header.Value);
break;
case Constants.HEADER_RATELIMIT_REMAINING:
result.Remaining = int.Parse((string)header.Value);
break;
case Constants.HEADER_RATELIMIT_COST:
result.Cost = int.Parse((string)header.Value);
break;
case Constants.HEADER_RATELIMIT_RESET:
result.Reset = long.Parse((string)header.Value);
break;
case Constants.HEADER_RATELIMIT_RESET_TTL:
result.ResetTtl = int.Parse((string)header.Value);
break;
}
}
return(result);
}
private async Task <RestResponse[]> Raw(string method, string url, IReadOnlyCollection <HttpStatusCode> allowedStatuses, RestSettings restSettings, SerializationContext?data = null)
{
Contract.EnsureWebhookIsNotBroken(_webhook.Status);
Contract.AssertArgumentNotTrue(string.IsNullOrEmpty(method), nameof(method));
Contract.AssertArgumentNotTrue(string.IsNullOrEmpty(url), nameof(url));
Contract.AssertNotNull(allowedStatuses, nameof(allowedStatuses));
Contract.CheckForNull(restSettings, nameof(restSettings));
List <RestResponse> responses = new List <RestResponse>();
uint currentAttimpts = 0;
// Used to prevent calls if something went wrong
bool forceStop = false;
do
{
if (responses.Count != 0)
{
await _webhook.ActionManager.FollowRateLimit(responses.Last().RateLimit).ConfigureAwait(false);
}
HttpWebRequest request = WebRequest.CreateHttp(url);
request.CachePolicy = _cachePolicy;
request.Method = method;
// Calling 'GetRequestStream()' after setting the request type
using var requestStream = request.GetRequestStream();
PrepareRequest(request, requestStream, data);
// Identify themselves
request.UserAgent = $"DSharp4Webhook ({WebhookProvider.LibraryUrl}, {WebhookProvider.LibraryVersion})";
// The content type is assigned in 'PrepareRequest'
// Uses it for accurate measurement RateLimit
request.Headers.Set("X-RateLimit-Precision", "millisecond");
// Disabling keep-alive, this is a one-time connection
request.KeepAlive = false;
// I noticed a memory leak on a stress test
// It wat because System.PinnableBufferCache not cleared
// If we use 'request.AllowWriteStreamBuffering = false', it just stops working and throwing an WebException
RestResponse restResponse;
using (HttpWebResponse response = request.GetResponseNoException())
{
RateLimitInfo rateLimitInfo = new RateLimitInfo(response.Headers.GetAsDictionary());
restResponse = new RestResponse(response, rateLimitInfo, currentAttimpts);
responses.Add(restResponse);
// Processing the necessary status codes
ProcessStatusCode(response.StatusCode, ref forceStop, allowedStatuses);
}
Log(new LogContext(LogSensitivity.VERBOSE, $"[A {currentAttimpts}] [SC {(int)responses.Last().StatusCode}] [RLR {restResponse.RateLimit.Reset:yyyy-MM-dd HH:mm:ss.fff zzz}] [RLMW {restResponse.RateLimit.MustWait}] Post request completed:{(restResponse.Content?.Length != 0 ? string.Concat(Environment.NewLine, restResponse.Content ?? string.Empty) : " No content")}", _webhook.Id));
// first of all we check the forceStop so that we don't go any further if
#pragma warning disable IDE0075 // Simplify conditional expression
} while (!forceStop && (!allowedStatuses.Contains(responses.Last().StatusCode) && (restSettings.Attempts > 0 ? ++currentAttimpts <= restSettings.Attempts : true)));
#pragma warning restore IDE0075 // Simplify conditional expression
return(responses.ToArray());
}
/// <summary>
/// </summary>
/// <param name="resp"></param>
public RateLimitExceededException(HttpResponseMessage resp)
{
RateLimitInfo?info = RateLimitInfo.FromHttp(resp);
if (info == null)
{
throw new Exception("Could not parse rate limit info");
}
else
{
RateLimit = (RateLimitInfo)info;
}
}
public void UpdateLimit(string bucketId, RateLimitInfo info)
{
throw new NotImplementedException();
}
public async Task <Stream> SendAsync(RestRequest request)
{
int id = Interlocked.Increment(ref nextId);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Start");
#endif
LastAttemptAt = DateTimeOffset.UtcNow;
while (true)
{
await _queue.EnterGlobalAsync(id, request).ConfigureAwait(false);
await EnterAsync(id, request).ConfigureAwait(false);
if (_redirectBucket != null)
{
return(await _redirectBucket.SendAsync(request));
}
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Sending...");
#endif
RateLimitInfo info = default(RateLimitInfo);
try
{
var response = await request.SendAsync().ConfigureAwait(false);
info = new RateLimitInfo(response.Headers);
if (response.StatusCode < (HttpStatusCode)200 || response.StatusCode >= (HttpStatusCode)300)
{
switch (response.StatusCode)
{
case (HttpStatusCode)429:
if (info.IsGlobal)
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 429 [Global]");
#endif
_queue.PauseGlobal(info);
}
else
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 429");
#endif
UpdateRateLimit(id, request, info, true);
}
await _queue.RaiseRateLimitTriggered(Id, info, $"{request.Method} {request.Endpoint}").ConfigureAwait(false);
continue; //Retry
case HttpStatusCode.BadGateway: //502
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 502");
#endif
if ((request.Options.RetryMode & RetryMode.Retry502) == 0)
{
throw new HttpException(HttpStatusCode.BadGateway, request, null);
}
continue; //Retry
default:
int? code = null;
string reason = null;
if (response.Stream != null)
{
try
{
using (var reader = new StreamReader(response.Stream))
using (var jsonReader = new JsonTextReader(reader))
{
try
{
var json = await JToken.LoadAsync(jsonReader).ConfigureAwait(false);
try { code = json.Value <int>("code"); } catch { }
try { reason = json.Value <string>("message"); } catch { }
}
catch
{
if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge)
{
code = 40005;
reason = "The server responded with error 40005: Request entity too large";
}
else
{
code = (int)response.StatusCode;
if (response.Stream is MemoryStream ms)
{
reason = Encoding.UTF8.GetString(ms.ToArray());
}
}
}
}
}
catch { }
}
throw new HttpException(response.StatusCode, request, code, reason);
}
}
else
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Success");
#endif
return(response.Stream);
}
}
//catch (HttpException) { throw; } //Pass through
catch (TimeoutException)
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Timeout");
#endif
if ((request.Options.RetryMode & RetryMode.RetryTimeouts) == 0)
{
throw;
}
await Task.Delay(500).ConfigureAwait(false);
continue; //Retry
}
/*catch (Exception)
* {
#if DEBUG_LIMITS
* Debug.WriteLine($"[{id}] Error");
#endif
* if ((request.Options.RetryMode & RetryMode.RetryErrors) == 0)
* throw;
*
* await Task.Delay(500);
* continue; //Retry
* }*/
finally
{
UpdateRateLimit(id, request, info, false);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Stop");
#endif
}
}
}
private void UpdateRateLimit(int id, IRequest request, RateLimitInfo info, bool is429, bool redirected = false)
{
if (WindowCount == 0)
{
return;
}
lock (_lock)
{
if (redirected)
{
Interlocked.Decrement(ref _semaphore); //we might still hit a real ratelimit if all tickets were already taken, can't do much about it since we didn't know they were the same
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Decrease Semaphore");
#endif
}
bool hasQueuedReset = _resetTick != null;
if (info.Bucket != null && !redirected)
{
(RequestBucket, BucketId)hashBucket = _queue.UpdateBucketHash(Id, info.Bucket);
if (!(hashBucket.Item1 is null) && !(hashBucket.Item2 is null))
{
if (hashBucket.Item1 == this) //this bucket got promoted to a hash queue
{
Id = hashBucket.Item2;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Promoted to Hash Bucket ({hashBucket.Item2})");
#endif
}
else
{
_redirectBucket = hashBucket.Item1; //this request should be part of another bucket, this bucket will be disabled, redirect everything
_redirectBucket.UpdateRateLimit(id, request, info, is429, redirected: true); //update the hash bucket ratelimit
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Redirected to {_redirectBucket.Id}");
#endif
return;
}
}
}
if (info.Limit.HasValue && WindowCount != info.Limit.Value)
{
WindowCount = info.Limit.Value;
_semaphore = info.Remaining.Value;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Upgraded Semaphore to {info.Remaining.Value}/{WindowCount}");
#endif
}
DateTimeOffset?resetTick = null;
//Using X-RateLimit-Remaining causes a race condition
/*if (info.Remaining.HasValue)
* {
* Debug.WriteLine($"[{id}] X-RateLimit-Remaining: " + info.Remaining.Value);
* _semaphore = info.Remaining.Value;
* }*/
if (info.RetryAfter.HasValue)
{
//RetryAfter is more accurate than Reset, where available
resetTick = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Retry-After: {info.RetryAfter.Value} ({info.RetryAfter.Value} ms)");
#endif
}
else if (info.ResetAfter.HasValue && (request.Options.UseSystemClock.HasValue ? !request.Options.UseSystemClock.Value : false))
{
resetTick = DateTimeOffset.UtcNow.Add(info.ResetAfter.Value);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Reset-After: {info.ResetAfter.Value} ({info.ResetAfter?.TotalMilliseconds} ms)");
#endif
}
else if (info.Reset.HasValue)
{
resetTick = info.Reset.Value.AddSeconds(info.Lag?.TotalSeconds ?? 1.0);
/* millisecond precision makes this unnecessary, retaining in case of regression
* if (request.Options.IsReactionBucket)
* resetTick = DateTimeOffset.Now.AddMilliseconds(250);
*/
int diff = (int)(resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] X-RateLimit-Reset: {info.Reset.Value.ToUnixTimeSeconds()} ({diff} ms, {info.Lag?.TotalMilliseconds} ms lag)");
#endif
}
else if (request.Options.IsClientBucket && Id != null)
{
resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(Id).WindowSeconds);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(Id).WindowSeconds * 1000} ms)");
#endif
}
else if (request.Options.IsGatewayBucket && request.Options.BucketId != null)
{
resetTick = DateTimeOffset.UtcNow.AddSeconds(GatewayBucket.Get(request.Options.BucketId).WindowSeconds);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Gateway Bucket ({GatewayBucket.Get(request.Options.BucketId).WindowSeconds * 1000} ms)");
#endif
if (!hasQueuedReset)
{
_resetTick = resetTick;
LastAttemptAt = resetTick.Value;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms");
#endif
var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds), request);
}
return;
}
if (resetTick == null)
{
WindowCount = 0; //No rate limit info, disable limits on this bucket
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Disabled Semaphore");
#endif
return;
}
if (!hasQueuedReset || resetTick > _resetTick)
{
_resetTick = resetTick;
LastAttemptAt = resetTick.Value; //Make sure we dont destroy this until after its been reset
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms");
#endif
if (!hasQueuedReset)
{
var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds), request);
}
}
}
}
private void UpdateRateLimit(int id, RestRequest request, RateLimitInfo info, bool is429)
{
if (WindowCount == 0)
{
return;
}
lock (_lock)
{
bool hasQueuedReset = _resetTick != null;
if (info.Limit.HasValue && WindowCount != info.Limit.Value)
{
WindowCount = info.Limit.Value;
_semaphore = info.Remaining.Value;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Upgraded Semaphore to {info.Remaining.Value}/{WindowCount}");
#endif
}
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
DateTimeOffset?resetTick = null;
//Using X-RateLimit-Remaining causes a race condition
/*if (info.Remaining.HasValue)
* {
* Debug.WriteLine($"[{id}] X-RateLimit-Remaining: " + info.Remaining.Value);
* _semaphore = info.Remaining.Value;
* }*/
if (info.RetryAfter.HasValue)
{
//RetryAfter is more accurate than Reset, where available
resetTick = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Retry-After: {info.RetryAfter.Value} ({info.RetryAfter.Value} ms)");
#endif
}
else if (info.Reset.HasValue)
{
resetTick = info.Reset.Value.AddSeconds(info.Lag?.TotalSeconds ?? 1.0);
if (request.Options.IsReactionBucket)
{
resetTick = DateTimeOffset.Now.AddMilliseconds(250);
}
int diff = (int)(resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds;
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] X-RateLimit-Reset: {info.Reset.Value.ToUnixTimeSeconds()} ({diff} ms, {info.Lag?.TotalMilliseconds} ms lag)");
#endif
}
else if (request.Options.IsClientBucket && request.Options.BucketId != null)
{
resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(request.Options.BucketId).WindowSeconds);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(request.Options.BucketId).WindowSeconds * 1000} ms)");
#endif
}
if (resetTick == null)
{
WindowCount = 0; //No rate limit info, disable limits on this bucket (should only ever happen with a user token)
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Disabled Semaphore");
#endif
return;
}
if (!hasQueuedReset || resetTick > _resetTick)
{
_resetTick = resetTick;
LastAttemptAt = resetTick.Value; //Make sure we dont destroy this until after its been reset
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms");
#endif
if (!hasQueuedReset)
{
var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds));
}
}
}
}
internal void PauseGlobal(RateLimitInfo info)
{
_waitUntil = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value + (info.Lag?.TotalMilliseconds ?? 0.0));
}
/// <summary>
/// Initializes a new instance of the <see cref="RateLimitExceededException"/> class with a reference to the
/// <see cref="PushoverResponse"/> and a reference to the inner exception that are the cause of this exception.
/// </summary>
/// <param name="response">The <see cref="PushoverResponse"/> that is the cause for the exception.</param>
/// <param name="innerException">
/// The exception that is the cause of the current exception, or a null reference if no inner exception.
/// </param>
public RateLimitExceededException(PushoverResponse response, Exception innerException)
: base("Rate limit exceeded", response, innerException)
{
this.RateLimitInfo = response.RateLimitInfo;
}
public async Task <Stream> SendAsync(RestRequest request)
{
int id = Interlocked.Increment(ref nextId);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Start");
#endif
LastAttemptAt = DateTimeOffset.UtcNow;
while (true)
{
await _queue.EnterGlobalAsync(id, request).ConfigureAwait(false);
await EnterAsync(id, request).ConfigureAwait(false);
if (_redirectBucket != null)
{
return(await _redirectBucket.SendAsync(request));
}
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Sending...");
#endif
RateLimitInfo info = default(RateLimitInfo);
try
{
var response = await request.SendAsync().ConfigureAwait(false);
info = new RateLimitInfo(response.Headers, request.Endpoint);
request.Options.ExecuteRatelimitCallback(info);
if (response.StatusCode < (HttpStatusCode)200 || response.StatusCode >= (HttpStatusCode)300)
{
switch (response.StatusCode)
{
case (HttpStatusCode)429:
if (info.IsGlobal)
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 429 [Global]");
#endif
_queue.PauseGlobal(info);
}
else
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 429");
#endif
UpdateRateLimit(id, request, info, true, body: response.Stream);
}
await _queue.RaiseRateLimitTriggered(Id, info, $"{request.Method} {request.Endpoint}").ConfigureAwait(false);
continue; //Retry
case HttpStatusCode.BadGateway: //502
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] (!) 502");
#endif
if ((request.Options.RetryMode & RetryMode.Retry502) == 0)
{
throw new HttpException(HttpStatusCode.BadGateway, request, null);
}
continue; //Retry
default:
API.DiscordError error = null;
if (response.Stream != null)
{
try
{
using var reader = new StreamReader(response.Stream);
using var jsonReader = new JsonTextReader(reader);
error = Discord.Rest.DiscordRestClient.Serializer.Deserialize <API.DiscordError>(jsonReader);
}
catch { }
}
throw new HttpException(
response.StatusCode,
request,
error?.Code,
error?.Message,
error?.Errors.IsSpecified == true ?
error.Errors.Value.Select(x => new DiscordJsonError(x.Name.GetValueOrDefault("root"), x.Errors.Select(y => new DiscordError(y.Code, y.Message)).ToArray())).ToArray() :
null
);
}
}
else
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Success");
#endif
return(response.Stream);
}
}
//catch (HttpException) { throw; } //Pass through
catch (TimeoutException)
{
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Timeout");
#endif
if ((request.Options.RetryMode & RetryMode.RetryTimeouts) == 0)
{
throw;
}
await Task.Delay(500).ConfigureAwait(false);
continue; //Retry
}
/*catch (Exception)
* {
#if DEBUG_LIMITS
* Debug.WriteLine($"[{id}] Error");
#endif
* if ((request.Options.RetryMode & RetryMode.RetryErrors) == 0)
* throw;
*
* await Task.Delay(500);
* continue; //Retry
* }*/
finally
{
UpdateRateLimit(id, request, info, false);
#if DEBUG_LIMITS
Debug.WriteLine($"[{id}] Stop");
#endif
}
}
}
/// <summary>
/// </summary>
/// <param name="info"></param>
public RateLimitExceededException(RateLimitInfo info)
{
RateLimit = info;
}
public async Task <IActionResult> GetIssuesByUserAsync(string repoSetName, string userName)
{
var repoSet = _dataSource.GetRepoDataSet().GetRepoSet(repoSetName);
var accessToken = await HttpContext.GetTokenAsync("access_token");
var gitHub = GitHubUtils.GetGitHubClient(accessToken);
// Issue the three queries simultaneously and wait for results
var assignedIssuesQuery = repoSet.GenerateQuery("is:open", "is:issue", $"assignee:{userName}");
var assignedPrsQuery = repoSet.GenerateQuery("is:open", "is:pr", $"assignee:{userName}");
var createdPrsQuery = repoSet.GenerateQuery("is:open", "is:pr", $"author:{userName}");
var assignedIssuesTask = _github.SearchIssuesAsync(assignedIssuesQuery, accessToken);
var assignedPrsTask = _github.SearchIssuesAsync(assignedPrsQuery, accessToken);
var createdPrsTask = _github.SearchIssuesAsync(createdPrsQuery, accessToken);
await Task.WhenAll(assignedIssuesTask, assignedPrsTask, createdPrsTask);
var assignedIssues = await assignedIssuesTask;
var assignedPrs = await assignedPrsTask;
var createdPrs = await createdPrsTask;
// Identify issues being worked on
var workingIssues = new List <IssueData>();
var otherIssues = new List <IssueData>();
foreach (var result in assignedIssues.Search)
{
if (result.Labels.Any(l => repoSet.WorkingLabels.Contains(l.Name)))
{
// We need to grab additional data about Working issues
result.Working = true;
result.WorkingStartedAt = await GetWorkingStartTime(result, repoSet.WorkingLabels, gitHub);
workingIssues.Add(result);
}
else
{
otherIssues.Add(result);
}
}
// Update rate limit information
var rateLimitCost = RateLimitInfo.Add(RateLimitInfo.Add(assignedIssues.RateLimit, assignedPrs.RateLimit), createdPrs.RateLimit);
_logger.LogDebug("Fetched issues for {User} in repo group {Group}. Total Rate Limit Cost: {Cost}", userName, repoSetName, rateLimitCost.Cost);
return(Json(new
{
working = SortWorkingIssues(workingIssues),
other = SortOtherAssignedIssues(otherIssues),
prs = SortPRs(Enumerable.Concat(assignedPrs.Search, createdPrs.Search)),
graphQlRateLimit = rateLimitCost,
restRateLimit = gitHub.GetLastApiInfo()?.RateLimit,
pages = assignedIssues.Pages + assignedPrs.Pages + createdPrs.Pages,
queries = new string[]
{
assignedIssuesQuery,
assignedPrsQuery,
createdPrsQuery
}
}));
}
public SearchResults(T search, RateLimitInfo rateLimit, int pages)
{
Search = search;
RateLimit = rateLimit;
Pages = pages;
}