public static RateLimitInfo ParseRateLimitHeaders(IList <Parameter> headers) { var result = new RateLimitInfo(); foreach (var header in headers) { switch (header.Name.ToLower()) { case Constants.HEADER_RATELIMIT_LIMIT: result.Limit = int.Parse((string)header.Value); break; case Constants.HEADER_RATELIMIT_REMAINING: result.Remaining = int.Parse((string)header.Value); break; case Constants.HEADER_RATELIMIT_COST: result.Cost = int.Parse((string)header.Value); break; case Constants.HEADER_RATELIMIT_RESET: result.Reset = long.Parse((string)header.Value); break; case Constants.HEADER_RATELIMIT_RESET_TTL: result.ResetTtl = int.Parse((string)header.Value); break; } } return(result); }
private async Task <RestResponse[]> Raw(string method, string url, IReadOnlyCollection <HttpStatusCode> allowedStatuses, RestSettings restSettings, SerializationContext?data = null) { Contract.EnsureWebhookIsNotBroken(_webhook.Status); Contract.AssertArgumentNotTrue(string.IsNullOrEmpty(method), nameof(method)); Contract.AssertArgumentNotTrue(string.IsNullOrEmpty(url), nameof(url)); Contract.AssertNotNull(allowedStatuses, nameof(allowedStatuses)); Contract.CheckForNull(restSettings, nameof(restSettings)); List <RestResponse> responses = new List <RestResponse>(); uint currentAttimpts = 0; // Used to prevent calls if something went wrong bool forceStop = false; do { if (responses.Count != 0) { await _webhook.ActionManager.FollowRateLimit(responses.Last().RateLimit).ConfigureAwait(false); } HttpWebRequest request = WebRequest.CreateHttp(url); request.CachePolicy = _cachePolicy; request.Method = method; // Calling 'GetRequestStream()' after setting the request type using var requestStream = request.GetRequestStream(); PrepareRequest(request, requestStream, data); // Identify themselves request.UserAgent = $"DSharp4Webhook ({WebhookProvider.LibraryUrl}, {WebhookProvider.LibraryVersion})"; // The content type is assigned in 'PrepareRequest' // Uses it for accurate measurement RateLimit request.Headers.Set("X-RateLimit-Precision", "millisecond"); // Disabling keep-alive, this is a one-time connection request.KeepAlive = false; // I noticed a memory leak on a stress test // It wat because System.PinnableBufferCache not cleared // If we use 'request.AllowWriteStreamBuffering = false', it just stops working and throwing an WebException RestResponse restResponse; using (HttpWebResponse response = request.GetResponseNoException()) { RateLimitInfo rateLimitInfo = new RateLimitInfo(response.Headers.GetAsDictionary()); restResponse = new RestResponse(response, rateLimitInfo, currentAttimpts); responses.Add(restResponse); // Processing the necessary status codes ProcessStatusCode(response.StatusCode, ref forceStop, allowedStatuses); } Log(new LogContext(LogSensitivity.VERBOSE, $"[A {currentAttimpts}] [SC {(int)responses.Last().StatusCode}] [RLR {restResponse.RateLimit.Reset:yyyy-MM-dd HH:mm:ss.fff zzz}] [RLMW {restResponse.RateLimit.MustWait}] Post request completed:{(restResponse.Content?.Length != 0 ? string.Concat(Environment.NewLine, restResponse.Content ?? string.Empty) : " No content")}", _webhook.Id)); // first of all we check the forceStop so that we don't go any further if #pragma warning disable IDE0075 // Simplify conditional expression } while (!forceStop && (!allowedStatuses.Contains(responses.Last().StatusCode) && (restSettings.Attempts > 0 ? ++currentAttimpts <= restSettings.Attempts : true))); #pragma warning restore IDE0075 // Simplify conditional expression return(responses.ToArray()); }
/// <summary> /// </summary> /// <param name="resp"></param> public RateLimitExceededException(HttpResponseMessage resp) { RateLimitInfo?info = RateLimitInfo.FromHttp(resp); if (info == null) { throw new Exception("Could not parse rate limit info"); } else { RateLimit = (RateLimitInfo)info; } }
public void UpdateLimit(string bucketId, RateLimitInfo info) { throw new NotImplementedException(); }
public async Task <Stream> SendAsync(RestRequest request) { int id = Interlocked.Increment(ref nextId); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Start"); #endif LastAttemptAt = DateTimeOffset.UtcNow; while (true) { await _queue.EnterGlobalAsync(id, request).ConfigureAwait(false); await EnterAsync(id, request).ConfigureAwait(false); if (_redirectBucket != null) { return(await _redirectBucket.SendAsync(request)); } #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sending..."); #endif RateLimitInfo info = default(RateLimitInfo); try { var response = await request.SendAsync().ConfigureAwait(false); info = new RateLimitInfo(response.Headers); if (response.StatusCode < (HttpStatusCode)200 || response.StatusCode >= (HttpStatusCode)300) { switch (response.StatusCode) { case (HttpStatusCode)429: if (info.IsGlobal) { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429 [Global]"); #endif _queue.PauseGlobal(info); } else { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429"); #endif UpdateRateLimit(id, request, info, true); } await _queue.RaiseRateLimitTriggered(Id, info, $"{request.Method} {request.Endpoint}").ConfigureAwait(false); continue; //Retry case HttpStatusCode.BadGateway: //502 #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 502"); #endif if ((request.Options.RetryMode & RetryMode.Retry502) == 0) { throw new HttpException(HttpStatusCode.BadGateway, request, null); } continue; //Retry default: int? code = null; string reason = null; if (response.Stream != null) { try { using (var reader = new StreamReader(response.Stream)) using (var jsonReader = new JsonTextReader(reader)) { try { var json = await JToken.LoadAsync(jsonReader).ConfigureAwait(false); try { code = json.Value <int>("code"); } catch { } try { reason = json.Value <string>("message"); } catch { } } catch { if (response.StatusCode == HttpStatusCode.RequestEntityTooLarge) { code = 40005; reason = "The server responded with error 40005: Request entity too large"; } else { code = (int)response.StatusCode; if (response.Stream is MemoryStream ms) { reason = Encoding.UTF8.GetString(ms.ToArray()); } } } } } catch { } } throw new HttpException(response.StatusCode, request, code, reason); } } else { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Success"); #endif return(response.Stream); } } //catch (HttpException) { throw; } //Pass through catch (TimeoutException) { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Timeout"); #endif if ((request.Options.RetryMode & RetryMode.RetryTimeouts) == 0) { throw; } await Task.Delay(500).ConfigureAwait(false); continue; //Retry } /*catch (Exception) * { #if DEBUG_LIMITS * Debug.WriteLine($"[{id}] Error"); #endif * if ((request.Options.RetryMode & RetryMode.RetryErrors) == 0) * throw; * * await Task.Delay(500); * continue; //Retry * }*/ finally { UpdateRateLimit(id, request, info, false); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Stop"); #endif } } }
private void UpdateRateLimit(int id, IRequest request, RateLimitInfo info, bool is429, bool redirected = false) { if (WindowCount == 0) { return; } lock (_lock) { if (redirected) { Interlocked.Decrement(ref _semaphore); //we might still hit a real ratelimit if all tickets were already taken, can't do much about it since we didn't know they were the same #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Decrease Semaphore"); #endif } bool hasQueuedReset = _resetTick != null; if (info.Bucket != null && !redirected) { (RequestBucket, BucketId)hashBucket = _queue.UpdateBucketHash(Id, info.Bucket); if (!(hashBucket.Item1 is null) && !(hashBucket.Item2 is null)) { if (hashBucket.Item1 == this) //this bucket got promoted to a hash queue { Id = hashBucket.Item2; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Promoted to Hash Bucket ({hashBucket.Item2})"); #endif } else { _redirectBucket = hashBucket.Item1; //this request should be part of another bucket, this bucket will be disabled, redirect everything _redirectBucket.UpdateRateLimit(id, request, info, is429, redirected: true); //update the hash bucket ratelimit #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Redirected to {_redirectBucket.Id}"); #endif return; } } } if (info.Limit.HasValue && WindowCount != info.Limit.Value) { WindowCount = info.Limit.Value; _semaphore = info.Remaining.Value; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Upgraded Semaphore to {info.Remaining.Value}/{WindowCount}"); #endif } DateTimeOffset?resetTick = null; //Using X-RateLimit-Remaining causes a race condition /*if (info.Remaining.HasValue) * { * Debug.WriteLine($"[{id}] X-RateLimit-Remaining: " + info.Remaining.Value); * _semaphore = info.Remaining.Value; * }*/ if (info.RetryAfter.HasValue) { //RetryAfter is more accurate than Reset, where available resetTick = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Retry-After: {info.RetryAfter.Value} ({info.RetryAfter.Value} ms)"); #endif } else if (info.ResetAfter.HasValue && (request.Options.UseSystemClock.HasValue ? !request.Options.UseSystemClock.Value : false)) { resetTick = DateTimeOffset.UtcNow.Add(info.ResetAfter.Value); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Reset-After: {info.ResetAfter.Value} ({info.ResetAfter?.TotalMilliseconds} ms)"); #endif } else if (info.Reset.HasValue) { resetTick = info.Reset.Value.AddSeconds(info.Lag?.TotalSeconds ?? 1.0); /* millisecond precision makes this unnecessary, retaining in case of regression * if (request.Options.IsReactionBucket) * resetTick = DateTimeOffset.Now.AddMilliseconds(250); */ int diff = (int)(resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] X-RateLimit-Reset: {info.Reset.Value.ToUnixTimeSeconds()} ({diff} ms, {info.Lag?.TotalMilliseconds} ms lag)"); #endif } else if (request.Options.IsClientBucket && Id != null) { resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(Id).WindowSeconds); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(Id).WindowSeconds * 1000} ms)"); #endif } else if (request.Options.IsGatewayBucket && request.Options.BucketId != null) { resetTick = DateTimeOffset.UtcNow.AddSeconds(GatewayBucket.Get(request.Options.BucketId).WindowSeconds); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Gateway Bucket ({GatewayBucket.Get(request.Options.BucketId).WindowSeconds * 1000} ms)"); #endif if (!hasQueuedReset) { _resetTick = resetTick; LastAttemptAt = resetTick.Value; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms"); #endif var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds), request); } return; } if (resetTick == null) { WindowCount = 0; //No rate limit info, disable limits on this bucket #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Disabled Semaphore"); #endif return; } if (!hasQueuedReset || resetTick > _resetTick) { _resetTick = resetTick; LastAttemptAt = resetTick.Value; //Make sure we dont destroy this until after its been reset #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms"); #endif if (!hasQueuedReset) { var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds), request); } } } }
private void UpdateRateLimit(int id, RestRequest request, RateLimitInfo info, bool is429) { if (WindowCount == 0) { return; } lock (_lock) { bool hasQueuedReset = _resetTick != null; if (info.Limit.HasValue && WindowCount != info.Limit.Value) { WindowCount = info.Limit.Value; _semaphore = info.Remaining.Value; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Upgraded Semaphore to {info.Remaining.Value}/{WindowCount}"); #endif } var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); DateTimeOffset?resetTick = null; //Using X-RateLimit-Remaining causes a race condition /*if (info.Remaining.HasValue) * { * Debug.WriteLine($"[{id}] X-RateLimit-Remaining: " + info.Remaining.Value); * _semaphore = info.Remaining.Value; * }*/ if (info.RetryAfter.HasValue) { //RetryAfter is more accurate than Reset, where available resetTick = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Retry-After: {info.RetryAfter.Value} ({info.RetryAfter.Value} ms)"); #endif } else if (info.Reset.HasValue) { resetTick = info.Reset.Value.AddSeconds(info.Lag?.TotalSeconds ?? 1.0); if (request.Options.IsReactionBucket) { resetTick = DateTimeOffset.Now.AddMilliseconds(250); } int diff = (int)(resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds; #if DEBUG_LIMITS Debug.WriteLine($"[{id}] X-RateLimit-Reset: {info.Reset.Value.ToUnixTimeSeconds()} ({diff} ms, {info.Lag?.TotalMilliseconds} ms lag)"); #endif } else if (request.Options.IsClientBucket && request.Options.BucketId != null) { resetTick = DateTimeOffset.UtcNow.AddSeconds(ClientBucket.Get(request.Options.BucketId).WindowSeconds); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Client Bucket ({ClientBucket.Get(request.Options.BucketId).WindowSeconds * 1000} ms)"); #endif } if (resetTick == null) { WindowCount = 0; //No rate limit info, disable limits on this bucket (should only ever happen with a user token) #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Disabled Semaphore"); #endif return; } if (!hasQueuedReset || resetTick > _resetTick) { _resetTick = resetTick; LastAttemptAt = resetTick.Value; //Make sure we dont destroy this until after its been reset #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Reset in {(int)Math.Ceiling((resetTick - DateTimeOffset.UtcNow).Value.TotalMilliseconds)} ms"); #endif if (!hasQueuedReset) { var _ = QueueReset(id, (int)Math.Ceiling((_resetTick.Value - DateTimeOffset.UtcNow).TotalMilliseconds)); } } } }
internal void PauseGlobal(RateLimitInfo info) { _waitUntil = DateTimeOffset.UtcNow.AddMilliseconds(info.RetryAfter.Value + (info.Lag?.TotalMilliseconds ?? 0.0)); }
/// <summary> /// Initializes a new instance of the <see cref="RateLimitExceededException"/> class with a reference to the /// <see cref="PushoverResponse"/> and a reference to the inner exception that are the cause of this exception. /// </summary> /// <param name="response">The <see cref="PushoverResponse"/> that is the cause for the exception.</param> /// <param name="innerException"> /// The exception that is the cause of the current exception, or a null reference if no inner exception. /// </param> public RateLimitExceededException(PushoverResponse response, Exception innerException) : base("Rate limit exceeded", response, innerException) { this.RateLimitInfo = response.RateLimitInfo; }
public async Task <Stream> SendAsync(RestRequest request) { int id = Interlocked.Increment(ref nextId); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Start"); #endif LastAttemptAt = DateTimeOffset.UtcNow; while (true) { await _queue.EnterGlobalAsync(id, request).ConfigureAwait(false); await EnterAsync(id, request).ConfigureAwait(false); if (_redirectBucket != null) { return(await _redirectBucket.SendAsync(request)); } #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Sending..."); #endif RateLimitInfo info = default(RateLimitInfo); try { var response = await request.SendAsync().ConfigureAwait(false); info = new RateLimitInfo(response.Headers, request.Endpoint); request.Options.ExecuteRatelimitCallback(info); if (response.StatusCode < (HttpStatusCode)200 || response.StatusCode >= (HttpStatusCode)300) { switch (response.StatusCode) { case (HttpStatusCode)429: if (info.IsGlobal) { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429 [Global]"); #endif _queue.PauseGlobal(info); } else { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 429"); #endif UpdateRateLimit(id, request, info, true, body: response.Stream); } await _queue.RaiseRateLimitTriggered(Id, info, $"{request.Method} {request.Endpoint}").ConfigureAwait(false); continue; //Retry case HttpStatusCode.BadGateway: //502 #if DEBUG_LIMITS Debug.WriteLine($"[{id}] (!) 502"); #endif if ((request.Options.RetryMode & RetryMode.Retry502) == 0) { throw new HttpException(HttpStatusCode.BadGateway, request, null); } continue; //Retry default: API.DiscordError error = null; if (response.Stream != null) { try { using var reader = new StreamReader(response.Stream); using var jsonReader = new JsonTextReader(reader); error = Discord.Rest.DiscordRestClient.Serializer.Deserialize <API.DiscordError>(jsonReader); } catch { } } throw new HttpException( response.StatusCode, request, error?.Code, error?.Message, error?.Errors.IsSpecified == true ? error.Errors.Value.Select(x => new DiscordJsonError(x.Name.GetValueOrDefault("root"), x.Errors.Select(y => new DiscordError(y.Code, y.Message)).ToArray())).ToArray() : null ); } } else { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Success"); #endif return(response.Stream); } } //catch (HttpException) { throw; } //Pass through catch (TimeoutException) { #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Timeout"); #endif if ((request.Options.RetryMode & RetryMode.RetryTimeouts) == 0) { throw; } await Task.Delay(500).ConfigureAwait(false); continue; //Retry } /*catch (Exception) * { #if DEBUG_LIMITS * Debug.WriteLine($"[{id}] Error"); #endif * if ((request.Options.RetryMode & RetryMode.RetryErrors) == 0) * throw; * * await Task.Delay(500); * continue; //Retry * }*/ finally { UpdateRateLimit(id, request, info, false); #if DEBUG_LIMITS Debug.WriteLine($"[{id}] Stop"); #endif } } }
/// <summary> /// </summary> /// <param name="info"></param> public RateLimitExceededException(RateLimitInfo info) { RateLimit = info; }
public async Task <IActionResult> GetIssuesByUserAsync(string repoSetName, string userName) { var repoSet = _dataSource.GetRepoDataSet().GetRepoSet(repoSetName); var accessToken = await HttpContext.GetTokenAsync("access_token"); var gitHub = GitHubUtils.GetGitHubClient(accessToken); // Issue the three queries simultaneously and wait for results var assignedIssuesQuery = repoSet.GenerateQuery("is:open", "is:issue", $"assignee:{userName}"); var assignedPrsQuery = repoSet.GenerateQuery("is:open", "is:pr", $"assignee:{userName}"); var createdPrsQuery = repoSet.GenerateQuery("is:open", "is:pr", $"author:{userName}"); var assignedIssuesTask = _github.SearchIssuesAsync(assignedIssuesQuery, accessToken); var assignedPrsTask = _github.SearchIssuesAsync(assignedPrsQuery, accessToken); var createdPrsTask = _github.SearchIssuesAsync(createdPrsQuery, accessToken); await Task.WhenAll(assignedIssuesTask, assignedPrsTask, createdPrsTask); var assignedIssues = await assignedIssuesTask; var assignedPrs = await assignedPrsTask; var createdPrs = await createdPrsTask; // Identify issues being worked on var workingIssues = new List <IssueData>(); var otherIssues = new List <IssueData>(); foreach (var result in assignedIssues.Search) { if (result.Labels.Any(l => repoSet.WorkingLabels.Contains(l.Name))) { // We need to grab additional data about Working issues result.Working = true; result.WorkingStartedAt = await GetWorkingStartTime(result, repoSet.WorkingLabels, gitHub); workingIssues.Add(result); } else { otherIssues.Add(result); } } // Update rate limit information var rateLimitCost = RateLimitInfo.Add(RateLimitInfo.Add(assignedIssues.RateLimit, assignedPrs.RateLimit), createdPrs.RateLimit); _logger.LogDebug("Fetched issues for {User} in repo group {Group}. Total Rate Limit Cost: {Cost}", userName, repoSetName, rateLimitCost.Cost); return(Json(new { working = SortWorkingIssues(workingIssues), other = SortOtherAssignedIssues(otherIssues), prs = SortPRs(Enumerable.Concat(assignedPrs.Search, createdPrs.Search)), graphQlRateLimit = rateLimitCost, restRateLimit = gitHub.GetLastApiInfo()?.RateLimit, pages = assignedIssues.Pages + assignedPrs.Pages + createdPrs.Pages, queries = new string[] { assignedIssuesQuery, assignedPrsQuery, createdPrsQuery } })); }
public SearchResults(T search, RateLimitInfo rateLimit, int pages) { Search = search; RateLimit = rateLimit; Pages = pages; }