// to allow proper rescheduling of the first request from a bucket private async Task ExecuteRequestAsync(BaseRestRequest request, RateLimitBucket bucket, TaskCompletionSource <bool> ratelimitTcs) { try { await this.GlobalRateLimitEvent.WaitAsync(); if (bucket == null) { bucket = request.RateLimitBucket; } if (ratelimitTcs == null) { ratelimitTcs = await this.WaitForInitialRateLimit(bucket); } if (ratelimitTcs == null) // ckeck rate limit only if we are not the probe request { var now = DateTimeOffset.UtcNow; await bucket.TryResetLimit(now); // Decrement the remaining number of requests as there can be other concurrent requests before this one finishes and has a chance to update the bucket #pragma warning disable 420 // interlocked access is always volatile if (Interlocked.Decrement(ref bucket._remaining) < 0) #pragma warning restore 420 // blaze it { request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Request for bucket {bucket}. Blocking.", DateTime.Now); var delay = bucket.Reset - now; if (delay < new TimeSpan(-TimeSpan.TicksPerMinute)) { request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", "Failed to retrieve ratelimits. Giving up and allowing next request for bucket.", DateTime.Now); bucket._remaining = 1; } if (delay < TimeSpan.Zero) { delay = TimeSpan.FromMilliseconds(100); } request.Discord?.DebugLogger?.LogMessage(LogLevel.Warning, "REST", $"Pre-emptive ratelimit triggered, waiting until {bucket.Reset:yyyy-MM-dd HH:mm:ss zzz} ({delay:c})", DateTime.Now); request.Discord?.DebugLogger?.LogTaskFault(Task.Delay(delay).ContinueWith(t => this.ExecuteRequestAsync(request, null, null)), LogLevel.Error, "RESET", "Error while executing request: "); return; } request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Request for bucket {bucket}. Allowing.", DateTime.Now); } else { request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Initial Request for bucket {bucket}. Allowing.", DateTime.Now); } var req = this.BuildRequest(request); var response = new RestResponse(); try { var res = await HttpClient.SendAsync(req, CancellationToken.None).ConfigureAwait(false); var bts = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(false); var txt = Utilities.UTF8.GetString(bts, 0, bts.Length); response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value)); response.Response = txt; response.ResponseCode = (int)res.StatusCode; } catch (HttpRequestException httpex) { request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an HttpException: {httpex.Message}", DateTime.Now); request.SetFaulted(httpex); this.FailInitialRateLimitTest(bucket, ratelimitTcs); return; } this.UpdateBucket(request, response, ratelimitTcs); Exception ex = null; switch (response.ResponseCode) { case 400: case 405: ex = new BadRequestException(request, response); break; case 401: case 403: ex = new UnauthorizedException(request, response); break; case 404: ex = new NotFoundException(request, response); break; case 429: ex = new RateLimitException(request, response); // check the limit info and requeue this.Handle429(response, out var wait, out var global); if (wait != null) { if (global) { request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", "Global ratelimit hit, cooling down", DateTime.Now); try { this.GlobalRateLimitEvent.Reset(); await wait.ConfigureAwait(false); } finally { // we don't want to wait here until all the blocked requests have been run, additionally Set can never throw an exception that could be suppressed here _ = this.GlobalRateLimitEvent.SetAsync(); } request.Discord?.DebugLogger?.LogTaskFault(ExecuteRequestAsync(request, bucket, ratelimitTcs), LogLevel.Error, "REST", "Error while retrying request: "); } else { request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Ratelimit hit, requeueing request to {request.Url}", DateTime.Now); await wait.ConfigureAwait(false); request.Discord?.DebugLogger?.LogTaskFault(this.ExecuteRequestAsync(request, bucket, ratelimitTcs), LogLevel.Error, "REST", "Error while retrying request: "); } return; } break; } if (ex != null) { request.SetFaulted(ex); } else { request.SetCompleted(response); } } catch (Exception ex) { request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}", DateTime.Now); // if something went wrong and we couldn't get rate limits for the first request here, allow the next request to run if (bucket != null && ratelimitTcs != null && bucket._limitTesting != 0) { this.FailInitialRateLimitTest(bucket, ratelimitTcs); } if (!request.TrySetFaulted(ex)) { throw; } } }
public async Task ExecuteRequestAsync(BaseRestRequest request) { if (request == null) { throw new ArgumentNullException(nameof(request)); } await this.RequestSemaphore.WaitAsync(); var bucket = request.RateLimitBucket; var now = DateTimeOffset.UtcNow; if (bucket.Remaining <= 0 && bucket.Maximum > 0 && now < bucket.Reset) { request.Discord.DebugLogger.LogMessage(LogLevel.Warning, "REST", $"Pre-emptive ratelimit triggered, waiting until {bucket.Reset.ToString("yyyy-MM-dd HH:mm:ss zzz")}", DateTime.Now); _ = Task.Delay(bucket.Reset - now).ContinueWith(t => this.ExecuteRequestAsync(request)); this.RequestSemaphore.Release(); return; } var req = this.BuildRequest(request); var response = new RestResponse(); try { var res = await HttpClient.SendAsync(req, HttpCompletionOption.ResponseContentRead); var bts = await res.Content.ReadAsByteArrayAsync(); var txt = UTF8.GetString(bts, 0, bts.Length); response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value)); response.Response = txt; response.ResponseCode = (int)res.StatusCode; } catch (HttpRequestException httpex) { request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an HttpException: {httpex.Message}", DateTime.Now); request.SetFaulted(httpex); this.RequestSemaphore.Release(); return; } Exception ex = null; switch (response.ResponseCode) { case 400: case 405: ex = new BadRequestException(request, response); break; case 401: case 403: ex = new UnauthorizedException(request, response); break; case 404: ex = new NotFoundException(request, response); break; case 429: ex = new RateLimitException(request, response); // check the limit info, if more than one minute, fault, otherwise requeue this.Handle429(response, out var wait, out var global); if (wait != null) { wait = wait.ContinueWith(t => this.ExecuteRequestAsync(request)); if (global) { request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", "Global ratelimit hit, cooling down", DateTime.Now); await wait; } else { request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", $"Ratelimit hit, requeueing request to {request.Url}", DateTime.Now); } this.RequestSemaphore.Release(); return; } break; } this.UpdateBucket(request, response); this.RequestSemaphore.Release(); if (ex != null) { request.SetFaulted(ex); } else { request.SetCompleted(response); } }
private static void Main() { Console.InputEncoding = Encoding.Unicode; Console.OutputEncoding = Encoding.Unicode; settings = new IniSettings(new FileInfo(ini_file)); Console.WriteLine("Loading login info..."); TwitterApi twitter = TwitterApi.Login(settings); if (twitter.OAuth?.User.Token == null) { return; } HashSet <string> blocklist = new HashSet <string>(); string readLine; if (!string.IsNullOrEmpty(twitter.MyUserInfo?.id_str)) { Console.WriteLine("Get My Block List... (Max 250000 per 15min)"); string cursor = "-1"; while (true) { try { UserIdsObject result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor)); while (result != null) { blocklist.UnionWith(result.ids); if (result.next_cursor == 0) { break; } result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor = result.next_cursor_str)); } break; } catch (RateLimitException) { if (Convert.ToBoolean(settings.GetValue("Configuration", "AutoRetry_GetBlockList", false)) == false) { Console.Write("Do you want retry get block list after 15min? (Yes/No/Auto)"); readLine = Console.ReadLine(); if (readLine != null) { if (readLine.ToUpper().Trim().StartsWith("N")) { break; } if (readLine.ToUpper().Trim().StartsWith("A")) { settings.SetValue("Configuration", "AutoRetry_GetBlockList", true); } } settings.Save(); } Console.WriteLine("Wait for 15min... The job will be resumed at : " + DateTime.Now.AddMinutes(15).ToString("hh:mm:ss")); Thread.Sleep(TimeSpan.FromMinutes(15)); } } } else { Console.WriteLine("Failed to get your info!"); Console.ReadKey(true); return; } Console.WriteLine($"Blocklist = {blocklist.Count}"); long count = 0; bool userStopped = false; RateLimitException rateLimit = null; foreach (string ids in blocklist) { count++; Console.WriteLine( $"Target= {(ids.Length < 18 ? ids : ids.Substring(0, 17) + "...")}, " + $"Progress= {count}/{blocklist.Count} ({Math.Round(count * 100 / (double)blocklist.Count, 2)}%)"); twitter.UnBlock(ids); if (!Console.KeyAvailable) { continue; } while (Console.KeyAvailable) { Console.ReadKey(true); } Console.WriteLine("Do you want stop reset blocklist?"); if (DialogResult.Yes != MessageBox.Show("Do you want stop reset blocklist?", "Stop ?", MessageBoxButtons.YesNo, MessageBoxIcon.Question)) { continue; } rateLimit = null; userStopped = true; break; } //Console.Write("Do you want export your block list? (Y/N) : "); //readLine = Console.ReadLine(); //if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y")) // File.WriteAllText($"blocklist_{DateTime.Now:yyyy-MM-dd_HHmm}.csv", string.Join(",", blocklist)); Console.Write("Finished !"); settings.SetValue("Authenticate", "AccessToken", ""); settings.SetValue("Authenticate", "AccessSecret", ""); settings.Save(); readLine = Console.ReadLine(); }
// to allow proper rescheduling of the first request from a bucket private async Task ExecuteRequestAsync(BaseRestRequest request, RateLimitBucket bucket, TaskCompletionSource <bool> ratelimitTcs) { if (this._disposed) { return; } HttpResponseMessage res = default; try { await this.GlobalRateLimitEvent.WaitAsync().ConfigureAwait(false); if (bucket == null) { bucket = request.RateLimitBucket; } if (ratelimitTcs == null) { ratelimitTcs = await this.WaitForInitialRateLimit(bucket).ConfigureAwait(false); } if (ratelimitTcs == null) // ckeck rate limit only if we are not the probe request { var now = DateTimeOffset.UtcNow; await bucket.TryResetLimitAsync(now).ConfigureAwait(false); // Decrement the remaining number of requests as there can be other concurrent requests before this one finishes and has a chance to update the bucket if (Interlocked.Decrement(ref bucket._remaining) < 0) { this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is blocked", bucket.ToString()); var delay = bucket.Reset - now; var resetDate = bucket.Reset; if (this.UseResetAfter) { delay = bucket.ResetAfter.Value; resetDate = bucket.ResetAfterOffset; } if (delay < new TimeSpan(-TimeSpan.TicksPerMinute)) { this.Logger.LogError(LoggerEvents.RatelimitDiag, "Failed to retrieve ratelimits - giving up and allowing next request for bucket"); bucket._remaining = 1; } if (delay < TimeSpan.Zero) { delay = TimeSpan.FromMilliseconds(100); } this.Logger.LogWarning(LoggerEvents.RatelimitPreemptive, "Pre-emptive ratelimit triggered - waiting until {0:yyyy-MM-dd HH:mm:ss zzz} ({1:c}).", resetDate, delay); Task.Delay(delay) .ContinueWith(_ => this.ExecuteRequestAsync(request, null, null)) .LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while executing request"); return; } this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is allowed", bucket.ToString()); } else { this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Initial request for {Bucket} is allowed", bucket.ToString()); } var req = this.BuildRequest(request); var response = new RestResponse(); try { if (this._disposed) { return; } res = await this.HttpClient.SendAsync(req, HttpCompletionOption.ResponseContentRead, CancellationToken.None).ConfigureAwait(false); var bts = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(false); var txt = Utilities.UTF8.GetString(bts, 0, bts.Length); this.Logger.LogTrace(LoggerEvents.RestRx, txt); response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value), StringComparer.OrdinalIgnoreCase); response.Response = txt; response.ResponseCode = (int)res.StatusCode; } catch (HttpRequestException httpex) { this.Logger.LogError(LoggerEvents.RestError, httpex, "Request to {Url} triggered an HttpException", request.Url); request.SetFaulted(httpex); this.FailInitialRateLimitTest(request, ratelimitTcs); return; } this.UpdateBucket(request, response, ratelimitTcs); Exception ex = null; switch (response.ResponseCode) { case 400: case 405: ex = new BadRequestException(request, response); break; case 401: case 403: ex = new UnauthorizedException(request, response); break; case 404: ex = new NotFoundException(request, response); break; case 413: ex = new RequestSizeException(request, response); break; case 429: ex = new RateLimitException(request, response); // check the limit info and requeue this.Handle429(response, out var wait, out var global); if (wait != null) { if (global) { this.Logger.LogError(LoggerEvents.RatelimitHit, "Global ratelimit hit, cooling down"); try { this.GlobalRateLimitEvent.Reset(); await wait.ConfigureAwait(false); } finally { // we don't want to wait here until all the blocked requests have been run, additionally Set can never throw an exception that could be suppressed here _ = this.GlobalRateLimitEvent.SetAsync(); } this.ExecuteRequestAsync(request, bucket, ratelimitTcs) .LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request"); } else { this.Logger.LogError(LoggerEvents.RatelimitHit, "Ratelimit hit, requeueing request to {Url}", request.Url); await wait.ConfigureAwait(false); this.ExecuteRequestAsync(request, bucket, ratelimitTcs) .LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request"); } return; } break; case 500: case 502: case 503: case 504: ex = new ServerErrorException(request, response); break; } if (ex != null) { request.SetFaulted(ex); } else { request.SetCompleted(response); } } catch (Exception ex) { this.Logger.LogError(LoggerEvents.RestError, ex, "Request to {Url} triggered an exception", request.Url); // if something went wrong and we couldn't get rate limits for the first request here, allow the next request to run if (bucket != null && ratelimitTcs != null && bucket._limitTesting != 0) { this.FailInitialRateLimitTest(request, ratelimitTcs); } if (!request.TrySetFaulted(ex)) { throw; } } finally { res?.Dispose(); // Get and decrement active requests in this bucket by 1. _ = this.RequestQueue.TryGetValue(bucket.BucketId, out var count); this.RequestQueue[bucket.BucketId] = Interlocked.Decrement(ref count); // If it's 0 or less, we can remove the bucket from the active request queue, // along with any of its past routes. if (count <= 0) { foreach (var r in bucket.RouteHashes) { if (this.RequestQueue.ContainsKey(r)) { _ = this.RequestQueue.TryRemove(r, out _); } } } } }
private static void Main() { Console.InputEncoding = Encoding.Unicode; Console.OutputEncoding = Encoding.Unicode; settings = new IniSettings(new FileInfo(ini_file)); Console.WriteLine("Loading login info..."); TwitterApi twitter = TwitterApi.Login(settings); if (twitter.OAuth?.User.Token == null) { return; } HashSet <string> whitelist = new HashSet <string>(); HashSet <string> blocklist = new HashSet <string>(); string readLine; if (!string.IsNullOrEmpty(twitter.MyUserInfo?.id_str)) { Console.WriteLine("Get My Friends..."); UserIdsObject result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFriends(twitter.MyUserInfo.id_str, "-1")); while (result != null) { whitelist.UnionWith(result.ids); if (result.next_cursor == 0) { break; } result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFriends(twitter.MyUserInfo.id_str, result.next_cursor_str)); } Console.Write("Do you want to include your Followers into whitelist? (Y/N)"); readLine = Console.ReadLine(); if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y")) { Console.WriteLine("Get My Followers..."); result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFollowers(twitter.MyUserInfo.id_str, "-1")); while (result != null) { whitelist.UnionWith(result.ids); if (result.next_cursor == 0) { break; } result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFollowers(twitter.MyUserInfo.id_str, result.next_cursor_str)); } } Console.Write("Do you have backup of blocklist? (Y/N)"); readLine = Console.ReadLine(); if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y")) { Console.Write("Enter path of your blocklist\n: "); string input = Console.ReadLine(); if ((input != null) && File.Exists(input.Replace("\"", ""))) { blocklist.UnionWith(File.ReadAllText(input.Replace("\"", "")).Split(',')); } } else { Console.WriteLine("Get My Block List... (Max 250000 per 15min)"); string cursor = "-1"; while (true) { try { result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor)); while (result != null) { blocklist.UnionWith(result.ids); if (result.next_cursor == 0) { break; } result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor = result.next_cursor_str)); } break; } catch (RateLimitException) { if (Convert.ToBoolean(settings.GetValue("Configuration", "AutoRetry_GetBlockList", false)) == false) { Console.Write("Do you want retry get block list after 15min? (Yes/No/Auto)"); readLine = Console.ReadLine(); if (readLine != null) { if (readLine.ToUpper().Trim().StartsWith("N")) { break; } if (readLine.ToUpper().Trim().StartsWith("A")) { settings.SetValue("Configuration", "AutoRetry_GetBlockList", true); } } settings.Save(); } Console.WriteLine("Wait for 15min... The job will be resumed at : " + DateTime.Now.AddMinutes(15).ToString("hh:mm:ss")); Thread.Sleep(TimeSpan.FromMinutes(15)); } } } } else { Console.WriteLine("Failed to get your info!"); Console.ReadKey(true); return; } Console.WriteLine($"Whitelist = {whitelist.Count}, Blocklist = {blocklist.Count}"); while (true) { Console.Write("Enter @usernames, search phase or filename to Block Them All\n: "); string input = Console.ReadLine(); if (input != null) { string[] targets = File.Exists(input.Replace("\"", "")) ? File.ReadAllText(input.Replace("\"", "")) .Split(new[] { ",", "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries) : input.Split(','); Console.WriteLine("Please check your input is correct!"); if (DialogResult.No == MessageBox.Show( string.Join("\n \n", "Please check your input is correct. ", string.Join("\n", targets), " Press Yes to go."), "Check your input is correct", MessageBoxButtons.YesNo, MessageBoxIcon.Question)) { continue; } while (Console.KeyAvailable) { Console.ReadKey(true); } bool userStopped = false; foreach (string target in targets) { if (userStopped) { break; } if (string.IsNullOrWhiteSpace(target)) { continue; } RateLimitException rateLimit = null; do { HashSet <string> targetLists = new HashSet <string>(); try { if (target.StartsWith("@")) { string username = target.Substring(1); blocklist.Add(twitter.Block(username, true)); GetTargetFollowers(twitter, username, rateLimit == null ? "-1" : rateLimit.cursor, targetLists); } else { if (rateLimit == null) { GetTargetSearchResult(twitter, target, true, targetLists); } else { GetTargetSearchResult(twitter, rateLimit.target, false, targetLists); } } rateLimit = null; } catch (RateLimitException e) { rateLimit = e; } Console.WriteLine("Processing list..."); long count = 0; foreach (string s in targetLists) { count++; if (whitelist.Contains(s) || blocklist.Contains(s)) { continue; } blocklist.Add(twitter.Block(s)); Console.WriteLine( $"Target= {(target.Length < 18 ? target : target.Substring(0, 17) + "...")}, " + $"Progress= {count}/{targetLists.Count} ({Math.Round(count * 100 / (double) targetLists.Count, 2)}%), Blocklist= {blocklist.Count}"); if (!Console.KeyAvailable) { continue; } while (Console.KeyAvailable) { Console.ReadKey(true); } Console.WriteLine("Do you want stop processing this list and targets?"); if (DialogResult.Yes != MessageBox.Show("Do you want stop processing this list and targets?", "Stop ?", MessageBoxButtons.YesNo, MessageBoxIcon.Question)) { continue; } rateLimit = null; userStopped = true; break; } if (rateLimit == null) { continue; } TimeSpan wait = rateLimit.thrownAt.AddMinutes(15) - DateTime.Now; if (wait < TimeSpan.Zero) { continue; } Console.WriteLine($"Wait {wait:g} for Rate limit..."); Thread.Sleep(wait); } while (rateLimit != null); } } Console.Write("Finished ! Do you want continue? (Y/N) : "); readLine = Console.ReadLine(); if ((readLine != null) && readLine.ToUpper().Trim().Equals("N")) { break; } } Console.Write("Do you want export your block list? (Y/N) : "); readLine = Console.ReadLine(); if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y")) { File.WriteAllText($"blocklist_{DateTime.Now:yyyy-MM-dd_HHmm}.csv", string.Join(",", blocklist)); } }