// to allow proper rescheduling of the first request from a bucket
private async Task ExecuteRequestAsync(BaseRestRequest request, RateLimitBucket bucket, TaskCompletionSource <bool> ratelimitTcs)
{
try
{
await this.GlobalRateLimitEvent.WaitAsync();
if (bucket == null)
{
bucket = request.RateLimitBucket;
}
if (ratelimitTcs == null)
{
ratelimitTcs = await this.WaitForInitialRateLimit(bucket);
}
if (ratelimitTcs == null) // ckeck rate limit only if we are not the probe request
{
var now = DateTimeOffset.UtcNow;
await bucket.TryResetLimit(now);
// Decrement the remaining number of requests as there can be other concurrent requests before this one finishes and has a chance to update the bucket
#pragma warning disable 420 // interlocked access is always volatile
if (Interlocked.Decrement(ref bucket._remaining) < 0)
#pragma warning restore 420 // blaze it
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Request for bucket {bucket}. Blocking.", DateTime.Now);
var delay = bucket.Reset - now;
if (delay < new TimeSpan(-TimeSpan.TicksPerMinute))
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", "Failed to retrieve ratelimits. Giving up and allowing next request for bucket.", DateTime.Now);
bucket._remaining = 1;
}
if (delay < TimeSpan.Zero)
{
delay = TimeSpan.FromMilliseconds(100);
}
request.Discord?.DebugLogger?.LogMessage(LogLevel.Warning, "REST", $"Pre-emptive ratelimit triggered, waiting until {bucket.Reset:yyyy-MM-dd HH:mm:ss zzz} ({delay:c})", DateTime.Now);
request.Discord?.DebugLogger?.LogTaskFault(Task.Delay(delay).ContinueWith(t => this.ExecuteRequestAsync(request, null, null)), LogLevel.Error, "RESET", "Error while executing request: ");
return;
}
request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Request for bucket {bucket}. Allowing.", DateTime.Now);
}
else
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Debug, "REST", $"Initial Request for bucket {bucket}. Allowing.", DateTime.Now);
}
var req = this.BuildRequest(request);
var response = new RestResponse();
try
{
var res = await HttpClient.SendAsync(req, CancellationToken.None).ConfigureAwait(false);
var bts = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
var txt = Utilities.UTF8.GetString(bts, 0, bts.Length);
response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value));
response.Response = txt;
response.ResponseCode = (int)res.StatusCode;
}
catch (HttpRequestException httpex)
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an HttpException: {httpex.Message}", DateTime.Now);
request.SetFaulted(httpex);
this.FailInitialRateLimitTest(bucket, ratelimitTcs);
return;
}
this.UpdateBucket(request, response, ratelimitTcs);
Exception ex = null;
switch (response.ResponseCode)
{
case 400:
case 405:
ex = new BadRequestException(request, response);
break;
case 401:
case 403:
ex = new UnauthorizedException(request, response);
break;
case 404:
ex = new NotFoundException(request, response);
break;
case 429:
ex = new RateLimitException(request, response);
// check the limit info and requeue
this.Handle429(response, out var wait, out var global);
if (wait != null)
{
if (global)
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", "Global ratelimit hit, cooling down", DateTime.Now);
try
{
this.GlobalRateLimitEvent.Reset();
await wait.ConfigureAwait(false);
}
finally
{
// we don't want to wait here until all the blocked requests have been run, additionally Set can never throw an exception that could be suppressed here
_ = this.GlobalRateLimitEvent.SetAsync();
}
request.Discord?.DebugLogger?.LogTaskFault(ExecuteRequestAsync(request, bucket, ratelimitTcs), LogLevel.Error, "REST", "Error while retrying request: ");
}
else
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Ratelimit hit, requeueing request to {request.Url}", DateTime.Now);
await wait.ConfigureAwait(false);
request.Discord?.DebugLogger?.LogTaskFault(this.ExecuteRequestAsync(request, bucket, ratelimitTcs), LogLevel.Error, "REST", "Error while retrying request: ");
}
return;
}
break;
}
if (ex != null)
{
request.SetFaulted(ex);
}
else
{
request.SetCompleted(response);
}
}
catch (Exception ex)
{
request.Discord?.DebugLogger?.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an {ex.GetType().Name}: {ex.Message}\n{ex.StackTrace}", DateTime.Now);
// if something went wrong and we couldn't get rate limits for the first request here, allow the next request to run
if (bucket != null && ratelimitTcs != null && bucket._limitTesting != 0)
{
this.FailInitialRateLimitTest(bucket, ratelimitTcs);
}
if (!request.TrySetFaulted(ex))
{
throw;
}
}
}
public async Task ExecuteRequestAsync(BaseRestRequest request)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
await this.RequestSemaphore.WaitAsync();
var bucket = request.RateLimitBucket;
var now = DateTimeOffset.UtcNow;
if (bucket.Remaining <= 0 && bucket.Maximum > 0 && now < bucket.Reset)
{
request.Discord.DebugLogger.LogMessage(LogLevel.Warning, "REST", $"Pre-emptive ratelimit triggered, waiting until {bucket.Reset.ToString("yyyy-MM-dd HH:mm:ss zzz")}", DateTime.Now);
_ = Task.Delay(bucket.Reset - now).ContinueWith(t => this.ExecuteRequestAsync(request));
this.RequestSemaphore.Release();
return;
}
var req = this.BuildRequest(request);
var response = new RestResponse();
try
{
var res = await HttpClient.SendAsync(req, HttpCompletionOption.ResponseContentRead);
var bts = await res.Content.ReadAsByteArrayAsync();
var txt = UTF8.GetString(bts, 0, bts.Length);
response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value));
response.Response = txt;
response.ResponseCode = (int)res.StatusCode;
}
catch (HttpRequestException httpex)
{
request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", $"Request to {request.Url} triggered an HttpException: {httpex.Message}", DateTime.Now);
request.SetFaulted(httpex);
this.RequestSemaphore.Release();
return;
}
Exception ex = null;
switch (response.ResponseCode)
{
case 400:
case 405:
ex = new BadRequestException(request, response);
break;
case 401:
case 403:
ex = new UnauthorizedException(request, response);
break;
case 404:
ex = new NotFoundException(request, response);
break;
case 429:
ex = new RateLimitException(request, response);
// check the limit info, if more than one minute, fault, otherwise requeue
this.Handle429(response, out var wait, out var global);
if (wait != null)
{
wait = wait.ContinueWith(t => this.ExecuteRequestAsync(request));
if (global)
{
request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", "Global ratelimit hit, cooling down", DateTime.Now);
await wait;
}
else
{
request.Discord.DebugLogger.LogMessage(LogLevel.Error, "REST", $"Ratelimit hit, requeueing request to {request.Url}", DateTime.Now);
}
this.RequestSemaphore.Release();
return;
}
break;
}
this.UpdateBucket(request, response);
this.RequestSemaphore.Release();
if (ex != null)
{
request.SetFaulted(ex);
}
else
{
request.SetCompleted(response);
}
}
private static void Main()
{
Console.InputEncoding = Encoding.Unicode;
Console.OutputEncoding = Encoding.Unicode;
settings = new IniSettings(new FileInfo(ini_file));
Console.WriteLine("Loading login info...");
TwitterApi twitter = TwitterApi.Login(settings);
if (twitter.OAuth?.User.Token == null)
{
return;
}
HashSet <string> blocklist = new HashSet <string>();
string readLine;
if (!string.IsNullOrEmpty(twitter.MyUserInfo?.id_str))
{
Console.WriteLine("Get My Block List... (Max 250000 per 15min)");
string cursor = "-1";
while (true)
{
try
{
UserIdsObject result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor));
while (result != null)
{
blocklist.UnionWith(result.ids);
if (result.next_cursor == 0)
{
break;
}
result =
JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor = result.next_cursor_str));
}
break;
}
catch (RateLimitException)
{
if (Convert.ToBoolean(settings.GetValue("Configuration", "AutoRetry_GetBlockList", false)) == false)
{
Console.Write("Do you want retry get block list after 15min? (Yes/No/Auto)");
readLine = Console.ReadLine();
if (readLine != null)
{
if (readLine.ToUpper().Trim().StartsWith("N"))
{
break;
}
if (readLine.ToUpper().Trim().StartsWith("A"))
{
settings.SetValue("Configuration", "AutoRetry_GetBlockList", true);
}
}
settings.Save();
}
Console.WriteLine("Wait for 15min... The job will be resumed at : " +
DateTime.Now.AddMinutes(15).ToString("hh:mm:ss"));
Thread.Sleep(TimeSpan.FromMinutes(15));
}
}
}
else
{
Console.WriteLine("Failed to get your info!");
Console.ReadKey(true);
return;
}
Console.WriteLine($"Blocklist = {blocklist.Count}");
long count = 0;
bool userStopped = false;
RateLimitException rateLimit = null;
foreach (string ids in blocklist)
{
count++;
Console.WriteLine(
$"Target= {(ids.Length < 18 ? ids : ids.Substring(0, 17) + "...")}, " +
$"Progress= {count}/{blocklist.Count} ({Math.Round(count * 100 / (double)blocklist.Count, 2)}%)");
twitter.UnBlock(ids);
if (!Console.KeyAvailable)
{
continue;
}
while (Console.KeyAvailable)
{
Console.ReadKey(true);
}
Console.WriteLine("Do you want stop reset blocklist?");
if (DialogResult.Yes != MessageBox.Show("Do you want stop reset blocklist?",
"Stop ?",
MessageBoxButtons.YesNo, MessageBoxIcon.Question))
{
continue;
}
rateLimit = null;
userStopped = true;
break;
}
//Console.Write("Do you want export your block list? (Y/N) : ");
//readLine = Console.ReadLine();
//if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y"))
// File.WriteAllText($"blocklist_{DateTime.Now:yyyy-MM-dd_HHmm}.csv", string.Join(",", blocklist));
Console.Write("Finished !");
settings.SetValue("Authenticate", "AccessToken", "");
settings.SetValue("Authenticate", "AccessSecret", "");
settings.Save();
readLine = Console.ReadLine();
}
// to allow proper rescheduling of the first request from a bucket
private async Task ExecuteRequestAsync(BaseRestRequest request, RateLimitBucket bucket, TaskCompletionSource <bool> ratelimitTcs)
{
if (this._disposed)
{
return;
}
HttpResponseMessage res = default;
try
{
await this.GlobalRateLimitEvent.WaitAsync().ConfigureAwait(false);
if (bucket == null)
{
bucket = request.RateLimitBucket;
}
if (ratelimitTcs == null)
{
ratelimitTcs = await this.WaitForInitialRateLimit(bucket).ConfigureAwait(false);
}
if (ratelimitTcs == null) // ckeck rate limit only if we are not the probe request
{
var now = DateTimeOffset.UtcNow;
await bucket.TryResetLimitAsync(now).ConfigureAwait(false);
// Decrement the remaining number of requests as there can be other concurrent requests before this one finishes and has a chance to update the bucket
if (Interlocked.Decrement(ref bucket._remaining) < 0)
{
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is blocked", bucket.ToString());
var delay = bucket.Reset - now;
var resetDate = bucket.Reset;
if (this.UseResetAfter)
{
delay = bucket.ResetAfter.Value;
resetDate = bucket.ResetAfterOffset;
}
if (delay < new TimeSpan(-TimeSpan.TicksPerMinute))
{
this.Logger.LogError(LoggerEvents.RatelimitDiag, "Failed to retrieve ratelimits - giving up and allowing next request for bucket");
bucket._remaining = 1;
}
if (delay < TimeSpan.Zero)
{
delay = TimeSpan.FromMilliseconds(100);
}
this.Logger.LogWarning(LoggerEvents.RatelimitPreemptive, "Pre-emptive ratelimit triggered - waiting until {0:yyyy-MM-dd HH:mm:ss zzz} ({1:c}).", resetDate, delay);
Task.Delay(delay)
.ContinueWith(_ => this.ExecuteRequestAsync(request, null, null))
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while executing request");
return;
}
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Request for {Bucket} is allowed", bucket.ToString());
}
else
{
this.Logger.LogDebug(LoggerEvents.RatelimitDiag, "Initial request for {Bucket} is allowed", bucket.ToString());
}
var req = this.BuildRequest(request);
var response = new RestResponse();
try
{
if (this._disposed)
{
return;
}
res = await this.HttpClient.SendAsync(req, HttpCompletionOption.ResponseContentRead, CancellationToken.None).ConfigureAwait(false);
var bts = await res.Content.ReadAsByteArrayAsync().ConfigureAwait(false);
var txt = Utilities.UTF8.GetString(bts, 0, bts.Length);
this.Logger.LogTrace(LoggerEvents.RestRx, txt);
response.Headers = res.Headers.ToDictionary(xh => xh.Key, xh => string.Join("\n", xh.Value), StringComparer.OrdinalIgnoreCase);
response.Response = txt;
response.ResponseCode = (int)res.StatusCode;
}
catch (HttpRequestException httpex)
{
this.Logger.LogError(LoggerEvents.RestError, httpex, "Request to {Url} triggered an HttpException", request.Url);
request.SetFaulted(httpex);
this.FailInitialRateLimitTest(request, ratelimitTcs);
return;
}
this.UpdateBucket(request, response, ratelimitTcs);
Exception ex = null;
switch (response.ResponseCode)
{
case 400:
case 405:
ex = new BadRequestException(request, response);
break;
case 401:
case 403:
ex = new UnauthorizedException(request, response);
break;
case 404:
ex = new NotFoundException(request, response);
break;
case 413:
ex = new RequestSizeException(request, response);
break;
case 429:
ex = new RateLimitException(request, response);
// check the limit info and requeue
this.Handle429(response, out var wait, out var global);
if (wait != null)
{
if (global)
{
this.Logger.LogError(LoggerEvents.RatelimitHit, "Global ratelimit hit, cooling down");
try
{
this.GlobalRateLimitEvent.Reset();
await wait.ConfigureAwait(false);
}
finally
{
// we don't want to wait here until all the blocked requests have been run, additionally Set can never throw an exception that could be suppressed here
_ = this.GlobalRateLimitEvent.SetAsync();
}
this.ExecuteRequestAsync(request, bucket, ratelimitTcs)
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request");
}
else
{
this.Logger.LogError(LoggerEvents.RatelimitHit, "Ratelimit hit, requeueing request to {Url}", request.Url);
await wait.ConfigureAwait(false);
this.ExecuteRequestAsync(request, bucket, ratelimitTcs)
.LogTaskFault(this.Logger, LogLevel.Error, LoggerEvents.RestError, "Error while retrying request");
}
return;
}
break;
case 500:
case 502:
case 503:
case 504:
ex = new ServerErrorException(request, response);
break;
}
if (ex != null)
{
request.SetFaulted(ex);
}
else
{
request.SetCompleted(response);
}
}
catch (Exception ex)
{
this.Logger.LogError(LoggerEvents.RestError, ex, "Request to {Url} triggered an exception", request.Url);
// if something went wrong and we couldn't get rate limits for the first request here, allow the next request to run
if (bucket != null && ratelimitTcs != null && bucket._limitTesting != 0)
{
this.FailInitialRateLimitTest(request, ratelimitTcs);
}
if (!request.TrySetFaulted(ex))
{
throw;
}
}
finally
{
res?.Dispose();
// Get and decrement active requests in this bucket by 1.
_ = this.RequestQueue.TryGetValue(bucket.BucketId, out var count);
this.RequestQueue[bucket.BucketId] = Interlocked.Decrement(ref count);
// If it's 0 or less, we can remove the bucket from the active request queue,
// along with any of its past routes.
if (count <= 0)
{
foreach (var r in bucket.RouteHashes)
{
if (this.RequestQueue.ContainsKey(r))
{
_ = this.RequestQueue.TryRemove(r, out _);
}
}
}
}
}
private static void Main()
{
Console.InputEncoding = Encoding.Unicode;
Console.OutputEncoding = Encoding.Unicode;
settings = new IniSettings(new FileInfo(ini_file));
Console.WriteLine("Loading login info...");
TwitterApi twitter = TwitterApi.Login(settings);
if (twitter.OAuth?.User.Token == null)
{
return;
}
HashSet <string> whitelist = new HashSet <string>();
HashSet <string> blocklist = new HashSet <string>();
string readLine;
if (!string.IsNullOrEmpty(twitter.MyUserInfo?.id_str))
{
Console.WriteLine("Get My Friends...");
UserIdsObject result =
JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFriends(twitter.MyUserInfo.id_str, "-1"));
while (result != null)
{
whitelist.UnionWith(result.ids);
if (result.next_cursor == 0)
{
break;
}
result =
JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFriends(twitter.MyUserInfo.id_str,
result.next_cursor_str));
}
Console.Write("Do you want to include your Followers into whitelist? (Y/N)");
readLine = Console.ReadLine();
if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y"))
{
Console.WriteLine("Get My Followers...");
result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFollowers(twitter.MyUserInfo.id_str, "-1"));
while (result != null)
{
whitelist.UnionWith(result.ids);
if (result.next_cursor == 0)
{
break;
}
result =
JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyFollowers(twitter.MyUserInfo.id_str,
result.next_cursor_str));
}
}
Console.Write("Do you have backup of blocklist? (Y/N)");
readLine = Console.ReadLine();
if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y"))
{
Console.Write("Enter path of your blocklist\n: ");
string input = Console.ReadLine();
if ((input != null) && File.Exists(input.Replace("\"", "")))
{
blocklist.UnionWith(File.ReadAllText(input.Replace("\"", "")).Split(','));
}
}
else
{
Console.WriteLine("Get My Block List... (Max 250000 per 15min)");
string cursor = "-1";
while (true)
{
try
{
result = JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor));
while (result != null)
{
blocklist.UnionWith(result.ids);
if (result.next_cursor == 0)
{
break;
}
result =
JsonConvert.DeserializeObject <UserIdsObject>(twitter.getMyBlockList(cursor = result.next_cursor_str));
}
break;
}
catch (RateLimitException)
{
if (Convert.ToBoolean(settings.GetValue("Configuration", "AutoRetry_GetBlockList", false)) == false)
{
Console.Write("Do you want retry get block list after 15min? (Yes/No/Auto)");
readLine = Console.ReadLine();
if (readLine != null)
{
if (readLine.ToUpper().Trim().StartsWith("N"))
{
break;
}
if (readLine.ToUpper().Trim().StartsWith("A"))
{
settings.SetValue("Configuration", "AutoRetry_GetBlockList", true);
}
}
settings.Save();
}
Console.WriteLine("Wait for 15min... The job will be resumed at : " +
DateTime.Now.AddMinutes(15).ToString("hh:mm:ss"));
Thread.Sleep(TimeSpan.FromMinutes(15));
}
}
}
}
else
{
Console.WriteLine("Failed to get your info!");
Console.ReadKey(true);
return;
}
Console.WriteLine($"Whitelist = {whitelist.Count}, Blocklist = {blocklist.Count}");
while (true)
{
Console.Write("Enter @usernames, search phase or filename to Block Them All\n: ");
string input = Console.ReadLine();
if (input != null)
{
string[] targets = File.Exists(input.Replace("\"", ""))
? File.ReadAllText(input.Replace("\"", ""))
.Split(new[] { ",", "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries)
: input.Split(',');
Console.WriteLine("Please check your input is correct!");
if (DialogResult.No ==
MessageBox.Show(
string.Join("\n \n", "Please check your input is correct. ", string.Join("\n", targets), " Press Yes to go."),
"Check your input is correct",
MessageBoxButtons.YesNo, MessageBoxIcon.Question))
{
continue;
}
while (Console.KeyAvailable)
{
Console.ReadKey(true);
}
bool userStopped = false;
foreach (string target in targets)
{
if (userStopped)
{
break;
}
if (string.IsNullOrWhiteSpace(target))
{
continue;
}
RateLimitException rateLimit = null;
do
{
HashSet <string> targetLists = new HashSet <string>();
try
{
if (target.StartsWith("@"))
{
string username = target.Substring(1);
blocklist.Add(twitter.Block(username, true));
GetTargetFollowers(twitter, username, rateLimit == null ? "-1" : rateLimit.cursor, targetLists);
}
else
{
if (rateLimit == null)
{
GetTargetSearchResult(twitter, target, true, targetLists);
}
else
{
GetTargetSearchResult(twitter, rateLimit.target, false, targetLists);
}
}
rateLimit = null;
}
catch (RateLimitException e)
{
rateLimit = e;
}
Console.WriteLine("Processing list...");
long count = 0;
foreach (string s in targetLists)
{
count++;
if (whitelist.Contains(s) || blocklist.Contains(s))
{
continue;
}
blocklist.Add(twitter.Block(s));
Console.WriteLine(
$"Target= {(target.Length < 18 ? target : target.Substring(0, 17) + "...")}, " +
$"Progress= {count}/{targetLists.Count} ({Math.Round(count * 100 / (double) targetLists.Count, 2)}%), Blocklist= {blocklist.Count}");
if (!Console.KeyAvailable)
{
continue;
}
while (Console.KeyAvailable)
{
Console.ReadKey(true);
}
Console.WriteLine("Do you want stop processing this list and targets?");
if (DialogResult.Yes != MessageBox.Show("Do you want stop processing this list and targets?",
"Stop ?",
MessageBoxButtons.YesNo, MessageBoxIcon.Question))
{
continue;
}
rateLimit = null;
userStopped = true;
break;
}
if (rateLimit == null)
{
continue;
}
TimeSpan wait = rateLimit.thrownAt.AddMinutes(15) - DateTime.Now;
if (wait < TimeSpan.Zero)
{
continue;
}
Console.WriteLine($"Wait {wait:g} for Rate limit...");
Thread.Sleep(wait);
} while (rateLimit != null);
}
}
Console.Write("Finished ! Do you want continue? (Y/N) : ");
readLine = Console.ReadLine();
if ((readLine != null) && readLine.ToUpper().Trim().Equals("N"))
{
break;
}
}
Console.Write("Do you want export your block list? (Y/N) : ");
readLine = Console.ReadLine();
if ((readLine != null) && readLine.ToUpper().Trim().Equals("Y"))
{
File.WriteAllText($"blocklist_{DateTime.Now:yyyy-MM-dd_HHmm}.csv", string.Join(",", blocklist));
}
}