public async Task Run_LongTasks_StopBeforeCompletion()
{
// In theory this should take 1000 * 100 / 10 = 10.000 ms = 10 seconds
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, 1000),
workFunction : longTask,
degreeOfParallelism : 10,
totalAmount : 1000,
skip : 0);
progressCount = 0;
completedFlag = false;
lastException = null;
parallelizer.ProgressChanged += OnProgress;
parallelizer.Completed += OnCompleted;
parallelizer.Error += OnException;
await parallelizer.Start();
await Task.Delay(250);
await parallelizer.Stop();
Assert.InRange(progressCount, 10, 50);
Assert.True(completedFlag);
Assert.Null(lastException);
}
public async Task Run_QuickTasks_CompleteAndCall()
{
var count = 100;
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, count),
workFunction : parityCheck,
degreeOfParallelism : 1,
totalAmount : count,
skip : 0);
progressCount = 0;
completedFlag = false;
lastException = null;
parallelizer.ProgressChanged += OnProgress;
parallelizer.NewResult += OnResult;
parallelizer.Completed += OnCompleted;
parallelizer.Error += OnException;
await parallelizer.Start();
var cts = new CancellationTokenSource();
cts.CancelAfter(10000);
await parallelizer.WaitCompletion(cts.Token);
Assert.Equal(100, progressCount);
Assert.True(completedFlag);
Assert.Null(lastException);
Assert.True(lastResult);
}
public async Task Run_DecreaseConcurrentThreads_CompleteSlower()
{
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, 12),
workFunction : longTask,
degreeOfParallelism : 3,
totalAmount : 12,
skip : 0);
var stopwatch = new Stopwatch();
// Start with 3 concurrent tasks
stopwatch.Start();
await parallelizer.Start();
// Wait for 1 round to complete (a.k.a 3 completed since there are 3 concurrent threads)
await Task.Delay(150);
// Remove 2 slots
await parallelizer.ChangeDegreeOfParallelism(1);
// Wait until finished
var cts = new CancellationTokenSource();
cts.CancelAfter(10000);
await parallelizer.WaitCompletion(cts.Token);
stopwatch.Stop();
// Make sure it took more than 12 * 100 / 3 = 400 ms (we'll say 600 to make sure)
Assert.True(stopwatch.ElapsedMilliseconds > 600);
}
public async Task Run_IncreaseConcurrentThreads_CompleteFaster()
{
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, 10),
workFunction : longTask,
degreeOfParallelism : 1,
totalAmount : 10,
skip : 0);
var stopwatch = new Stopwatch();
// Start with 1 concurrent task
stopwatch.Start();
await parallelizer.Start();
// Wait for 2 rounds to fully complete
await Task.Delay(250);
// Release 3 more slots
await parallelizer.ChangeDegreeOfParallelism(4);
// Wait until finished
var cts = new CancellationTokenSource();
cts.CancelAfter(10000);
await parallelizer.WaitCompletion(cts.Token);
stopwatch.Stop();
// Make sure it took less than 10 * 100 ms (let's say 800)
Assert.InRange(stopwatch.ElapsedMilliseconds, 0, 800);
}
public async Task Run_PauseAndResume_CompleteAll()
{
var count = 10;
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, count),
workFunction : longTask,
degreeOfParallelism : 1,
totalAmount : count,
skip : 0);
progressCount = 0;
completedFlag = false;
lastException = null;
parallelizer.ProgressChanged += OnProgress;
parallelizer.NewResult += OnResult;
parallelizer.Completed += OnCompleted;
parallelizer.Error += OnException;
await parallelizer.Start();
await Task.Delay(150);
await parallelizer.Pause();
// Make sure it's actually paused and nothing is going on
var progress = progressCount;
await Task.Delay(1000);
Assert.Equal(progress, progressCount);
await parallelizer.Resume();
var cts = new CancellationTokenSource();
cts.CancelAfter(10000);
await parallelizer.WaitCompletion(cts.Token);
Assert.Equal(count, progressCount);
Assert.True(completedFlag);
Assert.Null(lastException);
}
public async Task Run_QuickTasks_StopwatchStops()
{
var count = 100;
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, count),
workFunction : parityCheck,
degreeOfParallelism : 1,
totalAmount : count,
skip : 0);
await parallelizer.Start();
var cts = new CancellationTokenSource();
cts.CancelAfter(10000);
await parallelizer.WaitCompletion(cts.Token);
var elapsed = parallelizer.Elapsed;
await Task.Delay(1000);
Assert.Equal(elapsed, parallelizer.Elapsed);
}
public async Task Run_Pause_StopwatchStops()
{
var count = 10;
var parallelizer = ParallelizerFactory <int, bool> .Create(
type : type,
workItems : Enumerable.Range(1, count),
workFunction : longTask,
degreeOfParallelism : 1,
totalAmount : count,
skip : 0);
await parallelizer.Start();
await Task.Delay(150);
await parallelizer.Pause();
var elapsed = parallelizer.Elapsed;
await Task.Delay(1000);
Assert.Equal(elapsed, parallelizer.Elapsed);
await parallelizer.Abort();
}
public async Task <Checker> BuildAsync()
{
var quickBulletSettings = JsonConvert.DeserializeObject <QuickBulletSettings>(File.ReadAllText(settingsFile));
var loliScriptManager = new LoliScriptManager();
(var configSettings, var blocks) = loliScriptManager.Build(_configFile);
if (string.IsNullOrEmpty(configSettings.Name))
{
configSettings.Name = Path.GetFileNameWithoutExtension(_configFile);
}
if (!string.IsNullOrEmpty(configSettings.AdditionalInfo))
{
AnsiConsole.MarkupLine($"[grey]CONFIG INFO:[/] {configSettings.AdditionalInfo}");
}
if (configSettings.CustomInputs.Any())
{
AnsiConsole.Write(new Rule("[darkorange]Custom input[/]").RuleStyle("grey").LeftAligned());
foreach (var customInput in configSettings.CustomInputs)
{
customInput.Value = AnsiConsole.Ask <string>($"{customInput.Description}:");
}
}
var botInputs = File.ReadAllLines(_wordlistFile).Where(w => !string.IsNullOrEmpty(w)).Select(w => new BotInput(w));
if (configSettings.InputRules.Any())
{
botInputs = botInputs.Where(b => configSettings.InputRules.All(i => _checkInputRuleFunctions[i.Name].Invoke(b, i)));
}
var useProxy = _proxies.Any();
var proxyHttpClients = _proxies.Any() ? new List <ProxyHttpClient>(_proxies.Select(p => BuildProxy(p, _proxyType)).Select(p => new ProxyHttpClient(new HttpClientHandler()
{
UseCookies = false, Proxy = p
}, p)
{
Timeout = TimeSpan.FromSeconds(15)
})) : new List <ProxyHttpClient>()
{
new ProxyHttpClient(new HttpClientHandler()
{
UseCookies = false
}, null)
{
Timeout = TimeSpan.FromSeconds(15)
}
};
var proxyHttpClientManager = new ProxyHttpClientManager(proxyHttpClients);
if (useProxy)
{
_ = proxyHttpClientManager.StartValidateAllProxiesAsync();
}
var record = GetRecord(configSettings.Name);
Directory.CreateDirectory(Path.Combine(quickBulletSettings.OutputDirectory, configSettings.Name));
var skip = _skip == -1 ? record.Progress : _skip;
var checkerStats = new CheckerStats(skip)
{
DegreeOfParallelism = _bots
};
var statusesToBreak = new string[] { "toCheck", "failure", "retry", "ban", "error" };
var statusesToRecheck = new string[] { "retry", "ban", "error" };
var readerWriterLock = new ReaderWriterLock();
var handler = new HttpClientHandler()
{
UseCookies = false
};
var httpClient = new HttpClient(handler)
{
Timeout = TimeSpan.FromSeconds(15)
};
var playwright = await Playwright.CreateAsync();
Func <BotInput, CancellationToken, Task <bool> > check = new(async(input, cancellationToken) =>
{
BotData botData = null;
for (var attempts = 0; attempts < 8; attempts++)
{
var proxyHttpClient = proxyHttpClientManager.GetRandomProxyHttpClient();
botData = new BotData(quickBulletSettings, input, httpClient, proxyHttpClient, playwright)
{
UseProxy = useProxy
};
botData.Variables.Add("data.proxy", proxyHttpClient.Proxy is null ? string.Empty : proxyHttpClient.Proxy.ToString());
foreach (var customInput in configSettings.CustomInputs)
{
botData.Variables.Add(customInput.Name, customInput.Value);
}
foreach (var block in blocks)
{
try
{
await block.RunAsync(botData);
}
catch (HttpRequestException)
{
proxyHttpClient.IsValid = false;
botData.Variables["botStatus"] = "retry";
}
catch (Exception error)
{
if (error.Message.Contains("HttpClient.Timeout") || error.Message.Contains("ERR_TIMED_OUT"))
{
proxyHttpClient.IsValid = false;
}
else if (_verbose)
{
AnsiConsole.WriteException(error);
}
botData.Variables["botStatus"] = "retry";
}
if (statusesToBreak.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase))
{
break;
}
}
await botData.DisposeAsync();
if (statusesToRecheck.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase))
{
if (botData.Variables["botStatus"].Equals("ban", StringComparison.OrdinalIgnoreCase))
{
proxyHttpClient.IsValid = false;
}
checkerStats.Increment(botData.Variables["botStatus"]);
}
else
{
break;
}
}
var botStatus = statusesToRecheck.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase) ? "tocheck" : botData.Variables["botStatus"].ToLower();
if (botStatus.Equals("failure"))
{
checkerStats.Increment(botData.Variables["botStatus"]);
}
else
{
var outputPath = Path.Combine(quickBulletSettings.OutputDirectory, configSettings.Name, $"{botStatus}.txt");
var output = botData.Captures.Any() ? new StringBuilder().Append(botData.Input.ToString()).Append(quickBulletSettings.OutputSeparator).AppendJoin(quickBulletSettings.OutputSeparator, botData.Captures.Select(c => $"{c.Key} = {c.Value}")).ToString() : botData.Input.ToString();
try
{
readerWriterLock.AcquireWriterLock(int.MaxValue);
using var streamWriter = File.AppendText(outputPath);
await streamWriter.WriteLineAsync(output);
}
finally
{
readerWriterLock.ReleaseWriterLock();
}
switch (botStatus)
{
case "success":
AnsiConsole.MarkupLine($"[green4]SUCCESS:[/] {output}");
break;
case "tocheck":
AnsiConsole.MarkupLine($"[cyan3]TOCHECK:[/] {output}");
break;
default:
AnsiConsole.MarkupLine($"[orange3]{botStatus.ToUpper()}:[/] {output}");
break;
}
checkerStats.Increment(botStatus);
}
checkerStats.Increment("checked");
return(true);
});
var paradllelizer = ParallelizerFactory <BotInput, bool> .Create(type : ParallelizerType.TaskBased, workItems : botInputs, workFunction : check, degreeOfParallelism : _bots, totalAmount : botInputs.Skip(skip).Count(), skip : skip);
return(new Checker(paradllelizer, checkerStats, record));
}
