public async Task Run_LongTasks_StopBeforeCompletion() { // In theory this should take 1000 * 100 / 10 = 10.000 ms = 10 seconds var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, 1000), workFunction : longTask, degreeOfParallelism : 10, totalAmount : 1000, skip : 0); progressCount = 0; completedFlag = false; lastException = null; parallelizer.ProgressChanged += OnProgress; parallelizer.Completed += OnCompleted; parallelizer.Error += OnException; await parallelizer.Start(); await Task.Delay(250); await parallelizer.Stop(); Assert.InRange(progressCount, 10, 50); Assert.True(completedFlag); Assert.Null(lastException); }
public async Task Run_QuickTasks_CompleteAndCall() { var count = 100; var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, count), workFunction : parityCheck, degreeOfParallelism : 1, totalAmount : count, skip : 0); progressCount = 0; completedFlag = false; lastException = null; parallelizer.ProgressChanged += OnProgress; parallelizer.NewResult += OnResult; parallelizer.Completed += OnCompleted; parallelizer.Error += OnException; await parallelizer.Start(); var cts = new CancellationTokenSource(); cts.CancelAfter(10000); await parallelizer.WaitCompletion(cts.Token); Assert.Equal(100, progressCount); Assert.True(completedFlag); Assert.Null(lastException); Assert.True(lastResult); }
public async Task Run_DecreaseConcurrentThreads_CompleteSlower() { var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, 12), workFunction : longTask, degreeOfParallelism : 3, totalAmount : 12, skip : 0); var stopwatch = new Stopwatch(); // Start with 3 concurrent tasks stopwatch.Start(); await parallelizer.Start(); // Wait for 1 round to complete (a.k.a 3 completed since there are 3 concurrent threads) await Task.Delay(150); // Remove 2 slots await parallelizer.ChangeDegreeOfParallelism(1); // Wait until finished var cts = new CancellationTokenSource(); cts.CancelAfter(10000); await parallelizer.WaitCompletion(cts.Token); stopwatch.Stop(); // Make sure it took more than 12 * 100 / 3 = 400 ms (we'll say 600 to make sure) Assert.True(stopwatch.ElapsedMilliseconds > 600); }
public async Task Run_IncreaseConcurrentThreads_CompleteFaster() { var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, 10), workFunction : longTask, degreeOfParallelism : 1, totalAmount : 10, skip : 0); var stopwatch = new Stopwatch(); // Start with 1 concurrent task stopwatch.Start(); await parallelizer.Start(); // Wait for 2 rounds to fully complete await Task.Delay(250); // Release 3 more slots await parallelizer.ChangeDegreeOfParallelism(4); // Wait until finished var cts = new CancellationTokenSource(); cts.CancelAfter(10000); await parallelizer.WaitCompletion(cts.Token); stopwatch.Stop(); // Make sure it took less than 10 * 100 ms (let's say 800) Assert.InRange(stopwatch.ElapsedMilliseconds, 0, 800); }
public async Task Run_PauseAndResume_CompleteAll() { var count = 10; var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, count), workFunction : longTask, degreeOfParallelism : 1, totalAmount : count, skip : 0); progressCount = 0; completedFlag = false; lastException = null; parallelizer.ProgressChanged += OnProgress; parallelizer.NewResult += OnResult; parallelizer.Completed += OnCompleted; parallelizer.Error += OnException; await parallelizer.Start(); await Task.Delay(150); await parallelizer.Pause(); // Make sure it's actually paused and nothing is going on var progress = progressCount; await Task.Delay(1000); Assert.Equal(progress, progressCount); await parallelizer.Resume(); var cts = new CancellationTokenSource(); cts.CancelAfter(10000); await parallelizer.WaitCompletion(cts.Token); Assert.Equal(count, progressCount); Assert.True(completedFlag); Assert.Null(lastException); }
public async Task Run_QuickTasks_StopwatchStops() { var count = 100; var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, count), workFunction : parityCheck, degreeOfParallelism : 1, totalAmount : count, skip : 0); await parallelizer.Start(); var cts = new CancellationTokenSource(); cts.CancelAfter(10000); await parallelizer.WaitCompletion(cts.Token); var elapsed = parallelizer.Elapsed; await Task.Delay(1000); Assert.Equal(elapsed, parallelizer.Elapsed); }
public async Task Run_Pause_StopwatchStops() { var count = 10; var parallelizer = ParallelizerFactory <int, bool> .Create( type : type, workItems : Enumerable.Range(1, count), workFunction : longTask, degreeOfParallelism : 1, totalAmount : count, skip : 0); await parallelizer.Start(); await Task.Delay(150); await parallelizer.Pause(); var elapsed = parallelizer.Elapsed; await Task.Delay(1000); Assert.Equal(elapsed, parallelizer.Elapsed); await parallelizer.Abort(); }
public async Task <Checker> BuildAsync() { var quickBulletSettings = JsonConvert.DeserializeObject <QuickBulletSettings>(File.ReadAllText(settingsFile)); var loliScriptManager = new LoliScriptManager(); (var configSettings, var blocks) = loliScriptManager.Build(_configFile); if (string.IsNullOrEmpty(configSettings.Name)) { configSettings.Name = Path.GetFileNameWithoutExtension(_configFile); } if (!string.IsNullOrEmpty(configSettings.AdditionalInfo)) { AnsiConsole.MarkupLine($"[grey]CONFIG INFO:[/] {configSettings.AdditionalInfo}"); } if (configSettings.CustomInputs.Any()) { AnsiConsole.Write(new Rule("[darkorange]Custom input[/]").RuleStyle("grey").LeftAligned()); foreach (var customInput in configSettings.CustomInputs) { customInput.Value = AnsiConsole.Ask <string>($"{customInput.Description}:"); } } var botInputs = File.ReadAllLines(_wordlistFile).Where(w => !string.IsNullOrEmpty(w)).Select(w => new BotInput(w)); if (configSettings.InputRules.Any()) { botInputs = botInputs.Where(b => configSettings.InputRules.All(i => _checkInputRuleFunctions[i.Name].Invoke(b, i))); } var useProxy = _proxies.Any(); var proxyHttpClients = _proxies.Any() ? new List <ProxyHttpClient>(_proxies.Select(p => BuildProxy(p, _proxyType)).Select(p => new ProxyHttpClient(new HttpClientHandler() { UseCookies = false, Proxy = p }, p) { Timeout = TimeSpan.FromSeconds(15) })) : new List <ProxyHttpClient>() { new ProxyHttpClient(new HttpClientHandler() { UseCookies = false }, null) { Timeout = TimeSpan.FromSeconds(15) } }; var proxyHttpClientManager = new ProxyHttpClientManager(proxyHttpClients); if (useProxy) { _ = proxyHttpClientManager.StartValidateAllProxiesAsync(); } var record = GetRecord(configSettings.Name); Directory.CreateDirectory(Path.Combine(quickBulletSettings.OutputDirectory, configSettings.Name)); var skip = _skip == -1 ? record.Progress : _skip; var checkerStats = new CheckerStats(skip) { DegreeOfParallelism = _bots }; var statusesToBreak = new string[] { "toCheck", "failure", "retry", "ban", "error" }; var statusesToRecheck = new string[] { "retry", "ban", "error" }; var readerWriterLock = new ReaderWriterLock(); var handler = new HttpClientHandler() { UseCookies = false }; var httpClient = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(15) }; var playwright = await Playwright.CreateAsync(); Func <BotInput, CancellationToken, Task <bool> > check = new(async(input, cancellationToken) => { BotData botData = null; for (var attempts = 0; attempts < 8; attempts++) { var proxyHttpClient = proxyHttpClientManager.GetRandomProxyHttpClient(); botData = new BotData(quickBulletSettings, input, httpClient, proxyHttpClient, playwright) { UseProxy = useProxy }; botData.Variables.Add("data.proxy", proxyHttpClient.Proxy is null ? string.Empty : proxyHttpClient.Proxy.ToString()); foreach (var customInput in configSettings.CustomInputs) { botData.Variables.Add(customInput.Name, customInput.Value); } foreach (var block in blocks) { try { await block.RunAsync(botData); } catch (HttpRequestException) { proxyHttpClient.IsValid = false; botData.Variables["botStatus"] = "retry"; } catch (Exception error) { if (error.Message.Contains("HttpClient.Timeout") || error.Message.Contains("ERR_TIMED_OUT")) { proxyHttpClient.IsValid = false; } else if (_verbose) { AnsiConsole.WriteException(error); } botData.Variables["botStatus"] = "retry"; } if (statusesToBreak.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase)) { break; } } await botData.DisposeAsync(); if (statusesToRecheck.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase)) { if (botData.Variables["botStatus"].Equals("ban", StringComparison.OrdinalIgnoreCase)) { proxyHttpClient.IsValid = false; } checkerStats.Increment(botData.Variables["botStatus"]); } else { break; } } var botStatus = statusesToRecheck.Contains(botData.Variables["botStatus"], StringComparer.OrdinalIgnoreCase) ? "tocheck" : botData.Variables["botStatus"].ToLower(); if (botStatus.Equals("failure")) { checkerStats.Increment(botData.Variables["botStatus"]); } else { var outputPath = Path.Combine(quickBulletSettings.OutputDirectory, configSettings.Name, $"{botStatus}.txt"); var output = botData.Captures.Any() ? new StringBuilder().Append(botData.Input.ToString()).Append(quickBulletSettings.OutputSeparator).AppendJoin(quickBulletSettings.OutputSeparator, botData.Captures.Select(c => $"{c.Key} = {c.Value}")).ToString() : botData.Input.ToString(); try { readerWriterLock.AcquireWriterLock(int.MaxValue); using var streamWriter = File.AppendText(outputPath); await streamWriter.WriteLineAsync(output); } finally { readerWriterLock.ReleaseWriterLock(); } switch (botStatus) { case "success": AnsiConsole.MarkupLine($"[green4]SUCCESS:[/] {output}"); break; case "tocheck": AnsiConsole.MarkupLine($"[cyan3]TOCHECK:[/] {output}"); break; default: AnsiConsole.MarkupLine($"[orange3]{botStatus.ToUpper()}:[/] {output}"); break; } checkerStats.Increment(botStatus); } checkerStats.Increment("checked"); return(true); }); var paradllelizer = ParallelizerFactory <BotInput, bool> .Create(type : ParallelizerType.TaskBased, workItems : botInputs, workFunction : check, degreeOfParallelism : _bots, totalAmount : botInputs.Skip(skip).Count(), skip : skip); return(new Checker(paradllelizer, checkerStats, record)); }