public async Task ParallelTestAsync(int degree) { var tasks = Enumerable.Range(0, 2000).Select(i => 2).ToList(); tasks.Add(200); tasks.Add(100); tasks.Add(300); var cts = new CancellationTokenSource(); int count = 0; int max = 0; var option = new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = degree }; await tasks.ParallelizeAsync(async (i, ct) => { int loopMax = Interlocked.Increment(ref count); max = Math.Max(max, loopMax); await Task.Delay(i, ct); Interlocked.Decrement(ref count); Assert.True(count <= degree); return(true); }, option, cts.Token); Assert.Equal(degree, max); }
public async Task ExceptionAsync(Fail failMode) { var tasks = Enumerable.Range(0, 10).ToList(); int index = 0; bool badBehavior = false; var option = new ParallelizeOption { FailMode = failMode, MaxDegreeOfParallelism = 8 }; var pTask = tasks.ParallelizeAsync(async(i, ct) => { int ix = Interlocked.Increment(ref index); if (ix == 4) { throw new TestException(); } await Task.Delay(1000, ct); if (ct.IsCancellationRequested) { return(false); } if (failMode == Fail.Fast) { badBehavior = true; } return(true); }, option, CancellationToken.None); if (failMode == Fail.Default) { await Assert.ThrowsAsync <TestException>(async() => { await pTask; }); Assert.Equal(10, index); } else { ParallelizedSummary result = await pTask.GetExceptionsAsync(); Assert.False(result.IsSuccess); Assert.False(result.IsCanceled); Assert.NotEmpty(result.Exceptions); } if (badBehavior) { Assert.True(false); } }
public async Task <OperationSummary> Compress(CompressOption option) { IEnumerable <Block> blocks = BlockHelper.PreprareCompressBlocks(option); var sw = Stopwatch.StartNew(); var po = new ParallelizeOption { FailMode = option.FailFast ? Fail.Fast : Fail.Smart, MaxDegreeOfParallelism = option.Threads }; var operationBlocks = await blocks .AsAsyncStream(_cancellationTokenSource.Token) .CountAsync(out var counter) // Prepare encyption .ParallelizeStreamAsync(async(b, token) => { await _encryptionCommands.PrepareEncryptionKey(b, option, token); return(b); }, po) // Core loop .ParallelizeStreamAsync(async(block, token) => { _logger.LogInformation($"Starting {block.Source}"); CommandResult result = null; try { string cmd = _compressionCommands.CompressCommand(block, option); result = await cmd.Run(block.OperationFolder, token); _logger.LogInformation($"Finished {block.Source} on {result?.ElapsedMilliseconds} ms"); } catch (Exception e) { _logger.LogError(e, $"Error on {block.Source}"); } return(new OperationBlock(block, result)); }, po) // Cleanup loop .ParallelizeStreamAsync(async(opb, token) => { await _encryptionCommands.CleanupKey(opb.Block, option, opb.CommandResult, Mode.Compress); return(opb); }, po) .ForEachAsync((i, ct) => { _blockListener.OnBlockReport(new BlockReport(i.Item.CommandResult, i.Item.Block, counter.Count)); return(Task.CompletedTask); }) .AsEnumerableAsync(); sw.Stop(); return(new OperationSummary(operationBlocks, option.Threads, sw)); }
public static async Task RestoreAsync(RestoreCommand restoreCommand, ILogger logger, CancellationToken cancellationToken) { await using var connection = restoreCommand.CreateConnectionMars(); var state = new RestoreState(logger, restoreCommand, connection); state.LogStarting(); await state.EnsuredIndexsExistsAsync(); await Task.WhenAll( state.LoadServerInfosAsync(), state.LoadDatabasesAsync(), state.LoadRestoreHistoryAsync(), state.PrepareRestoreJobAsync()); logger.Information($"{state.Directories.Count} directories found"); logger.Information("Starting..."); var po = new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = restoreCommand.Threads }; IRestoreMethod method = restoreCommand.Brentozar ? (IRestoreMethod) new BrentozarRestoreMethod(state, connection) : new NativeRestoreMethod(state); await state.Directories .ParallelizeAsync((item, cancel) => RestoreBackupAsync(method, item, state), po, cancellationToken) .ForEachAsync((value, token) => { state.Accumulated = state.Accumulated.Add(value.Item.Elapsed); return(Task.CompletedTask); }); var report = await state.GetReportStateAsync(); state.LogFinished(report); var slack = new SlackSender(new SlackClient(logger)); await slack.ReportAsync(report, state.RestoreCommand.SlackChannel, state.RestoreCommand.SlackSecret, state.RestoreCommand.SlackOnlyOnError, state.RestoreCommand.SlackTitle); await report.SendMailAsync(state.RestoreCommand.Email, state.RestoreCommand.Smtp, cancellationToken); if (report.Status.HasFlag(ReportStatus.Error)) { Environment.ExitCode = 1; } }
public async Task CancellationAsync(Fail failMode) { var tasks = Enumerable.Range(0, 10).ToList(); var cts = new CancellationTokenSource(); int index = 0; var option = new ParallelizeOption { FailMode = failMode, MaxDegreeOfParallelism = 8 }; var pTask = tasks.ParallelizeAsync(async(i, ct) => { int ix = Interlocked.Increment(ref index); if (ix == 4) { cts.Cancel(); } else { await Task.Delay(1000, ct); if (ct.IsCancellationRequested) { return(false); } Assert.False(true); } return(true); }, option, cts.Token); if (failMode == Fail.Default) { await Assert.ThrowsAsync <TaskCanceledException>(async() => { await pTask; }); } else { var result = await pTask.GetExceptionsAsync(); Assert.True(result.IsCanceled); } }
private static void ThrowOnErrors(ParallelizeOption option, ParallelizeCore core) { if (option.FailMode != Fail.Default) { return; } if (core.IsFaulted) { if (core.Exceptions.Count() == 1) { throw core.Exceptions.First(); } throw new AggregateException(core.Exceptions); } if (core.IsCanceled) { throw new TaskCanceledException(); } }
internal static async Task DropDatabasesAsync(DropDatabaseCommand drop, ILogger logger, CancellationToken token) { await using var connection = drop.CreateConnectionMars(); var serverInfos = await connection.GetServerInfosAsync(); var databases = (await connection.GetUserDatabasesAsync()).ToList(); if (drop.DuplicatesIgnored != null && drop.DuplicatesIgnored.Any()) { var exclude = drop.DuplicatesIgnored.ToHashSet(StringComparer.InvariantCultureIgnoreCase); databases = databases.Where(d => !exclude.Contains(d.Name)).ToList(); } if (databases.Count == 0) { logger.Information("Nothing to drop"); return; } if (!drop.Force) { await Console.Out.WriteLineAsync($"Are you sure to drop all {databases.Count} databases on server {serverInfos.ServerName}? Please type the server name"); var read = await Console.In.ReadLineAsync(); if (!string.Equals(read, serverInfos.ServerName, StringComparison.OrdinalIgnoreCase)) { logger.Error("Drop operation aborded, bad confimation input"); return; } else { logger.Information("Drop confirmation for " + read); } } else { logger.Warning("Drop mode forced "); } var sw = Stopwatch.StartNew(); var po = new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = drop.Threads }; int counter = 0; int total = databases.Count; var state = databases.ToDictionary(i => i.Name); await databases .ParallelizeAsync(async (item, cancel) => { int i = Interlocked.Increment(ref counter); try { if (state.TryGetValue(item.Name, out var databaseInfo) && databaseInfo.State != DatabaseState.RESTORING) { await connection.ExecuteAsync($"ALTER DATABASE [{item.Name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", commandTimeout: _timeout); } await connection.ExecuteAsync($"DROP DATABASE [{item.Name}]", commandTimeout: _timeout); await connection.ExecuteAsync($"EXEC msdb.dbo.sp_delete_database_backuphistory @database_name = N'{item.Name}'", commandTimeout: _timeout); } catch (Exception e) { logger.Error(e, $"[{i}/{total}]Error while dropping " + item.Name); return(false); } logger.Information($"[{i}/{total}] : {item.Name} dropped successfully"); return(true); }, po, token); sw.Stop(); logger.Information("Drop finished in " + sw.Elapsed.HumanizedTimeSpan()); }
private static AsyncStream <TResult> ParallelizeStreamInternalAsync <T, TResult>(this AsyncStream <T> source, Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option) { var core = new ParallelizeCore(source.CancellationToken, option); var monitor = new ParallelMonitor <T>(option.MaxDegreeOfParallelism); var channel = Channel.CreateUnbounded <StreamedValue <TResult> >(); var task = Task.Run(async() => { try { using (core) { var parallelTasks = Enumerable.Range(0, option.MaxDegreeOfParallelism) .Select(i => ParallelizeCoreStreamAsync(core, actionAsync, source, channel, i, monitor)) .ToArray(); await Task.WhenAll(parallelTasks); } } catch (Exception e) { channel.Writer.Complete(e); throw; } channel.Writer.Complete(); ThrowOnErrors(option, core); }); return(new AsyncStream <TResult>(channel, task, source.CancellationToken)); }
public static AsyncStream <TResult> ParallelizeStreamAsync <T, TResult>(this AsyncStream <T> source, Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option) { if (actionAsync == null) { throw new ArgumentNullException(nameof(actionAsync)); } return(ParallelizeStreamInternalAsync(source, actionAsync, option)); }
public static AsyncStream <TResult> ParallelizeAsync <T, TResult>(this IEnumerable <T> source, Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option, CancellationToken cancellationToken) { if (source == null) { throw new ArgumentNullException(nameof(source)); } if (actionAsync == null) { throw new ArgumentNullException(nameof(actionAsync)); } return(source.AsAsyncStream(cancellationToken).ParallelizeStreamAsync(actionAsync, option)); }