public async Task ParallelTestAsync(int degree)
{
var tasks = Enumerable.Range(0, 2000).Select(i => 2).ToList();
tasks.Add(200);
tasks.Add(100);
tasks.Add(300);
var cts = new CancellationTokenSource();
int count = 0;
int max = 0;
var option = new ParallelizeOption
{
FailMode = Fail.Smart,
MaxDegreeOfParallelism = degree
};
await tasks.ParallelizeAsync(async (i, ct) =>
{
int loopMax = Interlocked.Increment(ref count);
max = Math.Max(max, loopMax);
await Task.Delay(i, ct);
Interlocked.Decrement(ref count);
Assert.True(count <= degree);
return(true);
}, option, cts.Token);
Assert.Equal(degree, max);
}
public async Task ExceptionAsync(Fail failMode)
{
var tasks = Enumerable.Range(0, 10).ToList();
int index = 0;
bool badBehavior = false;
var option = new ParallelizeOption
{
FailMode = failMode,
MaxDegreeOfParallelism = 8
};
var pTask = tasks.ParallelizeAsync(async(i, ct) =>
{
int ix = Interlocked.Increment(ref index);
if (ix == 4)
{
throw new TestException();
}
await Task.Delay(1000, ct);
if (ct.IsCancellationRequested)
{
return(false);
}
if (failMode == Fail.Fast)
{
badBehavior = true;
}
return(true);
}, option, CancellationToken.None);
if (failMode == Fail.Default)
{
await Assert.ThrowsAsync <TestException>(async() =>
{
await pTask;
});
Assert.Equal(10, index);
}
else
{
ParallelizedSummary result = await pTask.GetExceptionsAsync();
Assert.False(result.IsSuccess);
Assert.False(result.IsCanceled);
Assert.NotEmpty(result.Exceptions);
}
if (badBehavior)
{
Assert.True(false);
}
}
public async Task <OperationSummary> Compress(CompressOption option)
{
IEnumerable <Block> blocks = BlockHelper.PreprareCompressBlocks(option);
var sw = Stopwatch.StartNew();
var po = new ParallelizeOption
{
FailMode = option.FailFast ? Fail.Fast : Fail.Smart,
MaxDegreeOfParallelism = option.Threads
};
var operationBlocks = await blocks
.AsAsyncStream(_cancellationTokenSource.Token)
.CountAsync(out var counter)
// Prepare encyption
.ParallelizeStreamAsync(async(b, token) =>
{
await _encryptionCommands.PrepareEncryptionKey(b, option, token);
return(b);
}, po)
// Core loop
.ParallelizeStreamAsync(async(block, token) =>
{
_logger.LogInformation($"Starting {block.Source}");
CommandResult result = null;
try
{
string cmd = _compressionCommands.CompressCommand(block, option);
result = await cmd.Run(block.OperationFolder, token);
_logger.LogInformation($"Finished {block.Source} on {result?.ElapsedMilliseconds} ms");
}
catch (Exception e)
{
_logger.LogError(e, $"Error on {block.Source}");
}
return(new OperationBlock(block, result));
}, po)
// Cleanup loop
.ParallelizeStreamAsync(async(opb, token) =>
{
await _encryptionCommands.CleanupKey(opb.Block, option, opb.CommandResult, Mode.Compress);
return(opb);
}, po)
.ForEachAsync((i, ct) =>
{
_blockListener.OnBlockReport(new BlockReport(i.Item.CommandResult, i.Item.Block, counter.Count));
return(Task.CompletedTask);
})
.AsEnumerableAsync();
sw.Stop();
return(new OperationSummary(operationBlocks, option.Threads, sw));
}
public static async Task RestoreAsync(RestoreCommand restoreCommand, ILogger logger, CancellationToken cancellationToken)
{
await using var connection = restoreCommand.CreateConnectionMars();
var state = new RestoreState(logger, restoreCommand, connection);
state.LogStarting();
await state.EnsuredIndexsExistsAsync();
await Task.WhenAll(
state.LoadServerInfosAsync(),
state.LoadDatabasesAsync(),
state.LoadRestoreHistoryAsync(),
state.PrepareRestoreJobAsync());
logger.Information($"{state.Directories.Count} directories found");
logger.Information("Starting...");
var po = new ParallelizeOption
{
FailMode = Fail.Smart,
MaxDegreeOfParallelism = restoreCommand.Threads
};
IRestoreMethod method = restoreCommand.Brentozar
? (IRestoreMethod) new BrentozarRestoreMethod(state, connection)
: new NativeRestoreMethod(state);
await state.Directories
.ParallelizeAsync((item, cancel) => RestoreBackupAsync(method, item, state), po, cancellationToken)
.ForEachAsync((value, token) =>
{
state.Accumulated = state.Accumulated.Add(value.Item.Elapsed);
return(Task.CompletedTask);
});
var report = await state.GetReportStateAsync();
state.LogFinished(report);
var slack = new SlackSender(new SlackClient(logger));
await slack.ReportAsync(report, state.RestoreCommand.SlackChannel, state.RestoreCommand.SlackSecret, state.RestoreCommand.SlackOnlyOnError, state.RestoreCommand.SlackTitle);
await report.SendMailAsync(state.RestoreCommand.Email, state.RestoreCommand.Smtp, cancellationToken);
if (report.Status.HasFlag(ReportStatus.Error))
{
Environment.ExitCode = 1;
}
}
public async Task CancellationAsync(Fail failMode)
{
var tasks = Enumerable.Range(0, 10).ToList();
var cts = new CancellationTokenSource();
int index = 0;
var option = new ParallelizeOption
{
FailMode = failMode,
MaxDegreeOfParallelism = 8
};
var pTask = tasks.ParallelizeAsync(async(i, ct) =>
{
int ix = Interlocked.Increment(ref index);
if (ix == 4)
{
cts.Cancel();
}
else
{
await Task.Delay(1000, ct);
if (ct.IsCancellationRequested)
{
return(false);
}
Assert.False(true);
}
return(true);
}, option, cts.Token);
if (failMode == Fail.Default)
{
await Assert.ThrowsAsync <TaskCanceledException>(async() =>
{
await pTask;
});
}
else
{
var result = await pTask.GetExceptionsAsync();
Assert.True(result.IsCanceled);
}
}
private static void ThrowOnErrors(ParallelizeOption option, ParallelizeCore core)
{
if (option.FailMode != Fail.Default)
{
return;
}
if (core.IsFaulted)
{
if (core.Exceptions.Count() == 1)
{
throw core.Exceptions.First();
}
throw new AggregateException(core.Exceptions);
}
if (core.IsCanceled)
{
throw new TaskCanceledException();
}
}
internal static async Task DropDatabasesAsync(DropDatabaseCommand drop, ILogger logger, CancellationToken token)
{
await using var connection = drop.CreateConnectionMars();
var serverInfos = await connection.GetServerInfosAsync();
var databases = (await connection.GetUserDatabasesAsync()).ToList();
if (drop.DuplicatesIgnored != null && drop.DuplicatesIgnored.Any())
{
var exclude = drop.DuplicatesIgnored.ToHashSet(StringComparer.InvariantCultureIgnoreCase);
databases = databases.Where(d => !exclude.Contains(d.Name)).ToList();
}
if (databases.Count == 0)
{
logger.Information("Nothing to drop");
return;
}
if (!drop.Force)
{
await Console.Out.WriteLineAsync($"Are you sure to drop all {databases.Count} databases on server {serverInfos.ServerName}? Please type the server name");
var read = await Console.In.ReadLineAsync();
if (!string.Equals(read, serverInfos.ServerName, StringComparison.OrdinalIgnoreCase))
{
logger.Error("Drop operation aborded, bad confimation input");
return;
}
else
{
logger.Information("Drop confirmation for " + read);
}
}
else
{
logger.Warning("Drop mode forced ");
}
var sw = Stopwatch.StartNew();
var po = new ParallelizeOption
{
FailMode = Fail.Smart,
MaxDegreeOfParallelism = drop.Threads
};
int counter = 0;
int total = databases.Count;
var state = databases.ToDictionary(i => i.Name);
await databases
.ParallelizeAsync(async (item, cancel) =>
{
int i = Interlocked.Increment(ref counter);
try
{
if (state.TryGetValue(item.Name, out var databaseInfo) && databaseInfo.State != DatabaseState.RESTORING)
{
await connection.ExecuteAsync($"ALTER DATABASE [{item.Name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", commandTimeout: _timeout);
}
await connection.ExecuteAsync($"DROP DATABASE [{item.Name}]", commandTimeout: _timeout);
await connection.ExecuteAsync($"EXEC msdb.dbo.sp_delete_database_backuphistory @database_name = N'{item.Name}'", commandTimeout: _timeout);
}
catch (Exception e)
{
logger.Error(e, $"[{i}/{total}]Error while dropping " + item.Name);
return(false);
}
logger.Information($"[{i}/{total}] : {item.Name} dropped successfully");
return(true);
}, po, token);
sw.Stop();
logger.Information("Drop finished in " + sw.Elapsed.HumanizedTimeSpan());
}
private static AsyncStream <TResult> ParallelizeStreamInternalAsync <T, TResult>(this AsyncStream <T> source,
Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option)
{
var core = new ParallelizeCore(source.CancellationToken, option);
var monitor = new ParallelMonitor <T>(option.MaxDegreeOfParallelism);
var channel = Channel.CreateUnbounded <StreamedValue <TResult> >();
var task = Task.Run(async() =>
{
try
{
using (core)
{
var parallelTasks =
Enumerable.Range(0, option.MaxDegreeOfParallelism)
.Select(i => ParallelizeCoreStreamAsync(core, actionAsync, source, channel, i, monitor))
.ToArray();
await Task.WhenAll(parallelTasks);
}
}
catch (Exception e)
{
channel.Writer.Complete(e);
throw;
}
channel.Writer.Complete();
ThrowOnErrors(option, core);
});
return(new AsyncStream <TResult>(channel, task, source.CancellationToken));
}
public static AsyncStream <TResult> ParallelizeStreamAsync <T, TResult>(this AsyncStream <T> source,
Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option)
{
if (actionAsync == null)
{
throw new ArgumentNullException(nameof(actionAsync));
}
return(ParallelizeStreamInternalAsync(source, actionAsync, option));
}
public static AsyncStream <TResult> ParallelizeAsync <T, TResult>(this IEnumerable <T> source,
Func <T, CancellationToken, Task <TResult> > actionAsync, ParallelizeOption option, CancellationToken cancellationToken)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (actionAsync == null)
{
throw new ArgumentNullException(nameof(actionAsync));
}
return(source.AsAsyncStream(cancellationToken).ParallelizeStreamAsync(actionAsync, option));
}
