private void SequentialExecutionStrategy(ReplicateInBulkCommand command, IReadOnlyDictionary <TableName, Type[]> tableTypesDictionary)
{
ExecuteInTransactionScope(
command,
(targetConnection, schemaManagenentActor) =>
{
var schemaChangedEvents = schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCommands(command.DbManagementMode));
using (var sourceConnection = CreateTransactionlessDataConnection(command.SourceStorageDescriptor))
{
foreach (var tableTypesPair in tableTypesDictionary)
{
try
{
var replicationCommands = CreateReplicationCommands(tableTypesPair.Key, command.BulkCopyTimeout, command.DbManagementMode);
_bulkReplicator.Replicate(tableTypesPair.Value, sourceConnection, targetConnection, replicationCommands);
}
catch (Exception ex)
{
throw new Exception($"Failed to replicate using sequential strategy {tableTypesPair.Key}", ex);
}
}
}
schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCompensationalCommands(schemaChangedEvents));
});
}
private void ShadowParallelExecutionStrategy(ReplicateInBulkCommand command, IReadOnlyDictionary <TableName, Type[]> tableTypesDictionary)
{
Parallel.ForEach(
tableTypesDictionary,
new ParallelOptions {
MaxDegreeOfParallelism = 4
},
tableTypesPair =>
{
using (var connection = CreateDataConnection(command.TargetStorageDescriptor))
{
try
{
var replicationCommands = CreateShadowReplicationCommands(tableTypesPair.Key, command.BulkCopyTimeout, command.DbManagementMode);
ReplaceInBulk(tableTypesPair.Value, command.SourceStorageDescriptor, connection, replicationCommands);
}
catch (Exception ex)
{
throw new Exception($"Failed to replicate using shadow parallel strategy {tableTypesPair.Key}", ex);
}
}
});
ExecuteInTransactionScope(
command,
(targetConnection, schemaManagenentActor) =>
{
var schemaChangedEvents = schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCommands(command.DbManagementMode));
// Delete existed tables then rename newly created ones (remove prefix):
schemaManagenentActor.ExecuteCommands(CreateTablesReplacingCommands(tableTypesDictionary.Keys));
schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCompensationalCommands(schemaChangedEvents));
});
}
public IDataObjectTypesProvider Create(ReplicateInBulkCommand command)
{
if (command.TargetStorageDescriptor.ConnectionStringIdentity is FactsConnectionStringIdentity)
{
if (command.SourceStorageDescriptor.ConnectionStringIdentity is AmsConnectionStringIdentity)
{
return(new DataObjectTypesProvider(AmsFactTypes));
}
else if (command.SourceStorageDescriptor.ConnectionStringIdentity is RulesetConnectionStringIdentity)
{
return(new DataObjectTypesProvider(RulesetFactTypes));
}
return(new CommandRegardlessDataObjectTypesProvider(ErmFactTypes));
}
if (command.TargetStorageDescriptor.ConnectionStringIdentity is AggregatesConnectionStringIdentity)
{
return(new CommandRegardlessDataObjectTypesProvider(AggregateTypes));
}
if (command.TargetStorageDescriptor.ConnectionStringIdentity is MessagesConnectionStringIdentity)
{
return(new CommandRegardlessDataObjectTypesProvider(MessagesTypes));
}
throw new ArgumentException($"Instance of type IDataObjectTypesProvider cannot be created for connection string name {command.TargetStorageDescriptor.MappingSchema}");
}
private void Replicate(ReplicateInBulkCommand command, TableName tableName, IReadOnlyCollection <Type> types)
{
using var targetConnection = CreateDataConnection(command.TargetStorageDescriptor);
using var sourceConnection = CreateSourceDataConnection(command.SourceStorageDescriptor);
try
{
var replicationCommands = CreateReplicationCommands(
tableName,
command.BulkCopyTimeout,
command.DbManagementMode,
command.SourceStorageDescriptor.Tenant);
Console.WriteLine($"Replicating {tableName}");
_bulkReplicator.Replicate(
types,
sourceConnection,
targetConnection,
replicationCommands,
command.SourceStorageDescriptor.Tenant);
}
catch (Exception ex)
{
throw new Exception($"Failed to replicate using parallel strategy {tableName}", ex);
}
}
private void ParallelExecutionStrategy(ReplicateInBulkCommand command)
{
var tableTypesDictionary = command.TypesToReplicate
.ToLookup(x => command.TargetStorageDescriptor.MappingSchema.GetTableName(x));
IReadOnlyCollection <IEvent> schemaChangedEvents = null;
ExecuteInTransactionScope(
command.TargetStorageDescriptor,
(targetConnection, schemaManagementActor) =>
{
schemaChangedEvents =
schemaManagementActor.ExecuteCommands(CreateSchemaChangesCommands(command.DbManagementMode));
});
Parallel.ForEach(
tableTypesDictionary,
command.ExecutionMode.ParallelOptions,
tableTypesPair => Replicate(command, tableTypesPair.Key, tableTypesPair.ToList()));
ExecuteInTransactionScope(
command.TargetStorageDescriptor,
(targetConnection, schemaManagenentActor) =>
{
schemaManagenentActor.ExecuteCommands(
CreateSchemaChangesCompensationalCommands(schemaChangedEvents));
});
}
private void ExecuteInTransactionScope(ReplicateInBulkCommand command, Action <DataConnection, SequentialPipelineActor> action)
{
using (var transation = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionOptions))
{
using (var targetConnection = CreateDataConnection(command.TargetStorageDescriptor))
{
var schemaManagenentActor = CreateDbSchemaManagementActor((SqlConnection)targetConnection.Connection, command.TargetStorageDescriptor.CommandTimeout);
action(targetConnection, schemaManagenentActor);
transation.Complete();
}
}
}
private void ParallelExecutionStrategy(ReplicateInBulkCommand command, IReadOnlyDictionary <TableName, Type[]> tableTypesDictionary, ParallelOptions options)
{
IReadOnlyCollection <IEvent> schemaChangedEvents = null;
ExecuteInTransactionScope(
command,
(targetConnection, schemaManagenentActor) =>
{
schemaChangedEvents = schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCommands(command.DbManagementMode));
});
Parallel.ForEach(
tableTypesDictionary,
options,
tableTypesPair =>
{
using (var targetConnection = CreateDataConnection(command.TargetStorageDescriptor))
using (var sourceConnection = CreateTransactionlessDataConnection(command.SourceStorageDescriptor))
{
try
{
var replicationCommands = CreateReplicationCommands(tableTypesPair.Key, command.BulkCopyTimeout, command.DbManagementMode);
_bulkReplicator.Replicate(tableTypesPair.Value, sourceConnection, targetConnection, replicationCommands);
}
catch (Exception ex)
{
throw new Exception($"Failed to replicate using parallel strategy {tableTypesPair.Key}", ex);
}
}
});
ExecuteInTransactionScope(
command,
(targetConnection, schemaManagenentActor) =>
{
schemaManagenentActor.ExecuteCommands(CreateSchemaChangesCompensationalCommands(schemaChangedEvents));
});
}
private IReadOnlyCollection <Type> GetDataObjectTypes(ReplicateInBulkCommand command) => ((ICommandRegardlessDataObjectTypesProvider)_dataObjectTypesProviderFactory.Create(command)).Get();
private IReadOnlyCollection <CreateTableCopyCommand> CreateCopyTableCommands(ReplicateInBulkCommand command) => GetDataObjectTypes(command) .Select(t => command.TargetStorageDescriptor.MappingSchema.GetTableName(t)) .Distinct(TableNameComparer.Instance) .Select(CreateCopyCommand) .ToArray();
public KafkaReplicationCommand(IMessageFlow messageFlow, ReplicateInBulkCommand replicateInBulkCommand, int batchSize = 5000)
{
MessageFlow = messageFlow;
ReplicateInBulkCommand = replicateInBulkCommand;
BatchSize = batchSize;
}
private Action <ReplicateInBulkCommand, IReadOnlyDictionary <TableName, Type[]> > DetermineExecutionStrategy(ReplicateInBulkCommand command)
{
switch (command.ExecutionMode)
{
case ExecutionMode.Parallel:
return(ParallelExecutionStrategy);
case ExecutionMode.Sequential:
return(SequentialExecutionStrategy);
case ExecutionMode.ShadowParallel:
return(ShadowParallelExecutionStrategy);
default:
throw new ArgumentException($"Execution mode {command.ExecutionMode} is not supported", nameof(command));
}
}
private IReadOnlyCollection <Type> GetDataObjectTypes(ReplicateInBulkCommand command)
{
var dataObjectTypesProvider = (DataObjectTypesProviderFactory.DataObjectTypesProvider)_dataObjectTypesProviderFactory.Create(command);
return(dataObjectTypesProvider.DataObjectTypes);
}
private Action <ReplicateInBulkCommand, IReadOnlyDictionary <TableName, Type[]> > DetermineExecutionStrategy(ReplicateInBulkCommand command)
{
if (command.ExecutionMode == ExecutionMode.Sequential)
{
return(SequentialExecutionStrategy);
}
if (command.ExecutionMode.Shadow)
{
return((cmd, types) => ShadowParallelExecutionStrategy(cmd, types, command.ExecutionMode.ParallelOptions));
}
return((cmd, types) => ParallelExecutionStrategy(cmd, types, command.ExecutionMode.ParallelOptions));
}
public KafkaReplicationCommand(IMessageFlow messageFlow, ReplicateInBulkCommand replicateInBulkCommand)
{
MessageFlow = messageFlow;
ReplicateInBulkCommand = replicateInBulkCommand;
}
private void ExecuteCommand(ReplicateInBulkCommand command)
{
ParallelExecutionStrategy(command);
}
