public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
var tasksDeploymentStatus = new Dictionary <string, bool>();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
tasksDeploymentStatus.Add(tasksIds[i], false);
}
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 &&
tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0)
{
isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[i]] = true;
}
}
}
for (int i = 0; i < tasks.Length; i++)
{
isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
return(isSuccessful);
}
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
if (tasks[i].OperationType == OperatorType.Produce)
{
isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]);
}
else if (tasks[i].OperationType == OperatorType.Subscribe)
{
isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology);
}
else if (tasks[i].OperationType == OperatorType.Move)
{
isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology);
}
if (!isSuccessful)
{
break;
}
}
return(isSuccessful);
}