public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); var tasksDeploymentStatus = new Dictionary <string, bool>(); for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; tasksDeploymentStatus.Add(tasksIds[i], false); } bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 && tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0) { isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[i]] = true; } } } for (int i = 0; i < tasks.Length; i++) { isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology); } return(isSuccessful); }
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; if (tasks[i].OperationType == OperatorType.Produce) { isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]); } else if (tasks[i].OperationType == OperatorType.Subscribe) { isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology); } else if (tasks[i].OperationType == OperatorType.Move) { isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology); } if (!isSuccessful) { break; } } return(isSuccessful); }