private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap( CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology) { ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >(); for (int i = 0; i < tasks.Length; i++) { int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap()); tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), false); verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } foreach (string secondaryFromInputId in tasks[i].EndpointsDescriptor.SecondaryFromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, secondaryFromInputId, topology.OperatorsEndpointsDescriptors[secondaryFromInputId], tasksDictionary[secondaryFromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), true); verticesConnectionsMap.AddOrUpdate(secondaryFromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } } return(verticesConnectionsMap); }
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task, ref DetachedVertex clientTerminal, OperatorsToplogy topology) { try { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor); string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator( task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]); int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap()); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < inputEndpoints.Length; j++) { clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]); } } } return(true); } catch (Exception e) { Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString()); return(false); } }
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); var tasksDeploymentStatus = new Dictionary <string, bool>(); for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; tasksDeploymentStatus.Add(tasksIds[i], false); } bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 && tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0) { isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[i]] = true; } } } for (int i = 0; i < tasks.Length; i++) { isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology); } return(isSuccessful); }
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; if (tasks[i].OperationType == OperatorType.Produce) { isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]); } else if (tasks[i].OperationType == OperatorType.Subscribe) { isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology); } else if (tasks[i].OperationType == OperatorType.Move) { isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology); } if (!isSuccessful) { break; } } return(isSuccessful); }
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap( CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology) { ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >(); for (int i = 0; i < tasks.Length; i++) { int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap()); tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; if (tasks[i].OperationType == OperatorType.Move) { var shuffleTask = (ShuffleTask)tasks[i]; foreach (string fromInputId in shuffleTask.SecondaryEndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(fromInputId + shuffleTask.OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } var fromInput = shuffleTask.EndpointsDescriptor.FromInputs.First().Key; var newFromInputId = fromInput.Substring(0, fromInput.Length - 1); var shuffleFromToConnections = PrepareShuffleConnectionsMap(client, shardsCount, newFromInputId, topology.OperatorsEndpointsDescriptors[newFromInputId], tasksDictionary[newFromInputId].DeployDescriptor.InstancesMap(), shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(newFromInputId + shuffleTask.ReducerVertexName, shuffleFromToConnections, (key, value) => shuffleFromToConnections); } else { foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } } } return(verticesConnectionsMap); }