private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap(
CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology)
{
ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >();
for (int i = 0; i < tasks.Length; i++)
{
int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap());
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), false);
verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
foreach (string secondaryFromInputId in tasks[i].EndpointsDescriptor.SecondaryFromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
secondaryFromInputId, topology.OperatorsEndpointsDescriptors[secondaryFromInputId], tasksDictionary[secondaryFromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), true);
verticesConnectionsMap.AddOrUpdate(secondaryFromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
}
return(verticesConnectionsMap);
}
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task,
ref DetachedVertex clientTerminal, OperatorsToplogy topology)
{
try
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor);
string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]);
int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap());
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < inputEndpoints.Length; j++)
{
clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
var tasksDeploymentStatus = new Dictionary <string, bool>();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
tasksDeploymentStatus.Add(tasksIds[i], false);
}
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 &&
tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0)
{
isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[i]] = true;
}
}
}
for (int i = 0; i < tasks.Length; i++)
{
isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
return(isSuccessful);
}
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
if (tasks[i].OperationType == OperatorType.Produce)
{
isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]);
}
else if (tasks[i].OperationType == OperatorType.Subscribe)
{
isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology);
}
else if (tasks[i].OperationType == OperatorType.Move)
{
isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology);
}
if (!isSuccessful)
{
break;
}
}
return(isSuccessful);
}
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap(
CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology)
{
ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >();
for (int i = 0; i < tasks.Length; i++)
{
int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap());
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
if (tasks[i].OperationType == OperatorType.Move)
{
var shuffleTask = (ShuffleTask)tasks[i];
foreach (string fromInputId in shuffleTask.SecondaryEndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(fromInputId + shuffleTask.OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
var fromInput = shuffleTask.EndpointsDescriptor.FromInputs.First().Key;
var newFromInputId = fromInput.Substring(0, fromInput.Length - 1);
var shuffleFromToConnections = PrepareShuffleConnectionsMap(client, shardsCount,
newFromInputId, topology.OperatorsEndpointsDescriptors[newFromInputId], tasksDictionary[newFromInputId].DeployDescriptor.InstancesMap(),
shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(newFromInputId + shuffleTask.ReducerVertexName,
shuffleFromToConnections, (key, value) => shuffleFromToConnections);
}
else
{
foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
}
}
return(verticesConnectionsMap);
}