private static bool DeployShuffleReduceTask(CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology)
{
try
{
client.DefineVertex(typeof(ShuffleOperator).Name.ToLower(), () => new ShuffleOperator());
CRAErrorCode status = client.InstantiateShardedVertex(task.ReducerVertexName, typeof(ShuffleOperator).Name.ToLower(),
task, task.DeployDescriptor.InstancesMap());
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.SecondaryEndpointsDescriptor.FromInputs.Keys)
{
client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]);
}
var fromInput = task.EndpointsDescriptor.FromInputs.First().Key;
var newFromInputId = fromInput.Substring(0, fromInput.Length - 1);
client.ConnectShardedVertices(task.VerticesConnectionsMap[newFromInputId + task.ReducerVertexName]);
return(true);
}
else
{
return(false);
}
}
catch (Exception)
{
Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations");
return(false);
}
}
internal void UpdateShuffleOperatorTask(string taskId, ShuffleTask task)
{
if (_operatorsIds.Contains(taskId))
{
int index = _operatorsIds.IndexOf(taskId);
_operatorsTasks[index] = task;
}
}
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
var tasksDeploymentStatus = new Dictionary <string, bool>();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
tasksDeploymentStatus.Add(tasksIds[i], false);
}
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 &&
tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0)
{
isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[i]] = true;
}
}
}
for (int i = 0; i < tasks.Length; i++)
{
isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
return(isSuccessful);
}
private object ApplyMergerOnInputs(ShuffleTask shuffleTask, BinaryOperatorTypes mergeTypes, object[] inputSplitDatasets)
{
MethodInfo method = typeof(MoveUtils).GetMethod("ApplyMerger");
MethodInfo generic = method.MakeGenericMethod(
new Type[] { mergeTypes.SecondaryKeyType, mergeTypes.SecondaryPayloadType,
mergeTypes.SecondaryDatasetType, mergeTypes.OutputKeyType,
mergeTypes.OutputPayloadType, mergeTypes.OutputDatasetType });
object[] arguments = new Object[] { inputSplitDatasets, shuffleTask.ShuffleDescriptor, shuffleTask.ShuffleTransforms[1] };
return(generic.Invoke(this, arguments));
}
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
if (tasks[i].OperationType == OperatorType.Produce)
{
isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]);
}
else if (tasks[i].OperationType == OperatorType.Subscribe)
{
isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology);
}
else if (tasks[i].OperationType == OperatorType.Move)
{
isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology);
}
if (!isSuccessful)
{
break;
}
}
return(isSuccessful);
}
internal void AddShuffleOperator(string mapperTaskId, string reducerTaskId, ShuffleTask task)
{
_operatorsEndpointsDescriptors.AddOrUpdate(mapperTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor());
_operatorsEndpointsDescriptors.AddOrUpdate(reducerTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor());
if (_operatorsIds.Contains(reducerTaskId))
{
int index = _operatorsIds.IndexOf(reducerTaskId);
_operatorsTasks[index] = task;
}
else
{
_operatorsIds.Add(reducerTaskId);
_operatorsTasks.Add(task);
}
}
private static async Task <bool> DeployShuffleReduceTask(
CRAClientLibrary client,
ShuffleTask task,
OperatorsToplogy topology)
{
try
{
if (!_isShuffleOperatorDefined)
{
await client.DefineVertexAsync(typeof(ShardedShuffleOperator).Name.ToLower(), () => new ShardedShuffleOperator());
_isShuffleOperatorDefined = true;
}
var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.ReducerVertexName, typeof(ShardedShuffleOperator).Name.ToLower(), task, 1);
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.ReducerVertexName][0];
await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint);
}
foreach (string fromSecondaryInputId in task.EndpointsDescriptor.SecondaryFromInputs.Keys)
{
var fromToConnection = task.VerticesConnectionsMap[fromSecondaryInputId + task.ReducerVertexName][0];
await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint);
}
return(true);
}
else
{
return(false);
}
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
TaskBase shuffleTask = new ShuffleTask(_moveDescriptor);
shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
shuffleTask.IsRightOperandInput = false;
OperatorTransforms shuffleInputTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms);
shuffleTask.PrepareTaskTransformations(shuffleInputTransforms);
(shuffleTask as ShuffleTask).MapperVertexName = "shufflemapper" + Guid.NewGuid().ToString();
(shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString();
shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1);
shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2);
shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName;
OperatorTransforms shuffleTransforms = new OperatorTransforms();
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter),
OperatorType.MoveSplit.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger),
OperatorType.MoveMerge.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms);
topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask);
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
if (shuffleTask.Transforms != null)
{
foreach (OperatorInputs inputs in shuffleTask.TransformsInputs)
{
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2);
topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
}
}
// Update the inputs and types for the next operation
task.InputIds.SetInputId1(shuffleTask.OutputId);
task.OperationTypes.SetInputKeyType(typeof(TKeyO));
task.OperationTypes.SetInputPayloadType(typeof(TPayloadO));
task.OperationTypes.SetInputDatasetType(typeof(TDataSetO));
if (isRightOperandInput)
{
temporaryInputs.InputId2 = shuffleTask.OutputId;
}
else
{
temporaryInputs.InputId1 = shuffleTask.OutputId;
}
task.NextInputIds = temporaryInputs;
}