private static bool DeployShuffleReduceTask(CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology) { try { client.DefineVertex(typeof(ShuffleOperator).Name.ToLower(), () => new ShuffleOperator()); CRAErrorCode status = client.InstantiateShardedVertex(task.ReducerVertexName, typeof(ShuffleOperator).Name.ToLower(), task, task.DeployDescriptor.InstancesMap()); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.SecondaryEndpointsDescriptor.FromInputs.Keys) { client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]); } var fromInput = task.EndpointsDescriptor.FromInputs.First().Key; var newFromInputId = fromInput.Substring(0, fromInput.Length - 1); client.ConnectShardedVertices(task.VerticesConnectionsMap[newFromInputId + task.ReducerVertexName]); return(true); } else { return(false); } } catch (Exception) { Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations"); return(false); } }
internal void UpdateShuffleOperatorTask(string taskId, ShuffleTask task) { if (_operatorsIds.Contains(taskId)) { int index = _operatorsIds.IndexOf(taskId); _operatorsTasks[index] = task; } }
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); var tasksDeploymentStatus = new Dictionary <string, bool>(); for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; tasksDeploymentStatus.Add(tasksIds[i], false); } bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 && tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0) { isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[i]] = true; } } } for (int i = 0; i < tasks.Length; i++) { isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology); } return(isSuccessful); }
private object ApplyMergerOnInputs(ShuffleTask shuffleTask, BinaryOperatorTypes mergeTypes, object[] inputSplitDatasets) { MethodInfo method = typeof(MoveUtils).GetMethod("ApplyMerger"); MethodInfo generic = method.MakeGenericMethod( new Type[] { mergeTypes.SecondaryKeyType, mergeTypes.SecondaryPayloadType, mergeTypes.SecondaryDatasetType, mergeTypes.OutputKeyType, mergeTypes.OutputPayloadType, mergeTypes.OutputDatasetType }); object[] arguments = new Object[] { inputSplitDatasets, shuffleTask.ShuffleDescriptor, shuffleTask.ShuffleTransforms[1] }; return(generic.Invoke(this, arguments)); }
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; if (tasks[i].OperationType == OperatorType.Produce) { isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]); } else if (tasks[i].OperationType == OperatorType.Subscribe) { isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology); } else if (tasks[i].OperationType == OperatorType.Move) { isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology); } if (!isSuccessful) { break; } } return(isSuccessful); }
internal void AddShuffleOperator(string mapperTaskId, string reducerTaskId, ShuffleTask task) { _operatorsEndpointsDescriptors.AddOrUpdate(mapperTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor()); _operatorsEndpointsDescriptors.AddOrUpdate(reducerTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor()); if (_operatorsIds.Contains(reducerTaskId)) { int index = _operatorsIds.IndexOf(reducerTaskId); _operatorsTasks[index] = task; } else { _operatorsIds.Add(reducerTaskId); _operatorsTasks.Add(task); } }
private static async Task <bool> DeployShuffleReduceTask( CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology) { try { if (!_isShuffleOperatorDefined) { await client.DefineVertexAsync(typeof(ShardedShuffleOperator).Name.ToLower(), () => new ShardedShuffleOperator()); _isShuffleOperatorDefined = true; } var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.ReducerVertexName, typeof(ShardedShuffleOperator).Name.ToLower(), task, 1); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.ReducerVertexName][0]; await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint); } foreach (string fromSecondaryInputId in task.EndpointsDescriptor.SecondaryFromInputs.Keys) { var fromToConnection = task.VerticesConnectionsMap[fromSecondaryInputId + task.ReducerVertexName][0]; await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint); } return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations: " + e.ToString()); return(false); } }
private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology) { var isRightOperandInput = task.IsRightOperandInput; OperatorInputs temporaryInputs = new OperatorInputs(); TaskBase shuffleTask = new ShuffleTask(_moveDescriptor); shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); shuffleTask.IsRightOperandInput = false; OperatorTransforms shuffleInputTransforms = new OperatorTransforms(); (_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms); shuffleTask.PrepareTaskTransformations(shuffleInputTransforms); (shuffleTask as ShuffleTask).MapperVertexName = "shufflemapper" + Guid.NewGuid().ToString(); (shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString(); shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1); shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2); shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName; OperatorTransforms shuffleTransforms = new OperatorTransforms(); shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter), OperatorType.MoveSplit.ToString(), TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), shuffleTask.InputIds); shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger), OperatorType.MoveMerge.ToString(), TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), shuffleTask.InputIds); ((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms); topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask); topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1); topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2); topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName); topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName); if (shuffleTask.Transforms != null) { foreach (OperatorInputs inputs in shuffleTask.TransformsInputs) { topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1); topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2); topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName); topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName); } } // Update the inputs and types for the next operation task.InputIds.SetInputId1(shuffleTask.OutputId); task.OperationTypes.SetInputKeyType(typeof(TKeyO)); task.OperationTypes.SetInputPayloadType(typeof(TPayloadO)); task.OperationTypes.SetInputDatasetType(typeof(TDataSetO)); if (isRightOperandInput) { temporaryInputs.InputId2 = shuffleTask.OutputId; } else { temporaryInputs.InputId1 = shuffleTask.OutputId; } task.NextInputIds = temporaryInputs; }