Example #1
0
        private static bool DeployShuffleReduceTask(CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology)
        {
            try
            {
                client.DefineVertex(typeof(ShuffleOperator).Name.ToLower(), () => new ShuffleOperator());
                CRAErrorCode status = client.InstantiateShardedVertex(task.ReducerVertexName, typeof(ShuffleOperator).Name.ToLower(),
                                                                      task, task.DeployDescriptor.InstancesMap());
                if (status == CRAErrorCode.Success)
                {
                    foreach (string fromInputId in task.SecondaryEndpointsDescriptor.FromInputs.Keys)
                    {
                        client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]);
                    }

                    var fromInput      = task.EndpointsDescriptor.FromInputs.First().Key;
                    var newFromInputId = fromInput.Substring(0, fromInput.Length - 1);
                    client.ConnectShardedVertices(task.VerticesConnectionsMap[newFromInputId + task.ReducerVertexName]);

                    return(true);
                }
                else
                {
                    return(false);
                }
            }
            catch (Exception)
            {
                Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations");
                return(false);
            }
        }
Example #2
0
 internal void UpdateShuffleOperatorTask(string taskId, ShuffleTask task)
 {
     if (_operatorsIds.Contains(taskId))
     {
         int index = _operatorsIds.IndexOf(taskId);
         _operatorsTasks[index] = task;
     }
 }
Example #3
0
        public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
        {
            topology.PrepareFinalOperatorsTasks();

            string[]   tasksIds = topology.OperatorsIds.ToArray();
            TaskBase[] tasks    = topology.OperatorsTasks.ToArray();

            for (int i = 0; i < tasks.Length; i++)
            {
                if (tasks[i].OperationType == OperatorType.Move)
                {
                    ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
                    shuffleTask.SecondaryEndpointsDescriptor            = new OperatorEndpointsDescriptor();
                    shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
                    topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
                    int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
                    topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
                    topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
                }
            }

            var tasksDictionary       = PrepareTasksDictionary(tasks);
            var connectionsMap        = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
            var tasksDeploymentStatus = new Dictionary <string, bool>();

            for (int i = 0; i < tasks.Length; i++)
            {
                tasks[i].EndpointsDescriptor    = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
                tasks[i].VerticesConnectionsMap = connectionsMap;
                tasksDeploymentStatus.Add(tasksIds[i], false);
            }

            bool isSuccessful = true;

            for (int i = 0; i < tasks.Length; i++)
            {
                if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 &&
                    tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0)
                {
                    isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology);

                    if (isSuccessful)
                    {
                        tasksDeploymentStatus[tasksIds[i]] = true;
                    }
                }
            }

            for (int i = 0; i < tasks.Length; i++)
            {
                isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology);
            }

            return(isSuccessful);
        }
Example #4
0
        private object ApplyMergerOnInputs(ShuffleTask shuffleTask, BinaryOperatorTypes mergeTypes, object[] inputSplitDatasets)
        {
            MethodInfo method  = typeof(MoveUtils).GetMethod("ApplyMerger");
            MethodInfo generic = method.MakeGenericMethod(
                new Type[] { mergeTypes.SecondaryKeyType, mergeTypes.SecondaryPayloadType,
                             mergeTypes.SecondaryDatasetType, mergeTypes.OutputKeyType,
                             mergeTypes.OutputPayloadType, mergeTypes.OutputDatasetType });

            object[] arguments = new Object[] { inputSplitDatasets, shuffleTask.ShuffleDescriptor, shuffleTask.ShuffleTransforms[1] };
            return(generic.Invoke(this, arguments));
        }
Example #5
0
        public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
        {
            topology.PrepareFinalOperatorsTasks();

            string[]   tasksIds = topology.OperatorsIds.ToArray();
            TaskBase[] tasks    = topology.OperatorsTasks.ToArray();

            for (int i = 0; i < tasks.Length; i++)
            {
                if (tasks[i].OperationType == OperatorType.Move)
                {
                    ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
                    shuffleTask.SecondaryEndpointsDescriptor            = new OperatorEndpointsDescriptor();
                    shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
                    topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
                    int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
                    topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
                    topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
                }
            }

            var tasksDictionary = PrepareTasksDictionary(tasks);
            var connectionsMap  = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);

            bool isSuccessful = true;

            for (int i = 0; i < tasks.Length; i++)
            {
                tasks[i].EndpointsDescriptor    = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
                tasks[i].VerticesConnectionsMap = connectionsMap;
                if (tasks[i].OperationType == OperatorType.Produce)
                {
                    isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]);
                }
                else if (tasks[i].OperationType == OperatorType.Subscribe)
                {
                    isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology);
                }
                else if (tasks[i].OperationType == OperatorType.Move)
                {
                    isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology);
                }

                if (!isSuccessful)
                {
                    break;
                }
            }

            return(isSuccessful);
        }
Example #6
0
 internal void AddShuffleOperator(string mapperTaskId, string reducerTaskId, ShuffleTask task)
 {
     _operatorsEndpointsDescriptors.AddOrUpdate(mapperTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor());
     _operatorsEndpointsDescriptors.AddOrUpdate(reducerTaskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor());
     if (_operatorsIds.Contains(reducerTaskId))
     {
         int index = _operatorsIds.IndexOf(reducerTaskId);
         _operatorsTasks[index] = task;
     }
     else
     {
         _operatorsIds.Add(reducerTaskId);
         _operatorsTasks.Add(task);
     }
 }
Example #7
0
        private static async Task <bool> DeployShuffleReduceTask(
            CRAClientLibrary client,
            ShuffleTask task,
            OperatorsToplogy topology)
        {
            try
            {
                if (!_isShuffleOperatorDefined)
                {
                    await client.DefineVertexAsync(typeof(ShardedShuffleOperator).Name.ToLower(), () => new ShardedShuffleOperator());

                    _isShuffleOperatorDefined = true;
                }

                var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.ReducerVertexName, typeof(ShardedShuffleOperator).Name.ToLower(), task, 1);

                if (status == CRAErrorCode.Success)
                {
                    foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
                    {
                        var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.ReducerVertexName][0];
                        await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint);
                    }

                    foreach (string fromSecondaryInputId in task.EndpointsDescriptor.SecondaryFromInputs.Keys)
                    {
                        var fromToConnection = task.VerticesConnectionsMap[fromSecondaryInputId + task.ReducerVertexName][0];
                        await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint);
                    }
                    return(true);
                }
                else
                {
                    return(false);
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations: " + e.ToString());
                return(false);
            }
        }
Example #8
0
        private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology)
        {
            var            isRightOperandInput = task.IsRightOperandInput;
            OperatorInputs temporaryInputs     = new OperatorInputs();

            TaskBase shuffleTask = new ShuffleTask(_moveDescriptor);

            shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
                typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
            shuffleTask.IsRightOperandInput = false;
            OperatorTransforms shuffleInputTransforms = new OperatorTransforms();

            (_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms);
            shuffleTask.PrepareTaskTransformations(shuffleInputTransforms);
            (shuffleTask as ShuffleTask).MapperVertexName  = "shufflemapper" + Guid.NewGuid().ToString();
            (shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString();
            shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1);
            shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2);
            shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName;
            OperatorTransforms shuffleTransforms = new OperatorTransforms();

            shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter),
                                           OperatorType.MoveSplit.ToString(),
                                           TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                   typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                                                                                   typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
                                           shuffleTask.InputIds);
            shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger),
                                           OperatorType.MoveMerge.ToString(),
                                           TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                   typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                                                                                   typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
                                           shuffleTask.InputIds);
            ((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms);

            topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask);
            topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1);
            topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2);
            topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
            topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);

            if (shuffleTask.Transforms != null)
            {
                foreach (OperatorInputs inputs in shuffleTask.TransformsInputs)
                {
                    topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1);
                    topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2);
                    topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
                    topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
                }
            }

            // Update the inputs and types for the next operation
            task.InputIds.SetInputId1(shuffleTask.OutputId);
            task.OperationTypes.SetInputKeyType(typeof(TKeyO));
            task.OperationTypes.SetInputPayloadType(typeof(TPayloadO));
            task.OperationTypes.SetInputDatasetType(typeof(TDataSetO));
            if (isRightOperandInput)
            {
                temporaryInputs.InputId2 = shuffleTask.OutputId;
            }
            else
            {
                temporaryInputs.InputId1 = shuffleTask.OutputId;
            }
            task.NextInputIds = temporaryInputs;
        }