Beispiel #1
0
        private void DeployUnaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
        {
            var            isRightOperandInput = task.IsRightOperandInput;
            OperatorInputs temporaryInputs     = new OperatorInputs();

            // Prepare the transformation of the operation unary (left) operand
            task.IsRightOperandInput = false;
            OperatorTransforms unaryTransforms = new OperatorTransforms();

            (_input1 as IDeployable).Deploy(ref task, ref topology, ref unaryTransforms);
            temporaryInputs.InputId1 = task.NextInputIds.InputId1;

            // Update the inputs for the next operation
            if (isRightOperandInput)
            {
                task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
            }
            else
            {
                task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
            }

            // Add the transformations from this operand to the parent operand
            unaryTransforms.AddTransform(SerializationHelper.Serialize(_unaryTransformer),
                                         OperatorType.UnaryTransform.ToString(),
                                         TransformUtils.FillUnaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs);
            parentTransforms = unaryTransforms;
        }
Beispiel #2
0
        public TaskBase(OperatorType operationType)
        {
            _taskId             = NextTaskID();
            _operationType      = operationType;
            IsRightOperandInput = false;

            OperationTypes = new BinaryOperatorTypes();
            InputIds       = new OperatorInputs();
            NextInputIds   = new OperatorInputs();
        }
Beispiel #3
0
        public static void PrepareTransformInputs(OperatorInputs transformInputInfo, ref Object dataset1, ref string dataset1Id,
                                                  ref Object dataset2, ref string dataset2Id, Dictionary <string, object> cachedDatasets)
        {
            string inputId1 = transformInputInfo.InputId1;
            string inputId2 = transformInputInfo.InputId2;

            if (inputId1 != "$" && inputId1 != null && cachedDatasets.ContainsKey(inputId1))
            {
                dataset1   = cachedDatasets[inputId1];
                dataset1Id = inputId1;
            }

            if (dataset1 == null)
            {
                if (inputId2 != "$" && inputId2 != null && cachedDatasets.ContainsKey(inputId2))
                {
                    dataset1   = cachedDatasets[inputId2];
                    dataset1Id = inputId2;
                }
                else
                {
                    dataset1   = null;
                    dataset1Id = "$";
                }
                dataset2   = null;
                dataset2Id = "$";
            }
            else
            {
                if (inputId2 != "$" && inputId2 != null && cachedDatasets.ContainsKey(inputId2))
                {
                    dataset2   = cachedDatasets[inputId2];
                    dataset2Id = inputId2;
                }
                else
                {
                    dataset2   = null;
                    dataset2Id = "$";
                }
            }
        }
Beispiel #4
0
        private void DeployBinaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
        {
            var            isRightOperandInput = task.IsRightOperandInput;
            OperatorInputs temporaryInputs     = new OperatorInputs();

            // Prepare the transformations of the operation left operand
            task.IsRightOperandInput = false;
            OperatorTransforms leftTransforms = new OperatorTransforms();

            (_input1 as IDeployable).Deploy(ref task, ref topology, ref leftTransforms);
            temporaryInputs.InputId1 = task.NextInputIds.InputId1;

            // Prepare the transformations of the operation right operand
            task.IsRightOperandInput = true;
            OperatorTransforms rightTransforms = new OperatorTransforms();

            (_input2 as IDeployable).Deploy(ref task, ref topology, ref rightTransforms);
            temporaryInputs.InputId2 = task.NextInputIds.InputId2;

            // Update the inputs for the next operation
            if (isRightOperandInput)
            {
                task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
            }
            else
            {
                task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
            }

            // Merge the tranformations from the two operands, and assign them to the parent operation
            parentTransforms = TransformUtils.MergeTwoSetsOfTransforms(leftTransforms, rightTransforms);
            parentTransforms.AddTransform(SerializationHelper.Serialize(_binaryTransformer),
                                          OperatorType.BinaryTransform.ToString(),
                                          TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                  typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                                                                                  typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs);
        }
Beispiel #5
0
        private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology)
        {
            var            isRightOperandInput = task.IsRightOperandInput;
            OperatorInputs temporaryInputs     = new OperatorInputs();

            TaskBase shuffleTask = new ShuffleTask(_moveDescriptor);

            shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
                typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
            shuffleTask.IsRightOperandInput = false;
            OperatorTransforms shuffleInputTransforms = new OperatorTransforms();

            (_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms);
            shuffleTask.PrepareTaskTransformations(shuffleInputTransforms);
            (shuffleTask as ShuffleTask).MapperVertexName  = "shufflemapper" + Guid.NewGuid().ToString();
            (shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString();
            shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1);
            shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2);
            shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName;
            OperatorTransforms shuffleTransforms = new OperatorTransforms();

            shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter),
                                           OperatorType.MoveSplit.ToString(),
                                           TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                   typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                                                                                   typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
                                           shuffleTask.InputIds);
            shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger),
                                           OperatorType.MoveMerge.ToString(),
                                           TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
                                                                                   typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
                                                                                   typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
                                           shuffleTask.InputIds);
            ((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms);

            topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask);
            topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1);
            topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2);
            topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
            topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);

            if (shuffleTask.Transforms != null)
            {
                foreach (OperatorInputs inputs in shuffleTask.TransformsInputs)
                {
                    topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1);
                    topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2);
                    topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
                    topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
                }
            }

            // Update the inputs and types for the next operation
            task.InputIds.SetInputId1(shuffleTask.OutputId);
            task.OperationTypes.SetInputKeyType(typeof(TKeyO));
            task.OperationTypes.SetInputPayloadType(typeof(TPayloadO));
            task.OperationTypes.SetInputDatasetType(typeof(TDataSetO));
            if (isRightOperandInput)
            {
                temporaryInputs.InputId2 = shuffleTask.OutputId;
            }
            else
            {
                temporaryInputs.InputId1 = shuffleTask.OutputId;
            }
            task.NextInputIds = temporaryInputs;
        }
Beispiel #6
0
 public void AddTransform(string transform, string transformOperation, string transformType, OperatorInputs transformInput)
 {
     Transforms.Add(transform);
     TransformsOperations.Add(transformOperation);
     TransformsTypes.Add(transformType);
     TransformsInputs.Add(transformInput);
 }