internal void AddOperatorBase(string taskId, TaskBase task)
{
_operatorsEndpointsDescriptors.AddOrUpdate(taskId, new OperatorEndpointsDescriptor(), (operatorId, descriptor) => new OperatorEndpointsDescriptor());
if (_operatorsIds.Contains(taskId))
{
int index = _operatorsIds.IndexOf(taskId);
_operatorsTasks[index] = task;
}
else
{
_operatorsIds.Add(taskId);
_operatorsTasks.Add(task);
}
}
public void Deploy(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
{
switch (OperationType)
{
case OperatorType.Move:
DeployMove(ref task, ref topology);
break;
case OperatorType.BinaryTransform:
DeployBinaryTransform(ref task, ref topology, ref parentTransforms);
break;
default:
DeployUnaryTransform(ref task, ref topology, ref parentTransforms);
break;
}
}
private void DeployBinaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
// Prepare the transformations of the operation left operand
task.IsRightOperandInput = false;
OperatorTransforms leftTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref task, ref topology, ref leftTransforms);
temporaryInputs.InputId1 = task.NextInputIds.InputId1;
// Prepare the transformations of the operation right operand
task.IsRightOperandInput = true;
OperatorTransforms rightTransforms = new OperatorTransforms();
(_input2 as IDeployable).Deploy(ref task, ref topology, ref rightTransforms);
temporaryInputs.InputId2 = task.NextInputIds.InputId2;
// Update the inputs for the next operation
if (isRightOperandInput)
{
task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
}
else
{
task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
}
// Merge the tranformations from the two operands, and assign them to the parent operation
parentTransforms = TransformUtils.MergeTwoSetsOfTransforms(leftTransforms, rightTransforms);
parentTransforms.AddTransform(SerializationHelper.Serialize(_binaryTransformer),
OperatorType.BinaryTransform.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs);
}
private void DeployUnaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
// Prepare the transformation of the operation unary (left) operand
task.IsRightOperandInput = false;
OperatorTransforms unaryTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref task, ref topology, ref unaryTransforms);
temporaryInputs.InputId1 = task.NextInputIds.InputId1;
// Update the inputs for the next operation
if (isRightOperandInput)
{
task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
}
else
{
task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
}
// Add the transformations from this operand to the parent operand
unaryTransforms.AddTransform(
SerializationHelper.Serialize(_unaryTransformer),
OperatorType.UnaryTransform.ToString(),
TransformUtils.FillUnaryTransformTypes(
typeof(TKeyI1),
typeof(TPayloadI1),
typeof(TDataSetI1),
typeof(TKeyO),
typeof(TPayloadO),
typeof(TDataSetO)).ToString(),
temporaryInputs);
parentTransforms = unaryTransforms;
}
private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
TaskBase shuffleTask = new ShuffleTask(_moveDescriptor);
shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
shuffleTask.IsRightOperandInput = false;
OperatorTransforms shuffleInputTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms);
shuffleTask.PrepareTaskTransformations(shuffleInputTransforms);
(shuffleTask as ShuffleTask).MapperVertexName = "shufflemapper" + Guid.NewGuid().ToString();
(shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString();
shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1);
shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2);
shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName;
OperatorTransforms shuffleTransforms = new OperatorTransforms();
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter),
OperatorType.MoveSplit.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger),
OperatorType.MoveMerge.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms);
topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask);
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
if (shuffleTask.Transforms != null)
{
foreach (OperatorInputs inputs in shuffleTask.TransformsInputs)
{
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2);
topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
}
}
// Update the inputs and types for the next operation
task.InputIds.SetInputId1(shuffleTask.OutputId);
task.OperationTypes.SetInputKeyType(typeof(TKeyO));
task.OperationTypes.SetInputPayloadType(typeof(TPayloadO));
task.OperationTypes.SetInputDatasetType(typeof(TDataSetO));
if (isRightOperandInput)
{
temporaryInputs.InputId2 = shuffleTask.OutputId;
}
else
{
temporaryInputs.InputId1 = shuffleTask.OutputId;
}
task.NextInputIds = temporaryInputs;
}