private void DeployUnaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
// Prepare the transformation of the operation unary (left) operand
task.IsRightOperandInput = false;
OperatorTransforms unaryTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref task, ref topology, ref unaryTransforms);
temporaryInputs.InputId1 = task.NextInputIds.InputId1;
// Update the inputs for the next operation
if (isRightOperandInput)
{
task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
}
else
{
task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
}
// Add the transformations from this operand to the parent operand
unaryTransforms.AddTransform(SerializationHelper.Serialize(_unaryTransformer),
OperatorType.UnaryTransform.ToString(),
TransformUtils.FillUnaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs);
parentTransforms = unaryTransforms;
}
public TaskBase(OperatorType operationType)
{
_taskId = NextTaskID();
_operationType = operationType;
IsRightOperandInput = false;
OperationTypes = new BinaryOperatorTypes();
InputIds = new OperatorInputs();
NextInputIds = new OperatorInputs();
}
public static void PrepareTransformInputs(OperatorInputs transformInputInfo, ref Object dataset1, ref string dataset1Id,
ref Object dataset2, ref string dataset2Id, Dictionary <string, object> cachedDatasets)
{
string inputId1 = transformInputInfo.InputId1;
string inputId2 = transformInputInfo.InputId2;
if (inputId1 != "$" && inputId1 != null && cachedDatasets.ContainsKey(inputId1))
{
dataset1 = cachedDatasets[inputId1];
dataset1Id = inputId1;
}
if (dataset1 == null)
{
if (inputId2 != "$" && inputId2 != null && cachedDatasets.ContainsKey(inputId2))
{
dataset1 = cachedDatasets[inputId2];
dataset1Id = inputId2;
}
else
{
dataset1 = null;
dataset1Id = "$";
}
dataset2 = null;
dataset2Id = "$";
}
else
{
if (inputId2 != "$" && inputId2 != null && cachedDatasets.ContainsKey(inputId2))
{
dataset2 = cachedDatasets[inputId2];
dataset2Id = inputId2;
}
else
{
dataset2 = null;
dataset2Id = "$";
}
}
}
private void DeployBinaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
// Prepare the transformations of the operation left operand
task.IsRightOperandInput = false;
OperatorTransforms leftTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref task, ref topology, ref leftTransforms);
temporaryInputs.InputId1 = task.NextInputIds.InputId1;
// Prepare the transformations of the operation right operand
task.IsRightOperandInput = true;
OperatorTransforms rightTransforms = new OperatorTransforms();
(_input2 as IDeployable).Deploy(ref task, ref topology, ref rightTransforms);
temporaryInputs.InputId2 = task.NextInputIds.InputId2;
// Update the inputs for the next operation
if (isRightOperandInput)
{
task.NextInputIds.SetInputId2(temporaryInputs.InputId1);
}
else
{
task.NextInputIds.SetInputId1(temporaryInputs.InputId1);
}
// Merge the tranformations from the two operands, and assign them to the parent operation
parentTransforms = TransformUtils.MergeTwoSetsOfTransforms(leftTransforms, rightTransforms);
parentTransforms.AddTransform(SerializationHelper.Serialize(_binaryTransformer),
OperatorType.BinaryTransform.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs);
}
private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology)
{
var isRightOperandInput = task.IsRightOperandInput;
OperatorInputs temporaryInputs = new OperatorInputs();
TaskBase shuffleTask = new ShuffleTask(_moveDescriptor);
shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
shuffleTask.IsRightOperandInput = false;
OperatorTransforms shuffleInputTransforms = new OperatorTransforms();
(_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms);
shuffleTask.PrepareTaskTransformations(shuffleInputTransforms);
(shuffleTask as ShuffleTask).MapperVertexName = "shufflemapper" + Guid.NewGuid().ToString();
(shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString();
shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1);
shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2);
shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName;
OperatorTransforms shuffleTransforms = new OperatorTransforms();
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter),
OperatorType.MoveSplit.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger),
OperatorType.MoveMerge.ToString(),
TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1),
typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(),
shuffleTask.InputIds);
((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms);
topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask);
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
if (shuffleTask.Transforms != null)
{
foreach (OperatorInputs inputs in shuffleTask.TransformsInputs)
{
topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1);
topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2);
topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName);
topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName);
}
}
// Update the inputs and types for the next operation
task.InputIds.SetInputId1(shuffleTask.OutputId);
task.OperationTypes.SetInputKeyType(typeof(TKeyO));
task.OperationTypes.SetInputPayloadType(typeof(TPayloadO));
task.OperationTypes.SetInputDatasetType(typeof(TDataSetO));
if (isRightOperandInput)
{
temporaryInputs.InputId2 = shuffleTask.OutputId;
}
else
{
temporaryInputs.InputId1 = shuffleTask.OutputId;
}
task.NextInputIds = temporaryInputs;
}
public void AddTransform(string transform, string transformOperation, string transformType, OperatorInputs transformInput)
{
Transforms.Add(transform);
TransformsOperations.Add(transformOperation);
TransformsTypes.Add(transformType);
TransformsInputs.Add(transformInput);
}