private void DeployUnaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms) { var isRightOperandInput = task.IsRightOperandInput; OperatorInputs temporaryInputs = new OperatorInputs(); // Prepare the transformation of the operation unary (left) operand task.IsRightOperandInput = false; OperatorTransforms unaryTransforms = new OperatorTransforms(); (_input1 as IDeployable).Deploy(ref task, ref topology, ref unaryTransforms); temporaryInputs.InputId1 = task.NextInputIds.InputId1; // Update the inputs for the next operation if (isRightOperandInput) { task.NextInputIds.SetInputId2(temporaryInputs.InputId1); } else { task.NextInputIds.SetInputId1(temporaryInputs.InputId1); } // Add the transformations from this operand to the parent operand unaryTransforms.AddTransform(SerializationHelper.Serialize(_unaryTransformer), OperatorType.UnaryTransform.ToString(), TransformUtils.FillUnaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs); parentTransforms = unaryTransforms; }
internal void PrepareTaskTransformations(OperatorTransforms nodeMergedOperand) { if (nodeMergedOperand.Transforms.Count > 0) { Transforms = nodeMergedOperand.Transforms.ToArray(); TransformsOperations = nodeMergedOperand.TransformsOperations.ToArray(); TransformsTypes = nodeMergedOperand.TransformsTypes.ToArray(); TransformsInputs = nodeMergedOperand.TransformsInputs.ToArray(); } }
public void Deploy(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms) { switch (OperationType) { case OperatorType.Move: DeployMove(ref task, ref topology); break; case OperatorType.BinaryTransform: DeployBinaryTransform(ref task, ref topology, ref parentTransforms); break; default: DeployUnaryTransform(ref task, ref topology, ref parentTransforms); break; } }
private void DeployBinaryTransform(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms parentTransforms) { var isRightOperandInput = task.IsRightOperandInput; OperatorInputs temporaryInputs = new OperatorInputs(); // Prepare the transformations of the operation left operand task.IsRightOperandInput = false; OperatorTransforms leftTransforms = new OperatorTransforms(); (_input1 as IDeployable).Deploy(ref task, ref topology, ref leftTransforms); temporaryInputs.InputId1 = task.NextInputIds.InputId1; // Prepare the transformations of the operation right operand task.IsRightOperandInput = true; OperatorTransforms rightTransforms = new OperatorTransforms(); (_input2 as IDeployable).Deploy(ref task, ref topology, ref rightTransforms); temporaryInputs.InputId2 = task.NextInputIds.InputId2; // Update the inputs for the next operation if (isRightOperandInput) { task.NextInputIds.SetInputId2(temporaryInputs.InputId1); } else { task.NextInputIds.SetInputId1(temporaryInputs.InputId1); } // Merge the tranformations from the two operands, and assign them to the parent operation parentTransforms = TransformUtils.MergeTwoSetsOfTransforms(leftTransforms, rightTransforms); parentTransforms.AddTransform(SerializationHelper.Serialize(_binaryTransformer), OperatorType.BinaryTransform.ToString(), TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), temporaryInputs); }
public static OperatorTransforms MergeTwoSetsOfTransforms(OperatorTransforms leftOperand, OperatorTransforms rightOperand) { List <string> transforms = new List <string>(); transforms.AddRange(leftOperand.Transforms); transforms.AddRange(rightOperand.Transforms); List <string> transformsOperations = new List <string>(); transformsOperations.AddRange(leftOperand.TransformsOperations); transformsOperations.AddRange(rightOperand.TransformsOperations); List <string> transformsTypes = new List <string>(); transformsTypes.AddRange(leftOperand.TransformsTypes); transformsTypes.AddRange(rightOperand.TransformsTypes); List <OperatorInputs> transformsInputs = new List <OperatorInputs>(); transformsInputs.AddRange(leftOperand.TransformsInputs); transformsInputs.AddRange(rightOperand.TransformsInputs); return(new OperatorTransforms(transforms, transformsOperations, transformsTypes, transformsInputs)); }
private void DeployMove(ref TaskBase task, ref OperatorsToplogy topology) { var isRightOperandInput = task.IsRightOperandInput; OperatorInputs temporaryInputs = new OperatorInputs(); TaskBase shuffleTask = new ShuffleTask(_moveDescriptor); shuffleTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); shuffleTask.IsRightOperandInput = false; OperatorTransforms shuffleInputTransforms = new OperatorTransforms(); (_input1 as IDeployable).Deploy(ref shuffleTask, ref topology, ref shuffleInputTransforms); shuffleTask.PrepareTaskTransformations(shuffleInputTransforms); (shuffleTask as ShuffleTask).MapperVertexName = "shufflemapper" + Guid.NewGuid().ToString(); (shuffleTask as ShuffleTask).ReducerVertexName = typeof(ShuffleOperator).Name.ToLower() + Guid.NewGuid().ToString(); shuffleTask.InputIds.SetInputId1(shuffleTask.NextInputIds.InputId1); shuffleTask.InputIds.SetInputId2(shuffleTask.NextInputIds.InputId2); shuffleTask.OutputId = (shuffleTask as ShuffleTask).ReducerVertexName; OperatorTransforms shuffleTransforms = new OperatorTransforms(); shuffleTransforms.AddTransform(SerializationHelper.Serialize(_splitter), OperatorType.MoveSplit.ToString(), TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), shuffleTask.InputIds); shuffleTransforms.AddTransform(SerializationHelper.Serialize(_merger), OperatorType.MoveMerge.ToString(), TransformUtils.FillBinaryTransformTypes(typeof(TKeyI1), typeof(TPayloadI1), typeof(TDataSetI1), typeof(TKeyI2), typeof(TPayloadI2), typeof(TDataSetI2), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)).ToString(), shuffleTask.InputIds); ((ShuffleTask)shuffleTask).PrepareShuffleTransformations(shuffleTransforms); topology.AddShuffleOperator((shuffleTask as ShuffleTask).MapperVertexName, (shuffleTask as ShuffleTask).ReducerVertexName, shuffleTask as ShuffleTask); topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId1); topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, shuffleTask.InputIds.InputId2); topology.AddOperatorOutput(shuffleTask.InputIds.InputId1, (shuffleTask as ShuffleTask).MapperVertexName); topology.AddOperatorOutput(shuffleTask.InputIds.InputId2, (shuffleTask as ShuffleTask).MapperVertexName); if (shuffleTask.Transforms != null) { foreach (OperatorInputs inputs in shuffleTask.TransformsInputs) { topology.AddOperatorInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId1); topology.AddOperatorSecondaryInput((shuffleTask as ShuffleTask).MapperVertexName, inputs.InputId2); topology.AddOperatorOutput(inputs.InputId1, (shuffleTask as ShuffleTask).MapperVertexName); topology.AddOperatorOutput(inputs.InputId2, (shuffleTask as ShuffleTask).MapperVertexName); } } // Update the inputs and types for the next operation task.InputIds.SetInputId1(shuffleTask.OutputId); task.OperationTypes.SetInputKeyType(typeof(TKeyO)); task.OperationTypes.SetInputPayloadType(typeof(TPayloadO)); task.OperationTypes.SetInputDatasetType(typeof(TDataSetO)); if (isRightOperandInput) { temporaryInputs.InputId2 = shuffleTask.OutputId; } else { temporaryInputs.InputId1 = shuffleTask.OutputId; } task.NextInputIds = temporaryInputs; }
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } } ClientTerminalTask clientTerminalTask = new ClientTerminalTask(); clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString(); _craClient = new CRAClientLibrary(); _clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId); toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId); _isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { _isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy); } else { return(null); } } return(this); }
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1)) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); } } } _clientTerminalTask = new ClientTerminalTask(); _clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); _clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString(); _clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); _craClient = new CRAClientLibrary(_dataProvider); toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId); _isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString(); _craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000); _craWorker.DisableDynamicLoading(); _craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower()); new Thread(() => _craWorker.Start()).Start(); Thread.Sleep(1000); _isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy); } else { return(null); } } return(this); }
public void Deploy(ref TaskBase task, ref OperatorsToplogy operatorsTopology, ref OperatorTransforms operandTransforms) { GenerateProduceTask(ref operatorsTopology); var isRightOperandInput = task.IsRightOperandInput; if (isRightOperandInput) { task.InputIds.SetInputId2(_shardedDatasetId); task.NextInputIds.SetInputId2(_shardedDatasetId); task.OperationTypes.SetSecondaryKeyType(typeof(TKey)); task.OperationTypes.SetSecondaryPayloadType(typeof(TPayload)); task.OperationTypes.SetSecondaryDatasetType(typeof(TDataset)); } else { task.InputIds.SetInputId1(_shardedDatasetId); task.NextInputIds.SetInputId1(_shardedDatasetId); task.OperationTypes.SetInputKeyType(typeof(TKey)); task.OperationTypes.SetInputPayloadType(typeof(TPayload)); task.OperationTypes.SetInputDatasetType(typeof(TDataset)); } task.IsRightOperandInput = false; }