public override IShardedDataset <TKey, TPayload, TDataset> Deploy() { if (!_isDeployed) { OperatorsToplogy operatorsTopology = OperatorsToplogy.GetInstance(); GenerateProduceTask(ref operatorsTopology); _craClient = new CRAClientLibrary(); _isDeployed = DeploymentUtils.DeployOperators(_craClient, operatorsTopology); if (!_isDeployed) { return(null); } } return(this); }
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } } ClientTerminalTask clientTerminalTask = new ClientTerminalTask(); clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString(); _craClient = new CRAClientLibrary(); _clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId); toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId); _isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { _isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy); } else { return(null); } } return(this); }
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1)) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); } } } _clientTerminalTask = new ClientTerminalTask(); _clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); _clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString(); _clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); _craClient = new CRAClientLibrary(_dataProvider); toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId); _isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString(); _craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000); _craWorker.DisableDynamicLoading(); _craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower()); new Thread(() => _craWorker.Start()).Start(); Thread.Sleep(1000); _isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy); } else { return(null); } } return(this); }
internal void PrepareFinalOperatorsTasks() { string[] operatorsIds = _operatorsIds.ToArray(); TaskBase[] tasks = _operatorsTasks.ToArray(); List <string>[] tmpTransforms = new List <string> [tasks.Length]; List <string>[] tmpTransformsOperations = new List <string> [tasks.Length]; List <string>[] tmpTransformsTypes = new List <string> [tasks.Length]; List <OperatorInputs>[] tmpTransformsInputs = new List <OperatorInputs> [tasks.Length]; for (int i = 0; i < tasks.Length; i++) { tmpTransforms[i] = new List <string>(); tmpTransformsOperations[i] = new List <string>(); tmpTransformsTypes[i] = new List <string>(); tmpTransformsInputs[i] = new List <OperatorInputs>(); } if (tasks.Length >= 2) { for (int i = 0; i < operatorsIds.Length; i++) { if (tasks[i].Transforms != null) { int lastProcessedTransformIndex = 0; for (int j = 0; j < tasks[i].Transforms.Length; j++) { var currentInput = tasks[i].TransformsInputs[j].InputId1; int inputTaskIndex = DeploymentUtils.RetrieveTaskIndexOfOperator(currentInput, operatorsIds); if (tmpTransforms[inputTaskIndex].Count > 0 && tmpTransformsOperations[inputTaskIndex][tmpTransforms[inputTaskIndex].Count - 1] == OperatorType.MoveSplit.ToString()) { break; } else { tmpTransforms[inputTaskIndex].Add(tasks[i].Transforms[j]); tmpTransformsOperations[inputTaskIndex].Add(tasks[i].TransformsOperations[j]); tmpTransformsTypes[inputTaskIndex].Add(tasks[i].TransformsTypes[j]); tmpTransformsInputs[inputTaskIndex].Add(tasks[i].TransformsInputs[j]); if (tasks[i].TransformsOperations[j] == OperatorType.BinaryTransform.ToString()) { var currentSecondaryInput = tasks[i].TransformsInputs[j].InputId2; if (tasks[inputTaskIndex].OperationType != OperatorType.Move && tasks[i].OperationType != OperatorType.Move) { UpdateOperatorsSecondaryInput(operatorsIds[inputTaskIndex], operatorsIds[i], currentSecondaryInput); } else if (tasks[inputTaskIndex].OperationType != OperatorType.Move && tasks[i].OperationType == OperatorType.Move) { UpdateOperatorsSecondaryInput(operatorsIds[inputTaskIndex], ((ShuffleTask)tasks[i]).MapperVertexName, currentSecondaryInput); } else if (tasks[inputTaskIndex].OperationType == OperatorType.Move && tasks[i].OperationType != OperatorType.Move) { UpdateOperatorsSecondaryInput(((ShuffleTask)tasks[inputTaskIndex]).ReducerVertexName, operatorsIds[i], currentSecondaryInput); } else { UpdateOperatorsSecondaryInput(((ShuffleTask)tasks[inputTaskIndex]).ReducerVertexName, ((ShuffleTask)tasks[i]).MapperVertexName, currentSecondaryInput); } } lastProcessedTransformIndex++; } } for (int k = lastProcessedTransformIndex; k < tasks[i].Transforms.Length; k++) { tmpTransforms[i].Add(tasks[i].Transforms[k]); tmpTransformsOperations[i].Add(tasks[i].TransformsOperations[k]); tmpTransformsTypes[i].Add(tasks[i].TransformsTypes[k]); tmpTransformsInputs[i].Add(tasks[i].TransformsInputs[k]); } } if (tasks[i].OperationType == OperatorType.Move) { var currentInput = ((ShuffleTask)tasks[i]).ShuffleTransformsInputs[0].InputId1; int inputTaskIndex = DeploymentUtils.RetrieveTaskIndexOfOperator(currentInput, operatorsIds); if (!(tmpTransforms[inputTaskIndex].Count > 0 && tmpTransformsOperations[inputTaskIndex][tmpTransforms[inputTaskIndex].Count - 1] == OperatorType.MoveSplit.ToString())) { tmpTransforms[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransforms[0]); tmpTransformsOperations[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsOperations[0]); tmpTransformsTypes[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsTypes[0]); tmpTransformsInputs[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsInputs[0]); tasks[inputTaskIndex].SecondaryShuffleDescriptor = ((ShuffleTask)tasks[i]).ShuffleDescriptor; } } } for (int i = 0; i < tasks.Length; i++) { tasks[i].Transforms = tmpTransforms[i].ToArray(); tasks[i].TransformsOperations = tmpTransformsOperations[i].ToArray(); tasks[i].TransformsTypes = tmpTransformsTypes[i].ToArray(); tasks[i].TransformsInputs = tmpTransformsInputs[i].ToArray(); } _operatorsTasks = new List <TaskBase>(tasks); } }