public static async Task <bool> DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology) { try { if (!_isSubscribeOperatorDefined) { await client.DefineVertexAsync(typeof(ShardedSubscribeOperator).Name.ToLower(), () => new ShardedSubscribeOperator()); _isSubscribeOperatorDefined = true; } var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.OutputId, typeof(ShardedSubscribeOperator).Name.ToLower(), task, 1); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.OutputId][0]; await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint); } return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString()); return(false); } }
public static bool DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology) { try { client.DefineVertex(typeof(SubscribeOperator).Name.ToLower(), () => new SubscribeOperator()); CRAErrorCode status = client.InstantiateShardedVertex(task.OutputId, typeof(SubscribeOperator).Name.ToLower(), task, task.DeployDescriptor.InstancesMap()); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]); } return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString()); return(false); } }
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } } ClientTerminalTask clientTerminalTask = new ClientTerminalTask(); clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString(); _craClient = new CRAClientLibrary(); _clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId); toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId); _isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { _isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy); } else { return(null); } } return(this); }
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1)) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); } } } _clientTerminalTask = new ClientTerminalTask(); _clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); _clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString(); _clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); _craClient = new CRAClientLibrary(_dataProvider); toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId); _isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString(); _craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000); _craWorker.DisableDynamicLoading(); _craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower()); new Thread(() => _craWorker.Start()).Start(); Thread.Sleep(1000); _isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy); } else { return(null); } } return(this); }