public override IShardedDataset <TKey, TPayload, TDataset> Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy operatorsTopology = OperatorsToplogy.GetInstance();
GenerateProduceTask(ref operatorsTopology);
_craClient = new CRAClientLibrary();
_isDeployed = DeploymentUtils.DeployOperators(_craClient, operatorsTopology);
if (!_isDeployed)
{
return(null);
}
}
return(this);
}
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy toplogy = OperatorsToplogy.GetInstance();
TaskBase subscribeTask = new SubscribeTask();
subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
subscribeTask.IsRightOperandInput = false;
OperatorTransforms subscribeInputTransforms = new OperatorTransforms();
Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms);
subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1);
subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2);
subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString();
subscribeTask.PrepareTaskTransformations(subscribeInputTransforms);
toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask);
toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId);
if (subscribeTask.Transforms != null)
{
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2);
toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId);
}
}
ClientTerminalTask clientTerminalTask = new ClientTerminalTask();
clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId);
clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString();
_craClient = new CRAClientLibrary();
_clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId);
toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask);
toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1);
toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2);
toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId);
toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId);
_isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy);
if (_isDeployed)
{
_isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy);
}
else
{
return(null);
}
}
return(this);
}
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy toplogy = OperatorsToplogy.GetInstance();
TaskBase subscribeTask = new SubscribeTask();
subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
subscribeTask.IsRightOperandInput = false;
OperatorTransforms subscribeInputTransforms = new OperatorTransforms();
Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms);
subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1);
subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2);
subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString();
subscribeTask.PrepareTaskTransformations(subscribeInputTransforms);
toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask);
toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId);
if (subscribeTask.Transforms != null)
{
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2);
toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId);
}
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1))
{
toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1);
toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId);
}
}
}
_clientTerminalTask = new ClientTerminalTask();
_clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId);
_clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString();
_clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
_craClient = new CRAClientLibrary(_dataProvider);
toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask);
toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1);
toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2);
toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId);
toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId);
_isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy);
if (_isDeployed)
{
string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString();
_craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000);
_craWorker.DisableDynamicLoading();
_craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower());
new Thread(() => _craWorker.Start()).Start();
Thread.Sleep(1000);
_isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy);
}
else
{
return(null);
}
}
return(this);
}
internal void PrepareFinalOperatorsTasks()
{
string[] operatorsIds = _operatorsIds.ToArray();
TaskBase[] tasks = _operatorsTasks.ToArray();
List <string>[] tmpTransforms = new List <string> [tasks.Length];
List <string>[] tmpTransformsOperations = new List <string> [tasks.Length];
List <string>[] tmpTransformsTypes = new List <string> [tasks.Length];
List <OperatorInputs>[] tmpTransformsInputs = new List <OperatorInputs> [tasks.Length];
for (int i = 0; i < tasks.Length; i++)
{
tmpTransforms[i] = new List <string>();
tmpTransformsOperations[i] = new List <string>();
tmpTransformsTypes[i] = new List <string>();
tmpTransformsInputs[i] = new List <OperatorInputs>();
}
if (tasks.Length >= 2)
{
for (int i = 0; i < operatorsIds.Length; i++)
{
if (tasks[i].Transforms != null)
{
int lastProcessedTransformIndex = 0;
for (int j = 0; j < tasks[i].Transforms.Length; j++)
{
var currentInput = tasks[i].TransformsInputs[j].InputId1;
int inputTaskIndex = DeploymentUtils.RetrieveTaskIndexOfOperator(currentInput, operatorsIds);
if (tmpTransforms[inputTaskIndex].Count > 0 &&
tmpTransformsOperations[inputTaskIndex][tmpTransforms[inputTaskIndex].Count - 1] == OperatorType.MoveSplit.ToString())
{
break;
}
else
{
tmpTransforms[inputTaskIndex].Add(tasks[i].Transforms[j]);
tmpTransformsOperations[inputTaskIndex].Add(tasks[i].TransformsOperations[j]);
tmpTransformsTypes[inputTaskIndex].Add(tasks[i].TransformsTypes[j]);
tmpTransformsInputs[inputTaskIndex].Add(tasks[i].TransformsInputs[j]);
if (tasks[i].TransformsOperations[j] == OperatorType.BinaryTransform.ToString())
{
var currentSecondaryInput = tasks[i].TransformsInputs[j].InputId2;
if (tasks[inputTaskIndex].OperationType != OperatorType.Move && tasks[i].OperationType != OperatorType.Move)
{
UpdateOperatorsSecondaryInput(operatorsIds[inputTaskIndex], operatorsIds[i], currentSecondaryInput);
}
else if (tasks[inputTaskIndex].OperationType != OperatorType.Move && tasks[i].OperationType == OperatorType.Move)
{
UpdateOperatorsSecondaryInput(operatorsIds[inputTaskIndex], ((ShuffleTask)tasks[i]).MapperVertexName, currentSecondaryInput);
}
else if (tasks[inputTaskIndex].OperationType == OperatorType.Move && tasks[i].OperationType != OperatorType.Move)
{
UpdateOperatorsSecondaryInput(((ShuffleTask)tasks[inputTaskIndex]).ReducerVertexName, operatorsIds[i], currentSecondaryInput);
}
else
{
UpdateOperatorsSecondaryInput(((ShuffleTask)tasks[inputTaskIndex]).ReducerVertexName, ((ShuffleTask)tasks[i]).MapperVertexName, currentSecondaryInput);
}
}
lastProcessedTransformIndex++;
}
}
for (int k = lastProcessedTransformIndex; k < tasks[i].Transforms.Length; k++)
{
tmpTransforms[i].Add(tasks[i].Transforms[k]);
tmpTransformsOperations[i].Add(tasks[i].TransformsOperations[k]);
tmpTransformsTypes[i].Add(tasks[i].TransformsTypes[k]);
tmpTransformsInputs[i].Add(tasks[i].TransformsInputs[k]);
}
}
if (tasks[i].OperationType == OperatorType.Move)
{
var currentInput = ((ShuffleTask)tasks[i]).ShuffleTransformsInputs[0].InputId1;
int inputTaskIndex = DeploymentUtils.RetrieveTaskIndexOfOperator(currentInput, operatorsIds);
if (!(tmpTransforms[inputTaskIndex].Count > 0 &&
tmpTransformsOperations[inputTaskIndex][tmpTransforms[inputTaskIndex].Count - 1] == OperatorType.MoveSplit.ToString()))
{
tmpTransforms[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransforms[0]);
tmpTransformsOperations[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsOperations[0]);
tmpTransformsTypes[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsTypes[0]);
tmpTransformsInputs[inputTaskIndex].Add(((ShuffleTask)tasks[i]).ShuffleTransformsInputs[0]);
tasks[inputTaskIndex].SecondaryShuffleDescriptor = ((ShuffleTask)tasks[i]).ShuffleDescriptor;
}
}
}
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].Transforms = tmpTransforms[i].ToArray();
tasks[i].TransformsOperations = tmpTransformsOperations[i].ToArray();
tasks[i].TransformsTypes = tmpTransformsTypes[i].ToArray();
tasks[i].TransformsInputs = tmpTransformsInputs[i].ToArray();
}
_operatorsTasks = new List <TaskBase>(tasks);
}
}