internal void AddOperatorOutput(string taskId, string toOutputId, int outputsCount = 1)
{
if (taskId != null && taskId != "$" && toOutputId != null && toOutputId != "$")
{
OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId];
operatorDescriptor.ToOutputs.AddOrUpdate(toOutputId, outputsCount, (outputId, currentOutputs) => outputsCount);
_operatorsEndpointsDescriptors[taskId] = operatorDescriptor;
}
}
internal void AddOperatorSecondaryInput(string taskId, string secondaryFromInputId, int inputsCount = 1)
{
if (taskId != null && taskId != "$" && secondaryFromInputId != null && secondaryFromInputId != "$")
{
OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId];
operatorDescriptor.SecondaryFromInputs.AddOrUpdate(secondaryFromInputId, inputsCount, (inputId, currentInputs) => inputsCount);
_operatorsEndpointsDescriptors[taskId] = operatorDescriptor;
}
}
private void UpdateOperatorsSecondaryInputs(string firstOperatorId, string secondOperatorId)
{
OperatorEndpointsDescriptor secondOperatorDescriptor = _operatorsEndpointsDescriptors[secondOperatorId];
var secondaryInputs = secondOperatorDescriptor.SecondaryFromInputs;
foreach (string inputId in secondaryInputs.Keys)
{
RemoveOperatorOutput(inputId, secondOperatorId);
AddOperatorInput(firstOperatorId, inputId, secondaryInputs[inputId]);
AddOperatorOutput(inputId, firstOperatorId, secondaryInputs[inputId]);
}
}
private int RemoveOperatorOutput(string taskId, string toOutputId)
{
int updatedOutputs = 0;
if (taskId != null && toOutputId != null)
{
OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId];
operatorDescriptor.ToOutputs.TryRemove(toOutputId, out updatedOutputs);
_operatorsEndpointsDescriptors[taskId] = operatorDescriptor;
}
return(updatedOutputs);
}
private int RemoveOperatorInput(string taskId, string fromInputId)
{
int updatedInputs = 0;
if (taskId != null && fromInputId != null)
{
OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId];
operatorDescriptor.FromInputs.TryRemove(fromInputId, out updatedInputs);
_operatorsEndpointsDescriptors[taskId] = operatorDescriptor;
}
return(updatedInputs);
}
internal bool ContainsSecondaryOperatorInput(string taskId, string operatorId)
{
OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId];
if (operatorDescriptor.SecondaryFromInputs.ContainsKey(operatorId))
{
return(true);
}
else
{
return(false);
}
}
private void UpdateOperatorsSecondaryInput(string firstOperatorId, string secondOperatorId, string secondaryInputId)
{
OperatorEndpointsDescriptor secondOperatorDescriptor = _operatorsEndpointsDescriptors[secondOperatorId];
var secondaryInputs = secondOperatorDescriptor.SecondaryFromInputs;
var secondOperatorInputs = secondOperatorDescriptor.FromInputs;
if (!secondOperatorInputs.ContainsKey(secondaryInputId))
{
RemoveOperatorOutput(secondaryInputId, secondOperatorId);
}
AddOperatorInput(firstOperatorId, secondaryInputId, secondaryInputs[secondaryInputId]);
AddOperatorOutput(secondaryInputId, firstOperatorId, secondaryInputs[secondaryInputId]);
}
protected string[] ToEndpointsIds(OperatorEndpointsDescriptor endpointDescriptor, bool isToOutput)
{
List <string> endpointsIds = new List <string>();
if (isToOutput)
{
foreach (string endpointId in endpointDescriptor.ToOutputs.Keys)
{
endpointsIds.AddRange(
OperatorUtils.PrepareOutputEndpointsIdsForOperator(endpointId, endpointDescriptor));
}
}
else
{
foreach (string endpointId in endpointDescriptor.FromInputs.Keys)
{
endpointsIds.AddRange(
OperatorUtils.PrepareInputEndpointsIdsForOperator(endpointId, endpointDescriptor));
}
}
return(endpointsIds.ToArray());
}
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor);
string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput);
bool hasSameCRAInstances = true;
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < shardsCount; j++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + j;
hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
}
}
fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput));
return(fromToConnections);
}
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < shardsCount; j++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + j;
string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex + j, fromVertexDescriptor);
string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex + i, toVertexDescriptor);
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int k = 0; k < currentOutputs.Length; k++)
{
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances));
}
}
}
return(fromToConnections);
}
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex, fromVertexDescriptor);
string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex, toVertexDescriptor);
for (int i = 0; i < shardsCount; i++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + i;
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int j = 0; j < outputs.Length; j++)
{
var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]);
var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]);
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances));
}
}
return(fromToConnections);
}
