private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < shardsCount; j++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + j;
string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex + j, fromVertexDescriptor);
string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex + i, toVertexDescriptor);
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int k = 0; k < currentOutputs.Length; k++)
{
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances));
}
}
}
return(fromToConnections);
}
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task,
ref DetachedVertex clientTerminal, OperatorsToplogy topology)
{
try
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor);
string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]);
int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap());
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < inputEndpoints.Length; j++)
{
clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
public static async Task <bool> DeployClientTerminal(
CRAClientLibrary client,
string workerName,
ClientTerminalTask task,
OperatorsToplogy topology)
{
try
{
bool result = true;
client.DisableArtifactUploading();
if (!_isSubscribeClientOperatorDefined)
{
await client.DefineVertexAsync(typeof(ShardedSubscribeClientOperator).Name.ToLower(), () => new ShardedSubscribeClientOperator());
_isSubscribeClientOperatorDefined = true;
}
var status = await client.InstantiateVertexAsync(new string[] { workerName }, task.OutputId, typeof(ShardedSubscribeClientOperator).Name.ToLower(), task, 1);
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
string outputEndpoint = OperatorUtils.PrepareOutputEndpointIdForOperator(
task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]);
string inputEndpoint = OperatorUtils.PrepareInputEndpointIdForOperator(fromInputId, task.EndpointsDescriptor, false);
await client.ConnectAsync(fromInputId, outputEndpoint, task.OutputId, inputEndpoint);
}
result = true;
}
else
{
result = false;
}
client.EnableArtifactUploading();
return(result);
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor);
string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput);
bool hasSameCRAInstances = true;
for (int i = 0; i < shardsCount; i++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + i;
hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
}
fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput));
return(fromToConnections);
}
protected string[] ToEndpointsIds(OperatorEndpointsDescriptor endpointDescriptor, bool isToOutput)
{
List <string> endpointsIds = new List <string>();
if (isToOutput)
{
foreach (string endpointId in endpointDescriptor.ToOutputs.Keys)
{
endpointsIds.AddRange(
OperatorUtils.PrepareOutputEndpointsIdsForOperator(endpointId, endpointDescriptor));
}
}
else
{
foreach (string endpointId in endpointDescriptor.FromInputs.Keys)
{
endpointsIds.AddRange(
OperatorUtils.PrepareInputEndpointsIdsForOperator(endpointId, endpointDescriptor));
}
}
return(endpointsIds.ToArray());
}
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex, fromVertexDescriptor);
string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex, toVertexDescriptor);
for (int i = 0; i < shardsCount; i++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + i;
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int j = 0; j < outputs.Length; j++)
{
var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]);
var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]);
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances));
}
}
return(fromToConnections);
}
