private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < shardsCount; j++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + j; string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex + j, fromVertexDescriptor); string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex + i, toVertexDescriptor); bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int k = 0; k < currentOutputs.Length; k++) { fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances)); } } } return(fromToConnections); }
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task, ref DetachedVertex clientTerminal, OperatorsToplogy topology) { try { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor); string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator( task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]); int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap()); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < inputEndpoints.Length; j++) { clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]); } } } return(true); } catch (Exception e) { Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString()); return(false); } }
public static async Task <bool> DeployClientTerminal( CRAClientLibrary client, string workerName, ClientTerminalTask task, OperatorsToplogy topology) { try { bool result = true; client.DisableArtifactUploading(); if (!_isSubscribeClientOperatorDefined) { await client.DefineVertexAsync(typeof(ShardedSubscribeClientOperator).Name.ToLower(), () => new ShardedSubscribeClientOperator()); _isSubscribeClientOperatorDefined = true; } var status = await client.InstantiateVertexAsync(new string[] { workerName }, task.OutputId, typeof(ShardedSubscribeClientOperator).Name.ToLower(), task, 1); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { string outputEndpoint = OperatorUtils.PrepareOutputEndpointIdForOperator( task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]); string inputEndpoint = OperatorUtils.PrepareInputEndpointIdForOperator(fromInputId, task.EndpointsDescriptor, false); await client.ConnectAsync(fromInputId, outputEndpoint, task.OutputId, inputEndpoint); } result = true; } else { result = false; } client.EnableArtifactUploading(); return(result); } catch (Exception e) { Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString()); return(false); } }
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor); string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput); bool hasSameCRAInstances = true; for (int i = 0; i < shardsCount; i++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + i; hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); } fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput)); return(fromToConnections); }
protected string[] ToEndpointsIds(OperatorEndpointsDescriptor endpointDescriptor, bool isToOutput) { List <string> endpointsIds = new List <string>(); if (isToOutput) { foreach (string endpointId in endpointDescriptor.ToOutputs.Keys) { endpointsIds.AddRange( OperatorUtils.PrepareOutputEndpointsIdsForOperator(endpointId, endpointDescriptor)); } } else { foreach (string endpointId in endpointDescriptor.FromInputs.Keys) { endpointsIds.AddRange( OperatorUtils.PrepareInputEndpointsIdsForOperator(endpointId, endpointDescriptor)); } } return(endpointsIds.ToArray()); }
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex, fromVertexDescriptor); string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex, toVertexDescriptor); for (int i = 0; i < shardsCount; i++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + i; bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int j = 0; j < outputs.Length; j++) { var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]); var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]); fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances)); } } return(fromToConnections); }