internal void AddOperatorOutput(string taskId, string toOutputId, int outputsCount = 1) { if (taskId != null && taskId != "$" && toOutputId != null && toOutputId != "$") { OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId]; operatorDescriptor.ToOutputs.AddOrUpdate(toOutputId, outputsCount, (outputId, currentOutputs) => outputsCount); _operatorsEndpointsDescriptors[taskId] = operatorDescriptor; } }
internal void AddOperatorSecondaryInput(string taskId, string secondaryFromInputId, int inputsCount = 1) { if (taskId != null && taskId != "$" && secondaryFromInputId != null && secondaryFromInputId != "$") { OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId]; operatorDescriptor.SecondaryFromInputs.AddOrUpdate(secondaryFromInputId, inputsCount, (inputId, currentInputs) => inputsCount); _operatorsEndpointsDescriptors[taskId] = operatorDescriptor; } }
private void UpdateOperatorsSecondaryInputs(string firstOperatorId, string secondOperatorId) { OperatorEndpointsDescriptor secondOperatorDescriptor = _operatorsEndpointsDescriptors[secondOperatorId]; var secondaryInputs = secondOperatorDescriptor.SecondaryFromInputs; foreach (string inputId in secondaryInputs.Keys) { RemoveOperatorOutput(inputId, secondOperatorId); AddOperatorInput(firstOperatorId, inputId, secondaryInputs[inputId]); AddOperatorOutput(inputId, firstOperatorId, secondaryInputs[inputId]); } }
private int RemoveOperatorOutput(string taskId, string toOutputId) { int updatedOutputs = 0; if (taskId != null && toOutputId != null) { OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId]; operatorDescriptor.ToOutputs.TryRemove(toOutputId, out updatedOutputs); _operatorsEndpointsDescriptors[taskId] = operatorDescriptor; } return(updatedOutputs); }
private int RemoveOperatorInput(string taskId, string fromInputId) { int updatedInputs = 0; if (taskId != null && fromInputId != null) { OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId]; operatorDescriptor.FromInputs.TryRemove(fromInputId, out updatedInputs); _operatorsEndpointsDescriptors[taskId] = operatorDescriptor; } return(updatedInputs); }
internal bool ContainsSecondaryOperatorInput(string taskId, string operatorId) { OperatorEndpointsDescriptor operatorDescriptor = _operatorsEndpointsDescriptors[taskId]; if (operatorDescriptor.SecondaryFromInputs.ContainsKey(operatorId)) { return(true); } else { return(false); } }
private void UpdateOperatorsSecondaryInput(string firstOperatorId, string secondOperatorId, string secondaryInputId) { OperatorEndpointsDescriptor secondOperatorDescriptor = _operatorsEndpointsDescriptors[secondOperatorId]; var secondaryInputs = secondOperatorDescriptor.SecondaryFromInputs; var secondOperatorInputs = secondOperatorDescriptor.FromInputs; if (!secondOperatorInputs.ContainsKey(secondaryInputId)) { RemoveOperatorOutput(secondaryInputId, secondOperatorId); } AddOperatorInput(firstOperatorId, secondaryInputId, secondaryInputs[secondaryInputId]); AddOperatorOutput(secondaryInputId, firstOperatorId, secondaryInputs[secondaryInputId]); }
protected string[] ToEndpointsIds(OperatorEndpointsDescriptor endpointDescriptor, bool isToOutput) { List <string> endpointsIds = new List <string>(); if (isToOutput) { foreach (string endpointId in endpointDescriptor.ToOutputs.Keys) { endpointsIds.AddRange( OperatorUtils.PrepareOutputEndpointsIdsForOperator(endpointId, endpointDescriptor)); } } else { foreach (string endpointId in endpointDescriptor.FromInputs.Keys) { endpointsIds.AddRange( OperatorUtils.PrepareInputEndpointsIdsForOperator(endpointId, endpointDescriptor)); } } return(endpointsIds.ToArray()); }
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor); string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput); bool hasSameCRAInstances = true; for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < shardsCount; j++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + j; hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); } } fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput)); return(fromToConnections); }
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < shardsCount; j++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + j; string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex + j, fromVertexDescriptor); string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex + i, toVertexDescriptor); bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int k = 0; k < currentOutputs.Length; k++) { fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances)); } } } return(fromToConnections); }
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex, fromVertexDescriptor); string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex, toVertexDescriptor); for (int i = 0; i < shardsCount; i++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + i; bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int j = 0; j < outputs.Length; j++) { var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]); var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]); fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances)); } } return(fromToConnections); }