public static async Task <bool> DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology) { try { if (!_isSubscribeOperatorDefined) { await client.DefineVertexAsync(typeof(ShardedSubscribeOperator).Name.ToLower(), () => new ShardedSubscribeOperator()); _isSubscribeOperatorDefined = true; } var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.OutputId, typeof(ShardedSubscribeOperator).Name.ToLower(), task, 1); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.OutputId][0]; await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint); } return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString()); return(false); } }
static void Main(string[] args) { var client = new CRAClientLibrary(new FileDataProvider()); int chunkSize = 10 * 1024 * 1024; Console.ReadLine(); client.DefineVertexAsync("bandwidthtestvertex", () => new BandwidthTestVertex()).Wait(); client.InstantiateVertexAsync("crainst01", "bwvertex1", "bandwidthtestvertex", chunkSize).Wait(); client.InstantiateVertexAsync("crainst02", "bwvertex2", "bandwidthtestvertex", chunkSize).Wait(); // client.InstantiateVertexAsync("crainst03", "bwvertex3", "bandwidthtestvertex", chunkSize).Wait(); client.ConnectAsync("bwvertex1", "output1", "bwvertex2", "input1").Wait(); client.ConnectAsync("bwvertex1", "output2", "bwvertex2", "input2").Wait(); client.ConnectAsync("bwvertex1", "output3", "bwvertex2", "input3").Wait(); client.ConnectAsync("bwvertex1", "output4", "bwvertex2", "input4").Wait(); client.ConnectAsync("bwvertex1", "output5", "bwvertex2", "input5").Wait(); client.ConnectAsync("bwvertex1", "output6", "bwvertex2", "input6").Wait(); client.ConnectAsync("bwvertex1", "output7", "bwvertex2", "input7").Wait(); client.ConnectAsync("bwvertex1", "output8", "bwvertex2", "input8").Wait(); client.ConnectAsync("bwvertex2", "output9", "bwvertex1", "input9").Wait(); client.ConnectAsync("bwvertex2", "output10", "bwvertex1", "input10").Wait(); client.ConnectAsync("bwvertex2", "output11", "bwvertex1", "input11").Wait(); client.ConnectAsync("bwvertex2", "output12", "bwvertex1", "input12").Wait(); client.ConnectAsync("bwvertex2", "output13", "bwvertex1", "input13").Wait(); client.ConnectAsync("bwvertex2", "output14", "bwvertex1", "input14").Wait(); client.ConnectAsync("bwvertex2", "output15", "bwvertex1", "input15").Wait(); client.ConnectAsync("bwvertex2", "output16", "bwvertex1", "input16").Wait(); Console.ReadLine(); client.ResetAsync().Wait(); }
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task, ref DetachedVertex clientTerminal, OperatorsToplogy topology) { try { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor); string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator( task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]); int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap()); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < inputEndpoints.Length; j++) { clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]); } } } return(true); } catch (Exception e) { Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString()); return(false); } }
private static bool DeployShuffleReduceTask(CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology) { try { client.DefineVertex(typeof(ShuffleOperator).Name.ToLower(), () => new ShuffleOperator()); CRAErrorCode status = client.InstantiateShardedVertex(task.ReducerVertexName, typeof(ShuffleOperator).Name.ToLower(), task, task.DeployDescriptor.InstancesMap()); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.SecondaryEndpointsDescriptor.FromInputs.Keys) { client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]); } var fromInput = task.EndpointsDescriptor.FromInputs.First().Key; var newFromInputId = fromInput.Substring(0, fromInput.Length - 1); client.ConnectShardedVertices(task.VerticesConnectionsMap[newFromInputId + task.ReducerVertexName]); return(true); } else { return(false); } } catch (Exception) { Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations"); return(false); } }
public static bool DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology) { try { client.DefineVertex(typeof(SubscribeOperator).Name.ToLower(), () => new SubscribeOperator()); CRAErrorCode status = client.InstantiateShardedVertex(task.OutputId, typeof(SubscribeOperator).Name.ToLower(), task, task.DeployDescriptor.InstancesMap()); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]); } return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString()); return(false); } }
static void Main(string[] args) { if (args.Length != 1) { Console.WriteLine("UnsafeDeregisterInstance InstanceName"); Console.WriteLine("WARNING: This is a metadata hacking tool that should NEVER be used on a real deployment"); Console.WriteLine("This tool is a convenience for developers who want to more easily test certain application modfications"); Console.WriteLine("Usage: UnsafeDeregisterInstance InstanceName"); return; } var client = new CRAClientLibrary(Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING")); var serviceName = args[0]; foreach (var endpt in client.GetInputEndpoints(serviceName)) { client.DeleteEndpoint(serviceName, endpt); } foreach (var endpt in client.GetOutputEndpoints(serviceName)) { client.DeleteEndpoint(serviceName, endpt); } foreach (var conn in client.GetConnectionsFromVertex(serviceName)) { client.DeleteConnectionInfo(conn); } foreach (var conn in client.GetConnectionsToVertex(serviceName)) { client.DeleteConnectionInfo(conn); } try { client.DeleteVertex(serviceName); } catch { } }
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap( CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology) { ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >(); for (int i = 0; i < tasks.Length; i++) { int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap()); tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), false); verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } foreach (string secondaryFromInputId in tasks[i].EndpointsDescriptor.SecondaryFromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, secondaryFromInputId, topology.OperatorsEndpointsDescriptors[secondaryFromInputId], tasksDictionary[secondaryFromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), true); verticesConnectionsMap.AddOrUpdate(secondaryFromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } } return(verticesConnectionsMap); }
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); var tasksDeploymentStatus = new Dictionary <string, bool>(); for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; tasksDeploymentStatus.Add(tasksIds[i], false); } bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 && tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0) { isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[i]] = true; } } } for (int i = 0; i < tasks.Length; i++) { isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology); } return(isSuccessful); }
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology) { topology.PrepareFinalOperatorsTasks(); string[] tasksIds = topology.OperatorsIds.ToArray(); TaskBase[] tasks = topology.OperatorsTasks.ToArray(); for (int i = 0; i < tasks.Length; i++) { if (tasks[i].OperationType == OperatorType.Move) { ShuffleTask shuffleTask = (ShuffleTask)tasks[i]; shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs; topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>(); int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap()); topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount); topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask); } } var tasksDictionary = PrepareTasksDictionary(tasks); var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology); bool isSuccessful = true; for (int i = 0; i < tasks.Length; i++) { tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; tasks[i].VerticesConnectionsMap = connectionsMap; if (tasks[i].OperationType == OperatorType.Produce) { isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]); } else if (tasks[i].OperationType == OperatorType.Subscribe) { isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology); } else if (tasks[i].OperationType == OperatorType.Move) { isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology); } if (!isSuccessful) { break; } } return(isSuccessful); }
static void Main(string[] args) { var client = new CRAClientLibrary(); client.ResetAsync().Wait(); client.DefineVertexAsync("fusableconnectionpairvertex", () => new FusableConnectionPairVertex()).Wait(); client.InstantiateVertexAsync("crainst01", "fvertex1", "fusableconnectionpairvertex", null).Wait(); client.InstantiateVertexAsync("crainst01", "fvertex2", "fusableconnectionpairvertex", null).Wait(); client.ConnectAsync("fvertex1", "output", "fvertex2", "input").Wait(); Console.ReadLine(); }
public static async Task <bool> DeployClientTerminal( CRAClientLibrary client, string workerName, ClientTerminalTask task, OperatorsToplogy topology) { try { bool result = true; client.DisableArtifactUploading(); if (!_isSubscribeClientOperatorDefined) { await client.DefineVertexAsync(typeof(ShardedSubscribeClientOperator).Name.ToLower(), () => new ShardedSubscribeClientOperator()); _isSubscribeClientOperatorDefined = true; } var status = await client.InstantiateVertexAsync(new string[] { workerName }, task.OutputId, typeof(ShardedSubscribeClientOperator).Name.ToLower(), task, 1); if (status == CRAErrorCode.Success) { foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys) { string outputEndpoint = OperatorUtils.PrepareOutputEndpointIdForOperator( task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]); string inputEndpoint = OperatorUtils.PrepareInputEndpointIdForOperator(fromInputId, task.EndpointsDescriptor, false); await client.ConnectAsync(fromInputId, outputEndpoint, task.OutputId, inputEndpoint); } result = true; } else { result = false; } client.EnableArtifactUploading(); return(result); } catch (Exception e) { Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString()); return(false); } }
public ClientSideShardedDataset( IDataProvider dataProvider, Expression <Func <int, TDataset> > producer) : base(dataProvider) { if (producer != null) { _producer = new ClosureEliminator().Visit(producer) as Expression <Func <int, TDataset> >; } else { Console.WriteLine("The producer expression of the ShardedDataset should be provided !!"); } _dataProvider = dataProvider; _craClient = new CRAClientLibrary(_dataProvider); }
public override IShardedDataset <TKey, TPayload, TDataset> Deploy() { if (!_isDeployed) { OperatorsToplogy operatorsTopology = OperatorsToplogy.GetInstance(); GenerateProduceTask(ref operatorsTopology); _craClient = new CRAClientLibrary(); _isDeployed = DeploymentUtils.DeployOperators(_craClient, operatorsTopology); if (!_isDeployed) { return(null); } } return(this); }
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap( CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology) { ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >(); for (int i = 0; i < tasks.Length; i++) { int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap()); tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]]; if (tasks[i].OperationType == OperatorType.Move) { var shuffleTask = (ShuffleTask)tasks[i]; foreach (string fromInputId in shuffleTask.SecondaryEndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(fromInputId + shuffleTask.OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } var fromInput = shuffleTask.EndpointsDescriptor.FromInputs.First().Key; var newFromInputId = fromInput.Substring(0, fromInput.Length - 1); var shuffleFromToConnections = PrepareShuffleConnectionsMap(client, shardsCount, newFromInputId, topology.OperatorsEndpointsDescriptors[newFromInputId], tasksDictionary[newFromInputId].DeployDescriptor.InstancesMap(), shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(newFromInputId + shuffleTask.ReducerVertexName, shuffleFromToConnections, (key, value) => shuffleFromToConnections); } else { foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys) { var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount, fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(), tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap()); verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId, flatFromToConnections, (key, value) => flatFromToConnections); } } } return(verticesConnectionsMap); }
static void Main(string[] args) { var client = new CRAClientLibrary(); client.DefineVertex("connectionpairvertex", () => new ConnectionPairVertex()); client.InstantiateVertex("crainst01", "vertex1", "connectionpairvertex", new MyParam { field1 = 33, field2 = "foo" }); client.InstantiateVertex("crainst02", "vertex2", "connectionpairvertex", new MyParam { field1 = 34 }); client.Connect("vertex1", "output", "vertex2", "input"); client.Connect("vertex2", "output", "vertex1", "input"); Console.ReadLine(); }
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor); string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput); bool hasSameCRAInstances = true; for (int i = 0; i < shardsCount; i++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + i; hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); } fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput)); return(fromToConnections); }
public static bool DeployProduceTask(CRAClientLibrary client, ProduceTask task) { try { client.DefineVertex(typeof(ProducerOperator).Name.ToLower(), () => new ProducerOperator()); CRAErrorCode status = client.InstantiateShardedVertex(task.OutputId, typeof(ProducerOperator).Name.ToLower(), task, task.DeployDescriptor.InstancesMap()); if (status == CRAErrorCode.Success) { return(true); } else { return(false); } } catch (Exception e) { Console.WriteLine("Error in deploying a sharded CRA produce task. Please, double check your task configurations: " + e.ToString()); return(false); } }
static void Main(string[] args) { var client = new CRAClientLibrary(); client.DefineVertex("shardedconnectionpairvertex", () => new ShardedConnectionPairVertex()); client.InstantiateVertex( new[] { "crainst01", "crainst02" }, "vertex1", "shardedconnectionpairvertex", null, 1, e => e % 2); client.InstantiateVertex( new[] { "crainst01", "crainst02" }, "vertex2", "shardedconnectionpairvertex", null, 1, e => e % 2); client.Connect("vertex1", "output", "vertex2", "input"); client.Connect("vertex2", "output", "vertex1", "input"); Console.ReadLine(); }
static void Main(string[] args) { var client = new CRAClientLibrary(); client.DefineVertexAsync("shardedconnectionpairvertex", (System.Linq.Expressions.Expression <Func <IVertex> >)(() => new ShardedConnectionPairVertex())).Wait(); client.InstantiateVertexAsync( new[] { "crainst01", "crainst02" }, "vertex1", "shardedconnectionpairvertex", null, 1, e => e % 2).Wait(); client.InstantiateVertexAsync( new[] { "crainst01", "crainst02" }, "vertex2", "shardedconnectionpairvertex", null, 1, e => e % 2).Wait(); client.ConnectAsync("vertex1", "output", "vertex2", "input").Wait(); client.ConnectAsync("vertex2", "output", "vertex1", "input").Wait(); Console.ReadLine(); }
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex, fromVertexDescriptor); string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex, toVertexDescriptor); for (int i = 0; i < shardsCount; i++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + i; bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int j = 0; j < outputs.Length; j++) { var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]); var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]); fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances)); } } return(fromToConnections); }
public OperatorBase(IDataProvider dataProvider) : base() { _craClient = new CRAClientLibrary(dataProvider); }
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount, string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards, string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards) { List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>(); for (int i = 0; i < shardsCount; i++) { for (int j = 0; j < shardsCount; j++) { string currentFromVertex = fromVertex + "$" + i; string currentToVertex = toVertex + "$" + j; string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator( toVertex + j, fromVertexDescriptor); string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator( fromVertex + i, toVertexDescriptor); bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards); for (int k = 0; k < currentOutputs.Length; k++) { fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances)); } } } return(fromToConnections); }
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } } ClientTerminalTask clientTerminalTask = new ClientTerminalTask(); clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString(); _craClient = new CRAClientLibrary(); _clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId); toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId); toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId); _isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { _isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy); } else { return(null); } } return(this); }
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy() { if (!_isDeployed) { OperatorsToplogy toplogy = OperatorsToplogy.GetInstance(); TaskBase subscribeTask = new SubscribeTask(); subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); subscribeTask.IsRightOperandInput = false; OperatorTransforms subscribeInputTransforms = new OperatorTransforms(); Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms); subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1); subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2); subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString(); subscribeTask.PrepareTaskTransformations(subscribeInputTransforms); toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask); toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1); toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId); toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId); if (subscribeTask.Transforms != null) { foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2); toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId); } foreach (OperatorInputs inputs in subscribeTask.TransformsInputs) { if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1)) { toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1); toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId); } } } _clientTerminalTask = new ClientTerminalTask(); _clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId); _clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString(); _clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes( typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO), typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO)); _craClient = new CRAClientLibrary(_dataProvider); toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1); toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId); toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId); _isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy); if (_isDeployed) { string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString(); _craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000); _craWorker.DisableDynamicLoading(); _craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower()); new Thread(() => _craWorker.Start()).Start(); Thread.Sleep(1000); _isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy); } else { return(null); } } return(this); }
private static async Task <bool> DeployTask(int taskIndex, TaskBase[] tasks, string[] tasksIds, Dictionary <string, bool> tasksDeploymentStatus, CRAClientLibrary client, OperatorsToplogy topology) { if (!tasksDeploymentStatus[tasksIds[taskIndex]]) { bool isSuccessful = true; foreach (var fromInput in tasks[taskIndex].EndpointsDescriptor.FromInputs.Keys) { int fromInputIndex = RetrieveTaskIndexOfOperator(fromInput, tasksIds); isSuccessful = isSuccessful & await DeployTask(fromInputIndex, tasks, tasksIds, tasksDeploymentStatus, client, topology); } foreach (var fromSecondaryInput in tasks[taskIndex].EndpointsDescriptor.SecondaryFromInputs.Keys) { int fromSecondaryInputIndex = RetrieveTaskIndexOfOperator(fromSecondaryInput, tasksIds); isSuccessful = isSuccessful & await DeployTask(fromSecondaryInputIndex, tasks, tasksIds, tasksDeploymentStatus, client, topology); } if (isSuccessful) { if (tasks[taskIndex].OperationType == OperatorType.Produce) { isSuccessful = isSuccessful & await DeployProduceTask(client, (ProduceTask)tasks[taskIndex], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[taskIndex]] = true; } } else if (tasks[taskIndex].OperationType == OperatorType.Subscribe) { isSuccessful = isSuccessful & await DeploySubscribeTask(client, (SubscribeTask)tasks[taskIndex], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[taskIndex]] = true; } } else if (tasks[taskIndex].OperationType == OperatorType.Move) { isSuccessful = isSuccessful & await DeployShuffleReduceTask(client, (ShuffleTask)tasks[taskIndex], topology); if (isSuccessful) { tasksDeploymentStatus[tasksIds[taskIndex]] = true; } } } return(isSuccessful); } else { return(true); } }
static void Main(string[] args) { GenericLogsInterface.SetToGenericLogs(); ParseAndValidateOptions(args); Trace.Listeners.Add(new TextWriterTraceListener(Console.Out)); switch (_runtimeMode) { case LocalAmbrosiaRuntimeModes.DebugInstance: var myRuntime = new AmbrosiaRuntime(); myRuntime.InitializeRepro(_instanceName, _serviceLogPath, _checkpointToLoad, _currentVersion, _isTestingUpgrade, _serviceReceiveFromPort, _serviceSendToPort); return; case LocalAmbrosiaRuntimeModes.AddReplica: case LocalAmbrosiaRuntimeModes.RegisterInstance: if (_runtimeMode == LocalAmbrosiaRuntimeModes.AddReplica) { _isActiveActive = true; } var dataProvider = new CRA.DataProvider.Azure.AzureDataProvider(Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING")); var client = new CRAClientLibrary(dataProvider); client.DisableArtifactUploading(); var replicaName = $"{_instanceName}{_replicaNumber}"; AmbrosiaRuntimeParams param = new AmbrosiaRuntimeParams(); param.createService = _recoveryMode == AmbrosiaRecoveryModes.A ? (bool?)null : (_recoveryMode != AmbrosiaRecoveryModes.N); param.pauseAtStart = _isPauseAtStart; param.persistLogs = _isPersistLogs; param.logTriggerSizeMB = _logTriggerSizeMB; param.activeActive = _isActiveActive; param.upgradeToVersion = _upgradeVersion; param.currentVersion = _currentVersion; param.serviceReceiveFromPort = _serviceReceiveFromPort; param.serviceSendToPort = _serviceSendToPort; param.serviceName = _instanceName; param.serviceLogPath = _serviceLogPath; param.AmbrosiaBinariesLocation = _binariesLocation; param.storageConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING"); try { if (client.DefineVertexAsync(param.AmbrosiaBinariesLocation, () => new AmbrosiaRuntime()).GetAwaiter().GetResult() != CRAErrorCode.Success) { throw new Exception(); } // Workaround because of limitation in parameter serialization in CRA XmlSerializer xmlSerializer = new XmlSerializer(param.GetType()); string serializedParams; using (StringWriter textWriter = new StringWriter()) { xmlSerializer.Serialize(textWriter, param); serializedParams = textWriter.ToString(); } if (client.InstantiateVertexAsync(replicaName, param.serviceName, param.AmbrosiaBinariesLocation, serializedParams).GetAwaiter().GetResult() != CRAErrorCode.Success) { throw new Exception(); } client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true).Wait(); client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true).Wait(); client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaControlInputsName, true, true).Wait(); client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaControlOutputsName, false, true).Wait(); } catch (Exception e) { Console.WriteLine("Error trying to upload service. Exception: " + e.Message); } return; default: throw new NotSupportedException($"Runtime mode: {_runtimeMode} not supported."); } }
public OperatorBase() : base() { _craClient = new CRAClientLibrary(); }