public void TestBroadcastOperatorWithDefaultCodec() { INameServer nameServer = NameServerTests.BuildNameServer(); string groupName = "group1"; string operatorName = "broadcast"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 10; int value = 1337; int fanOut = 3; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); var commGroup = groupCommDriver.DefaultGroup .AddBroadcast(operatorName, masterTaskId) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName); IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); sender.Send(value); Assert.Equal(value, receiver1.Receive()); Assert.Equal(value, receiver2.Receive()); }
public void TestLookupList() { using (INameServer server = BuildNameServer()) { using (INameClient client = BuildNameClient(server.LocalEndpoint)) { IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100); IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200); IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300); // Register endpoints client.Register("a", endpoint1); client.Register("b", endpoint2); client.Register("c", endpoint3); // Look up both at the same time List <string> ids = new List <string> { "a", "b", "c", "d" }; List <NameAssignment> assignments = client.Lookup(ids); // Check that a, b, and c are registered Assert.Equal("a", assignments[0].Identifier); Assert.Equal(endpoint1, assignments[0].Endpoint); Assert.Equal("b", assignments[1].Identifier); Assert.Equal(endpoint2, assignments[1].Endpoint); Assert.Equal("c", assignments[2].Identifier); Assert.Equal(endpoint3, assignments[2].Endpoint); // Check that d is not registered Assert.Equal(3, assignments.Count); } } }
public void TestRegister() { using (INameServer server = BuildNameServer()) { using (INameClient client = BuildNameClient(server.LocalEndpoint)) { IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100); IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200); IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300); // Check that no endpoints have been registered Assert.Null(client.Lookup("a")); Assert.Null(client.Lookup("b")); Assert.Null(client.Lookup("c")); // Register endpoints client.Register("a", endpoint1); client.Register("b", endpoint2); client.Register("c", endpoint3); // Check that they can be looked up correctly Assert.Equal(endpoint1, client.Lookup("a")); Assert.Equal(endpoint2, client.Lookup("b")); Assert.Equal(endpoint3, client.Lookup("c")); } } }
private GroupCommDriver( [Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId, [Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId, [Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut, [Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName, [Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks, AvroConfigurationSerializer configSerializer, INameServer nameServer) { _driverId = driverId; _contextIds = -1; _fanOut = fanOut; MasterTaskId = masterTaskId; _groupName = groupName; _configSerializer = configSerializer; _commGroups = new Dictionary <string, ICommunicationGroupDriver>(); IPEndPoint localEndpoint = nameServer.LocalEndpoint; _nameServerAddr = localEndpoint.Address.ToString(); _nameServerPort = localEndpoint.Port; NewCommunicationGroup(groupName, numberOfTasks); }
public GroupCommDriver( [Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId, [Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId, [Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut, [Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName, [Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks, AvroConfigurationSerializer configSerializer, INameServer nameServer) { _driverId = driverId; _contextIds = -1; _fanOut = fanOut; MasterTaskId = masterTaskId; _groupName = groupName; _configSerializer = configSerializer; _commGroups = new Dictionary<string, ICommunicationGroupDriver>(); _nameServer = nameServer; IPEndPoint localEndpoint = _nameServer.LocalEndpoint; _nameServerAddr = localEndpoint.Address.ToString(); _nameServerPort = localEndpoint.Port; NewCommunicationGroup(groupName, numberOfTasks); }
private IMRUDriver(IPartitionedInputDataSet dataSet, [Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs, ConfigurationManager configurationManager, IEvaluatorRequestor evaluatorRequestor, [Parameter(typeof(CoresPerMapper))] int coresPerMapper, [Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask, [Parameter(typeof(MemoryPerMapper))] int memoryPerMapper, [Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask, [Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction, [Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery, [Parameter(typeof(InvokeGC))] bool invokeGC, IGroupCommDriver groupCommDriver, INameServer nameServer) { _configurationManager = configurationManager; _groupCommDriver = groupCommDriver; _nameServer = nameServer; _perMapperConfigs = perMapperConfigs; _totalMappers = dataSet.Count; _invokeGC = invokeGC; _maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery; _contextManager = new ActiveContextManager(_totalMappers + 1); _contextManager.Subscribe(this); var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask); var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper); var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers); _evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec); _systemState = new SystemStateMachine(); _serviceAndContextConfigurationProvider = new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager); var msg = string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.", memoryPerMapper, memoryForUpdateTask, coresPerMapper, coresForUpdateTask, _maxRetryNumberForFaultTolerant, allowedFailedEvaluators); Logger.Log(Level.Info, msg); }
public void TestConstructorInjection() { int port = 6666; using (INameServer server = BuildNameServer(port)) { IConfiguration nameClientConfiguration = NamingConfiguration.ConfigurationModule .Set(NamingConfiguration.NameServerAddress, server.LocalEndpoint.Address.ToString()) .Set(NamingConfiguration.NameServerPort, port + string.Empty) .Build(); ConstructorInjection c = TangFactory.GetTang() .NewInjector(nameClientConfiguration) .GetInstance <ConstructorInjection>(); Assert.NotNull(c); } }
public void TestLookup() { using (INameServer server = BuildNameServer()) { using (INameClient client = BuildNameClient(server.LocalEndpoint)) { IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100); IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200); // Register endpoint1 client.Register("a", endpoint1); Assert.Equal(endpoint1, client.Lookup("a")); // Reregister identifer a client.Register("a", endpoint2); Assert.Equal(endpoint2, client.Lookup("a")); } } }
public void TestUnregister() { using (INameServer server = BuildNameServer()) { using (INameClient client = BuildNameClient(server.LocalEndpoint)) { IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100); // Register endpoint client.Register("a", endpoint1); // Check that it can be looked up correctly Assert.Equal(endpoint1, client.Lookup("a")); // Unregister endpoints client.Unregister("a"); Thread.Sleep(1000); // Make sure they were unregistered correctly Assert.Null(client.Lookup("a")); } } }
public void TestNameClientRestart() { int oldPort = 6666; int newPort = 6662; INameServer server = BuildNameServer(oldPort); using (INameClient client = BuildNameClient(server.LocalEndpoint)) { IPEndPoint endpoint = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100); client.Register("a", endpoint); Assert.Equal(endpoint, client.Lookup("a")); server.Dispose(); server = BuildNameServer(newPort); client.Restart(server.LocalEndpoint); client.Register("b", endpoint); Assert.Equal(endpoint, client.Lookup("b")); server.Dispose(); } }