public void TestBroadcastOperatorWithDefaultCodec()
{
INameServer nameServer = NameServerTests.BuildNameServer();
string groupName = "group1";
string operatorName = "broadcast";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 10;
int value = 1337;
int fanOut = 3;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddBroadcast(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
sender.Send(value);
Assert.Equal(value, receiver1.Receive());
Assert.Equal(value, receiver2.Receive());
}
public void TestLookupList()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300);
// Register endpoints
client.Register("a", endpoint1);
client.Register("b", endpoint2);
client.Register("c", endpoint3);
// Look up both at the same time
List <string> ids = new List <string> {
"a", "b", "c", "d"
};
List <NameAssignment> assignments = client.Lookup(ids);
// Check that a, b, and c are registered
Assert.Equal("a", assignments[0].Identifier);
Assert.Equal(endpoint1, assignments[0].Endpoint);
Assert.Equal("b", assignments[1].Identifier);
Assert.Equal(endpoint2, assignments[1].Endpoint);
Assert.Equal("c", assignments[2].Identifier);
Assert.Equal(endpoint3, assignments[2].Endpoint);
// Check that d is not registered
Assert.Equal(3, assignments.Count);
}
}
}
public void TestRegister()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300);
// Check that no endpoints have been registered
Assert.Null(client.Lookup("a"));
Assert.Null(client.Lookup("b"));
Assert.Null(client.Lookup("c"));
// Register endpoints
client.Register("a", endpoint1);
client.Register("b", endpoint2);
client.Register("c", endpoint3);
// Check that they can be looked up correctly
Assert.Equal(endpoint1, client.Lookup("a"));
Assert.Equal(endpoint2, client.Lookup("b"));
Assert.Equal(endpoint3, client.Lookup("c"));
}
}
}
private GroupCommDriver(
[Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId,
[Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId,
[Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut,
[Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks,
AvroConfigurationSerializer configSerializer,
INameServer nameServer)
{
_driverId = driverId;
_contextIds = -1;
_fanOut = fanOut;
MasterTaskId = masterTaskId;
_groupName = groupName;
_configSerializer = configSerializer;
_commGroups = new Dictionary <string, ICommunicationGroupDriver>();
IPEndPoint localEndpoint = nameServer.LocalEndpoint;
_nameServerAddr = localEndpoint.Address.ToString();
_nameServerPort = localEndpoint.Port;
NewCommunicationGroup(groupName, numberOfTasks);
}
public GroupCommDriver(
[Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId,
[Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId,
[Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut,
[Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks,
AvroConfigurationSerializer configSerializer,
INameServer nameServer)
{
_driverId = driverId;
_contextIds = -1;
_fanOut = fanOut;
MasterTaskId = masterTaskId;
_groupName = groupName;
_configSerializer = configSerializer;
_commGroups = new Dictionary<string, ICommunicationGroupDriver>();
_nameServer = nameServer;
IPEndPoint localEndpoint = _nameServer.LocalEndpoint;
_nameServerAddr = localEndpoint.Address.ToString();
_nameServerPort = localEndpoint.Port;
NewCommunicationGroup(groupName, numberOfTasks);
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver,
INameServer nameServer)
{
_configurationManager = configurationManager;
_groupCommDriver = groupCommDriver;
_nameServer = nameServer;
_perMapperConfigs = perMapperConfigs;
_totalMappers = dataSet.Count;
_invokeGC = invokeGC;
_maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery;
_contextManager = new ActiveContextManager(_totalMappers + 1);
_contextManager.Subscribe(this);
var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask);
var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper);
var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers);
_evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec);
_systemState = new SystemStateMachine();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager);
var msg =
string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.",
memoryPerMapper,
memoryForUpdateTask,
coresPerMapper,
coresForUpdateTask,
_maxRetryNumberForFaultTolerant,
allowedFailedEvaluators);
Logger.Log(Level.Info, msg);
}
public void TestConstructorInjection()
{
int port = 6666;
using (INameServer server = BuildNameServer(port))
{
IConfiguration nameClientConfiguration = NamingConfiguration.ConfigurationModule
.Set(NamingConfiguration.NameServerAddress, server.LocalEndpoint.Address.ToString())
.Set(NamingConfiguration.NameServerPort, port + string.Empty)
.Build();
ConstructorInjection c = TangFactory.GetTang()
.NewInjector(nameClientConfiguration)
.GetInstance <ConstructorInjection>();
Assert.NotNull(c);
}
}
public void TestLookup()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
// Register endpoint1
client.Register("a", endpoint1);
Assert.Equal(endpoint1, client.Lookup("a"));
// Reregister identifer a
client.Register("a", endpoint2);
Assert.Equal(endpoint2, client.Lookup("a"));
}
}
}
public void TestUnregister()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
// Register endpoint
client.Register("a", endpoint1);
// Check that it can be looked up correctly
Assert.Equal(endpoint1, client.Lookup("a"));
// Unregister endpoints
client.Unregister("a");
Thread.Sleep(1000);
// Make sure they were unregistered correctly
Assert.Null(client.Lookup("a"));
}
}
}
public void TestNameClientRestart()
{
int oldPort = 6666;
int newPort = 6662;
INameServer server = BuildNameServer(oldPort);
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
client.Register("a", endpoint);
Assert.Equal(endpoint, client.Lookup("a"));
server.Dispose();
server = BuildNameServer(newPort);
client.Restart(server.LocalEndpoint);
client.Register("b", endpoint);
Assert.Equal(endpoint, client.Lookup("b"));
server.Dispose();
}
}
