public void TestLookupList()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300);
// Register endpoints
client.Register("a", endpoint1);
client.Register("b", endpoint2);
client.Register("c", endpoint3);
// Look up both at the same time
List <string> ids = new List <string> {
"a", "b", "c", "d"
};
List <NameAssignment> assignments = client.Lookup(ids);
// Check that a, b, and c are registered
Assert.Equal("a", assignments[0].Identifier);
Assert.Equal(endpoint1, assignments[0].Endpoint);
Assert.Equal("b", assignments[1].Identifier);
Assert.Equal(endpoint2, assignments[1].Endpoint);
Assert.Equal("c", assignments[2].Identifier);
Assert.Equal(endpoint3, assignments[2].Endpoint);
// Check that d is not registered
Assert.Equal(3, assignments.Count);
}
}
}
private OperatorTopology(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
[Parameter(typeof(GroupCommConfigurationOptions.Timeout))] int timeout,
[Parameter(typeof(GroupCommConfigurationOptions.RetryCountWaitingForRegistration))] int retryCount,
[Parameter(typeof(GroupCommConfigurationOptions.SleepTimeWaitingForRegistration))] int sleepTime,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyRootTaskId))] string rootId,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyChildTaskIds))] ISet <string> childIds,
StreamingNetworkService <GeneralGroupCommunicationMessage> networkService,
Sender sender)
{
_operatorName = operatorName;
_groupName = groupName;
_selfId = taskId;
_timeout = timeout;
_retryCount = retryCount;
_sleepTime = sleepTime;
_nameClient = networkService.NamingClient;
_sender = sender;
_parent = _selfId.Equals(rootId) ? null : new NodeStruct <T>(rootId);
foreach (var childId in childIds)
{
_childNodeContainer.PutNode(new NodeStruct <T>(childId));
}
}
public void TestRegister()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
IPEndPoint endpoint3 = new IPEndPoint(IPAddress.Parse("100.0.0.3"), 300);
// Check that no endpoints have been registered
Assert.Null(client.Lookup("a"));
Assert.Null(client.Lookup("b"));
Assert.Null(client.Lookup("c"));
// Register endpoints
client.Register("a", endpoint1);
client.Register("b", endpoint2);
client.Register("c", endpoint3);
// Check that they can be looked up correctly
Assert.Equal(endpoint1, client.Lookup("a"));
Assert.Equal(endpoint2, client.Lookup("b"));
Assert.Equal(endpoint3, client.Lookup("c"));
}
}
}
private StreamingNetworkService(
IObserver <NsMessage <T> > universalObserver,
IObserver <IRemoteMessage <NsMessage <T> > > remoteMessageUniversalObserver,
INameClient nameClient,
StreamingRemoteManagerFactory remoteManagerFactory,
NsMessageStreamingCodec <T> codec,
ILocalAddressProvider localAddressProvider)
{
_remoteManager = remoteManagerFactory.GetInstance(localAddressProvider.LocalAddress, codec);
if (universalObserver != null)
{
// Create and register incoming message handler
// TODO[REEF-419] This should use the TcpPortProvider mechanism
var anyEndpoint = new IPEndPoint(IPAddress.Any, 0);
_universalObserverDisposable = _remoteManager.RegisterObserver(anyEndpoint, universalObserver);
}
else
{
_universalObserverDisposable = null;
}
_remoteMessageUniversalObserver = remoteMessageUniversalObserver != null?
_remoteManager.RegisterObserver(remoteMessageUniversalObserver) : null;
_nameClient = nameClient;
_connectionMap = new Dictionary <IIdentifier, IConnection <T> >();
Logger.Log(Level.Verbose, "Started network service");
}
private StreamingNetworkService(
IObserver <IRemoteMessage <NsMessage <T> > > remoteMessageUniversalObserver,
INameClient nameClient,
StreamingRemoteManagerFactory remoteManagerFactory,
NsMessageStreamingCodec <T> codec,
ILocalAddressProvider localAddressProvider)
: this(null, remoteMessageUniversalObserver, nameClient, remoteManagerFactory, codec, localAddressProvider)
{
}
/// <summary>
/// Creates a new NsConnection between two hosts.
/// </summary>
/// <param name="sourceId">The identifier of the sender</param>
/// <param name="destId">The identifier of the receiver</param>
/// <param name="nameClient">The NameClient used for naming lookup</param>
/// <param name="remoteManager">The remote manager used for network communication</param>
/// <param name="connectionMap">A cache of opened connections. Will remove itself from
/// the cache when the NsConnection is disposed.</param>
public WritableNsConnection(
IIdentifier sourceId,
IIdentifier destId,
INameClient nameClient,
IRemoteManager <WritableNsMessage <T> > remoteManager,
Dictionary <IIdentifier, IConnection <T> > connectionMap)
{
_sourceId = sourceId;
_destId = destId;
_nameClient = nameClient;
_remoteManager = remoteManager;
_connectionMap = connectionMap;
}
private WritableNetworkService(
[Parameter(typeof(NetworkServiceOptions.NetworkServicePort))] int nsPort,
IObserver <WritableNsMessage <T> > messageHandler,
IIdentifierFactory idFactory,
INameClient nameClient,
WritableRemoteManagerFactory remoteManagerFactory)
{
IPAddress localAddress = NetworkUtils.LocalIPAddress;
_remoteManager = remoteManagerFactory.GetInstance <WritableNsMessage <T> >(localAddress, nsPort);
_messageHandler = messageHandler;
_nameClient = nameClient;
_connectionMap = new Dictionary <IIdentifier, IConnection <T> >();
Logger.Log(Level.Info, "Started network service");
}
public void TestLookup()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
IPEndPoint endpoint2 = new IPEndPoint(IPAddress.Parse("100.0.0.2"), 200);
// Register endpoint1
client.Register("a", endpoint1);
Assert.Equal(endpoint1, client.Lookup("a"));
// Reregister identifer a
client.Register("a", endpoint2);
Assert.Equal(endpoint2, client.Lookup("a"));
}
}
}
private OperatorTopology(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
[Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId,
[Parameter(typeof(GroupCommConfigurationOptions.Timeout))] int timeout,
[Parameter(typeof(GroupCommConfigurationOptions.RetryCount))] int retryCount,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyRootTaskId))] string rootId,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyChildTaskIds))] ISet <string> childIds,
WritableNetworkService <GeneralGroupCommunicationMessage> networkService,
Sender sender,
IStreamingCodec <T> codec)
{
_operatorName = operatorName;
_groupName = groupName;
_selfId = taskId;
_driverId = driverId;
_timeout = timeout;
_retryCount = retryCount;
_nameClient = networkService.NamingClient;
_sender = sender;
_nodesWithData = new BlockingCollection <NodeStruct <T> >();
_children = new List <NodeStruct <T> >();
_idToNodeMap = new Dictionary <string, NodeStruct <T> >();
_codec = codec;
if (_selfId.Equals(rootId))
{
_parent = null;
}
else
{
_parent = new NodeStruct <T>(rootId);
_idToNodeMap[rootId] = _parent;
}
foreach (var childId in childIds)
{
var node = new NodeStruct <T>(childId);
_children.Add(node);
_idToNodeMap[childId] = node;
}
}
public NetworkService(
[Parameter(typeof(NetworkServiceOptions.NetworkServicePort))] int nsPort,
IObserver <NsMessage <T> > messageHandler,
IIdentifierFactory idFactory,
ICodec <T> codec,
INameClient nameClient,
IRemoteManagerFactory remoteManagerFactory)
{
_codec = new NsMessageCodec <T>(codec, idFactory);
IPAddress localAddress = NetworkUtils.LocalIPAddress;
_remoteManager = remoteManagerFactory.GetInstance(localAddress, nsPort, _codec);
_messageHandler = messageHandler;
NamingClient = nameClient;
_connectionMap = new Dictionary <IIdentifier, IConnection <T> >();
LOGGER.Log(Level.Info, "Started network service");
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
IClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
INameClient nameClient)
{
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
OperationState = EvaluatorOperationState.OPERATIONAL;
NameClient = nameClient;
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
IClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
INameClient nameClient)
{
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
OperationState = EvaluatorOperationState.OPERATIONAL;
NameClient = nameClient;
}
public TaskRuntime(IInjector taskInjector, string contextId, string taskId, HeartBeatManager heartBeatManager, string memento = null)
{
_injector = taskInjector;
_heartBeatManager = heartBeatManager;
Optional <ISet <ITaskMessageSource> > messageSources = Optional <ISet <ITaskMessageSource> > .Empty();
try
{
_task = _injector.GetInstance <ITask>();
}
catch (Exception e)
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.CaughtAndThrow(new InvalidOperationException("Unable to inject task.", e), Level.Error, "Unable to inject task.", LOGGER);
}
try
{
ITaskMessageSource taskMessageSource = _injector.GetInstance <ITaskMessageSource>();
messageSources = Optional <ISet <ITaskMessageSource> > .Of(new HashSet <ITaskMessageSource>() { taskMessageSource });
}
catch (Exception e)
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, "Cannot inject task message source with error: " + e.StackTrace, LOGGER);
// do not rethrow since this is benign
}
try
{
_nameClient = _injector.GetInstance <INameClient>();
_heartBeatManager.EvaluatorSettings.NameClient = _nameClient;
}
catch (InjectionException)
{
LOGGER.Log(Level.Warning, "Cannot inject name client from task configuration.");
// do not rethrow since user is not required to provide name client
}
LOGGER.Log(Level.Info, "task message source injected");
_currentStatus = new TaskStatus(_heartBeatManager, contextId, taskId, messageSources);
_memento = memento == null ?
Optional <byte[]> .Empty() : Optional <byte[]> .Of(ByteUtilities.StringToByteArrays(memento));
}
public void TestUnregister()
{
using (INameServer server = BuildNameServer())
{
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint1 = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
// Register endpoint
client.Register("a", endpoint1);
// Check that it can be looked up correctly
Assert.Equal(endpoint1, client.Lookup("a"));
// Unregister endpoints
client.Unregister("a");
Thread.Sleep(1000);
// Make sure they were unregistered correctly
Assert.Null(client.Lookup("a"));
}
}
}
private StreamingNetworkService(
IObserver <NsMessage <T> > messageHandler,
IIdentifierFactory idFactory,
INameClient nameClient,
StreamingRemoteManagerFactory remoteManagerFactory,
NsMessageStreamingCodec <T> codec,
IInjector injector)
{
IPAddress localAddress = NetworkUtils.LocalIPAddress;
_remoteManager = remoteManagerFactory.GetInstance(localAddress, codec);
// Create and register incoming message handler
// TODO[REEF-419] This should use the TcpPortProvider mechanism
var anyEndpoint = new IPEndPoint(IPAddress.Any, 0);
_messageHandlerDisposable = _remoteManager.RegisterObserver(anyEndpoint, messageHandler);
_nameClient = nameClient;
_connectionMap = new Dictionary <IIdentifier, IConnection <T> >();
Logger.Log(Level.Info, "Started network service");
}
private OperatorTopology(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
[Parameter(typeof(GroupCommConfigurationOptions.Timeout))] int timeout,
[Parameter(typeof(GroupCommConfigurationOptions.RetryCountWaitingForRegistration))] int retryCount,
[Parameter(typeof(GroupCommConfigurationOptions.SleepTimeWaitingForRegistration))] int sleepTime,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyRootTaskId))] string rootId,
[Parameter(typeof(GroupCommConfigurationOptions.TopologyChildTaskIds))] ISet <string> childIds,
GroupCommNetworkObserver networkObserver,
StreamingNetworkService <GeneralGroupCommunicationMessage> networkService,
Sender sender)
{
_operatorName = operatorName;
_groupName = groupName;
_selfId = taskId;
_timeout = timeout;
_retryCount = retryCount;
_sleepTime = sleepTime;
_nameClient = networkService.NamingClient;
_sender = sender;
_parent = _selfId.Equals(rootId) ? null : new NodeStruct <T>(rootId, groupName, operatorName);
// Register the observers for Task IDs and nodes adjacent to the current node
// in the group communication graph.
if (_parent != null)
{
networkObserver.RegisterAndGetForTask(_parent.Identifier).RegisterNodeObserver(new NodeMessageObserver <T>(_parent));
}
foreach (var childId in childIds)
{
var childNode = new NodeStruct <T>(childId, groupName, operatorName);
_childNodeContainer.PutNode(childNode);
networkObserver.RegisterAndGetForTask(childId).RegisterNodeObserver(new NodeMessageObserver <T>(childNode));
}
}
public void TestNameClientRestart()
{
int oldPort = 6666;
int newPort = 6662;
INameServer server = BuildNameServer(oldPort);
using (INameClient client = BuildNameClient(server.LocalEndpoint))
{
IPEndPoint endpoint = new IPEndPoint(IPAddress.Parse("100.0.0.1"), 100);
client.Register("a", endpoint);
Assert.Equal(endpoint, client.Lookup("a"));
server.Dispose();
server = BuildNameServer(newPort);
client.Restart(server.LocalEndpoint);
client.Register("b", endpoint);
Assert.Equal(endpoint, client.Lookup("b"));
server.Dispose();
}
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector,
INameClient nameClient)
{
_serializer = serializer;
_injector = injector;
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
if (string.IsNullOrWhiteSpace(rootContextConfigString))
{
Utilities.Diagnostics.Exceptions.Throw(
new ArgumentException("empty or null rootContextConfigString"), Logger);
}
_rootContextConfig = _serializer.FromString(rootContextConfigString);
_rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
_rootTaskConfiguration = CreateTaskConfiguration();
_rootServiceConfiguration = CreateRootServiceConfiguration();
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
_operationState = EvaluatorOperationState.OPERATIONAL;
_nameClient = nameClient;
}
public HelloTask(HelloService service, INameClient nameClient)
{
Console.WriteLine("HelloTask constructor 2");
Service = service;
_nameClient = nameClient;
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector,
INameClient nameClient)
{
_serializer = serializer;
_injector = injector;
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
if (string.IsNullOrWhiteSpace(rootContextConfigString))
{
Utilities.Diagnostics.Exceptions.Throw(
new ArgumentException("empty or null rootContextConfigString"), Logger);
}
_rootContextConfig = _serializer.FromString(rootContextConfigString);
try
{
_rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
}
catch (InjectionException)
{
Logger.Log(Level.Info, "Using deprecated ContextConfiguration.");
// TODO[JIRA REEF-1167]: Remove this catch.
var deprecatedContextConfig = new Context.ContextConfiguration(rootContextConfigString);
_rootContextConfig = deprecatedContextConfig;
_rootContextId = deprecatedContextConfig.Id;
}
_rootTaskConfiguration = CreateTaskConfiguration();
_rootServiceConfiguration = CreateRootServiceConfiguration();
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
_operationState = EvaluatorOperationState.OPERATIONAL;
_nameClient = nameClient;
}
public HelloTask(HelloService service, INameClient nameClient)
{
Console.WriteLine("HelloTask constructor 2");
Service = service;
_nameClient = nameClient;
}
