/// <summary>
/// Reads the value from the link synchronously
/// </summary>
public T Read()
{
if (_disposed)
{
Exceptions.Throw(new IllegalStateException("Link has been disposed."), Logger);
}
string dataType = _reader.ReadString();
if (dataType == null)
{
return(default(T));
}
try
{
T value = (T)_injector.ForkInjector().GetInstance(dataType);
value.Read(_reader);
return(value);
}
catch (InjectionException)
{
return(default(T));
}
}
public void TestForkWorks()
{
Type checkChildIfaceType = typeof(CheckChildIface);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { FileNames.Examples });
cb.BindImplementation(GenericType <CheckChildIface> .Class, GenericType <CheckChildImpl> .Class);
IInjector i = TangFactory.GetTang().NewInjector(cb.Build());
IInjector i1 = i.ForkInjector();
CheckChildIface c1 = (CheckChildIface)i1.GetInstance(checkChildIfaceType);
IInjector i2 = i.ForkInjector();
CheckChildIface c2 = (CheckChildIface)i2.GetInstance(checkChildIfaceType);
Assert.AreNotEqual(c1, c2);
}
public ForksInjectorInConstructor(IInjector i)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { @"Org.Apache.REEF.Tang.Examples" });
//// cb.BindImplementation(Number.class, typeof(Int32));
i.ForkInjector(cb.Build());
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional <ContextRuntime> parentContext)
{
ContextConfiguration config = contextConfiguration as ContextConfiguration;
if (config == null)
{
var e = new ArgumentException("contextConfiguration is not of type ContextConfiguration");
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
_contextLifeCycle = new ContextLifeCycle(config.Id);
_serviceInjector = serviceInjector;
_parentContext = parentContext;
try
{
_contextInjector = serviceInjector.ForkInjector();
}
catch (Exception e)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
Optional <string> parentId = ParentContext.IsPresent() ?
Optional <string> .Of(ParentContext.Value.Id) :
Optional <string> .Empty();
ContextClientCodeException ex = new ContextClientCodeException(ContextClientCodeException.GetId(contextConfiguration), parentId, "Unable to spawn context", e);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(ex, LOGGER);
}
// Trigger the context start events on contextInjector.
_contextLifeCycle.Start();
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional <ContextRuntime> parentContext)
{
_serviceInjector = serviceInjector;
// Note that for Service objects and handlers, we are not merging them into a separate
// class (e.g. ServiceContainer) due to the inability to allow service stacking if an instance
// of such a class were to be materialized. i.e. if a ServiceContainer object were initialized
// and a child ServiceConfiguration is submitted, when the child service injector tries to
// get the relevant handlers and services set, it will get the same set of handlers as
// previously instantiated by the parent injector, and thus will not allow the stacking
// of ServiceConfigurations.
_injectedServices = serviceInjector.GetNamedInstance <ServicesSet, ISet <object> >();
_serviceContextStartHandlers =
serviceInjector.GetNamedInstance <ContextConfigurationOptions.StartHandlers, ISet <IObserver <IContextStart> > >();
_serviceContextStopHandlers =
serviceInjector.GetNamedInstance <ContextConfigurationOptions.StopHandlers, ISet <IObserver <IContextStop> > >();
_serviceTaskStartHandlers =
serviceInjector.GetNamedInstance <TaskConfigurationOptions.StartHandlers, ISet <IObserver <ITaskStart> > >();
_serviceTaskStopHandlers =
serviceInjector.GetNamedInstance <TaskConfigurationOptions.StopHandlers, ISet <IObserver <ITaskStop> > >();
_contextInjector = serviceInjector.ForkInjector(contextConfiguration);
_contextLifeCycle = _contextInjector.GetInstance <ContextLifeCycle>();
_parentContext = parentContext;
_contextLifeCycle.Start();
}
public GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
StreamingNetworkService<GeneralGroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary<string, ICommunicationGroupClientInternal>();
_networkService = networkService;
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance<ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
networkService.Register(new StringIdentifier(taskId));
foreach (var group in _commGroups.Values)
{
group.WaitingForRegistration();
}
}
public ForksInjectorInConstructor(IInjector i)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { @"Com.Microsoft.Tang.Examples" });
//cb.BindImplementation(Number.class, typeof(Int32));
i.ForkInjector(new IConfiguration[] { cb.Build() });
}
private string GetCommand(JobParameters jobParameters)
{
var commandProviderConfigModule = AzureBatchCommandBuilderConfiguration.ConfigurationModule;
if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
{
commandProviderConfigModule = commandProviderConfigModule
.Set(AzureBatchCommandBuilderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
}
if (jobParameters.StdoutFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(AzureBatchCommandBuilderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
}
if (jobParameters.StderrFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(AzureBatchCommandBuilderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
}
var azureBatchJobCommandBuilder = _injector.ForkInjector(commandProviderConfigModule.Build())
.GetInstance <ICommandBuilder>();
var command = azureBatchJobCommandBuilder.BuildDriverCommand(jobParameters.DriverMemoryInMB);
return(command);
}
public GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet <string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
StreamingNetworkService <GeneralGroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary <string, ICommunicationGroupClientInternal>();
_networkService = networkService;
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance <ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
networkService.Register(new StringIdentifier(taskId));
try
{
foreach (var group in _commGroups.Values)
{
group.WaitingForRegistration();
}
}
catch (SystemException e)
{
networkService.Unregister();
networkService.Dispose();
Exceptions.CaughtAndThrow(e, Level.Error, "In GroupCommClient, exception from WaitingForRegistration.", Logger);
}
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional<ContextRuntime> parentContext)
{
ContextConfiguration config = contextConfiguration as ContextConfiguration;
if (config == null)
{
var e = new ArgumentException("contextConfiguration is not of type ContextConfiguration");
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
_contextLifeCycle = new ContextLifeCycle(config.Id);
_serviceInjector = serviceInjector;
_parentContext = parentContext;
try
{
_contextInjector = serviceInjector.ForkInjector();
}
catch (Exception e)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
Optional<string> parentId = ParentContext.IsPresent() ?
Optional<string>.Of(ParentContext.Value.Id) :
Optional<string>.Empty();
ContextClientCodeException ex = new ContextClientCodeException(ContextClientCodeException.GetId(contextConfiguration), parentId, "Unable to spawn context", e);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(ex, LOGGER);
}
// Trigger the context start events on contextInjector.
_contextLifeCycle.Start();
}
public GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet <string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
StreamingNetworkService <GeneralGroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary <string, ICommunicationGroupClientInternal>();
_networkService = networkService;
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance <ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
networkService.Register(new StringIdentifier(taskId));
foreach (var group in _commGroups.Values)
{
group.WaitingForRegistration();
}
}
private CommunicationGroupClient(
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.SerializedOperatorConfigs))] ISet <string> operatorConfigs,
IGroupCommNetworkObserver groupCommNetworkObserver,
AvroConfigurationSerializer configSerializer,
ICommunicationGroupNetworkObserver commGroupNetworkHandler,
IInjector injector)
{
_operators = new Dictionary <string, object>();
GroupName = groupName;
groupCommNetworkObserver.Register(groupName, commGroupNetworkHandler);
foreach (string operatorConfigStr in operatorConfigs)
{
IConfiguration operatorConfig = configSerializer.FromString(operatorConfigStr);
IInjector operatorInjector = injector.ForkInjector(operatorConfig);
string operatorName = operatorInjector.GetNamedInstance <GroupCommConfigurationOptions.OperatorName, string>(
GenericType <GroupCommConfigurationOptions.OperatorName> .Class);
string msgType = operatorInjector.GetNamedInstance <GroupCommConfigurationOptions.MessageType, string>(
GenericType <GroupCommConfigurationOptions.MessageType> .Class);
Type groupCommOperatorGenericInterface = typeof(IGroupCommOperator <>);
Type groupCommOperatorInterface = groupCommOperatorGenericInterface.MakeGenericType(Type.GetType(msgType));
var operatorObj = operatorInjector.GetInstance(groupCommOperatorInterface);
_operators.Add(operatorName, operatorObj);
}
}
public void TestForkWorks()
{
Type checkChildIfaceType = typeof(Com.Microsoft.Tang.Examples.CheckChildIface);
Type checkChildImplType = typeof(Com.Microsoft.Tang.Examples.CheckChildImpl);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { file });
cb.Bind(checkChildIfaceType, checkChildImplType);
IInjector i = TangFactory.GetTang().NewInjector(cb.Build());
IInjector i1 = i.ForkInjector();
CheckChildIface c1 = (CheckChildIface)i1.GetInstance(checkChildIfaceType);
IInjector i2 = i.ForkInjector();
CheckChildIface c2 = (CheckChildIface)i2.GetInstance(checkChildIfaceType);
Assert.AreNotEqual(c1, c2);
}
/// <summary>
/// Launches an Task on this context.
/// </summary>
/// <param name="taskConfiguration"></param>
public Thread StartTaskOnNewThread(IConfiguration taskConfiguration)
{
lock (_contextLifeCycle)
{
LOGGER.Log(Level.Info, "ContextRuntime::StartTask(TaskConfiguration) task is present: " + _task.IsPresent());
if (_task.IsPresent())
{
LOGGER.Log(Level.Info, "Task state: " + _task.Value.GetTaskState());
LOGGER.Log(Level.Info, "ContextRuntime::StartTask(TaskConfiguration) task has ended: " + _task.Value.HasEnded());
if (_task.Value.HasEnded())
{
// clean up state
_task = Optional <TaskRuntime> .Empty();
}
else
{
// note: java code is putting thread id here
var e = new InvalidOperationException(
string.Format(CultureInfo.InvariantCulture, "Attempting to spawn a child context when an Task with id '{0}' is running", _task.Value.TaskId));
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
AssertChildContextNotPresent("Attempting to instantiate a child context on a context that is not the topmost active context.");
var taskInjector = _contextInjector.ForkInjector(taskConfiguration);
try
{
var taskRuntime = taskInjector.GetInstance <TaskRuntime>();
_task = Optional <TaskRuntime> .Of(taskRuntime);
return(taskRuntime.StartTaskOnNewThread());
}
catch (InjectionException e)
{
var taskId = string.Empty;
try
{
taskId = taskInjector.GetNamedInstance <TaskConfigurationOptions.Identifier, string>();
}
catch (Exception)
{
LOGGER.Log(Level.Error, "Unable to get Task ID from TaskConfiguration.");
}
var ex = TaskClientCodeException.Create(taskId, Id, "Unable to run the new task", e);
Utilities.Diagnostics.Exceptions.CaughtAndThrow(ex, Level.Error, "Task start error.", LOGGER);
return(null);
}
catch (Exception e)
{
throw new SystemException("System error in starting Task.", e);
}
}
}
/// <summary>
/// Spawns a new context.
/// The new context will have a serviceInjector that is created by forking the one in this object with the given
/// serviceConfiguration. The contextConfiguration is used to fork the contextInjector from that new serviceInjector.
/// </summary>
/// <param name="childContextConfiguration">the new context's context (local) Configuration.</param>
/// <param name="childServiceConfiguration">the new context's service Configuration.</param>
/// <returns>a child context.</returns>
public ContextRuntime SpawnChildContext(IConfiguration childContextConfiguration, IConfiguration childServiceConfiguration)
{
lock (_contextLifeCycle)
{
if (_task.IsPresent())
{
var message =
string.Format(CultureInfo.InvariantCulture, "Attempting to spawn a child context when an Task with id '{0}' is running", _task.Value.TaskId);
var e = new InvalidOperationException(message);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
AssertChildContextNotPresent("Attempting to instantiate a child context on a context that is not the topmost active context.");
try
{
var childServiceInjector = _serviceInjector.ForkInjector(childServiceConfiguration);
var childContext = new ContextRuntime(childServiceInjector, childContextConfiguration, Optional <ContextRuntime> .Of(this));
_childContext = Optional <ContextRuntime> .Of(childContext);
return(childContext);
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
var childContextId = string.Empty;
try
{
var injector = TangFactory.GetTang().NewInjector(childContextConfiguration);
childContextId = injector.GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
}
catch (InjectionException)
{
Utilities.Diagnostics.Exceptions.Caught(
e, Level.Error, "Unable to get Context ID from child ContextConfiguration. Using empty string.", LOGGER);
}
throw new ContextClientCodeException(childContextId, Optional <string> .Of(Id), "Unable to spawn context", e);
}
}
}
private MapFunctions <TMapInput, TMapOutput> MakeMapFunctions(IInjector injector)
{
ISet <IMapFunction <TMapInput, TMapOutput> > mappers = new HashSet <IMapFunction <TMapInput, TMapOutput> >();
for (var i = 0; i < _numberOfMappers; ++i)
{
mappers.Add(injector.ForkInjector().GetInstance <IMapFunction <TMapInput, TMapOutput> >());
}
return(new MapFunctions <TMapInput, TMapOutput>(mappers));
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional<ContextRuntime> parentContext)
{
_serviceInjector = serviceInjector;
_contextInjector = serviceInjector.ForkInjector(contextConfiguration);
_contextLifeCycle = _contextInjector.GetInstance<ContextLifeCycle>();
_parentContext = parentContext;
_contextLifeCycle.Start();
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional<ContextRuntime> parentContext)
{
_serviceInjector = serviceInjector;
_injectedServices = Optional<ISet<object>>.Of(serviceInjector.GetNamedInstance<ServicesSet, ISet<object>>());
_contextInjector = serviceInjector.ForkInjector(contextConfiguration);
_contextLifeCycle = _contextInjector.GetInstance<ContextLifeCycle>();
_parentContext = parentContext;
_contextLifeCycle.Start();
}
/// <summary>
/// Spawns a new context.
/// The new context will have a serviceInjector that is created by forking the one in this object with the given
/// serviceConfiguration. The contextConfiguration is used to fork the contextInjector from that new serviceInjector.
/// </summary>
/// <param name="contextConfiguration">the new context's context (local) Configuration.</param>
/// <param name="serviceConfiguration">the new context's service Configuration.</param>
/// <returns>a child context.</returns>
public ContextRuntime SpawnChildContext(IConfiguration contextConfiguration, IConfiguration serviceConfiguration)
{
ContextRuntime childContext = null;
lock (_contextLifeCycle)
{
if (_task.IsPresent())
{
var e = new InvalidOperationException(
string.Format(CultureInfo.InvariantCulture, "Attempting to spawn a child context when an Task with id '{0}' is running", _task.Value.TaskId)); // note: java code is putting thread id here
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
if (_childContext.IsPresent())
{
var e = new InvalidOperationException("Attempting to instantiate a child context on a context that is not the topmost active context.");
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
try
{
IInjector childServiceInjector = _serviceInjector.ForkInjector(serviceConfiguration);
childContext = new ContextRuntime(childServiceInjector, contextConfiguration, Optional <ContextRuntime> .Of(this));
_childContext = Optional <ContextRuntime> .Of(childContext);
return(childContext);
}
catch (Exception e)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
Optional <string> parentId = ParentContext.IsPresent() ?
Optional <string> .Of(ParentContext.Value.Id) :
Optional <string> .Empty();
ContextClientCodeException ex = new ContextClientCodeException(ContextClientCodeException.GetId(contextConfiguration), parentId, "Unable to spawn context", e);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(ex, LOGGER);
}
}
return(childContext);
}
/// <summary>
/// Spawns a new context.
/// The new context will have a serviceInjector that is created by forking the one in this object with the given
/// serviceConfiguration. The contextConfiguration is used to fork the contextInjector from that new serviceInjector.
/// </summary>
/// <param name="childContextConfiguration">the new context's context (local) Configuration.</param>
/// <param name="childServiceConfiguration">the new context's service Configuration.</param>
/// <returns>a child context.</returns>
public ContextRuntime SpawnChildContext(IConfiguration childContextConfiguration, IConfiguration childServiceConfiguration)
{
lock (_contextLifeCycle)
{
if (_task.IsPresent())
{
var message =
string.Format(CultureInfo.InvariantCulture, "Attempting to spawn a child context when an Task with id '{0}' is running", _task.Value.TaskId);
var e = new InvalidOperationException(message);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
AssertChildContextNotPresent("Attempting to instantiate a child context on a context that is not the topmost active context.");
try
{
var childServiceInjector = _serviceInjector.ForkInjector(childServiceConfiguration);
var childContext = new ContextRuntime(childServiceInjector, childContextConfiguration, Optional <ContextRuntime> .Of(this));
_childContext = Optional <ContextRuntime> .Of(childContext);
return(childContext);
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
Optional <string> parentId = ParentContext.IsPresent() ?
Optional <string> .Of(ParentContext.Value.Id) :
Optional <string> .Empty();
ContextClientCodeException ex = new ContextClientCodeException(ContextClientCodeException.GetId(childContextConfiguration), parentId, "Unable to spawn context", e);
Utilities.Diagnostics.Exceptions.Throw(ex, LOGGER);
}
}
return(null);
}
/// <summary>
/// Read the class fields.
/// </summary>
/// <param name="reader">The reader from which to read </param>
public void Read(IDataReader reader)
{
SourceId = _factory.Create(reader.ReadString());
DestId = _factory.Create(reader.ReadString());
int messageCount = reader.ReadInt32();
string dataType = reader.ReadString();
Data = new List <T>();
for (int index = 0; index < messageCount; index++)
{
var dataPoint = (T)_injection.ForkInjector().GetInstance(Type.GetType(dataType));
if (null == dataPoint)
{
throw new Exception("T type instance cannot be created from the stream data in Network Service Message");
}
dataPoint.Read(reader);
Data.Add(dataPoint);
}
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional <ContextRuntime> parentContext)
{
_serviceInjector = serviceInjector;
_injectedServices = Optional <ISet <object> > .Of(serviceInjector.GetNamedInstance <ServicesSet, ISet <object> >());
_contextInjector = serviceInjector.ForkInjector(contextConfiguration);
_contextLifeCycle = _contextInjector.GetInstance <ContextLifeCycle>();
_parentContext = parentContext;
_contextLifeCycle.Start();
}
/// <summary>
/// Spawns a new context.
/// The new context will have a serviceInjector that is created by forking the one in this object with the given
/// serviceConfiguration. The contextConfiguration is used to fork the contextInjector from that new serviceInjector.
/// </summary>
/// <param name="childContextConfiguration">the new context's context (local) Configuration.</param>
/// <param name="childServiceConfiguration">the new context's service Configuration.</param>
/// <returns>a child context.</returns>
public ContextRuntime SpawnChildContext(
IConfiguration childContextConfiguration,
IConfiguration childServiceConfiguration = null)
{
lock (_contextLifeCycle)
{
if (_task.IsPresent())
{
throw new InvalidOperationException(
string.Format(
CultureInfo.InvariantCulture,
"Attempting to spawn a child context when an Task with id '{0}' is running",
_task.Value.TaskId));
}
AssertChildContextNotPresent("Attempting to instantiate a child context on a context that is not the topmost active context.");
try
{
var childServiceInjector = childServiceConfiguration == null
? _serviceInjector.ForkInjector()
: _serviceInjector.ForkInjector(childServiceConfiguration);
_childContext = Optional <ContextRuntime> .Of(
new ContextRuntime(childServiceInjector, childContextConfiguration, Optional <ContextRuntime> .Of(this)));
return(_childContext.Value);
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
var childContextId = GetChildContextId(childContextConfiguration);
throw new ContextClientCodeException(childContextId, Optional <string> .Of(Id), "Unable to spawn context", e);
}
}
}
private RootContextLauncher(
IConfiguration contextConfiguration,
IConfiguration rootServiceConfig,
Optional <IConfiguration> rootTaskConfig,
IInjector injector)
{
_rootContextConfiguration = contextConfiguration;
_rootServiceInjector = injector.ForkInjector(rootServiceConfig);
Id = _rootServiceInjector
.ForkInjector(contextConfiguration)
.GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
_services = _rootServiceInjector.GetNamedInstance <ServicesSet, ISet <object> >();
Logger.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "injected service(s)"));
RootTaskConfig = rootTaskConfig;
}
private RootContextLauncher(
IConfiguration contextConfiguration,
IConfiguration rootServiceConfig,
Optional<IConfiguration> rootTaskConfig,
IInjector injector)
{
_rootContextConfiguration = contextConfiguration;
_rootServiceInjector = injector.ForkInjector(rootServiceConfig);
Id = _rootServiceInjector
.ForkInjector(contextConfiguration)
.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
_services = _rootServiceInjector.GetNamedInstance<ServicesSet, ISet<object>>();
Logger.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "injected service(s)"));
RootTaskConfig = rootTaskConfig;
}
/// <summary>
/// Launches an Task on this context.
/// </summary>
/// <param name="taskConfiguration"></param>
/// <param name="contextId"></param>
/// <param name="heartBeatManager"></param>
public void StartTask(TaskConfiguration taskConfiguration, string contextId, HeartBeatManager heartBeatManager)
{
lock (_contextLifeCycle)
{
bool taskPresent = _task.IsPresent();
bool taskEnded = taskPresent && _task.Value.HasEnded();
LOGGER.Log(Level.Info, "ContextRuntime::StartTask(TaskConfiguration)" + "task is present: " + taskPresent + " task has ended: " + taskEnded);
if (taskPresent)
{
LOGGER.Log(Level.Info, "Task state: " + _task.Value.GetTaskState());
}
if (taskEnded)
{
// clean up state
_task = Optional <TaskRuntime> .Empty();
taskPresent = false;
}
if (taskPresent)
{
var e = new InvalidOperationException(
string.Format(CultureInfo.InvariantCulture, "Attempting to spawn a child context when an Task with id '{0}' is running", _task.Value.TaskId)); // note: java code is putting thread id here
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
if (_childContext.IsPresent())
{
var e = new InvalidOperationException("Attempting to instantiate a child context on a context that is not the topmost active context.");
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
try
{
IInjector taskInjector = _contextInjector.ForkInjector(taskConfiguration.TangConfig);
LOGGER.Log(Level.Info, "Trying to inject task with configuration" + taskConfiguration.ToString());
TaskRuntime taskRuntime = new TaskRuntime(taskInjector, contextId, taskConfiguration.TaskId, heartBeatManager); // taskInjector.getInstance(TaskRuntime.class);
taskRuntime.Initialize();
System.Threading.Tasks.Task.Run(new Action(taskRuntime.Start));
_task = Optional <TaskRuntime> .Of(taskRuntime);
}
catch (Exception e)
{
var ex = new TaskClientCodeException(taskConfiguration.TaskId, Id, "Unable to instantiate the new task", e);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.CaughtAndThrow(ex, Level.Error, "Task start error.", LOGGER);
}
}
}
public void TestDeserializationForServiceAndContext()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var config = serializer.FromFile("evaluatorWithService.conf", classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
string contextConfigString = evaluatorInjector.GetNamedInstance <RootContextConfiguration, string>();
string rootServiceConfigString = evaluatorInjector.GetNamedInstance <RootServiceConfiguration, string>();
var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
});
var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ServicesConfigurationOptions).Assembly.GetName().Name,
typeof(IStreamingCodec <>).Assembly.GetName().Name
});
var rootServiceConfig = serializer.FromString(rootServiceConfigString, serviceClassHierarchy);
var contextInjector = evaluatorInjector.ForkInjector(contextConfig);
string contextId = contextInjector.GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
Assert.IsTrue(contextId.StartsWith("MasterTaskContext"));
string serviceConfigString = TangFactory.GetTang().NewInjector(rootServiceConfig)
.GetNamedInstance <ServicesConfigurationOptions.ServiceConfigString, string>();
var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy);
var serviceInjector = contextInjector.ForkInjector(serviceConfig);
var tcpCountRange = serviceInjector.GetNamedInstance <TcpPortRangeStart, int>();
var tcpCountCount = serviceInjector.GetNamedInstance <TcpPortRangeCount, int>();
Assert.IsTrue(tcpCountRange > 0);
Assert.IsTrue(tcpCountCount > 0);
}
private GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
NetworkService<GroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary<string, ICommunicationGroupClient>();
_networkService = networkService;
networkService.Register(new StringIdentifier(taskId));
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
ICommunicationGroupClient commGroupClient = groupInjector.GetInstance<ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
}
public void testFutures2()
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
IInjector i = TangFactory.GetTang().NewInjector(cb.Build());
IInjector i2 = i.ForkInjector(new IConfiguration[] { });
FlyingCar c = (FlyingCar)i.GetInstance(typeof(FlyingCar));
Assert.IsTrue(c == c.getDriver().getMyCar());
Assert.IsTrue(c.getDriver() == c.getDriver().getMyCar().getDriver());
FlyingCar c2 = (FlyingCar)i2.GetInstance(typeof(FlyingCar));
Assert.IsTrue(c2 == c2.getDriver().getMyCar());
Assert.IsTrue(c2.getDriver() == c2.getDriver().getMyCar().getDriver());
Assert.IsTrue(c2 != c.getDriver().getMyCar());
Assert.IsTrue(c2.getDriver() != c.getDriver().getMyCar().getDriver());
}
public void TestDeserializationForContextAndTask()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var config = serializer.FromFile("evaluator.conf", classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
string taskConfigString = evaluatorInjector.GetNamedInstance <InitialTaskConfiguration, string>();
string contextConfigString = evaluatorInjector.GetNamedInstance <RootContextConfiguration, string>();
var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
});
var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ITask).Assembly.GetName().Name,
typeof(HelloTask).Assembly.GetName().Name
});
var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy);
var contextInjector = evaluatorInjector.ForkInjector(contextConfig);
string contextId = contextInjector.GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
Assert.IsTrue(contextId.StartsWith(ContextIdPrefix));
var taskInjector = contextInjector.ForkInjector(taskConfig);
string taskId = taskInjector.GetNamedInstance <TaskConfigurationOptions.Identifier, string>();
ITask task = taskInjector.GetInstance <ITask>();
Assert.IsTrue(taskId.StartsWith("HelloTask"));
Assert.IsTrue(task is HelloTask);
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector,
INameClient nameClient)
{
_serializer = serializer;
_injector = injector;
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
if (string.IsNullOrWhiteSpace(rootContextConfigString))
{
Utilities.Diagnostics.Exceptions.Throw(
new ArgumentException("empty or null rootContextConfigString"), Logger);
}
_rootContextConfig = _serializer.FromString(rootContextConfigString);
_rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance <ContextConfigurationOptions.ContextIdentifier, string>();
_rootTaskConfiguration = CreateTaskConfiguration();
_rootServiceConfiguration = CreateRootServiceConfiguration();
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
_operationState = EvaluatorOperationState.OPERATIONAL;
_nameClient = nameClient;
}
public void TestNonEmptyInjectionTransportServer()
{
int id = 5;
IConfiguration config = TangFactory.GetTang().NewConfigurationBuilder().BindNamedParameter <StringId, int>(
GenericType <StringId> .Class, id.ToString(CultureInfo.InvariantCulture)).Build();
IInjector injector = TangFactory.GetTang().NewInjector(config);
BlockingCollection <PrefixedStringWritable> queue = new BlockingCollection <PrefixedStringWritable>();
List <string> events = new List <string>();
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, 0);
var remoteHandler = Observer.Create <TransportEvent <PrefixedStringWritable> >(tEvent => queue.Add(tEvent.Data));
using (var server = new WritableTransportServer <PrefixedStringWritable>(endpoint, remoteHandler, _tcpPortProvider, injector.ForkInjector()))
{
server.Run();
IPEndPoint remoteEndpoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), server.LocalEndpoint.Port);
using (var client = new WritableTransportClient <PrefixedStringWritable>(remoteEndpoint, injector.ForkInjector()))
{
client.Send(new PrefixedStringWritable("Hello"));
client.Send(new PrefixedStringWritable(", "));
client.Send(new PrefixedStringWritable("World!"));
events.Add(queue.Take().Data);
events.Add(queue.Take().Data);
events.Add(queue.Take().Data);
}
}
Assert.AreEqual(3, events.Count);
Assert.AreEqual(events[0], "Hello_" + id);
Assert.AreEqual(events[1], ", _" + id);
Assert.AreEqual(events[2], "World!_" + id);
}
public ForksInjectorInConstructor(IInjector i)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { @"Org.Apache.REEF.Tang.Examples" });
//cb.BindImplementation(Number.class, typeof(Int32));
i.ForkInjector(cb.Build());
}
/// <summary>
/// Create a new ContextRuntime.
/// </summary>
/// <param name="serviceInjector"></param>
/// <param name="contextConfiguration">the Configuration for this context.</param>
/// <param name="parentContext"></param>
public ContextRuntime(
IInjector serviceInjector,
IConfiguration contextConfiguration,
Optional<ContextRuntime> parentContext)
{
_serviceInjector = serviceInjector;
// Note that for Service objects and handlers, we are not merging them into a separate
// class (e.g. ServiceContainer) due to the inability to allow service stacking if an instance
// of such a class were to be materialized. i.e. if a ServiceContainer object were initialized
// and a child ServiceConfiguration is submitted, when the child service injector tries to
// get the relevant handlers and services set, it will get the same set of handlers as
// previously instantiated by the parent injector, and thus will not allow the stacking
// of ServiceConfigurations.
_injectedServices = serviceInjector.GetNamedInstance<ServicesSet, ISet<object>>();
_serviceContextStartHandlers =
serviceInjector.GetNamedInstance<ContextConfigurationOptions.StartHandlers, ISet<IObserver<IContextStart>>>();
_serviceContextStopHandlers =
serviceInjector.GetNamedInstance<ContextConfigurationOptions.StopHandlers, ISet<IObserver<IContextStop>>>();
_serviceTaskStartHandlers =
serviceInjector.GetNamedInstance<TaskConfigurationOptions.StartHandlers, ISet<IObserver<ITaskStart>>>();
_serviceTaskStopHandlers =
serviceInjector.GetNamedInstance<TaskConfigurationOptions.StopHandlers, ISet<IObserver<ITaskStop>>>();
_contextInjector = serviceInjector.ForkInjector(contextConfiguration);
_contextLifeCycle = _contextInjector.GetInstance<ContextLifeCycle>();
_parentContext = parentContext;
try
{
_contextLifeCycle.Start();
}
catch (Exception e)
{
const string message = "Encountered Exception in ContextStartHandler.";
if (ParentContext.IsPresent())
{
throw new ContextStartHandlerException(
Id, Optional<string>.Of(ParentContext.Value.Id), message, e);
}
throw new ContextStartHandlerException(Id, Optional<string>.Empty(), message, e);
}
}
private SubmitApplication CreateApplicationSubmissionRequest(
JobParameters jobParameters,
string appId,
int maxApplicationSubmissions,
IReadOnlyCollection <JobResource> jobResources)
{
var commandProviderConfigModule = YarnCommandProviderConfiguration.ConfigurationModule;
if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
}
if (jobParameters.StdoutFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
}
if (jobParameters.StderrFilePath.IsPresent())
{
commandProviderConfigModule = commandProviderConfigModule
.Set(YarnCommandProviderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
}
var yarnJobCommandProvider = _injector.ForkInjector(commandProviderConfigModule.Build())
.GetInstance <IYarnJobCommandProvider>();
var command = yarnJobCommandProvider.GetJobSubmissionCommand();
Log.Log(Level.Verbose, "Command for YARN: {0}", command);
Log.Log(Level.Verbose, "ApplicationID: {0}", appId);
Log.Log(Level.Verbose, "MaxApplicationSubmissions: {0}", maxApplicationSubmissions);
foreach (var jobResource in jobResources)
{
Log.Log(Level.Verbose, "Remote file: {0}", jobResource.RemoteUploadPath);
}
var submitApplication = new SubmitApplication
{
ApplicationId = appId,
ApplicationName = jobParameters.JobIdentifier,
AmResource = new Resouce
{
MemoryMB = jobParameters.DriverMemoryInMB,
VCores = 1 // keeping parity with existing code
},
MaxAppAttempts = maxApplicationSubmissions,
ApplicationType = REEFApplicationType,
KeepContainersAcrossApplicationAttempts = true,
Queue = @"default", // keeping parity with existing code
Priority = 1, // keeping parity with existing code
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
LocalResources = CreateLocalResources(jobResources),
Commands = new Commands
{
Command = command
}
}
};
return(submitApplication);
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector,
INameClient nameClient)
{
_serializer = serializer;
_injector = injector;
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
if (string.IsNullOrWhiteSpace(rootContextConfigString))
{
Utilities.Diagnostics.Exceptions.Throw(
new ArgumentException("empty or null rootContextConfigString"), Logger);
}
_rootContextConfig = _serializer.FromString(rootContextConfigString);
try
{
_rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
}
catch (InjectionException)
{
Logger.Log(Level.Info, "Using deprecated ContextConfiguration.");
// TODO[JIRA REEF-1167]: Remove this catch.
var deprecatedContextConfig = new Context.ContextConfiguration(rootContextConfigString);
_rootContextConfig = deprecatedContextConfig;
_rootContextId = deprecatedContextConfig.Id;
}
_rootTaskConfiguration = CreateTaskConfiguration();
_rootServiceConfiguration = CreateRootServiceConfiguration();
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
_operationState = EvaluatorOperationState.OPERATIONAL;
_nameClient = nameClient;
}
/// <summary>
/// Read the class fields.
/// </summary>
/// <param name="reader">The reader from which to read </param>
public void Read(IDataReader reader)
{
Value = (T)_injector.ForkInjector().GetInstance(typeof(T));
Value.Read(reader);
}
public ForksInjectorInConstructor(IInjector i)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { @"Com.Microsoft.Tang.Examples" });
//cb.BindImplementation(Number.class, typeof(Int32));
i.ForkInjector(new IConfiguration[] { cb.Build() });
}
public ContextRuntime(
string id,
IInjector serviceInjector,
IConfiguration contextConfiguration)
{
// This should only be used at the root context to support backward compatibility.
LOGGER.Log(Level.Info, "Instantiating root context");
_contextLifeCycle = new ContextLifeCycle(id);
_serviceInjector = serviceInjector;
_parentContext = Optional<ContextRuntime>.Empty();
try
{
_contextInjector = serviceInjector.ForkInjector();
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
Optional<string> parentId = ParentContext.IsPresent() ?
Optional<string>.Of(ParentContext.Value.Id) :
Optional<string>.Empty();
ContextClientCodeException ex = new ContextClientCodeException(ContextClientCodeException.GetId(contextConfiguration), parentId, "Unable to spawn context", e);
Utilities.Diagnostics.Exceptions.Throw(ex, LOGGER);
}
// Trigger the context start events on contextInjector.
_contextLifeCycle.Start();
}