public GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
StreamingNetworkService<GeneralGroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary<string, ICommunicationGroupClientInternal>();
_networkService = networkService;
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance<ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
networkService.Register(new StringIdentifier(taskId));
foreach (var group in _commGroups.Values)
{
group.WaitingForRegistration();
}
}
public GroupCommDriver(
[Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId,
[Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId,
[Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut,
[Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks,
AvroConfigurationSerializer configSerializer,
INameServer nameServer)
{
_driverId = driverId;
_contextIds = -1;
_fanOut = fanOut;
MasterTaskId = masterTaskId;
_groupName = groupName;
_configSerializer = configSerializer;
_commGroups = new Dictionary<string, ICommunicationGroupDriver>();
_nameServer = nameServer;
IPEndPoint localEndpoint = _nameServer.LocalEndpoint;
_nameServerAddr = localEndpoint.Address.ToString();
_nameServerPort = localEndpoint.Port;
NewCommunicationGroup(groupName, numberOfTasks);
}
internal ActiveContext(IActiveContextClr2Java clr2Java)
{
InstanceId = Guid.NewGuid().ToString("N");
Clr2Java = clr2Java;
_serializer = new AvroConfigurationSerializer();
_parentId = Optional<string>.OfNullable(clr2Java.GetParentId());
}
public void TestServiceConfiguration()
{
string groupName = "group1";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var serializer = new AvroConfigurationSerializer();
var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
groupName, fanOut,
numTasks);
//driver side to prepar for service config
var codecConfig = CodecConfiguration<int>.Conf
.Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class)
.Build();
var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
//wrap it before serializing
var wrappedSeriveConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter<ServicesConfigurationOptions.ServiceConfigString, string>(
GenericType<ServicesConfigurationOptions.ServiceConfigString>.Class,
new AvroConfigurationSerializer().ToString(serviceConfig))
.Build();
var serviceConfigString = serializer.ToString(wrappedSeriveConfig);
//the configuration string is received at Evaluator side
var serviceConfig2 = new ServiceConfiguration(serviceConfigString);
Assert.AreEqual(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2.TangConfig));
}
public void TestServiceConfiguration()
{
string groupName = "group1";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var serializer = new AvroConfigurationSerializer();
var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
groupName, fanOut,
numTasks);
// driver side to prepar for service config
var codecConfig = CodecConfiguration<int>.Conf
.Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class)
.Build();
var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
var serviceConfigString = serializer.ToString(serviceConfig);
// the configuration string is received at Evaluator side
var serviceConfig2 = new AvroConfigurationSerializer().FromString(serviceConfigString);
Assert.Equal(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2));
}
public void TestDeserializationWithAlias()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
var language = avroConfiguration.language;
Assert.True(language.ToString().Equals(Language.Java.ToString()));
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var config = serializer.FromAvro(avroConfiguration, classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
string appid = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>();
string remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>();
string evaluatorIdentifier = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>();
string rid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>();
string launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>();
Assert.True(remoteId.StartsWith(RemoteIdPrefix));
Assert.True(appid.Equals(AppIdForTest));
Assert.True(evaluatorIdentifier.StartsWith(EvaluatorIdPrefix));
Assert.True(rid.StartsWith(RemoteIdPrefix));
Assert.True(launchId.Equals(AppIdForTest));
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
}
public void TestDeserializeEvaluatorContextServiceTaskWithAlias()
{
var serializer = new AvroConfigurationSerializer();
var config = DeserializeConfigWithAlias();
var evaluatorInjector = TangFactory.GetTang().NewInjector(config);
var evaluatorConfigString = evaluatorInjector.GetNamedInstance<EvaluatorConfiguration, string>();
var taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>();
var contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>();
var serviceConfigString = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>();
var evaluatorClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(DefaultLocalHttpDriverConnection).Assembly.GetName().Name
});
var evaluatorConfig = serializer.FromString(evaluatorConfigString, evaluatorClassHierarchy);
var fullEvaluatorInjector = evaluatorInjector.ForkInjector(evaluatorConfig);
Assert.True(fullEvaluatorInjector.GetInstance<IDriverConnection>() is DefaultLocalHttpDriverConnection);
var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(Common.Context.ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
});
var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ITask).Assembly.GetName().Name,
typeof(HelloTask).Assembly.GetName().Name
});
var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy);
var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ServiceConfiguration).Assembly.GetName().Name,
typeof(IStreamingCodec<>).Assembly.GetName().Name
});
var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy);
var contextInjector = fullEvaluatorInjector.ForkInjector(contextConfig);
string contextId = contextInjector.GetNamedInstance<Common.Context.ContextConfigurationOptions.ContextIdentifier, string>();
Assert.True(contextId.StartsWith(ContextIdPrefix));
var serviceInjector = contextInjector.ForkInjector(serviceConfig);
var service = serviceInjector.GetInstance<TestService>();
Assert.NotNull(service);
var taskInjector = serviceInjector.ForkInjector(taskConfig);
var taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>();
var task = taskInjector.GetInstance<ITask>();
Assert.True(taskId.StartsWith("HelloTask"));
Assert.True(task is HelloTask);
}
private ContextManager(
AvroConfigurationSerializer serializer,
IHeartBeatManager heartBeatManager,
RootContextLauncher rootContextLauncher)
{
_rootContextLauncher = rootContextLauncher;
_heartBeatManager = heartBeatManager;
_serializer = serializer;
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var evaluatorConfiguration = serializer.FromFile(configFile, classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(evaluatorConfiguration);
LOGGER.Log(Level.Info,
string.Format(CultureInfo.CurrentCulture, "Evaluator Configuration is deserialized from file {0}:", configFile));
try
{
_taskConfiguration = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Info, "InitialTaskConfiguration is not set in Evaluator.config.");
}
try
{
_rootContextConfiguration = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Warning, "RootContextConfiguration is not set in Evaluator.config.");
}
try
{
_rootServiceConfiguration = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Info, "RootServiceConfiguration is not set in Evaluator.config.");
}
_applicationId = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>();
_remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>();
_evaluatorId = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>();
_errorHandlerRid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>();
_launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>();
}
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets,
[Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet<IConfigurationProvider> driverConfigurationProviders)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
_driverConfigurationProviders = driverConfigurationProviders;
}
public AllocatedEvaluator(IAllocatedEvaluaotrClr2Java clr2Java, ISet<IConfigurationProvider> configurationProviders)
{
_configurationProviders = configurationProviders;
InstanceId = Guid.NewGuid().ToString("N");
_serializer = new AvroConfigurationSerializer();
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
private RootContextLauncher(
AvroConfigurationSerializer serializer,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfiguration,
[Parameter(typeof(RootServiceConfiguration))] string rootServiceConfiguration,
IInjector injector)
: this(serializer.FromString(rootContextConfiguration),
serializer.FromString(rootServiceConfiguration),
Optional<IConfiguration>.Empty(),
injector)
{
}
public void TestEvaluatorConfigurationFile()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf");
Assert.IsNotNull(avroConfiguration);
foreach (var b in avroConfiguration.Bindings)
{
Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value);
}
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector) :
this(applicationId, evaluatorId, heartbeatPeriodInMs, maxHeartbeatRetries, rootContextConfigString, serializer,
clock, remoteManagerFactory, reefMessageCodec, injector, null)
{
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector)
{
_serializer = serializer;
_injector = injector;
_applicationId = applicationId;
_evaluatorId = evaluatorId;
_heartBeatPeriodInMs = heartbeatPeriodInMs;
_maxHeartbeatRetries = maxHeartbeatRetries;
_clock = clock;
if (string.IsNullOrWhiteSpace(rootContextConfigString))
{
Utilities.Diagnostics.Exceptions.Throw(
new ArgumentException("empty or null rootContextConfigString"), Logger);
}
_rootContextConfig = _serializer.FromString(rootContextConfigString);
try
{
_rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
}
catch (InjectionException)
{
Logger.Log(Level.Info, "Using deprecated ContextConfiguration.");
// TODO[JIRA REEF-1167]: Remove this catch.
var deprecatedContextConfig = new Context.ContextConfiguration(rootContextConfigString);
_rootContextConfig = deprecatedContextConfig;
_rootContextId = deprecatedContextConfig.Id;
}
_rootTaskConfiguration = CreateTaskConfiguration();
_rootServiceConfiguration = CreateRootServiceConfiguration();
_remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec);
_operationState = EvaluatorOperationState.OPERATIONAL;
}
private ContextManager(
HeartBeatManager heartBeatManager,
EvaluatorSettings evaluatorSettings,
AvroConfigurationSerializer serializer)
{
using (LOGGER.LogFunction("ContextManager::ContextManager"))
{
_heartBeatManager = heartBeatManager;
_serializer = serializer;
_rootContextLauncher = new RootContextLauncher(
evaluatorSettings.RootContextId,
evaluatorSettings.RootContextConfig,
evaluatorSettings.RootServiceConfiguration,
evaluatorSettings.RootTaskConfiguration);
}
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeserializeFromFile(_configFile);
}
}
public AllocatedEvaluator(IAllocatedEvaluatorClr2Java clr2Java, ISet<IConfigurationProvider> configurationProviders)
{
_serializer = new AvroConfigurationSerializer();
var evaluatorConfig = TangFactory.GetTang().NewConfigurationBuilder().Build();
foreach (var configurationProvider in configurationProviders)
{
evaluatorConfig = Configurations.Merge(evaluatorConfig, configurationProvider.GetConfiguration());
}
_evaluatorConfigStr = _serializer.ToString(evaluatorConfig);
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
public void CanInjectAndExecuteTask()
{
// to enforce that shell task dll be copied to output directory.
ShellTask tmpTask = new ShellTask("invalid");
Assert.NotNull(tmpTask);
string tmp = Directory.GetCurrentDirectory();
Assert.NotNull(tmp);
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
AvroConfiguration avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
Assert.NotNull(avroConfiguration);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Test_CLRContext_task")
.Set(TaskConfiguration.Task, GenericType<ShellTask>.Class)
.Build());
cb.BindNamedParameter<ShellTask.Command, string>(GenericType<ShellTask.Command>.Class, "dir");
IConfiguration taskConfiguration = cb.Build();
string taskConfig = serializer.ToString(taskConfiguration);
ITask task = null;
TaskConfiguration config = new TaskConfiguration(taskConfig);
Assert.NotNull(config);
try
{
IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig);
task = (ITask)injector.GetInstance(typeof(ITask));
}
catch (Exception e)
{
throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e);
}
byte[] bytes = task.Call(null);
string result = System.Text.Encoding.Default.GetString(bytes);
// a dir command is executed in the container directory, which includes the file "evaluator.conf"
Assert.True(result.Contains("evaluator.conf"));
}
private ContextManager(
IHeartBeatManager heartBeatManager,
EvaluatorSettings evaluatorSettings,
AvroConfigurationSerializer serializer)
{
// TODO[JIRA REEF-217]: Inject base Injector and pass Injector to RootContextLauncher
using (LOGGER.LogFunction("ContextManager::ContextManager"))
{
_heartBeatManager = heartBeatManager;
_serializer = serializer;
_rootContextLauncher = new RootContextLauncher(
evaluatorSettings.RootContextId,
evaluatorSettings.RootContextConfig,
evaluatorSettings.RootServiceConfiguration,
evaluatorSettings.RootTaskConfiguration,
heartBeatManager);
}
}
private GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
NetworkService<GroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary<string, ICommunicationGroupClient>();
_networkService = networkService;
networkService.Register(new StringIdentifier(taskId));
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
ICommunicationGroupClient commGroupClient = groupInjector.GetInstance<ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeserializeFromFile(_configFile);
var language = _avroConfiguration.language;
LOGGER.Log(Level.Info, "The language that created the configFile is " + language);
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
_evaluatorConfiguration = serializer.FromAvro(_avroConfiguration, classHierarchy);
}
}
/// <summary>
/// Read evaluator configuration from evaluator.conf generated from Java
/// </summary>
/// <param name="evaluatorConfigFile"></param>
/// <returns></returns>
private static IConfiguration ReadEvaluatorConfiguration(string evaluatorConfigFile)
{
if (string.IsNullOrWhiteSpace(evaluatorConfigFile))
{
Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), logger);
}
if (!File.Exists(evaluatorConfigFile))
{
Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + evaluatorConfigFile), logger);
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var evaluatorConfiguration = serializer.FromFile(evaluatorConfigFile, classHierarchy);
logger.Log(Level.Info,
string.Format(CultureInfo.CurrentCulture, "Evaluator Configuration is deserialized from file {0}:", evaluatorConfigFile));
return evaluatorConfiguration;
}
public void TestDeserializationForContextAndTask()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var config = serializer.FromFile("evaluator.conf", classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
string taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>();
string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>();
var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
});
var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ITask).Assembly.GetName().Name,
typeof(HelloTask).Assembly.GetName().Name
});
var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy);
var contextInjector = evaluatorInjector.ForkInjector(contextConfig);
string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
Assert.True(contextId.StartsWith(ContextIdPrefix));
var taskInjector = contextInjector.ForkInjector(taskConfig);
string taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>();
ITask task = taskInjector.GetInstance<ITask>();
Assert.True(taskId.StartsWith("HelloTask"));
Assert.True(task is HelloTask);
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}
private ConfigurationManager(
AvroConfigurationSerializer configurationSerializer,
[Parameter(typeof(SerializedMapConfiguration))] string mapConfig,
[Parameter(typeof(SerializedReduceConfiguration))] string reduceConfig,
[Parameter(typeof(SerializedUpdateConfiguration))] string updateConfig,
[Parameter(typeof(SerializedMapInputCodecConfiguration))] string mapInputCodecConfig,
[Parameter(typeof(SerializedUpdateFunctionCodecsConfiguration))] string updateFunctionCodecsConfig,
[Parameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration))] string
mapOutputPipelineDataConverterConfiguration,
[Parameter(typeof(SerializedMapInputPipelineDataConverterConfiguration))] string
mapInputPipelineDataConverterConfiguration,
[Parameter(typeof(SerializedResultHandlerConfiguration))] string resultHandlerConfiguration)
{
try
{
_mapFunctionConfiguration = configurationSerializer.FromString(mapConfig);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize map function configuration", Logger);
}
try
{
_reduceFunctionConfiguration = configurationSerializer.FromString(reduceConfig);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize reduce function configuration", Logger);
}
try
{
_updateFunctionConfiguration = configurationSerializer.FromString(updateConfig);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize update function configuration", Logger);
}
try
{
_updateFunctionCodecsConfiguration = configurationSerializer.FromString(updateFunctionCodecsConfig);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize update function codec configuration", Logger);
}
try
{
_mapInputCodecConfiguration = configurationSerializer.FromString(mapInputCodecConfig);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize map input codec configuration", Logger);
}
try
{
_mapOutputPipelineDataConverterConfiguration =
configurationSerializer.FromString(mapOutputPipelineDataConverterConfiguration);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize map output pipeline data converter configuration", Logger);
}
try
{
_mapInputPipelineDataConverterConfiguration =
configurationSerializer.FromString(mapInputPipelineDataConverterConfiguration);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize map input pipeline data converter configuration", Logger);
}
try
{
_resultHandlerConfiguration =
configurationSerializer.FromString(resultHandlerConfiguration);
Logger.Log(Level.Verbose, "Serialized result handler is " + resultHandlerConfiguration);
}
catch (Exception e)
{
Exceptions.Throw(e, "Unable to deserialize map input pipeline data converter configuration", Logger);
}
}
public TaskConfiguration(string configString)
{
TangConfig = new AvroConfigurationSerializer().FromString(configString);
AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString);
foreach (ConfigurationEntry config in avroConfiguration.Bindings)
{
if (config.key.Contains(TaskIdentifier))
{
TaskId = config.value;
}
}
if (string.IsNullOrWhiteSpace(TaskId))
{
string msg = "Required parameter TaskId not provided.";
LOGGER.Log(Level.Error, msg);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER);
}
}
/// <summary>
/// Simulate evaluator configuration generated from Java for unit testing
/// </summary>
/// <param name="serializer"></param>
/// <returns></returns>
private static AvroConfiguration EvaluatorConfig(AvroConfigurationSerializer serializer)
{
var configurationEntries = new HashSet<ConfigurationEntry>();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.ApplicationIdentifier",
"REEF_LOCAL_RUNTIME"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.DriverRemoteIdentifier",
"socket://10.130.68.76:9723"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorIdentifier",
"Node-2-1447450298921"));
var evaluatorConfiguration = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType<IDriverConnection>.Class, GenericType<DefaultLocalHttpDriverConnection>.Class)
.Build();
var evaluatorString = serializer.ToString(evaluatorConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorConfiguration",
evaluatorString));
var taskConfiguration = TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "HelloTask")
.Set(TaskConfiguration.Task, GenericType<HelloTask>.Class)
.Build();
var taskString = serializer.ToString(taskConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.InitialTaskConfiguration",
taskString));
var contextConfig = Common.Context.ContextConfiguration.ConfigurationModule.Set(Common.Context.ContextConfiguration.Identifier, ContextIdPrefix).Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootContextConfiguration",
serializer.ToString(contextConfig)));
var serviceConfiguration = ServiceConfiguration.ConfigurationModule
.Set(ServiceConfiguration.Services, GenericType<TestService>.Class)
.Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootServiceConfiguration",
serializer.ToString(serviceConfiguration)));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID",
"socket://10.130.68.76:9723"));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.LaunchID",
"REEF_LOCAL_RUNTIME"));
return new AvroConfiguration(Language.Java.ToString(), configurationEntries);
}
/// <summary>
/// Deserialize evaluator configuration with alias
/// </summary>
/// <returns></returns>
private static IConfiguration DeserializeConfigWithAlias()
{
var serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var avroConfiguration = EvaluatorConfig(serializer);
return serializer.FromAvro(avroConfiguration, classHierarchy);
}
