public void TestServiceConfiguration()
{
string groupName = "group1";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var serializer = new AvroConfigurationSerializer();
var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
groupName, fanOut,
numTasks);
// driver side to prepar for service config
var codecConfig = CodecConfiguration<int>.Conf
.Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class)
.Build();
var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
var serviceConfigString = serializer.ToString(serviceConfig);
// the configuration string is received at Evaluator side
var serviceConfig2 = new AvroConfigurationSerializer().FromString(serviceConfigString);
Assert.Equal(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2));
}
public void TestServiceConfiguration()
{
string groupName = "group1";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var serializer = new AvroConfigurationSerializer();
var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
groupName, fanOut,
numTasks);
//driver side to prepar for service config
var codecConfig = CodecConfiguration<int>.Conf
.Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class)
.Build();
var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
//wrap it before serializing
var wrappedSeriveConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter<ServicesConfigurationOptions.ServiceConfigString, string>(
GenericType<ServicesConfigurationOptions.ServiceConfigString>.Class,
new AvroConfigurationSerializer().ToString(serviceConfig))
.Build();
var serviceConfigString = serializer.ToString(wrappedSeriveConfig);
//the configuration string is received at Evaluator side
var serviceConfig2 = new ServiceConfiguration(serviceConfigString);
Assert.AreEqual(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2.TangConfig));
}
public AllocatedEvaluator(IAllocatedEvaluatorClr2Java clr2Java, ISet<IConfigurationProvider> configurationProviders)
{
_serializer = new AvroConfigurationSerializer();
var evaluatorConfig = TangFactory.GetTang().NewConfigurationBuilder().Build();
foreach (var configurationProvider in configurationProviders)
{
evaluatorConfig = Configurations.Merge(evaluatorConfig, configurationProvider.GetConfiguration());
}
_evaluatorConfigStr = _serializer.ToString(evaluatorConfig);
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
public void CanInjectAndExecuteTask()
{
// to enforce that shell task dll be copied to output directory.
ShellTask tmpTask = new ShellTask("invalid");
Assert.NotNull(tmpTask);
string tmp = Directory.GetCurrentDirectory();
Assert.NotNull(tmp);
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
AvroConfiguration avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
Assert.NotNull(avroConfiguration);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Test_CLRContext_task")
.Set(TaskConfiguration.Task, GenericType<ShellTask>.Class)
.Build());
cb.BindNamedParameter<ShellTask.Command, string>(GenericType<ShellTask.Command>.Class, "dir");
IConfiguration taskConfiguration = cb.Build();
string taskConfig = serializer.ToString(taskConfiguration);
ITask task = null;
TaskConfiguration config = new TaskConfiguration(taskConfig);
Assert.NotNull(config);
try
{
IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig);
task = (ITask)injector.GetInstance(typeof(ITask));
}
catch (Exception e)
{
throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e);
}
byte[] bytes = task.Call(null);
string result = System.Text.Encoding.Default.GetString(bytes);
// a dir command is executed in the container directory, which includes the file "evaluator.conf"
Assert.True(result.Contains("evaluator.conf"));
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}
/// <summary>
/// Simulate evaluator configuration generated from Java for unit testing
/// </summary>
/// <param name="serializer"></param>
/// <returns></returns>
private static AvroConfiguration EvaluatorConfig(AvroConfigurationSerializer serializer)
{
var configurationEntries = new HashSet<ConfigurationEntry>();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.ApplicationIdentifier",
"REEF_LOCAL_RUNTIME"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.DriverRemoteIdentifier",
"socket://10.130.68.76:9723"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorIdentifier",
"Node-2-1447450298921"));
var evaluatorConfiguration = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType<IDriverConnection>.Class, GenericType<DefaultLocalHttpDriverConnection>.Class)
.Build();
var evaluatorString = serializer.ToString(evaluatorConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorConfiguration",
evaluatorString));
var taskConfiguration = TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "HelloTask")
.Set(TaskConfiguration.Task, GenericType<HelloTask>.Class)
.Build();
var taskString = serializer.ToString(taskConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.InitialTaskConfiguration",
taskString));
var contextConfig = Common.Context.ContextConfiguration.ConfigurationModule.Set(Common.Context.ContextConfiguration.Identifier, ContextIdPrefix).Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootContextConfiguration",
serializer.ToString(contextConfig)));
var serviceConfiguration = ServiceConfiguration.ConfigurationModule
.Set(ServiceConfiguration.Services, GenericType<TestService>.Class)
.Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootServiceConfiguration",
serializer.ToString(serviceConfiguration)));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID",
"socket://10.130.68.76:9723"));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.LaunchID",
"REEF_LOCAL_RUNTIME"));
return new AvroConfiguration(Language.Java.ToString(), configurationEntries);
}
public override IConfiguration RoundTrip(IConfiguration configuration)
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
return serializer.FromString(serializer.ToString(configuration));
}