public void TestDeserializationWithAlias()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
var language = avroConfiguration.language;
Assert.True(language.ToString().Equals(Language.Java.ToString()));
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var config = serializer.FromAvro(avroConfiguration, classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
string appid = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>();
string remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>();
string evaluatorIdentifier = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>();
string rid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>();
string launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>();
Assert.True(remoteId.StartsWith(RemoteIdPrefix));
Assert.True(appid.Equals(AppIdForTest));
Assert.True(evaluatorIdentifier.StartsWith(EvaluatorIdPrefix));
Assert.True(rid.StartsWith(RemoteIdPrefix));
Assert.True(launchId.Equals(AppIdForTest));
}
public void TestEvaluatorConfigurationFile()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
Assert.NotNull(avroConfiguration);
Assert.Equal(avroConfiguration.language, Language.Java.ToString());
foreach (var b in avroConfiguration.Bindings)
{
Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value);
}
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeserializeFromFile(_configFile);
}
}
public void CanInjectAndExecuteTask()
{
// to enforce that shell task dll be copied to output directory.
ShellTask tmpTask = new ShellTask("invalid");
Assert.NotNull(tmpTask);
string tmp = Directory.GetCurrentDirectory();
Assert.NotNull(tmp);
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
AvroConfiguration avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
Assert.NotNull(avroConfiguration);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Test_CLRContext_task")
.Set(TaskConfiguration.Task, GenericType<ShellTask>.Class)
.Build());
cb.BindNamedParameter<ShellTask.Command, string>(GenericType<ShellTask.Command>.Class, "dir");
IConfiguration taskConfiguration = cb.Build();
string taskConfig = serializer.ToString(taskConfiguration);
ITask task = null;
TaskConfiguration config = new TaskConfiguration(taskConfig);
Assert.NotNull(config);
try
{
IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig);
task = (ITask)injector.GetInstance(typeof(ITask));
}
catch (Exception e)
{
throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e);
}
byte[] bytes = task.Call(null);
string result = System.Text.Encoding.Default.GetString(bytes);
// a dir command is executed in the container directory, which includes the file "evaluator.conf"
Assert.True(result.Contains("evaluator.conf"));
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeserializeFromFile(_configFile);
var language = _avroConfiguration.language;
LOGGER.Log(Level.Info, "The language that created the configFile is " + language);
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
_evaluatorConfiguration = serializer.FromAvro(_avroConfiguration, classHierarchy);
}
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}