public ContextConfiguration(string configString)
{
using (LOGGER.LogFunction("ContextConfiguration::ContextConfigurationn"))
{
ContainerDirectory = Directory.GetCurrentDirectory();
_settings = new Dictionary <string, string>();
AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString);
foreach (ConfigurationEntry config in avroConfiguration.Bindings)
{
if (config.key.Contains(Reef.Evaluator.Constants.ContextIdentifier))
{
config.key = Reef.Evaluator.Constants.ContextIdentifier;
LOGGER.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "{0} detected for context id with value {1}", config.key, config.value));
}
_settings.Add(config.key, config.value);
}
if (!_settings.ContainsKey(Reef.Evaluator.Constants.ContextIdentifier))
{
string msg = "Required parameter ContextIdentifier not provided.";
LOGGER.Log(Level.Error, msg);
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER);
}
}
}
public AvroConfiguration AvroDeserializeFromFile(string fileName)
{
AvroConfiguration avroConf = null;
try
{
using (var buffer = new MemoryStream())
{
if (!ReadFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method : " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
buffer.Seek(0, SeekOrigin.Begin);
using (var reader = new SequentialReader <AvroConfiguration>(AvroContainer.CreateReader <AvroConfiguration>(buffer, true)))
{
var results = reader.Objects;
if (results != null)
{
avroConf = (AvroConfiguration)results.First();
}
}
}
}
catch (SerializationException ex)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
var e = new ApplicationException("Cannot deserialize the file: " + fileName, ex);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
return(avroConf);
}
public void TestDeserializedConfigMerge()
{
Type activityInterfaceType = typeof(ITask);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb1 = tang.NewConfigurationBuilder();
cb1.BindImplementation(GenericType <ITask> .Class, GenericType <HelloTask> .Class);
cb1.BindNamedParameter <TaskConfigurationOptions.Identifier, string>(
GenericType <TaskConfigurationOptions.Identifier> .Class, "Hello Task");
IConfiguration conf1 = cb1.Build();
var serializer = new AvroConfigurationSerializer();
serializer.ToFile(conf1, "task.config");
ICsConfigurationBuilder cb2 = tang.NewConfigurationBuilder();
cb2.BindNamedParameter <Timer.Seconds, int>(GenericType <Timer.Seconds> .Class, "2");
IConfiguration conf2 = cb2.Build();
serializer.ToFile(conf2, "timer.config");
ProtocolBufferClassHierarchy.Serialize("TaskTimer.bin", conf1.GetClassHierarchy());
IClassHierarchy ns = ProtocolBufferClassHierarchy.DeSerialize("TaskTimer.bin");
AvroConfiguration taskAvroconfiguration = serializer.AvroDeserializeFromFile("task.config");
IConfiguration taskConfiguration = serializer.FromAvro(taskAvroconfiguration, ns);
AvroConfiguration timerAvroconfiguration = serializer.AvroDeserializeFromFile("timer.config");
IConfiguration timerConfiguration = serializer.FromAvro(timerAvroconfiguration, ns);
IConfiguration merged = Configurations.MergeDeserializedConfs(taskConfiguration, timerConfiguration);
var b = merged.newBuilder().Build();
}
public IConfiguration FromFileStream(string fileName)
{
byte[] bytes = File.ReadAllBytes(fileName);
AvroConfiguration avroConf = AvroDeserialize(bytes);
return(FromAvro(avroConf));
}
public string ToString(IConfiguration c)
{
byte[] bytes = ToByteArray(c);
AvroConfiguration avroConf = AvroDeserialize(bytes);
string s = JsonConvert.SerializeObject(avroConf, Formatting.Indented);
return(s);
}
private byte[] AvroSerialize(AvroConfiguration obj)
{
var serializer = AvroSerializer.Create <AvroConfiguration>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
private IConfiguration AddFromAvro(IConfigurationBuilder cb, AvroConfiguration avroConfiguration)
{
IList <KeyValuePair <string, string> > settings = new List <KeyValuePair <string, string> >();
foreach (ConfigurationEntry e in avroConfiguration.Bindings)
{
settings.Add(new KeyValuePair <string, string>(e.key, e.value));
}
ConfigurationFile.ProcessConfigData(cb, settings, avroConfiguration.language);
return(cb.Build());
}
private AvroConfiguration ToAvroConfiguration()
{
HashSet<ConfigurationEntry> b = new HashSet<ConfigurationEntry>();
ConfigurationEntry e1 = new ConfigurationEntry();
e1.key = "a";
e1.value = "a1";
ConfigurationEntry e2 = new ConfigurationEntry();
e2.key = "b";
e2.value = "b1=b2";
b.Add(e1);
b.Add(e2);
var a = new AvroConfiguration(Language.Cs.ToString(), b);
return a;
}
public void CanInjectAndExecuteTask()
{
//To enforce that shell task dll be copied to output directory.
ShellTask tmpTask = new ShellTask("invalid");
Assert.IsNotNull(tmpTask);
string tmp = Directory.GetCurrentDirectory();
Assert.IsNotNull(tmp);
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
AvroConfiguration avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf");
Assert.IsNotNull(avroConfiguration);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Test_CLRContext_task")
.Set(TaskConfiguration.Task, GenericType <ShellTask> .Class)
.Build());
cb.BindNamedParameter <ShellTask.Command, string>(GenericType <ShellTask.Command> .Class, "dir");
IConfiguration taskConfiguration = cb.Build();
string taskConfig = serializer.ToString(taskConfiguration);
ITask task = null;
TaskConfiguration config = new TaskConfiguration(taskConfig);
Assert.IsNotNull(config);
try
{
IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig);
task = (ITask)injector.GetInstance(typeof(ITask));
}
catch (Exception e)
{
throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e);
}
byte[] bytes = task.Call(null);
string result = System.Text.Encoding.Default.GetString(bytes);
//a dir command is executed in the container directory, which includes the file "evaluator.conf"
Assert.IsTrue(result.Contains("evaluator.conf"));
}
private AvroConfiguration ToAvroConfiguration()
{
HashSet <ConfigurationEntry> b = new HashSet <ConfigurationEntry>();
ConfigurationEntry e1 = new ConfigurationEntry();
e1.key = "a";
e1.value = "a1";
ConfigurationEntry e2 = new ConfigurationEntry();
e2.key = "b";
e2.value = "b1=b2";
b.Add(e1);
b.Add(e2);
var a = new AvroConfiguration(b);
return(a);
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeseriaizeFromFile(_configFile);
}
}
public TaskConfiguration(string configString)
{
TangConfig = new AvroConfigurationSerializer().FromString(configString);
AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString);
foreach (ConfigurationEntry config in avroConfiguration.Bindings)
{
if (config.key.Contains(TaskIdentifier))
{
TaskId = config.value;
}
}
if (string.IsNullOrWhiteSpace(TaskId))
{
string msg = "Required parameter TaskId not provided.";
LOGGER.Log(Level.Error, msg);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER);
}
}
public byte[] ToByteArray(IConfiguration c)
{
AvroConfiguration obj = ToAvroConfiguration(c);
return(AvroSerialize(obj));
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}
public IConfiguration FromAvro(AvroConfiguration avroConfiguration, IClassHierarchy classHierarchy)
{
IConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(classHierarchy);
return(AddFromAvro(cb, avroConfiguration));
}
public IConfiguration FromAvro(AvroConfiguration avroConfiguration)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
return(AddFromAvro(cb, avroConfiguration));
}
public IConfiguration FromByteArray(byte[] bytes)
{
AvroConfiguration avroConf = AvroDeserialize(bytes);
return(FromAvro(avroConf));
}
public IConfiguration FromString(string josonString, IClassHierarchy ch)
{
AvroConfiguration avroConf = JsonConvert.DeserializeObject <AvroConfiguration>(josonString);
return(FromAvro(avroConf, ch));
}
public IConfiguration FromString(string jsonString)
{
AvroConfiguration avroConf = JsonConvert.DeserializeObject <AvroConfiguration>(jsonString);
return(FromAvro(avroConf));
}
public IConfiguration FromFile(string fileName, IClassHierarchy classHierarchy)
{
AvroConfiguration avroConf = AvroDeserializeFromFile(fileName);
return(FromAvro(avroConf, classHierarchy));
}
public IConfiguration FromFile(string fileName)
{
AvroConfiguration avroConf = AvroDeserializeFromFile(fileName);
return(FromAvro(avroConf));
}
public IConfiguration AddFromByteArray(ICsConfigurationBuilder cb, byte[] bytes)
{
AvroConfiguration avroConf = AvroDeserialize(bytes);
return(AddFromAvro(cb, avroConf));
}
