public void TestGeneratingDriverConfigFileWithoutNameServer()
{
DriverConfigurationSettings driverSubmissionSettings = new DriverConfigurationSettings()
{
DriverMemory = 1024,
DriverIdentifier = "juliaDriverId",
SubmissionDirectory = "reefClrBridgeTmp/job_" + Guid.NewGuid().ToString("N").Substring(0, 4),
IncludingHttpServer = true,
IncludingNameServer = false,
ClrFolder = ".",
JarFileFolder = ".\\"
};
DriverConfigGenerator.DriverConfigurationBuilder(driverSubmissionSettings);
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}