public void TestDeSerializeClassHierarchyAndBind()
{
Type streamTask1Type = typeof(StreamTask1);
Type helloTaskType = typeof(HelloTask);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(HelloTask).Assembly.GetName().Name });
IClassNode StreamTask1ClassNode = (IClassNode)ns.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode HelloTaskClassNode = (IClassNode)ns.GetNode(helloTaskType.AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize("task.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("task.bin");
IClassNode StreamTask1ClassNode2 = (IClassNode)ch.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode HelloTaskClassNode2 = (IClassNode)ch.GetNode(helloTaskType.AssemblyQualifiedName);
Assert.Equal(StreamTask1ClassNode.GetName(), StreamTask1ClassNode2.GetName());
Assert.Equal(HelloTaskClassNode.GetName(), HelloTaskClassNode2.GetName());
// have to use original class hierarchy for the merge. ClassHierarchy from ProtoBuffer doesn't support merge.
IConfigurationBuilder cb = TangFactory.GetTang()
.NewConfigurationBuilder(ns);
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Hello_From_Streaming1")
.Set(TaskConfiguration.Task, GenericType <StreamTask1> .Class)
.Build());
IConfiguration taskConfiguration = cb.Build();
StreamTask1 st = TangFactory.GetTang().NewInjector(taskConfiguration).GetInstance <StreamTask1>();
Assert.NotNull(st);
}
public void TestDeSerializeClassHierarchy()
{
Type timerType = typeof(Timer);
Type SecondType = typeof(Timer.Seconds);
Type simpleCOnstuctorType = typeof(SimpleConstructors);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(Timer).Assembly.GetName().Name });
IClassNode timerClassNode = (IClassNode)ns.GetNode(timerType.AssemblyQualifiedName);
INode secondNode = (INode)ns.GetNode(SecondType.AssemblyQualifiedName);
IClassNode SimpleConstructorsClassNode = (IClassNode)ns.GetNode(simpleCOnstuctorType.AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize("node.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("node.bin");
IClassNode timerClassNode2 = (IClassNode)ch.GetNode(timerType.AssemblyQualifiedName);
INode secondNode2 = ch.GetNode(SecondType.AssemblyQualifiedName);
IClassNode SimpleConstructorsClassNode2 = (IClassNode)ch.GetNode(simpleCOnstuctorType.AssemblyQualifiedName);
Assert.Equal(timerClassNode.GetFullName(), timerClassNode2.GetFullName());
Assert.Equal(secondNode.GetFullName(), secondNode2.GetFullName());
Assert.Equal(SimpleConstructorsClassNode.GetFullName(), SimpleConstructorsClassNode2.GetFullName());
Assert.True(SimpleConstructorsClassNode2.GetChildren().Count == 0);
IList <IConstructorDef> def = SimpleConstructorsClassNode2.GetInjectableConstructors();
Assert.Equal(3, def.Count);
}
public void TestDeserializedConfigMerge()
{
Type activityInterfaceType = typeof(ITask);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb1 = tang.NewConfigurationBuilder();
cb1.BindImplementation(GenericType <ITask> .Class, GenericType <HelloTask> .Class);
cb1.BindNamedParameter <TaskConfigurationOptions.Identifier, string>(
GenericType <TaskConfigurationOptions.Identifier> .Class, "Hello Task");
IConfiguration conf1 = cb1.Build();
var serializer = new AvroConfigurationSerializer();
serializer.ToFile(conf1, "task.config");
ICsConfigurationBuilder cb2 = tang.NewConfigurationBuilder();
cb2.BindNamedParameter <Timer.Seconds, int>(GenericType <Timer.Seconds> .Class, "2");
IConfiguration conf2 = cb2.Build();
serializer.ToFile(conf2, "timer.config");
ProtocolBufferClassHierarchy.Serialize("TaskTimer.bin", conf1.GetClassHierarchy());
IClassHierarchy ns = ProtocolBufferClassHierarchy.DeSerialize("TaskTimer.bin");
AvroConfiguration taskAvroconfiguration = serializer.AvroDeserializeFromFile("task.config");
IConfiguration taskConfiguration = serializer.FromAvro(taskAvroconfiguration, ns);
AvroConfiguration timerAvroconfiguration = serializer.AvroDeserializeFromFile("timer.config");
IConfiguration timerConfiguration = serializer.FromAvro(timerAvroconfiguration, ns);
IConfiguration merged = Configurations.MergeDeserializedConfs(taskConfiguration, timerConfiguration);
var b = merged.newBuilder().Build();
}
public void TestSerializeClassHierarchyForAvro()
{
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(Microsoft.Hadoop.Avro.AvroSerializer).Assembly.GetName().Name });
Assert.IsNotNull(ns);
ProtocolBufferClassHierarchy.Serialize("avro.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("avro.bin");
Assert.IsNotNull(ch);
}
public void TestDeSerializeClassHierarchyFromJava()
{
//the file comes from Java TestClassHierarchyRoundTrip SetUp3 testSimpleConstructors
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("simpleConstructorJavaProto.bin");
IClassNode simpleConstructorNode = (IClassNode)ch.GetNode("org.apache.reef.tang.implementation.SimpleConstructors");
Assert.AreEqual(simpleConstructorNode.GetChildren().Count, 0);
Assert.AreEqual(simpleConstructorNode.GetInjectableConstructors().Count, 3);
}
public void TestGenericClass()
{
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(Timer).Assembly.GetName().Name });
Type t = typeof(Timer);
IClassNode EventClassNode = (IClassNode)ns.GetNode(t.AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize("event.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("event.bin");
IClassNode EventClassNode1 = (IClassNode)ns.GetNode(t.AssemblyQualifiedName);
Assert.AreEqual(EventClassNode.GetName(), EventClassNode1.GetName());
}
/// <summary>
/// This program generates class hierarchy bin file for the list of dlls, plus a defalut list
/// The default list include: ITask, StreamTask1, HelloTask and ShellTask, please remove if not needed
/// </summary>
/// <param name="args"> additional dlls needed to build class hierarchy </param>
public static void Main(string[] args)
{
const string DllSubfix = ".dll";
const string ClassHierarchyBinFileName = "task.bin";
List <string> taskDlls = new List <string>();
foreach (string arg in args)
{
string assemblyName = arg;
if (!arg.EndsWith(DllSubfix, StringComparison.OrdinalIgnoreCase))
{
assemblyName += DllSubfix;
}
if (!File.Exists(assemblyName))
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "invalid argument: assembly {0} cannot be found", assemblyName));
}
taskDlls.Add(arg);
}
taskDlls.Add(GetAssemblyName(typeof(ITask)));
taskDlls.Add(GetAssemblyName(typeof(HelloTask)));
taskDlls.Add(GetAssemblyName(typeof(ShellTask)));
taskDlls.Add(GetAssemblyName(typeof(StreamTask1)));
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(taskDlls.ToArray());
// the following is verification only
// to verify that a class indeeded has been added to the class hierarchy, check the class name
IClassNode streamTaskClassNode = (IClassNode)ns.GetNode(typeof(StreamTask1).AssemblyQualifiedName);
IClassNode helloTaskClassNode = (IClassNode)ns.GetNode(typeof(HelloTask).AssemblyQualifiedName);
IClassNode shellTaskClassNode = (IClassNode)ns.GetNode(typeof(ShellTask).AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize(ClassHierarchyBinFileName, ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize(ClassHierarchyBinFileName);
IClassNode retrievedStreamTaskClassNode = (IClassNode)ch.GetNode(typeof(StreamTask1).AssemblyQualifiedName);
IClassNode retrievedHelloTaskClassNode = (IClassNode)ch.GetNode(typeof(HelloTask).AssemblyQualifiedName);
IClassNode retrievedShellTaskClassNode = (IClassNode)ch.GetNode(typeof(ShellTask).AssemblyQualifiedName);
if (!streamTaskClassNode.GetFullName().Equals(retrievedStreamTaskClassNode.GetFullName()) ||
!helloTaskClassNode.GetFullName().Equals(retrievedHelloTaskClassNode.GetFullName()) ||
!shellTaskClassNode.GetFullName().Equals(retrievedShellTaskClassNode.GetFullName()))
{
Console.WriteLine("Node deseriliazed is not equal");
}
else
{
Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Class hierarchy written to [{0}].", Directory.GetCurrentDirectory()));
}
}
private void setup3()
{
TangImpl.Reset();
try
{
ProtocolBufferClassHierarchy.Serialize("testProto.bin", ns);
ns = ProtocolBufferClassHierarchy.DeSerialize("testProto.bin");
}
catch (IOException e)
{
Assert.Fail(string.Format(CultureInfo.CurrentCulture, "IOException when serialize/deserialize proto buffer file", e));
}
}
public void TestDeSerializeClassHierarchyForActivity()
{
string[] s = new string[2];
s[0] = @"com.microsoft.reef.activity";
s[1] = @"com.microsoft.reef.activityInterface";
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(s);
IClassNode activityClassNode = (IClassNode)ns.GetNode("com.microsoft.reef.activity.HelloActivity");
ProtocolBufferClassHierarchy.Serialize("activity.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("activity.bin");
IClassNode activityClassNode2 = (IClassNode)ch.GetNode("com.microsoft.reef.activity.HelloActivity");
Assert.AreEqual(activityClassNode.GetFullName(), activityClassNode2.GetFullName());
}
public void TestDeSerializeClassHierarchyForTask()
{
Type streamTask1Type = typeof(StreamTask1);
Type helloTaskType = typeof(HelloTask);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(HelloTask).Assembly.GetName().Name });
IClassNode StreamTask1ClassNode = (IClassNode)ns.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode HelloTaskClassNode = (IClassNode)ns.GetNode(helloTaskType.AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize("task.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("task.bin");
IClassNode StreamTask1ClassNode2 = (IClassNode)ch.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode HelloTaskClassNode2 = (IClassNode)ch.GetNode(helloTaskType.AssemblyQualifiedName);
Assert.Equal(StreamTask1ClassNode.GetFullName(), StreamTask1ClassNode2.GetFullName());
Assert.Equal(HelloTaskClassNode.GetFullName(), HelloTaskClassNode2.GetFullName());
}
public void TestGetConfigFromProtoBufClassHierarchy()
{
Type iTaskType = typeof(ITask);
Type helloTaskType = typeof(HelloTask);
Type identifierType = typeof(TaskConfigurationOptions.Identifier);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { FileNames.Common, FileNames.Tasks });
ProtocolBufferClassHierarchy.Serialize("Task.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("Task.bin");
ITang tang = TangFactory.GetTang();
IConfigurationBuilder cb = tang.NewConfigurationBuilder(ch);
cb.Bind(iTaskType.AssemblyQualifiedName, helloTaskType.AssemblyQualifiedName);
cb.Bind(identifierType.AssemblyQualifiedName, "Hello Task");
IConfiguration conf = cb.Build();
ConfigurationFile.WriteConfigurationFile(conf, "taskConf2.txt");
}
public void TestGetConfgiFromProtoBufClassHierarchy()
{
Type activityInterfaceType = typeof(com.microsoft.reef.activity.IActivity);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { file2, file3 });
ProtocolBufferClassHierarchy.Serialize("activity.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("activity.bin");
ITang tang = TangFactory.GetTang();
IConfigurationBuilder cb = tang.NewConfigurationBuilder(ch);
cb.Bind("com.microsoft.reef.activity.IActivity", "com.microsoft.reef.activity.HelloActivity");
cb.Bind(@"com.microsoft.reef.driver.activity.ActivityConfigurationOptions+Identifier", "Hello Activity");
IConfiguration conf = cb.Build();
ConfigurationFile.WriteConfigurationFile(conf, "activityConf2.txt");
}
public void TestGenericArgument()
{
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(ClassWithGenericArgument<>).Assembly.GetName().Name });
Type t = typeof(ClassWithGenericArgument<>);
IClassNode classNode = (IClassNode)ns.GetNode(t.AssemblyQualifiedName);
var cons = classNode.GetAllConstructors();
foreach (var c in cons)
{
var args = c.GetArgs();
foreach (var a in args)
{
Assert.IsNotNull(a.GetName());
}
}
ProtocolBufferClassHierarchy.Serialize("generic.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("generic.bin");
IClassNode classNode1 = (IClassNode)ns.GetNode(t.AssemblyQualifiedName);
Assert.AreEqual(classNode.GetName(), classNode1.GetName());
}
public void TestDeSerializeClassHierarchy()
{
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { @"Com.Microsoft.Tang.Examples" });
IClassNode timerClassNode = (IClassNode)ns.GetNode("Com.Microsoft.Tang.Examples.Timer");
INode secondNode = (INode)ns.GetNode("Com.Microsoft.Tang.Examples.Timer+Seconds");
IClassNode SimpleConstructorsClassNode = (IClassNode)ns.GetNode("Com.Microsoft.Tang.Examples.SimpleConstructors");
ProtocolBufferClassHierarchy.Serialize("node.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("node.bin");
IClassNode timerClassNode2 = (IClassNode)ch.GetNode("Com.Microsoft.Tang.Examples.Timer");
INode secondNode2 = ch.GetNode("Com.Microsoft.Tang.Examples.Timer+Seconds");
IClassNode SimpleConstructorsClassNode2 = (IClassNode)ch.GetNode("Com.Microsoft.Tang.Examples.SimpleConstructors");
Assert.AreEqual(timerClassNode.GetFullName(), timerClassNode2.GetFullName());
Assert.AreEqual(secondNode.GetFullName(), secondNode.GetFullName());
Assert.AreEqual(SimpleConstructorsClassNode.GetFullName(), SimpleConstructorsClassNode2.GetFullName());
Assert.IsTrue(SimpleConstructorsClassNode2.GetChildren().Count == 0);
IList <IConstructorDef> def = SimpleConstructorsClassNode2.GetInjectableConstructors();
Assert.AreEqual(3, def.Count);
}
public void TestAnonymousTypeWithDictionary()
{
List <string> appDlls = new List <string>();
appDlls.Add(typeof(AnonymousType).Assembly.GetName().Name);
var c = TangFactory.GetTang().GetClassHierarchy(appDlls.ToArray());
c.GetNode(typeof(AnonymousType).AssemblyQualifiedName);
IConfiguration conf = TangFactory.GetTang().NewConfigurationBuilder(c).Build();
IInjector injector = TangFactory.GetTang().NewInjector(conf);
var obj = injector.GetInstance <AnonymousType>();
Assert.IsNotNull(obj);
var cd = Directory.GetCurrentDirectory();
Console.WriteLine(cd);
ProtocolBufferClassHierarchy.Serialize(ClassHierarchyBinFileName, c);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize(ClassHierarchyBinFileName);
ch.GetNode(typeof(AnonymousType).AssemblyQualifiedName);
}
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings)
{
ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder);
if (!File.Exists(DriverChFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(HttpServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(JobDriverConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
if (!File.Exists(NameServerConfigFile))
{
Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder));
return;
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile);
AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile);
IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy);
AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile);
IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy);
AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile);
IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy);
IConfiguration merged;
if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration);
}
else if (driverConfigurationSettings.IncludingHttpServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration);
}
else if (driverConfigurationSettings.IncludingNameServer)
{
merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration);
}
else
{
merged = jobDriverConfiguration;
}
var b = merged.newBuilder();
b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier);
b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture));
b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory);
// add for all the globallibaries
if (File.Exists(UserSuppliedGlobalLibraries))
{
var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries);
if (!string.IsNullOrEmpty(globalLibString))
{
foreach (string fname in globalLibString.Split(','))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname);
}
}
}
foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder))
{
b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f);
}
IConfiguration c = b.Build();
serializer.ToFile(c, DriverConfigFile);
Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile));
// additional file for easy to read
using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt"))
{
outfile.Write(serializer.ToString(c));
}
}