public ContextConfiguration(string configString) { using (LOGGER.LogFunction("ContextConfiguration::ContextConfigurationn")) { ContainerDirectory = Directory.GetCurrentDirectory(); _settings = new Dictionary <string, string>(); AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString); foreach (ConfigurationEntry config in avroConfiguration.Bindings) { if (config.key.Contains(Reef.Evaluator.Constants.ContextIdentifier)) { config.key = Reef.Evaluator.Constants.ContextIdentifier; LOGGER.Log(Level.Verbose, string.Format(CultureInfo.InvariantCulture, "{0} detected for context id with value {1}", config.key, config.value)); } _settings.Add(config.key, config.value); } if (!_settings.ContainsKey(Reef.Evaluator.Constants.ContextIdentifier)) { string msg = "Required parameter ContextIdentifier not provided."; LOGGER.Log(Level.Error, msg); Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER); } } }
public AvroConfiguration AvroDeserializeFromFile(string fileName) { AvroConfiguration avroConf = null; try { using (var buffer = new MemoryStream()) { if (!ReadFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method : " + fileName); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } buffer.Seek(0, SeekOrigin.Begin); using (var reader = new SequentialReader <AvroConfiguration>(AvroContainer.CreateReader <AvroConfiguration>(buffer, true))) { var results = reader.Objects; if (results != null) { avroConf = (AvroConfiguration)results.First(); } } } } catch (SerializationException ex) { Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER); var e = new ApplicationException("Cannot deserialize the file: " + fileName, ex); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } return(avroConf); }
public void TestDeserializedConfigMerge() { Type activityInterfaceType = typeof(ITask); ITang tang = TangFactory.GetTang(); ICsConfigurationBuilder cb1 = tang.NewConfigurationBuilder(); cb1.BindImplementation(GenericType <ITask> .Class, GenericType <HelloTask> .Class); cb1.BindNamedParameter <TaskConfigurationOptions.Identifier, string>( GenericType <TaskConfigurationOptions.Identifier> .Class, "Hello Task"); IConfiguration conf1 = cb1.Build(); var serializer = new AvroConfigurationSerializer(); serializer.ToFile(conf1, "task.config"); ICsConfigurationBuilder cb2 = tang.NewConfigurationBuilder(); cb2.BindNamedParameter <Timer.Seconds, int>(GenericType <Timer.Seconds> .Class, "2"); IConfiguration conf2 = cb2.Build(); serializer.ToFile(conf2, "timer.config"); ProtocolBufferClassHierarchy.Serialize("TaskTimer.bin", conf1.GetClassHierarchy()); IClassHierarchy ns = ProtocolBufferClassHierarchy.DeSerialize("TaskTimer.bin"); AvroConfiguration taskAvroconfiguration = serializer.AvroDeserializeFromFile("task.config"); IConfiguration taskConfiguration = serializer.FromAvro(taskAvroconfiguration, ns); AvroConfiguration timerAvroconfiguration = serializer.AvroDeserializeFromFile("timer.config"); IConfiguration timerConfiguration = serializer.FromAvro(timerAvroconfiguration, ns); IConfiguration merged = Configurations.MergeDeserializedConfs(taskConfiguration, timerConfiguration); var b = merged.newBuilder().Build(); }
public IConfiguration FromFileStream(string fileName) { byte[] bytes = File.ReadAllBytes(fileName); AvroConfiguration avroConf = AvroDeserialize(bytes); return(FromAvro(avroConf)); }
public string ToString(IConfiguration c) { byte[] bytes = ToByteArray(c); AvroConfiguration avroConf = AvroDeserialize(bytes); string s = JsonConvert.SerializeObject(avroConf, Formatting.Indented); return(s); }
private byte[] AvroSerialize(AvroConfiguration obj) { var serializer = AvroSerializer.Create <AvroConfiguration>(); using (MemoryStream stream = new MemoryStream()) { serializer.Serialize(stream, obj); return(stream.GetBuffer()); } }
private IConfiguration AddFromAvro(IConfigurationBuilder cb, AvroConfiguration avroConfiguration) { IList <KeyValuePair <string, string> > settings = new List <KeyValuePair <string, string> >(); foreach (ConfigurationEntry e in avroConfiguration.Bindings) { settings.Add(new KeyValuePair <string, string>(e.key, e.value)); } ConfigurationFile.ProcessConfigData(cb, settings, avroConfiguration.language); return(cb.Build()); }
private AvroConfiguration ToAvroConfiguration() { HashSet<ConfigurationEntry> b = new HashSet<ConfigurationEntry>(); ConfigurationEntry e1 = new ConfigurationEntry(); e1.key = "a"; e1.value = "a1"; ConfigurationEntry e2 = new ConfigurationEntry(); e2.key = "b"; e2.value = "b1=b2"; b.Add(e1); b.Add(e2); var a = new AvroConfiguration(Language.Cs.ToString(), b); return a; }
public void CanInjectAndExecuteTask() { //To enforce that shell task dll be copied to output directory. ShellTask tmpTask = new ShellTask("invalid"); Assert.IsNotNull(tmpTask); string tmp = Directory.GetCurrentDirectory(); Assert.IsNotNull(tmp); AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); AvroConfiguration avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf"); Assert.IsNotNull(avroConfiguration); ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(); cb.AddConfiguration(TaskConfiguration.ConfigurationModule .Set(TaskConfiguration.Identifier, "Test_CLRContext_task") .Set(TaskConfiguration.Task, GenericType <ShellTask> .Class) .Build()); cb.BindNamedParameter <ShellTask.Command, string>(GenericType <ShellTask.Command> .Class, "dir"); IConfiguration taskConfiguration = cb.Build(); string taskConfig = serializer.ToString(taskConfiguration); ITask task = null; TaskConfiguration config = new TaskConfiguration(taskConfig); Assert.IsNotNull(config); try { IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig); task = (ITask)injector.GetInstance(typeof(ITask)); } catch (Exception e) { throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e); } byte[] bytes = task.Call(null); string result = System.Text.Encoding.Default.GetString(bytes); //a dir command is executed in the container directory, which includes the file "evaluator.conf" Assert.IsTrue(result.Contains("evaluator.conf")); }
private AvroConfiguration ToAvroConfiguration() { HashSet <ConfigurationEntry> b = new HashSet <ConfigurationEntry>(); ConfigurationEntry e1 = new ConfigurationEntry(); e1.key = "a"; e1.value = "a1"; ConfigurationEntry e2 = new ConfigurationEntry(); e2.key = "b"; e2.value = "b1=b2"; b.Add(e1); b.Add(e2); var a = new AvroConfiguration(b); return(a); }
public EvaluatorConfigurations(string configFile) { using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations")) { if (string.IsNullOrWhiteSpace(configFile)) { Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER); } if (!File.Exists(configFile)) { Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER); } _configFile = configFile; AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); _avroConfiguration = serializer.AvroDeseriaizeFromFile(_configFile); } }
public TaskConfiguration(string configString) { TangConfig = new AvroConfigurationSerializer().FromString(configString); AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString); foreach (ConfigurationEntry config in avroConfiguration.Bindings) { if (config.key.Contains(TaskIdentifier)) { TaskId = config.value; } } if (string.IsNullOrWhiteSpace(TaskId)) { string msg = "Required parameter TaskId not provided."; LOGGER.Log(Level.Error, msg); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER); } }
public byte[] ToByteArray(IConfiguration c) { AvroConfiguration obj = ToAvroConfiguration(c); return(AvroSerialize(obj)); }
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings) { ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder); if (!File.Exists(DriverChFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(HttpServerConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(JobDriverConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(NameServerConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder)); return; } AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile); AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile); IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy); AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile); IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy); AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile); IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy); IConfiguration merged; if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration); } else if (driverConfigurationSettings.IncludingHttpServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration); } else if (driverConfigurationSettings.IncludingNameServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration); } else { merged = jobDriverConfiguration; } var b = merged.newBuilder(); b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier); b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture)); b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory); // add for all the globallibaries if (File.Exists(UserSuppliedGlobalLibraries)) { var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries); if (!string.IsNullOrEmpty(globalLibString)) { foreach (string fname in globalLibString.Split(',')) { b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname); } } } foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder)) { b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f); } IConfiguration c = b.Build(); serializer.ToFile(c, DriverConfigFile); Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile)); // additional file for easy to read using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt")) { outfile.Write(serializer.ToString(c)); } }
public IConfiguration FromAvro(AvroConfiguration avroConfiguration, IClassHierarchy classHierarchy) { IConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(classHierarchy); return(AddFromAvro(cb, avroConfiguration)); }
public IConfiguration FromAvro(AvroConfiguration avroConfiguration) { ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(); return(AddFromAvro(cb, avroConfiguration)); }
public IConfiguration FromByteArray(byte[] bytes) { AvroConfiguration avroConf = AvroDeserialize(bytes); return(FromAvro(avroConf)); }
public IConfiguration FromString(string josonString, IClassHierarchy ch) { AvroConfiguration avroConf = JsonConvert.DeserializeObject <AvroConfiguration>(josonString); return(FromAvro(avroConf, ch)); }
public IConfiguration FromString(string jsonString) { AvroConfiguration avroConf = JsonConvert.DeserializeObject <AvroConfiguration>(jsonString); return(FromAvro(avroConf)); }
public IConfiguration FromFile(string fileName, IClassHierarchy classHierarchy) { AvroConfiguration avroConf = AvroDeserializeFromFile(fileName); return(FromAvro(avroConf, classHierarchy)); }
public IConfiguration FromFile(string fileName) { AvroConfiguration avroConf = AvroDeserializeFromFile(fileName); return(FromAvro(avroConf)); }
public IConfiguration AddFromByteArray(ICsConfigurationBuilder cb, byte[] bytes) { AvroConfiguration avroConf = AvroDeserialize(bytes); return(AddFromAvro(cb, avroConf)); }