public GroupCommClient( [Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs, [Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId, StreamingNetworkService<GeneralGroupCommunicationMessage> networkService, AvroConfigurationSerializer configSerializer, IInjector injector) { _commGroups = new Dictionary<string, ICommunicationGroupClientInternal>(); _networkService = networkService; foreach (string serializedGroupConfig in groupConfigs) { IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig); IInjector groupInjector = injector.ForkInjector(groupConfig); var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance<ICommunicationGroupClient>(); _commGroups[commGroupClient.GroupName] = commGroupClient; } networkService.Register(new StringIdentifier(taskId)); foreach (var group in _commGroups.Values) { group.WaitingForRegistration(); } }
public GroupCommDriver( [Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId, [Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId, [Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut, [Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName, [Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks, AvroConfigurationSerializer configSerializer, INameServer nameServer) { _driverId = driverId; _contextIds = -1; _fanOut = fanOut; MasterTaskId = masterTaskId; _groupName = groupName; _configSerializer = configSerializer; _commGroups = new Dictionary<string, ICommunicationGroupDriver>(); _nameServer = nameServer; IPEndPoint localEndpoint = _nameServer.LocalEndpoint; _nameServerAddr = localEndpoint.Address.ToString(); _nameServerPort = localEndpoint.Port; NewCommunicationGroup(groupName, numberOfTasks); }
internal ActiveContext(IActiveContextClr2Java clr2Java) { InstanceId = Guid.NewGuid().ToString("N"); Clr2Java = clr2Java; _serializer = new AvroConfigurationSerializer(); _parentId = Optional<string>.OfNullable(clr2Java.GetParentId()); }
public void TestServiceConfiguration() { string groupName = "group1"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 3; int fanOut = 2; var serializer = new AvroConfigurationSerializer(); var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); //driver side to prepar for service config var codecConfig = CodecConfiguration<int>.Conf .Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class) .Build(); var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration(); var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig); //wrap it before serializing var wrappedSeriveConfig = TangFactory.GetTang().NewConfigurationBuilder() .BindNamedParameter<ServicesConfigurationOptions.ServiceConfigString, string>( GenericType<ServicesConfigurationOptions.ServiceConfigString>.Class, new AvroConfigurationSerializer().ToString(serviceConfig)) .Build(); var serviceConfigString = serializer.ToString(wrappedSeriveConfig); //the configuration string is received at Evaluator side var serviceConfig2 = new ServiceConfiguration(serviceConfigString); Assert.AreEqual(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2.TangConfig)); }
public void TestServiceConfiguration() { string groupName = "group1"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 3; int fanOut = 2; var serializer = new AvroConfigurationSerializer(); var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); // driver side to prepar for service config var codecConfig = CodecConfiguration<int>.Conf .Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class) .Build(); var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration(); var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig); var serviceConfigString = serializer.ToString(serviceConfig); // the configuration string is received at Evaluator side var serviceConfig2 = new AvroConfigurationSerializer().FromString(serviceConfigString); Assert.Equal(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2)); }
public void TestDeserializationWithAlias() { AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf"); var language = avroConfiguration.language; Assert.True(language.ToString().Equals(Language.Java.ToString())); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); var config = serializer.FromAvro(avroConfiguration, classHierarchy); IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config); string appid = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>(); string remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>(); string evaluatorIdentifier = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>(); string rid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>(); string launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>(); Assert.True(remoteId.StartsWith(RemoteIdPrefix)); Assert.True(appid.Equals(AppIdForTest)); Assert.True(evaluatorIdentifier.StartsWith(EvaluatorIdPrefix)); Assert.True(rid.StartsWith(RemoteIdPrefix)); Assert.True(launchId.Equals(AppIdForTest)); }
internal DriverFolderPreparationHelper( REEFFileNames fileNames, AvroConfigurationSerializer configurationSerializer, FileSets fileSets) { _fileNames = fileNames; _configurationSerializer = configurationSerializer; _fileSets = fileSets; }
public void TestDeserializeEvaluatorContextServiceTaskWithAlias() { var serializer = new AvroConfigurationSerializer(); var config = DeserializeConfigWithAlias(); var evaluatorInjector = TangFactory.GetTang().NewInjector(config); var evaluatorConfigString = evaluatorInjector.GetNamedInstance<EvaluatorConfiguration, string>(); var taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>(); var contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>(); var serviceConfigString = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>(); var evaluatorClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(DefaultLocalHttpDriverConnection).Assembly.GetName().Name }); var evaluatorConfig = serializer.FromString(evaluatorConfigString, evaluatorClassHierarchy); var fullEvaluatorInjector = evaluatorInjector.ForkInjector(evaluatorConfig); Assert.True(fullEvaluatorInjector.GetInstance<IDriverConnection>() is DefaultLocalHttpDriverConnection); var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(Common.Context.ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name }); var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy); var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(ITask).Assembly.GetName().Name, typeof(HelloTask).Assembly.GetName().Name }); var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy); var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(ServiceConfiguration).Assembly.GetName().Name, typeof(IStreamingCodec<>).Assembly.GetName().Name }); var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy); var contextInjector = fullEvaluatorInjector.ForkInjector(contextConfig); string contextId = contextInjector.GetNamedInstance<Common.Context.ContextConfigurationOptions.ContextIdentifier, string>(); Assert.True(contextId.StartsWith(ContextIdPrefix)); var serviceInjector = contextInjector.ForkInjector(serviceConfig); var service = serviceInjector.GetInstance<TestService>(); Assert.NotNull(service); var taskInjector = serviceInjector.ForkInjector(taskConfig); var taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>(); var task = taskInjector.GetInstance<ITask>(); Assert.True(taskId.StartsWith("HelloTask")); Assert.True(task is HelloTask); }
private ContextManager( AvroConfigurationSerializer serializer, IHeartBeatManager heartBeatManager, RootContextLauncher rootContextLauncher) { _rootContextLauncher = rootContextLauncher; _heartBeatManager = heartBeatManager; _serializer = serializer; }
public EvaluatorConfigurations(string configFile) { using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations")) { if (string.IsNullOrWhiteSpace(configFile)) { Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER); } if (!File.Exists(configFile)) { Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER); } AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); var evaluatorConfiguration = serializer.FromFile(configFile, classHierarchy); IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(evaluatorConfiguration); LOGGER.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "Evaluator Configuration is deserialized from file {0}:", configFile)); try { _taskConfiguration = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>(); } catch (InjectionException) { LOGGER.Log(Level.Info, "InitialTaskConfiguration is not set in Evaluator.config."); } try { _rootContextConfiguration = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>(); } catch (InjectionException) { LOGGER.Log(Level.Warning, "RootContextConfiguration is not set in Evaluator.config."); } try { _rootServiceConfiguration = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>(); } catch (InjectionException) { LOGGER.Log(Level.Info, "RootServiceConfiguration is not set in Evaluator.config."); } _applicationId = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>(); _remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>(); _evaluatorId = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>(); _errorHandlerRid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>(); _launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>(); } }
internal DriverFolderPreparationHelper( REEFFileNames fileNames, AvroConfigurationSerializer configurationSerializer, FileSets fileSets, [Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet<IConfigurationProvider> driverConfigurationProviders) { _fileNames = fileNames; _configurationSerializer = configurationSerializer; _fileSets = fileSets; _driverConfigurationProviders = driverConfigurationProviders; }
public AllocatedEvaluator(IAllocatedEvaluaotrClr2Java clr2Java, ISet<IConfigurationProvider> configurationProviders) { _configurationProviders = configurationProviders; InstanceId = Guid.NewGuid().ToString("N"); _serializer = new AvroConfigurationSerializer(); Clr2Java = clr2Java; Id = Clr2Java.GetId(); ProcessNewEvaluator(); NameServerInfo = Clr2Java.GetNameServerInfo(); }
private RootContextLauncher( AvroConfigurationSerializer serializer, [Parameter(typeof(RootContextConfiguration))] string rootContextConfiguration, [Parameter(typeof(RootServiceConfiguration))] string rootServiceConfiguration, IInjector injector) : this(serializer.FromString(rootContextConfiguration), serializer.FromString(rootServiceConfiguration), Optional<IConfiguration>.Empty(), injector) { }
public void TestEvaluatorConfigurationFile() { AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); var avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf"); Assert.IsNotNull(avroConfiguration); foreach (var b in avroConfiguration.Bindings) { Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value); } }
private EvaluatorSettings( [Parameter(typeof(ApplicationIdentifier))] string applicationId, [Parameter(typeof(EvaluatorIdentifier))] string evaluatorId, [Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs, [Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries, [Parameter(typeof(RootContextConfiguration))] string rootContextConfigString, AvroConfigurationSerializer serializer, RuntimeClock clock, IRemoteManagerFactory remoteManagerFactory, REEFMessageCodec reefMessageCodec, IInjector injector) : this(applicationId, evaluatorId, heartbeatPeriodInMs, maxHeartbeatRetries, rootContextConfigString, serializer, clock, remoteManagerFactory, reefMessageCodec, injector, null) { }
private EvaluatorSettings( [Parameter(typeof(ApplicationIdentifier))] string applicationId, [Parameter(typeof(EvaluatorIdentifier))] string evaluatorId, [Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs, [Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries, [Parameter(typeof(RootContextConfiguration))] string rootContextConfigString, AvroConfigurationSerializer serializer, RuntimeClock clock, IRemoteManagerFactory remoteManagerFactory, REEFMessageCodec reefMessageCodec, IInjector injector) { _serializer = serializer; _injector = injector; _applicationId = applicationId; _evaluatorId = evaluatorId; _heartBeatPeriodInMs = heartbeatPeriodInMs; _maxHeartbeatRetries = maxHeartbeatRetries; _clock = clock; if (string.IsNullOrWhiteSpace(rootContextConfigString)) { Utilities.Diagnostics.Exceptions.Throw( new ArgumentException("empty or null rootContextConfigString"), Logger); } _rootContextConfig = _serializer.FromString(rootContextConfigString); try { _rootContextId = injector.ForkInjector(_rootContextConfig).GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>(); } catch (InjectionException) { Logger.Log(Level.Info, "Using deprecated ContextConfiguration."); // TODO[JIRA REEF-1167]: Remove this catch. var deprecatedContextConfig = new Context.ContextConfiguration(rootContextConfigString); _rootContextConfig = deprecatedContextConfig; _rootContextId = deprecatedContextConfig.Id; } _rootTaskConfiguration = CreateTaskConfiguration(); _rootServiceConfiguration = CreateRootServiceConfiguration(); _remoteManager = remoteManagerFactory.GetInstance(reefMessageCodec); _operationState = EvaluatorOperationState.OPERATIONAL; }
private ContextManager( HeartBeatManager heartBeatManager, EvaluatorSettings evaluatorSettings, AvroConfigurationSerializer serializer) { using (LOGGER.LogFunction("ContextManager::ContextManager")) { _heartBeatManager = heartBeatManager; _serializer = serializer; _rootContextLauncher = new RootContextLauncher( evaluatorSettings.RootContextId, evaluatorSettings.RootContextConfig, evaluatorSettings.RootServiceConfiguration, evaluatorSettings.RootTaskConfiguration); } }
public EvaluatorConfigurations(string configFile) { using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations")) { if (string.IsNullOrWhiteSpace(configFile)) { Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER); } if (!File.Exists(configFile)) { Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER); } _configFile = configFile; AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); _avroConfiguration = serializer.AvroDeserializeFromFile(_configFile); } }
public AllocatedEvaluator(IAllocatedEvaluatorClr2Java clr2Java, ISet<IConfigurationProvider> configurationProviders) { _serializer = new AvroConfigurationSerializer(); var evaluatorConfig = TangFactory.GetTang().NewConfigurationBuilder().Build(); foreach (var configurationProvider in configurationProviders) { evaluatorConfig = Configurations.Merge(evaluatorConfig, configurationProvider.GetConfiguration()); } _evaluatorConfigStr = _serializer.ToString(evaluatorConfig); Clr2Java = clr2Java; Id = Clr2Java.GetId(); ProcessNewEvaluator(); NameServerInfo = Clr2Java.GetNameServerInfo(); }
public void CanInjectAndExecuteTask() { // to enforce that shell task dll be copied to output directory. ShellTask tmpTask = new ShellTask("invalid"); Assert.NotNull(tmpTask); string tmp = Directory.GetCurrentDirectory(); Assert.NotNull(tmp); AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); AvroConfiguration avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf"); Assert.NotNull(avroConfiguration); ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(); cb.AddConfiguration(TaskConfiguration.ConfigurationModule .Set(TaskConfiguration.Identifier, "Test_CLRContext_task") .Set(TaskConfiguration.Task, GenericType<ShellTask>.Class) .Build()); cb.BindNamedParameter<ShellTask.Command, string>(GenericType<ShellTask.Command>.Class, "dir"); IConfiguration taskConfiguration = cb.Build(); string taskConfig = serializer.ToString(taskConfiguration); ITask task = null; TaskConfiguration config = new TaskConfiguration(taskConfig); Assert.NotNull(config); try { IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig); task = (ITask)injector.GetInstance(typeof(ITask)); } catch (Exception e) { throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e); } byte[] bytes = task.Call(null); string result = System.Text.Encoding.Default.GetString(bytes); // a dir command is executed in the container directory, which includes the file "evaluator.conf" Assert.True(result.Contains("evaluator.conf")); }
private ContextManager( IHeartBeatManager heartBeatManager, EvaluatorSettings evaluatorSettings, AvroConfigurationSerializer serializer) { // TODO[JIRA REEF-217]: Inject base Injector and pass Injector to RootContextLauncher using (LOGGER.LogFunction("ContextManager::ContextManager")) { _heartBeatManager = heartBeatManager; _serializer = serializer; _rootContextLauncher = new RootContextLauncher( evaluatorSettings.RootContextId, evaluatorSettings.RootContextConfig, evaluatorSettings.RootServiceConfiguration, evaluatorSettings.RootTaskConfiguration, heartBeatManager); } }
private GroupCommClient( [Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet<string> groupConfigs, [Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId, NetworkService<GroupCommunicationMessage> networkService, AvroConfigurationSerializer configSerializer, IInjector injector) { _commGroups = new Dictionary<string, ICommunicationGroupClient>(); _networkService = networkService; networkService.Register(new StringIdentifier(taskId)); foreach (string serializedGroupConfig in groupConfigs) { IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig); IInjector groupInjector = injector.ForkInjector(groupConfig); ICommunicationGroupClient commGroupClient = groupInjector.GetInstance<ICommunicationGroupClient>(); _commGroups[commGroupClient.GroupName] = commGroupClient; } }
public EvaluatorConfigurations(string configFile) { using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations")) { if (string.IsNullOrWhiteSpace(configFile)) { Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER); } if (!File.Exists(configFile)) { Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER); } _configFile = configFile; AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); _avroConfiguration = serializer.AvroDeserializeFromFile(_configFile); var language = _avroConfiguration.language; LOGGER.Log(Level.Info, "The language that created the configFile is " + language); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); _evaluatorConfiguration = serializer.FromAvro(_avroConfiguration, classHierarchy); } }
/// <summary> /// Read evaluator configuration from evaluator.conf generated from Java /// </summary> /// <param name="evaluatorConfigFile"></param> /// <returns></returns> private static IConfiguration ReadEvaluatorConfiguration(string evaluatorConfigFile) { if (string.IsNullOrWhiteSpace(evaluatorConfigFile)) { Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), logger); } if (!File.Exists(evaluatorConfigFile)) { Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + evaluatorConfigFile), logger); } AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); var evaluatorConfiguration = serializer.FromFile(evaluatorConfigFile, classHierarchy); logger.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "Evaluator Configuration is deserialized from file {0}:", evaluatorConfigFile)); return evaluatorConfiguration; }
public void TestDeserializationForContextAndTask() { AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); var config = serializer.FromFile("evaluator.conf", classHierarchy); IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config); string taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>(); string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>(); var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name }); var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy); var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(ITask).Assembly.GetName().Name, typeof(HelloTask).Assembly.GetName().Name }); var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy); var contextInjector = evaluatorInjector.ForkInjector(contextConfig); string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>(); Assert.True(contextId.StartsWith(ContextIdPrefix)); var taskInjector = contextInjector.ForkInjector(taskConfig); string taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>(); ITask task = taskInjector.GetInstance<ITask>(); Assert.True(taskId.StartsWith("HelloTask")); Assert.True(task is HelloTask); }
public static void DriverConfigurationBuilder(DriverConfigurationSettings driverConfigurationSettings) { ExtractConfigFromJar(driverConfigurationSettings.JarFileFolder); if (!File.Exists(DriverChFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", DriverChFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(HttpServerConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", HttpServerConfigFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(JobDriverConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", JobDriverConfigFile, driverConfigurationSettings.JarFileFolder)); return; } if (!File.Exists(NameServerConfigFile)) { Log.Log(Level.Warning, string.Format(CultureInfo.CurrentCulture, "There is no file {0} extracted from the jar file at {1}.", NameServerConfigFile, driverConfigurationSettings.JarFileFolder)); return; } AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); IClassHierarchy driverClassHierarchy = ProtocolBufferClassHierarchy.DeSerialize(DriverChFile); AvroConfiguration jobDriverAvroconfiguration = serializer.AvroDeserializeFromFile(JobDriverConfigFile); IConfiguration jobDriverConfiguration = serializer.FromAvro(jobDriverAvroconfiguration, driverClassHierarchy); AvroConfiguration httpAvroconfiguration = serializer.AvroDeserializeFromFile(HttpServerConfigFile); IConfiguration httpConfiguration = serializer.FromAvro(httpAvroconfiguration, driverClassHierarchy); AvroConfiguration nameAvroconfiguration = serializer.AvroDeserializeFromFile(NameServerConfigFile); IConfiguration nameConfiguration = serializer.FromAvro(nameAvroconfiguration, driverClassHierarchy); IConfiguration merged; if (driverConfigurationSettings.IncludingHttpServer && driverConfigurationSettings.IncludingNameServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration, nameConfiguration); } else if (driverConfigurationSettings.IncludingHttpServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, httpConfiguration); } else if (driverConfigurationSettings.IncludingNameServer) { merged = Configurations.MergeDeserializedConfs(jobDriverConfiguration, nameConfiguration); } else { merged = jobDriverConfiguration; } var b = merged.newBuilder(); b.BindSetEntry("org.apache.reef.driver.parameters.DriverIdentifier", driverConfigurationSettings.DriverIdentifier); b.Bind("org.apache.reef.driver.parameters.DriverMemory", driverConfigurationSettings.DriverMemory.ToString(CultureInfo.CurrentCulture)); b.Bind("org.apache.reef.driver.parameters.DriverJobSubmissionDirectory", driverConfigurationSettings.SubmissionDirectory); // add for all the globallibaries if (File.Exists(UserSuppliedGlobalLibraries)) { var globalLibString = File.ReadAllText(UserSuppliedGlobalLibraries); if (!string.IsNullOrEmpty(globalLibString)) { foreach (string fname in globalLibString.Split(',')) { b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalLibraries", fname); } } } foreach (string f in Directory.GetFiles(driverConfigurationSettings.ClrFolder)) { b.BindSetEntry("org.apache.reef.driver.parameters.JobGlobalFiles", f); } IConfiguration c = b.Build(); serializer.ToFile(c, DriverConfigFile); Log.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "driver.config is written to: {0} {1}.", Directory.GetCurrentDirectory(), DriverConfigFile)); // additional file for easy to read using (StreamWriter outfile = new StreamWriter(DriverConfigFile + ".txt")) { outfile.Write(serializer.ToString(c)); } }
private ConfigurationManager( AvroConfigurationSerializer configurationSerializer, [Parameter(typeof(SerializedMapConfiguration))] string mapConfig, [Parameter(typeof(SerializedReduceConfiguration))] string reduceConfig, [Parameter(typeof(SerializedUpdateConfiguration))] string updateConfig, [Parameter(typeof(SerializedMapInputCodecConfiguration))] string mapInputCodecConfig, [Parameter(typeof(SerializedUpdateFunctionCodecsConfiguration))] string updateFunctionCodecsConfig, [Parameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration))] string mapOutputPipelineDataConverterConfiguration, [Parameter(typeof(SerializedMapInputPipelineDataConverterConfiguration))] string mapInputPipelineDataConverterConfiguration, [Parameter(typeof(SerializedResultHandlerConfiguration))] string resultHandlerConfiguration) { try { _mapFunctionConfiguration = configurationSerializer.FromString(mapConfig); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize map function configuration", Logger); } try { _reduceFunctionConfiguration = configurationSerializer.FromString(reduceConfig); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize reduce function configuration", Logger); } try { _updateFunctionConfiguration = configurationSerializer.FromString(updateConfig); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize update function configuration", Logger); } try { _updateFunctionCodecsConfiguration = configurationSerializer.FromString(updateFunctionCodecsConfig); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize update function codec configuration", Logger); } try { _mapInputCodecConfiguration = configurationSerializer.FromString(mapInputCodecConfig); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize map input codec configuration", Logger); } try { _mapOutputPipelineDataConverterConfiguration = configurationSerializer.FromString(mapOutputPipelineDataConverterConfiguration); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize map output pipeline data converter configuration", Logger); } try { _mapInputPipelineDataConverterConfiguration = configurationSerializer.FromString(mapInputPipelineDataConverterConfiguration); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize map input pipeline data converter configuration", Logger); } try { _resultHandlerConfiguration = configurationSerializer.FromString(resultHandlerConfiguration); Logger.Log(Level.Verbose, "Serialized result handler is " + resultHandlerConfiguration); } catch (Exception e) { Exceptions.Throw(e, "Unable to deserialize map input pipeline data converter configuration", Logger); } }
public TaskConfiguration(string configString) { TangConfig = new AvroConfigurationSerializer().FromString(configString); AvroConfiguration avroConfiguration = AvroConfiguration.GetAvroConfigurationFromEmbeddedString(configString); foreach (ConfigurationEntry config in avroConfiguration.Bindings) { if (config.key.Contains(TaskIdentifier)) { TaskId = config.value; } } if (string.IsNullOrWhiteSpace(TaskId)) { string msg = "Required parameter TaskId not provided."; LOGGER.Log(Level.Error, msg); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException(msg), LOGGER); } }
/// <summary> /// Simulate evaluator configuration generated from Java for unit testing /// </summary> /// <param name="serializer"></param> /// <returns></returns> private static AvroConfiguration EvaluatorConfig(AvroConfigurationSerializer serializer) { var configurationEntries = new HashSet<ConfigurationEntry>(); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.ApplicationIdentifier", "REEF_LOCAL_RUNTIME")); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.DriverRemoteIdentifier", "socket://10.130.68.76:9723")); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorIdentifier", "Node-2-1447450298921")); var evaluatorConfiguration = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType<IDriverConnection>.Class, GenericType<DefaultLocalHttpDriverConnection>.Class) .Build(); var evaluatorString = serializer.ToString(evaluatorConfiguration); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorConfiguration", evaluatorString)); var taskConfiguration = TaskConfiguration.ConfigurationModule .Set(TaskConfiguration.Identifier, "HelloTask") .Set(TaskConfiguration.Task, GenericType<HelloTask>.Class) .Build(); var taskString = serializer.ToString(taskConfiguration); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.InitialTaskConfiguration", taskString)); var contextConfig = Common.Context.ContextConfiguration.ConfigurationModule.Set(Common.Context.ContextConfiguration.Identifier, ContextIdPrefix).Build(); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootContextConfiguration", serializer.ToString(contextConfig))); var serviceConfiguration = ServiceConfiguration.ConfigurationModule .Set(ServiceConfiguration.Services, GenericType<TestService>.Class) .Build(); configurationEntries.Add( new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootServiceConfiguration", serializer.ToString(serviceConfiguration))); configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID", "socket://10.130.68.76:9723")); configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.LaunchID", "REEF_LOCAL_RUNTIME")); return new AvroConfiguration(Language.Java.ToString(), configurationEntries); }
/// <summary> /// Deserialize evaluator configuration with alias /// </summary> /// <returns></returns> private static IConfiguration DeserializeConfigWithAlias() { var serializer = new AvroConfigurationSerializer(); var classHierarchy = TangFactory.GetTang() .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name }); var avroConfiguration = EvaluatorConfig(serializer); return serializer.FromAvro(avroConfiguration, classHierarchy); }