public override IConfiguration RoundTrip(IConfiguration configuration)
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
byte[] theBytes = serializer.ToByteArray(configuration);
return(serializer.FromByteArray(theBytes));
}
private static void InitInjector()
{
string clrRuntimeConfigurationFile = Path.Combine(Directory.GetCurrentDirectory(), "reef", "global",
Common.Constants.ClrBridgeRuntimeConfiguration);
if (!File.Exists(clrRuntimeConfigurationFile))
{
var e =
new InvalidOperationException("Cannot find clrRuntimeConfiguration from " +
clrRuntimeConfigurationFile);
Exceptions.Throw(e, _logger);
}
try
{
IConfiguration clrBridgeConfiguration =
new AvroConfigurationSerializer().FromFile(clrRuntimeConfigurationFile);
_injector = TangFactory.GetTang().NewInjector(clrBridgeConfiguration);
}
catch (Exception e)
{
Exceptions.Caught(e, Level.Error, "Cannot obtain injector from clr bridge configuration.", _logger);
Exceptions.Throw(
new InvalidOperationException("Cannot obtain injector from clr bridge configuration.", e),
_logger);
}
}
private CommunicationGroupClient(
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.SerializedOperatorConfigs))] ISet <string> operatorConfigs,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_operators = new Dictionary <string, object>();
GroupName = groupName;
foreach (string operatorConfigStr in operatorConfigs)
{
IConfiguration operatorConfig = configSerializer.FromString(operatorConfigStr);
IInjector operatorInjector = injector.ForkInjector(operatorConfig);
string operatorName = operatorInjector.GetNamedInstance <GroupCommConfigurationOptions.OperatorName, string>(
GenericType <GroupCommConfigurationOptions.OperatorName> .Class);
string msgType = operatorInjector.GetNamedInstance <GroupCommConfigurationOptions.MessageType, string>(
GenericType <GroupCommConfigurationOptions.MessageType> .Class);
Type groupCommOperatorGenericInterface = typeof(IGroupCommOperator <>);
Type groupCommOperatorInterface = groupCommOperatorGenericInterface.MakeGenericType(Type.GetType(msgType));
var operatorObj = operatorInjector.GetInstance(groupCommOperatorInterface);
_operators.Add(operatorName, operatorObj);
}
}
public void TestDeserializedConfigMerge()
{
Type activityInterfaceType = typeof(ITask);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb1 = tang.NewConfigurationBuilder();
cb1.BindImplementation(GenericType <ITask> .Class, GenericType <HelloTask> .Class);
cb1.BindNamedParameter <TaskConfigurationOptions.Identifier, string>(
GenericType <TaskConfigurationOptions.Identifier> .Class, "Hello Task");
IConfiguration conf1 = cb1.Build();
var serializer = new AvroConfigurationSerializer();
serializer.ToFile(conf1, "task.config");
ICsConfigurationBuilder cb2 = tang.NewConfigurationBuilder();
cb2.BindNamedParameter <Timer.Seconds, Int32>(GenericType <Timer.Seconds> .Class, "2");
IConfiguration conf2 = cb2.Build();
serializer.ToFile(conf2, "timer.config");
ProtocolBufferClassHierarchy.Serialize("TaskTimer.bin", conf1.GetClassHierarchy());
IClassHierarchy ns = ProtocolBufferClassHierarchy.DeSerialize("TaskTimer.bin");
AvroConfiguration taskAvroconfiguration = serializer.AvroDeseriaizeFromFile("task.config");
IConfiguration taskConfiguration = serializer.FromAvro(taskAvroconfiguration, ns);
AvroConfiguration timerAvroconfiguration = serializer.AvroDeseriaizeFromFile("timer.config");
IConfiguration timerConfiguration = serializer.FromAvro(timerAvroconfiguration, ns);
IConfiguration merged = Configurations.MergeDeserializedConfs(taskConfiguration, timerConfiguration);
var b = merged.newBuilder().Build();
}
public void TestDeserializeEvaluatorContextServiceTaskWithAlias()
{
var serializer = new AvroConfigurationSerializer();
var config = DeserializeConfigWithAlias();
var evaluatorInjector = TangFactory.GetTang().NewInjector(config);
var evaluatorConfigString = evaluatorInjector.GetNamedInstance <EvaluatorConfiguration, string>();
var taskConfigString = evaluatorInjector.GetNamedInstance <InitialTaskConfiguration, string>();
var contextConfigString = evaluatorInjector.GetNamedInstance <RootContextConfiguration, string>();
var serviceConfigString = evaluatorInjector.GetNamedInstance <RootServiceConfiguration, string>();
var evaluatorClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(DefaultLocalHttpDriverConnection).Assembly.GetName().Name
});
var evaluatorConfig = serializer.FromString(evaluatorConfigString, evaluatorClassHierarchy);
var fullEvaluatorInjector = evaluatorInjector.ForkInjector(evaluatorConfig);
Assert.True(fullEvaluatorInjector.GetInstance <IDriverConnection>() is DefaultLocalHttpDriverConnection);
var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(Common.Context.ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
});
var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ITask).Assembly.GetName().Name,
typeof(HelloTask).Assembly.GetName().Name
});
var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy);
var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
{
typeof(ServiceConfiguration).Assembly.GetName().Name,
typeof(IStreamingCodec <>).Assembly.GetName().Name
});
var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy);
var contextInjector = fullEvaluatorInjector.ForkInjector(contextConfig);
string contextId = contextInjector.GetNamedInstance <Common.Context.ContextConfigurationOptions.ContextIdentifier, string>();
Assert.True(contextId.StartsWith(ContextIdPrefix));
var serviceInjector = contextInjector.ForkInjector(serviceConfig);
var service = serviceInjector.GetInstance <TestService>();
Assert.NotNull(service);
var taskInjector = serviceInjector.ForkInjector(taskConfig);
var taskId = taskInjector.GetNamedInstance <TaskConfigurationOptions.Identifier, string>();
var task = taskInjector.GetInstance <ITask>();
Assert.True(taskId.StartsWith("HelloTask"));
Assert.True(task is HelloTask);
}
private GroupCommDriver(
[Parameter(typeof(GroupCommConfigurationOptions.DriverId))] string driverId,
[Parameter(typeof(GroupCommConfigurationOptions.MasterTaskId))] string masterTaskId,
[Parameter(typeof(GroupCommConfigurationOptions.FanOut))] int fanOut,
[Parameter(typeof(GroupCommConfigurationOptions.GroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.NumberOfTasks))] int numberOfTasks,
AvroConfigurationSerializer configSerializer,
INameServer nameServer)
{
_driverId = driverId;
_contextIds = -1;
_fanOut = fanOut;
MasterTaskId = masterTaskId;
_groupName = groupName;
_configSerializer = configSerializer;
_commGroups = new Dictionary <string, ICommunicationGroupDriver>();
IPEndPoint localEndpoint = nameServer.LocalEndpoint;
_nameServerAddr = localEndpoint.Address.ToString();
_nameServerPort = localEndpoint.Port;
NewCommunicationGroup(groupName, numberOfTasks);
}
public GroupCommClient(
[Parameter(typeof(GroupCommConfigurationOptions.SerializedGroupConfigs))] ISet <string> groupConfigs,
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
StreamingNetworkService <GeneralGroupCommunicationMessage> networkService,
AvroConfigurationSerializer configSerializer,
IInjector injector)
{
_commGroups = new Dictionary <string, ICommunicationGroupClientInternal>();
_networkService = networkService;
foreach (string serializedGroupConfig in groupConfigs)
{
IConfiguration groupConfig = configSerializer.FromString(serializedGroupConfig);
IInjector groupInjector = injector.ForkInjector(groupConfig);
var commGroupClient = (ICommunicationGroupClientInternal)groupInjector.GetInstance <ICommunicationGroupClient>();
_commGroups[commGroupClient.GroupName] = commGroupClient;
}
networkService.Register(new StringIdentifier(taskId));
try
{
foreach (var group in _commGroups.Values)
{
group.WaitingForRegistration();
}
}
catch (SystemException e)
{
networkService.Unregister();
networkService.Dispose();
Exceptions.CaughtAndThrow(e, Level.Error, "In GroupCommClient, exception from WaitingForRegistration.", Logger);
}
}
public static ISet <string> GetCommandLineArguments()
{
using (LOGGER.LogFunction("ClrHandlerHelper::GetCommandLineArguments"))
{
string bridgeConfiguration = Path.Combine(Directory.GetCurrentDirectory(), "reef", "global",
Constants.DriverBridgeConfiguration);
if (!File.Exists(bridgeConfiguration))
{
string error = "Configuraiton file not found: " + bridgeConfiguration;
LOGGER.Log(Level.Error, error);
Exceptions.Throw(new InvalidOperationException(error), LOGGER);
}
CommandLineArguments arguments;
IInjector injector;
try
{
IConfiguration driverBridgeConfiguration =
new AvroConfigurationSerializer().FromFile(bridgeConfiguration);
injector = TangFactory.GetTang().NewInjector(driverBridgeConfiguration);
arguments = injector.GetInstance <CommandLineArguments>();
}
catch (InjectionException e)
{
string error = "Cannot inject command line arguments from driver bridge configuration. ";
Exceptions.Caught(e, Level.Error, error, LOGGER);
// return empty string set
return(new HashSet <string>());
}
return(arguments.Arguments);
}
}
public void TestServiceConfiguration()
{
string groupName = "group1";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var serializer = new AvroConfigurationSerializer();
var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
groupName, fanOut,
numTasks);
// driver side to prepar for service config
var codecConfig = CodecConfiguration <int> .Conf
.Set(CodecConfiguration <int> .Codec, GenericType <IntCodec> .Class)
.Build();
var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
// wrap it before serializing
var wrappedSeriveConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <ServicesConfigurationOptions.ServiceConfigString, string>(
GenericType <ServicesConfigurationOptions.ServiceConfigString> .Class,
new AvroConfigurationSerializer().ToString(serviceConfig))
.Build();
var serviceConfigString = serializer.ToString(wrappedSeriveConfig);
// the configuration string is received at Evaluator side
var serviceConfig2 = new ServiceConfiguration(serviceConfigString);
Assert.Equal(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2.TangConfig));
}
public override IConfiguration RoundTrip(IConfiguration configuration)
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
serializer.ToFile(configuration, "TangTest.avroconf");
return(serializer.FromFile("TangTest.avroconf"));
}
private FailedTaskDriver(
IEvaluatorRequestor requestor,
[Parameter(typeof(TaskConfigurationString))] string taskConfigString,
AvroConfigurationSerializer avroConfigurationSerializer)
{
_requestor = requestor;
_taskConfiguration = avroConfigurationSerializer.FromString(taskConfigString);
}
private CloseTaskTestDriver(IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(DisposeMessage))] string disposeMessage,
[Parameter(typeof(TaskConfigurationString))] string taskConfigString,
AvroConfigurationSerializer avroConfigurationSerializer)
{
_requestor = evaluatorRequestor;
_disposeMessage = disposeMessage;
_taskConfiguration = avroConfigurationSerializer.FromString(taskConfigString);
}
private REEFIMRUClient(
IREEFClient reefClient,
AvroConfigurationSerializer configurationSerializer,
JobRequestBuilder jobRequestBuilder)
{
_reefClient = reefClient;
_configurationSerializer = configurationSerializer;
_jobRequestBuilder = jobRequestBuilder;
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
}
private ContextManager(
AvroConfigurationSerializer serializer,
IHeartBeatManager heartBeatManager,
RootContextLauncher rootContextLauncher)
{
_rootContextLauncher = rootContextLauncher;
_heartBeatManager = heartBeatManager;
_serializer = serializer;
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var evaluatorConfiguration = serializer.FromFile(configFile, classHierarchy);
IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(evaluatorConfiguration);
LOGGER.Log(Level.Info,
string.Format(CultureInfo.CurrentCulture, "Evaluator Configuration is deserialized from file {0}:", configFile));
try
{
_taskConfiguration = evaluatorInjector.GetNamedInstance <InitialTaskConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Info, "InitialTaskConfiguration is not set in Evaluator.config.");
}
try
{
_rootContextConfiguration = evaluatorInjector.GetNamedInstance <RootContextConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Warning, "RootContextConfiguration is not set in Evaluator.config.");
}
try
{
_rootServiceConfiguration = evaluatorInjector.GetNamedInstance <RootServiceConfiguration, string>();
}
catch (InjectionException)
{
LOGGER.Log(Level.Info, "RootServiceConfiguration is not set in Evaluator.config.");
}
_applicationId = evaluatorInjector.GetNamedInstance <ApplicationIdentifier, string>();
_remoteId = evaluatorInjector.GetNamedInstance <DriverRemoteIdentifier, string>();
_evaluatorId = evaluatorInjector.GetNamedInstance <EvaluatorIdentifier, string>();
_errorHandlerRid = evaluatorInjector.GetNamedInstance <ErrorHandlerRid, string>();
_launchId = evaluatorInjector.GetNamedInstance <LaunchId, string>();
}
}
public ServiceConfiguration(string config)
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
string serviceConfigString = (string)TangFactory.GetTang()
.NewInjector(serializer.FromString(config))
.GetNamedInstance(typeof(ServicesConfigurationOptions.ServiceConfigString));
TangConfig = serializer.FromString(serviceConfigString);
}
public AllocatedEvaluator(IAllocatedEvaluaotrClr2Java clr2Java)
{
InstanceId = Guid.NewGuid().ToString("N");
_serializer = new AvroConfigurationSerializer();
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
/// <summary>
/// Build driver configuration
/// </summary>
/// <typeparam name="TMapInput"></typeparam>
/// <typeparam name="TMapOutput"></typeparam>
/// <typeparam name="TResult"></typeparam>
/// <typeparam name="TPartitionType"></typeparam>
/// <param name="jobDefinition"></param>
/// <param name="driverHandlerConfig"></param>
/// <returns></returns>
private IConfiguration DriverConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>(
IMRUJobDefinition jobDefinition,
IConfiguration driverHandlerConfig)
{
string driverId = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
IConfiguration overallPerMapConfig = null;
var configurationSerializer = new AvroConfigurationSerializer();
try
{
overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
}
catch (Exception e)
{
Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
}
var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
{
driverHandlerConfig,
CreateGroupCommunicationConfiguration <TMapInput, TMapOutput, TResult, TPartitionType>(jobDefinition.NumberOfMappers + 1,
driverId),
jobDefinition.PartitionedDatasetConfiguration,
overallPerMapConfig
})
.BindNamedParameter(typeof(SerializedMapConfiguration),
configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
.BindNamedParameter(typeof(SerializedUpdateConfiguration),
configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
.BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
.BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
.BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
.BindNamedParameter(typeof(SerializedReduceConfiguration),
configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
.BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
.BindNamedParameter(typeof(MemoryPerMapper),
jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(MemoryForUpdateTask),
jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresPerMapper),
jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(CoresForUpdateTask),
jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter(typeof(InvokeGC),
jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
.Build();
return(imruDriverConfiguration);
}
/// <summary>
/// Simulate evaluator configuration generated from Java for unit testing
/// </summary>
/// <param name="serializer"></param>
/// <returns></returns>
private static AvroConfiguration EvaluatorConfig(AvroConfigurationSerializer serializer)
{
var configurationEntries = new HashSet <ConfigurationEntry>();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.ApplicationIdentifier",
"REEF_LOCAL_RUNTIME"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.DriverRemoteIdentifier",
"socket://10.130.68.76:9723"));
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorIdentifier",
"Node-2-1447450298921"));
var evaluatorConfiguration = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType <IDriverConnection> .Class, GenericType <DefaultLocalHttpDriverConnection> .Class)
.Build();
var evaluatorString = serializer.ToString(evaluatorConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.EvaluatorConfiguration",
evaluatorString));
var taskConfiguration = TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "HelloTask")
.Set(TaskConfiguration.Task, GenericType <JavaTask> .Class)
.Build();
var taskString = serializer.ToString(taskConfiguration);
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.InitialTaskConfiguration",
taskString));
var contextConfig = Common.Context.ContextConfiguration.ConfigurationModule.Set(Common.Context.ContextConfiguration.Identifier, ContextIdPrefix).Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootContextConfiguration",
serializer.ToString(contextConfig)));
var serviceConfiguration = ServiceConfiguration.ConfigurationModule
.Set(ServiceConfiguration.Services, GenericType <TestService> .Class)
.Build();
configurationEntries.Add(
new ConfigurationEntry("org.apache.reef.runtime.common.evaluator.parameters.RootServiceConfiguration",
serializer.ToString(serviceConfiguration)));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID",
"socket://10.130.68.76:9723"));
configurationEntries.Add(new ConfigurationEntry("org.apache.reef.runtime.common.launch.parameters.LaunchID",
"REEF_LOCAL_RUNTIME"));
return(new AvroConfiguration(Language.Java.ToString(), configurationEntries));
}
/// <summary>
/// Deserialize evaluator configuration with alias
/// </summary>
/// <returns></returns>
private static IConfiguration DeserializeConfigWithAlias()
{
var serializer = new AvroConfigurationSerializer();
var classHierarchy = TangFactory.GetTang()
.GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
var avroConfiguration = EvaluatorConfig(serializer);
return(serializer.FromAvro(avroConfiguration, classHierarchy));
}
internal DriverFolderPreparationHelper(
REEFFileNames fileNames,
AvroConfigurationSerializer configurationSerializer,
FileSets fileSets,
[Parameter(typeof(EnvironmentDriverConfigurationProviders))] ISet <IConfigurationProvider> driverConfigurationProviders)
{
_fileNames = fileNames;
_configurationSerializer = configurationSerializer;
_fileSets = fileSets;
_driverConfigurationProviders = driverConfigurationProviders;
}
public AllocatedEvaluator(IAllocatedEvaluatorClr2Java clr2Java, ISet <IConfigurationProvider> configurationProviders)
{
_configurationProviders = configurationProviders;
InstanceId = Guid.NewGuid().ToString("N");
_serializer = new AvroConfigurationSerializer();
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
private RootContextLauncher(
AvroConfigurationSerializer serializer,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfiguration,
[Parameter(typeof(RootServiceConfiguration))] string rootServiceConfiguration,
IInjector injector)
: this(serializer.FromString(rootContextConfiguration),
serializer.FromString(rootServiceConfiguration),
Optional <IConfiguration> .Empty(),
injector)
{
}
public void TestEvaluatorConfigurationFile()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
Assert.IsNotNull(avroConfiguration);
Assert.AreEqual(avroConfiguration.language, Language.Java.ToString());
foreach (var b in avroConfiguration.Bindings)
{
Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value);
}
}
public AllocatedEvaluator(IAllocatedEvaluatorClr2Java clr2Java, string configuration)
{
using (LOGGER.LogFunction("AllocatedEvaluator::AllocatedEvaluator:", clr2Java.GetId()))
{
_serializer = TangFactory.GetTang().NewInjector().GetInstance <AvroConfigurationSerializer>();
_evaluatorConfigStr = configuration;
Clr2Java = clr2Java;
Id = Clr2Java.GetId();
ProcessNewEvaluator();
NameServerInfo = Clr2Java.GetNameServerInfo();
}
}
private EvaluatorSettings(
[Parameter(typeof(ApplicationIdentifier))] string applicationId,
[Parameter(typeof(EvaluatorIdentifier))] string evaluatorId,
[Parameter(typeof(EvaluatorHeartbeatPeriodInMs))] int heartbeatPeriodInMs,
[Parameter(typeof(HeartbeatMaxRetry))] int maxHeartbeatRetries,
[Parameter(typeof(RootContextConfiguration))] string rootContextConfigString,
AvroConfigurationSerializer serializer,
RuntimeClock clock,
IRemoteManagerFactory remoteManagerFactory,
REEFMessageCodec reefMessageCodec,
IInjector injector) :
this(applicationId, evaluatorId, heartbeatPeriodInMs, maxHeartbeatRetries, rootContextConfigString, serializer,
clock, remoteManagerFactory, reefMessageCodec, injector, null)
{
}
private IConfiguration DriverConfigurations(IConfiguration taskConfig)
{
var driverConfig = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnDriverStarted, GenericType <FailedTaskDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <FailedTaskDriver> .Class)
.Set(DriverConfiguration.OnTaskFailed, GenericType <FailedTaskDriver> .Class)
.Set(DriverConfiguration.OnTaskCompleted, GenericType <FailedTaskDriver> .Class)
.Build();
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
return(TangFactory.GetTang().NewConfigurationBuilder(driverConfig)
.BindStringNamedParam <TaskConfigurationString>(serializer.ToString(taskConfig))
.Build());
}
public void CanInjectAndExecuteTask()
{
//To enforce that shell task dll be copied to output directory.
ShellTask tmpTask = new ShellTask("invalid");
Assert.IsNotNull(tmpTask);
string tmp = Directory.GetCurrentDirectory();
Assert.IsNotNull(tmp);
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
AvroConfiguration avroConfiguration = serializer.AvroDeseriaizeFromFile("evaluator.conf");
Assert.IsNotNull(avroConfiguration);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
cb.AddConfiguration(TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, "Test_CLRContext_task")
.Set(TaskConfiguration.Task, GenericType <ShellTask> .Class)
.Build());
cb.BindNamedParameter <ShellTask.Command, string>(GenericType <ShellTask.Command> .Class, "dir");
IConfiguration taskConfiguration = cb.Build();
string taskConfig = serializer.ToString(taskConfiguration);
ITask task = null;
TaskConfiguration config = new TaskConfiguration(taskConfig);
Assert.IsNotNull(config);
try
{
IInjector injector = TangFactory.GetTang().NewInjector(config.TangConfig);
task = (ITask)injector.GetInstance(typeof(ITask));
}
catch (Exception e)
{
throw new InvalidOperationException("unable to inject task with configuration: " + taskConfig, e);
}
byte[] bytes = task.Call(null);
string result = System.Text.Encoding.Default.GetString(bytes);
//a dir command is executed in the container directory, which includes the file "evaluator.conf"
Assert.IsTrue(result.Contains("evaluator.conf"));
}
public EvaluatorConfigurations(string configFile)
{
using (LOGGER.LogFunction("EvaluatorConfigurations::EvaluatorConfigurations"))
{
if (string.IsNullOrWhiteSpace(configFile))
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("configFile"), LOGGER);
}
if (!File.Exists(configFile))
{
Org.Apache.Reef.Utilities.Diagnostics.Exceptions.Throw(new FileNotFoundException("cannot find file " + configFile), LOGGER);
}
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeseriaizeFromFile(_configFile);
}
}
