/// <summary>
/// Master test function for testing types of arrays in the ArrayPipelineDataConverter
/// </summary>
/// <typeparam name="T">The type of array to test</typeparam>
/// <param name="originalArray">An array to use in the test</param>
private static void TestArrayPipelineDataConverter <T>(T[] originalArray) where T : new()
{
// Verify that the constructor has the proper restrictions
AssertPositivePipelinePackageElementsRequired <T[], ArrayPipelineDataConverter <T> >();
// Verify that the FullMessage method properly handles null and zero-length arrays
AssertNullAndZeroLengthArraysHandledByFullMessage <T, ArrayPipelineDataConverter <T> >();
// Test the valid case where we break up the array into smaller pieces
// First determine how many messages to create from originalArray
int pipelineMessageSize;
int nMessages;
if (originalArray == null)
{
nMessages = 0;
pipelineMessageSize = 1;
}
else if (originalArray.Length == 0 || originalArray.Length == 1)
{
nMessages = 1;
pipelineMessageSize = 1; // Necessary to instantiate the ArrayPipelineDataConverterConfig
}
else
{
nMessages = 2;
pipelineMessageSize = (int)Math.Ceiling(originalArray.Length / (double)nMessages);
}
// Test that the valid configuration can be injected
IConfiguration config = GetPipelineDataConverterConfig(pipelineMessageSize);
IPipelineDataConverter <T[]> dataConverter = TangFactory.GetTang().NewInjector(config).GetInstance <ArrayPipelineDataConverter <T> >();
var pipelineData = dataConverter.PipelineMessage(originalArray);
// Validate that the pipeline constructed the correct number of messages
Assert.Equal <int>(pipelineData.Count, nMessages);
T[] deserializedArray = dataConverter.FullMessage(pipelineData);
// Validate that the array is unaffected by the serialization / deserialization
AssertArrayEquality(originalArray, deserializedArray);
// Verify that the "IsLast" property is set correctly
AssertIsLastFlag(pipelineData);
}
internal static bool TestWithByteDeserializer()
{
string remoteFilePath1 = MakeRemoteTestFile(new byte[] { 111, 112, 113 });
string remoteFilePath2 = MakeRemoteTestFile(new byte[] { 114, 115, 116, 117 });
var serializerConf = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation <IFileDeSerializer <IEnumerable <byte> >, ByteSerializer>(GenericType <IFileDeSerializer <IEnumerable <byte> > > .Class,
GenericType <ByteSerializer> .Class)
.Build();
var serializerConfString = (new AvroConfigurationSerializer()).ToString(serializerConf);
var dataSet = TangFactory.GetTang()
.NewInjector(FileSystemInputPartitionConfiguration <IEnumerable <byte> > .ConfigurationModule
.Set(FileSystemInputPartitionConfiguration <IEnumerable <byte> > .FilePathForPartitions, remoteFilePath1)
.Set(FileSystemInputPartitionConfiguration <IEnumerable <byte> > .FilePathForPartitions, remoteFilePath2)
.Set(FileSystemInputPartitionConfiguration <IEnumerable <byte> > .FileSerializerConfig, serializerConfString)
.Build(),
HadoopFileSystemConfiguration.ConfigurationModule.Build())
.GetInstance <IPartitionedInputDataSet>();
Logger.Log(Level.Info, "IPartitionedDataSet created.");
int count = 0;
foreach (var partitionDescriptor in dataSet)
{
var partition = TangFactory.GetTang()
.NewInjector(partitionDescriptor.GetPartitionConfiguration(), GetHadoopFileSystemConfiguration())
.GetInstance <IInputPartition <IEnumerable <byte> > >();
Logger.Log(Level.Info, "GetInstance of partition.");
using (partition as IDisposable)
{
Logger.Log(Level.Info, "get partition instance.");
var e = partition.GetPartitionHandle();
foreach (var v in e)
{
Logger.Log(Level.Info, string.Format(CultureInfo.CurrentCulture, "Data read {0}: ", v));
count++;
}
}
}
Logger.Log(Level.Info, "Total count returend: " + count);
return(true);
}
/// <summary>
/// Get the configuration for a particular task.
/// The task may belong to many Communication Groups, so each one is serialized
/// in the configuration as a SerializedGroupConfig.
/// The user must merge their part of task configuration (task id, task class)
/// with this returned Group Communication task configuration.
/// </summary>
/// <param name="taskId">The id of the task Configuration to generate</param>
/// <returns>The Group Communication task configuration with communication group and
/// operator configuration set.</returns>
public IConfiguration GetGroupCommTaskConfiguration(string taskId)
{
var confBuilder = TangFactory.GetTang().NewConfigurationBuilder();
foreach (ICommunicationGroupDriver commGroup in _commGroups.Values)
{
var taskConf = commGroup.GetGroupTaskConfiguration(taskId);
if (taskConf != null)
{
confBuilder.BindSetEntry <GroupCommConfigurationOptions.SerializedGroupConfigs, string>(
GenericType <GroupCommConfigurationOptions.SerializedGroupConfigs> .Class,
_configSerializer.ToString(taskConf));
}
}
return(confBuilder.Build());
}
public void TestCantGetInstanceOfNamedParameter()
{
string msg = null;
try
{
IConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
IInjector i = TangFactory.GetTang().NewInjector(cb.Build());
i.GetInstance <IfaceWithDefaultName>();
msg = "getInstance() called on Name IfaceWithDefaultName Did you mean to call getNamedInstance() instead?";
}
catch (InjectionException e)
{
System.Diagnostics.Debug.WriteLine(e);
}
Assert.IsNull(msg);
}
public void OnNext(IActiveContext value)
{
var taskConfig = TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, TaskId)
.Set(TaskConfiguration.Task, GenericType <SleepTask> .Class)
.Set(TaskConfiguration.OnClose, GenericType <SleepTask> .Class)
.Set(TaskConfiguration.OnTaskStart, GenericType <PoisonedEventHandler <ITaskStart> > .Class)
.Build();
var poisonConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindIntNamedParam <CrashTimeout>("500")
.BindIntNamedParam <CrashMinDelay>("100")
.BindNamedParameter <CrashProbability, double>(GenericType <CrashProbability> .Class, "1.0")
.Build();
value.SubmitTask(Configurations.Merge(taskConfig, poisonConfig));
}
public void TestStringInjectDefault()
{
Box b = (Box)TangFactory.GetTang().NewInjector().GetInstance(typeof(Box));
ISet <string> actual = b.Numbers;
ISet <string> expected = new HashSet <string>();
expected.Add("one");
expected.Add("two");
expected.Add("three");
Assert.IsTrue(actual.Contains("one"));
Assert.IsTrue(actual.Contains("two"));
Assert.IsTrue(actual.Contains("three"));
//Assert.AreEqual(expected, actual);
}
public IConfiguration DriverConfigurationsWithEvaluatorRequest()
{
IConfiguration driverConfig = DriverConfiguration.ConfigurationModule
.Set(DriverConfiguration.OnDriverStarted, GenericType <EvaluatorRequestingDriver> .Class)
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <EvaluatorRequestingDriver> .Class)
.Set(DriverConfiguration.OnTaskRunning, GenericType <EvaluatorRequestingDriver> .Class)
.Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
.Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
.Build();
IConfiguration taskConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(RuntimeNameTask).Assembly.GetName().Name)
.BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name)
.Build();
return(Configurations.Merge(driverConfig, taskConfig));
}
public void TestForkWorks()
{
Type checkChildIfaceType = typeof(Com.Microsoft.Tang.Examples.CheckChildIface);
Type checkChildImplType = typeof(Com.Microsoft.Tang.Examples.CheckChildImpl);
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(new string[] { file });
cb.Bind(checkChildIfaceType, checkChildImplType);
IInjector i = TangFactory.GetTang().NewInjector(cb.Build());
IInjector i1 = i.ForkInjector();
CheckChildIface c1 = (CheckChildIface)i1.GetInstance(checkChildIfaceType);
IInjector i2 = i.ForkInjector();
CheckChildIface c2 = (CheckChildIface)i2.GetInstance(checkChildIfaceType);
Assert.AreNotEqual(c1, c2);
}
public void TestActivityConfig()
{
Type activityInterfaceType = typeof(ITask);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb = tang.NewConfigurationBuilder(new string[] { FileNames.Examples, FileNames.Common, FileNames.Tasks });
cb.BindImplementation(GenericType <ITask> .Class, GenericType <HelloTask> .Class);
IConfiguration conf = cb.Build();
ConfigurationFile.WriteConfigurationFile(conf, "TaskConf.txt");
IDictionary <string, string> p = ConfigurationFile.FromFile("TaskConf.txt");
IInjector injector = tang.NewInjector(new string[] { FileNames.Common, FileNames.Tasks }, "TaskConf.txt");
var activityRef = (ITask)injector.GetInstance(activityInterfaceType);
Assert.NotNull(activityRef);
}
private TestSenderMapFunction(
[Parameter(typeof(TaskConfigurationOptions.Identifier))] string taskId,
IInputPartition <T> partition,
[Parameter(typeof(CopyToLocal))] bool copyToLocal)
{
var tempFileDir = TangFactory.GetTang().NewInjector().GetNamedInstance <TempFileFolder, string>();
var tmpFileFodler = Directory.GetCurrentDirectory() + tempFileDir.Substring(1, tempFileDir.Length - 1);
Assert.True(Directory.Exists(tmpFileFodler));
var directories = Directory.EnumerateDirectories(tmpFileFodler);
if (copyToLocal)
{
Assert.Equal(1, directories.Count());
var directory = directories.FirstOrDefault();
Assert.True(directory.Contains("-partition-"));
var files = Directory.EnumerateFiles(directory);
Assert.Equal(1, files.Count());
var file = files.FirstOrDefault();
var a = file.Split('\\');
var fileName = a[a.Length - 1];
Assert.Equal(8, fileName.Length);
var matchCounter = Regex.Matches(fileName, @"[a-zA-Z0-9]").Count;
Assert.Equal(8, matchCounter);
}
else
{
Assert.Equal(0, directories.Count());
}
int count = 0;
var e = (IEnumerable <Row>)partition.GetPartitionHandle();
foreach (var row in e)
{
Logger.Log(Level.Info, "Data read {0}: ", row.GetValue());
count++;
}
Logger.Log(Level.Info, "TestSenderMapFunction: TaskId: {0}, count {1}", taskId.Length, count);
Assert.Equal(count, DataCount);
}
public async Task TestErrorResponse()
{
const string wrongApplicationName = @"Something";
var client = TangFactory.GetTang().NewInjector().GetInstance <IYarnRMClient>();
try
{
await client.GetApplicationAsync(wrongApplicationName);
Assert.True(false, "Should throw YarnRestAPIException");
}
catch (AggregateException aggregateException)
{
Assert.IsType(typeof(YarnRestAPIException), aggregateException.GetBaseException());
}
}
public void TestDeSerializeClassHierarchyForTask()
{
Type streamTask1Type = typeof(StreamTask1);
Type helloTaskType = typeof(HelloTask);
IClassHierarchy ns = TangFactory.GetTang().GetClassHierarchy(new string[] { typeof(HelloTask).GetTypeInfo().Assembly.GetName().Name });
IClassNode streamTask1ClassNode = (IClassNode)ns.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode helloTaskClassNode = (IClassNode)ns.GetNode(helloTaskType.AssemblyQualifiedName);
ProtocolBufferClassHierarchy.Serialize("task.bin", ns);
IClassHierarchy ch = ProtocolBufferClassHierarchy.DeSerialize("task.bin");
IClassNode streamTask1ClassNode2 = (IClassNode)ch.GetNode(streamTask1Type.AssemblyQualifiedName);
IClassNode helloTaskClassNode2 = (IClassNode)ch.GetNode(helloTaskType.AssemblyQualifiedName);
Assert.Equal(streamTask1ClassNode.GetFullName(), streamTask1ClassNode2.GetFullName());
Assert.Equal(helloTaskClassNode.GetFullName(), helloTaskClassNode2.GetFullName());
}
public void TestSerirializeInjectionPlanForSimpleConstructor()
{
Type simpleConstructorType = typeof(SimpleConstructors);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb = tang.NewConfigurationBuilder(new string[] { FileNames.Examples });
IConfiguration conf = cb.Build();
IInjector injector = tang.NewInjector(conf);
Org.Apache.REEF.Tang.Implementations.InjectionPlan.InjectionPlan ip = injector.GetInjectionPlan(simpleConstructorType);
ProtocolBufferInjectionPlan.Serialize("plan.bin", ip);
var ch = conf.GetClassHierarchy();
var ipRecovered = ProtocolBufferInjectionPlan.DeSerialize("plan.bin", ch);
Assert.NotNull(ipRecovered);
}
public void TestSerirializeInjectionPlanForTimer()
{
Type timerType = typeof(Timer);
ITang tang = TangFactory.GetTang();
ICsConfigurationBuilder cb = tang.NewConfigurationBuilder(new string[] { FileNames.Examples });
cb.BindNamedParameter <Timer.Seconds, int>(GenericType <Timer.Seconds> .Class, "2");
IConfiguration conf = cb.Build();
IInjector injector = tang.NewInjector(conf);
Org.Apache.REEF.Tang.Implementations.InjectionPlan.InjectionPlan ip = injector.GetInjectionPlan(timerType);
ProtocolBufferInjectionPlan.Serialize("timerplan.bin", ip);
var ch = conf.GetClassHierarchy();
var ip1 = ProtocolBufferInjectionPlan.DeSerialize("timerplan.bin", ch);
Assert.NotNull(ip1);
}
public void ListParamTest()
{
IList <string> v = new List <string>();
v.Add("a");
v.Add("b");
IConfiguration c = ListConfigurationModule.CONF
.Set(ListConfigurationModule.P, v)
.Build();
IList <string> s = (IList <string>)TangFactory.GetTang().NewInjector(c).GetNamedInstance(typeof(ListName));
Assert.AreEqual(s.Count, 2);
Assert.IsTrue(s.Contains("a"));
Assert.IsTrue(s.Contains("b"));
}
public void HttpEventHanldersTest()
{
ConfigurationModule module =
new ConfigurationModuleBuilder()
.BindSetEntry <HttpEventHanlders, HttpServerReefEventHandler, IHttpHandler>(GenericType <HttpEventHanlders> .Class, GenericType <HttpServerReefEventHandler> .Class)
.BindSetEntry <HttpEventHanlders, HttpServerNrtEventHandler, IHttpHandler>(GenericType <HttpEventHanlders> .Class, GenericType <HttpServerNrtEventHandler> .Class)
.Build();
IConfiguration c = module.Build();
var service = TangFactory.GetTang().NewInjector(c).GetInstance <HttpServer>();
Assert.IsNotNull(service);
var j = TangFactory.GetTang().NewInjector(c).GetInstance <HttpRunTimeStartHandler>();
Assert.IsNotNull(j);
}
public void ListImplTest()
{
IList <Type> v = new List <Type>();
v.Add(typeof(ListSubA));
v.Add(typeof(ListSubB));
IConfiguration c = ListClassConfigurationModule.CONF
.Set(ListClassConfigurationModule.P, v)
.Build();
IList <IListSuper> s = (IList <IListSuper>)TangFactory.GetTang().NewInjector(c).GetNamedInstance(typeof(ListClass));
Assert.AreEqual(s.Count, 2);
Assert.IsTrue(s[0] is ListSubA);
Assert.IsTrue(s[1] is ListSubB);
}
/// <summary>
/// Build a test mapper function configuration
/// </summary>
/// <param name="maxRetryInRecovery">Number of retries done if first run failed.</param>
/// <param name="totalNumberOfForcedFailures">Number of forced failure times in recovery.</param>
private IConfiguration BuildMapperFunctionConfig(int maxRetryInRecovery, int totalNumberOfForcedFailures)
{
var c1 = IMRUMapConfiguration <int[], int[]> .ConfigurationModule
.Set(IMRUMapConfiguration <int[], int[]> .MapFunction,
GenericType <SenderMapFunctionFT> .Class)
.Build();
var c2 = TangFactory.GetTang().NewConfigurationBuilder()
.BindSetEntry <TaskIdsToFail, string>(GenericType <TaskIdsToFail> .Class, "IMRUMap-RandomInputPartition-2-")
.BindSetEntry <TaskIdsToFail, string>(GenericType <TaskIdsToFail> .Class, "IMRUMap-RandomInputPartition-3-")
.BindIntNamedParam <FailureType>(FailureType.EvaluatorFailureDuringTaskExecution.ToString())
.BindNamedParameter(typeof(MaxRetryNumberInRecovery), maxRetryInRecovery.ToString())
.BindNamedParameter(typeof(TotalNumberOfForcedFailures), totalNumberOfForcedFailures.ToString())
.Build();
return(Configurations.Merge(c1, c2));
}
/// <summary>
/// Mapper function configuration. Subclass can override it to have its own test function.
/// </summary>
/// <returns></returns>
protected override IConfiguration BuildMapperFunctionConfig()
{
var c1 = IMRUMapConfiguration <int[], int[]> .ConfigurationModule
.Set(IMRUMapConfiguration <int[], int[]> .MapFunction,
GenericType <PipelinedBroadcastAndReduceWithFaultTolerant.SenderMapFunctionFT> .Class)
.Build();
var c2 = TangFactory.GetTang().NewConfigurationBuilder()
.BindSetEntry <PipelinedBroadcastAndReduceWithFaultTolerant.TaskIdsToFail, string>(GenericType <PipelinedBroadcastAndReduceWithFaultTolerant.TaskIdsToFail> .Class, "IMRUMap-RandomInputPartition-2-")
.BindSetEntry <PipelinedBroadcastAndReduceWithFaultTolerant.TaskIdsToFail, string>(GenericType <PipelinedBroadcastAndReduceWithFaultTolerant.TaskIdsToFail> .Class, "IMRUMap-RandomInputPartition-3-")
.BindIntNamedParam <PipelinedBroadcastAndReduceWithFaultTolerant.FailureType>(PipelinedBroadcastAndReduceWithFaultTolerant.FailureType.EvaluatorFailureDuringTaskExecution.ToString())
.BindNamedParameter(typeof(MaxRetryNumberInRecovery), NumberOfRetry.ToString())
.BindNamedParameter(typeof(PipelinedBroadcastAndReduceWithFaultTolerant.TotalNumberOfForcedFailures), NumberOfRetry.ToString())
.Build();
return(Configurations.Merge(c1, c2, GetTcpConfiguration()));
}
private ContextAndServiceConfiguration GetMapTaskContextAndServiceConfiguration(IPartitionDescriptor partitionDescriptor)
{
var codecConfig =
TangFactory.GetTang()
.NewConfigurationBuilder(
StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Conf.Set(
StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Codec,
GenericType <MapInputWithControlMessageCodec <TMapInput> > .Class).Build(),
StreamingCodecConfigurationMinusMessage <TMapOutput> .Conf.Build(),
_configurationManager.MapInputCodecConfiguration)
.Build();
var contextConf = _groupCommDriver.GetContextConfiguration();
var serviceConf = Configurations.Merge(_groupCommDriver.GetServiceConfiguration(), codecConfig, partitionDescriptor.GetPartitionConfiguration());
return(new ContextAndServiceConfiguration(contextConf, serviceConf));
}
public void TestUnableToRunMultipleTasksAtTheSameTime()
{
var serviceInjector = TangFactory.GetTang().NewInjector();
var contextConfig = GetSimpleContextConfiguration();
using (var contextRuntime = new ContextRuntime(serviceInjector, contextConfig, Optional <ContextRuntime> .Empty()))
{
var taskConfig = TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Task, GenericType <TestTask> .Class)
.Set(TaskConfiguration.Identifier, "ID")
.Build();
Thread taskThread = null;
try
{
var hbMgr = Substitute.For <IHeartBeatManager>();
contextRuntime.ContextInjector.BindVolatileInstance(GenericType <IHeartBeatManager> .Class, hbMgr);
taskThread = contextRuntime.StartTaskOnNewThread(taskConfig);
Assert.True(contextRuntime.TaskRuntime.IsPresent());
Assert.True(contextRuntime.GetTaskStatus().IsPresent());
// wait for the task to start
var testTask = contextRuntime.TaskRuntime.Value.Task as TestTask;
testTask.StartEvent.Wait();
Assert.Equal(State.RUNNING, contextRuntime.GetTaskStatus().Value.state);
Assert.Throws <InvalidOperationException>(() => contextRuntime.StartTaskOnNewThread(taskConfig));
}
finally
{
var testTask = contextRuntime.TaskRuntime.Value.Task as TestTask;
if (testTask == null)
{
throw new Exception();
}
testTask.CountDownEvent.Signal();
if (taskThread != null)
{
taskThread.Join();
}
}
}
}
public void TestEvaluatorRuntime()
{
ConfigurationModule module =
new ConfigurationModuleBuilder()
.BindSetEntry <RuntimeStartHandler, EvaluatorRuntime, IObserver <RuntimeStart> >(GenericType <RuntimeStartHandler> .Class, GenericType <EvaluatorRuntime> .Class)
.Build();
IConfiguration clockConfiguraiton = module.Build();
RuntimeClock clock = TangFactory.GetTang().NewInjector(clockConfiguraiton).GetInstance <RuntimeClock>();
var r = clock.ClockRuntimeStartHandler.Get();
Assert.AreEqual(r.Count, 1);
foreach (var e in r)
{
Assert.IsTrue(e is EvaluatorRuntime);
}
}
/// <summary>
/// Runs the actual mapper count job
/// </summary>
/// <returns>The number of MapFunction instances that are part of the job.</returns>
public int Run(int numberofMappers, string outputFile, IConfiguration fileSystemConfig)
{
var results = _imruClient.Submit <int, int, int>(
new IMRUJobDefinitionBuilder()
.SetMapFunctionConfiguration(IMRUMapConfiguration <int, int> .ConfigurationModule
.Set(IMRUMapConfiguration <int, int> .MapFunction, GenericType <IdentityMapFunction> .Class)
.Build())
.SetUpdateFunctionConfiguration(
IMRUUpdateConfiguration <int, int, int> .ConfigurationModule
.Set(IMRUUpdateConfiguration <int, int, int> .UpdateFunction,
GenericType <MapperCountUpdateFunction> .Class)
.Build())
.SetMapInputCodecConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule
.Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build())
.SetUpdateFunctionCodecsConfiguration(IMRUCodecConfiguration <int> .ConfigurationModule
.Set(IMRUCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build())
.SetReduceFunctionConfiguration(IMRUReduceFunctionConfiguration <int> .ConfigurationModule
.Set(IMRUReduceFunctionConfiguration <int> .ReduceFunction,
GenericType <IntSumReduceFunction> .Class)
.Build())
.SetPartitionedDatasetConfiguration(
RandomInputDataConfiguration.ConfigurationModule.Set(
RandomInputDataConfiguration.NumberOfPartitions,
numberofMappers.ToString()).Build())
.SetResultHandlerConfiguration(
TangFactory.GetTang()
.NewConfigurationBuilder(fileSystemConfig)
.BindImplementation(GenericType <IIMRUResultHandler <int> > .Class,
GenericType <WriteResultHandler <int> > .Class)
.BindNamedParameter(typeof(ResultOutputLocation), outputFile)
.Build())
.SetMapTaskCores(2)
.SetUpdateTaskCores(3)
.SetJobName("MapperCount")
.SetNumberOfMappers(numberofMappers)
.Build());
if (results != null)
{
return(results.First());
}
return(-1);
}
public void OnNext(IActiveContext value)
{
Logger.Log(Level.Verbose, "ContextId: " + value.Id);
switch (value.Id)
{
case ContextOneId:
var contextConfig =
REEF.Common.Context.ContextConfiguration.ConfigurationModule
.Set(REEF.Common.Context.ContextConfiguration.Identifier, ContextTwoId)
.Set(REEF.Common.Context.ContextConfiguration.OnContextStart, GenericType <TestContextStackContextStartHandler> .Class)
.Build();
var stackingContextConfig =
TangFactory.GetTang()
.NewConfigurationBuilder()
.BindImplementation(GenericType <IInjectableInterface> .Class,
GenericType <InjectableInterfaceImpl> .Class)
.Build();
Assert.False(value.ParentId.IsPresent());
var stackingContextServiceConfig =
ServiceConfiguration.ConfigurationModule
.Set(ServiceConfiguration.Services, GenericType <TestService> .Class)
.Build();
value.SubmitContextAndService(
Configurations.Merge(stackingContextConfig, contextConfig), stackingContextServiceConfig);
break;
case ContextTwoId:
Assert.True(value.ParentId.IsPresent());
Assert.Equal(value.ParentId.Value, ContextOneId);
value.SubmitTask(
TaskConfiguration.ConfigurationModule.Set(TaskConfiguration.Identifier, "contextServiceStackTestTask")
.Set(TaskConfiguration.Task, GenericType <TestContextAndServiceStackTask> .Class)
.Build());
break;
default:
throw new Exception("Unexpected ContextId: " + value.Id);
}
}
private void Launch(IJobSubmission jobSubmission, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobSubmission, driverFolderPath);
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobSubmission.DriverConfigurations.ToArray());
var avroJobSubmissionParameters = new AvroJobSubmissionParameters
{
jobId = jobSubmission.JobIdentifier,
tcpBeginPort = paramInjector.GetNamedInstance <TcpPortRangeStart, int>(),
tcpRangeCount = paramInjector.GetNamedInstance <TcpPortRangeCount, int>(),
tcpTryCount = paramInjector.GetNamedInstance <TcpPortRangeTryCount, int>(),
jobSubmissionFolder = driverFolderPath
};
var avroYarnJobSubmissionParameters = new AvroYarnJobSubmissionParameters
{
driverMemory = jobSubmission.DriverMemory,
driverRecoveryTimeout = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds, int>(),
jobSubmissionDirectoryPrefix = _jobSubmissionPrefix,
sharedJobSubmissionParameters = avroJobSubmissionParameters
};
var avroYarnClusterJobSubmissionParameters = new AvroYarnClusterJobSubmissionParameters
{
maxApplicationSubmissions = paramInjector.GetNamedInstance <DriverBridgeConfigurationOptions.MaxApplicationSubmissions, int>(),
securityTokenKind = _securityTokenKind,
securityTokenService = _securityTokenService,
yarnJobSubmissionParameters = avroYarnJobSubmissionParameters
};
var submissionArgsFilePath = Path.Combine(driverFolderPath, _fileNames.GetJobSubmissionParametersFile());
using (var argsFileStream = new FileStream(submissionArgsFilePath, FileMode.CreateNew))
{
var serializedArgs = AvroJsonSerializer <AvroYarnClusterJobSubmissionParameters> .ToBytes(avroYarnClusterJobSubmissionParameters);
argsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
}
// Submit the driver
_javaClientLauncher.Launch(JavaClassName, submissionArgsFilePath);
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobSubmission.JobIdentifier);
}
public void TestConstructorInjection()
{
int port = 6666;
using (INameServer server = BuildNameServer(port))
{
IConfiguration nameClientConfiguration = NamingConfiguration.ConfigurationModule
.Set(NamingConfiguration.NameServerAddress, server.LocalEndpoint.Address.ToString())
.Set(NamingConfiguration.NameServerPort, port + string.Empty)
.Build();
ConstructorInjection c = TangFactory.GetTang()
.NewInjector(nameClientConfiguration)
.GetInstance <ConstructorInjection>();
Assert.NotNull(c);
}
}
public void TestInjectionWithGenericArguments()
{
var c = TangFactory.GetTang().NewConfigurationBuilder()
.BindImplementation(GenericType <IMyOperator <int> > .Class, GenericType <MyOperatorImpl <int> > .Class)
.BindImplementation(GenericType <IMyOperator <string> > .Class, GenericType <MyOperatorImpl <string> > .Class)
.Build();
var injector = TangFactory.GetTang().NewInjector(c);
// argument type must be specified in injection
var o1 = injector.GetInstance(typeof(IMyOperator <int>));
var o2 = injector.GetInstance(typeof(IMyOperator <string>));
var o3 = injector.GetInstance(typeof(MyOperatorTopology <int>));
Assert.IsTrue(o1 is MyOperatorImpl <int>);
Assert.IsTrue(o2 is MyOperatorImpl <string>);
Assert.IsTrue(o3 is MyOperatorTopology <int>);
}
public void TestContextBindServiceBoundInterfaceThrowsException()
{
var boundTestServiceConfig = TangFactory.GetTang()
.NewConfigurationBuilder()
.BindImplementation(GenericType <ITestService> .Class, GenericType <TestService> .Class)
.Build();
var serviceConfiguration = ServiceConfiguration.ConfigurationModule.Build();
var serviceInjector = TangFactory.GetTang().NewInjector(Configurations.Merge(serviceConfiguration, boundTestServiceConfig));
var contextConfig = Configurations.Merge(GetContextEventHandlerContextConfiguration(), boundTestServiceConfig);
var ex = Record.Exception(() => new ContextRuntime(serviceInjector, contextConfig, Optional <ContextRuntime> .Empty()));
// This should throw an Exception because we are binding the interface ITestService twice, once in serviceConfiguration
// and once in contextConfiguration. The Context injector is forked from the ServiceInjector, which already has the
// interface bound.
Assert.True(ex != null);
}
private void Launch(JobRequest jobRequest, string driverFolderPath)
{
_driverFolderPreparationHelper.PrepareDriverFolder(jobRequest.AppParameters, driverFolderPath);
_paramSerializer.WriteSecurityTokens();
// TODO: Remove this when we have a generalized way to pass config to java
var paramInjector = TangFactory.GetTang().NewInjector(jobRequest.DriverConfigurations.ToArray());
var submissionJobArgsFilePath = _paramSerializer.SerializeJobFile(jobRequest.JobParameters, paramInjector, driverFolderPath);
var submissionAppArgsFilePath = _paramSerializer.SerializeAppFile(jobRequest.AppParameters, paramInjector, driverFolderPath);
// Submit the driver
_javaClientLauncher.LaunchAsync(
jobRequest.JavaLogLevel, JavaClassName, submissionJobArgsFilePath, submissionAppArgsFilePath)
.GetAwaiter()
.GetResult();
Logger.Log(Level.Info, "Submitted the Driver for execution." + jobRequest.JobIdentifier);
}
public IFileSystem GetAzureFileSystem()
{
var conf = AzureBlockBlobFileSystemConfiguration.ConfigurationModule
.Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;")
.Build();
var injector = TangFactory.GetTang().NewInjector(conf);
injector.BindVolatileInstance(TestCloudBlobClient);
var fs = injector.GetInstance <AzureBlockBlobFileSystem>();
TestCloudBlobClient.BaseUri.ReturnsForAnyArgs(FakeUri);
TestCloudBlobClient.GetBlockBlobReference(FakeUri).ReturnsForAnyArgs(TestCloudBlockBlob);
TestCloudBlobClient.GetContainerReference("container").ReturnsForAnyArgs(TestCloudBlobContainer);
TestCloudBlobContainer.GetDirectoryReference("directory").ReturnsForAnyArgs(TestCloudBlobDirectory);
TestCloudBlobDirectory.GetDirectoryReference("directory").ReturnsForAnyArgs(TestCloudBlobDirectory);
return(fs);
}
