public static void ClassSetup(TestContext context)
{
TangImpl.Reset();
ns = TangFactory.GetTang().GetClassHierarchy(new string[] { FileNames.Examples, FileNames.Common, FileNames.Tasks });
}
public void TestCanGetStringDefaultedInterface()
{
Assert.NotNull(TangFactory.GetTang().NewInjector().GetInstance <IHaveDefaultStringImpl>());
}
public IConfiguration FromAvro(AvroConfiguration avroConfiguration)
{
ICsConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder();
return(AddFromAvro(cb, avroConfiguration));
}
public IConfiguration FromAvro(AvroConfiguration avroConfiguration, IClassHierarchy classHierarchy)
{
IConfigurationBuilder cb = TangFactory.GetTang().NewConfigurationBuilder(classHierarchy);
return(AddFromAvro(cb, avroConfiguration));
}
/// <summary>
/// Gets the task configuration for the operator topology.
/// </summary>
/// <param name="taskId">The task identifier</param>
/// <returns>The task configuration</returns>
public IConfiguration GetTaskConfiguration(string taskId)
{
ICsConfigurationBuilder confBuilder;
confBuilder = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <GroupCommConfigurationOptions.TopologyRootTaskId, string>(
GenericType <GroupCommConfigurationOptions.TopologyRootTaskId> .Class,
_rootId);
if (taskId.Equals(_rootId))
{
foreach (var tId in _nodes.Keys)
{
if (!tId.Equals(_rootId))
{
confBuilder.BindSetEntry <GroupCommConfigurationOptions.TopologyChildTaskIds, string>(
GenericType <GroupCommConfigurationOptions.TopologyChildTaskIds> .Class,
tId);
}
}
}
if (OperatorSpec is BroadcastOperatorSpec)
{
var broadcastSpec = OperatorSpec as BroadcastOperatorSpec;
if (taskId.Equals(broadcastSpec.SenderId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <BroadcastSender <T> > .Class);
SetMessageType(typeof(BroadcastSender <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <BroadcastReceiver <T> > .Class);
SetMessageType(typeof(BroadcastReceiver <T>), confBuilder);
}
}
else if (OperatorSpec is ReduceOperatorSpec)
{
var reduceSpec = OperatorSpec as ReduceOperatorSpec;
if (taskId.Equals(reduceSpec.ReceiverId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ReduceReceiver <T> > .Class);
SetMessageType(typeof(ReduceReceiver <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ReduceSender <T> > .Class);
SetMessageType(typeof(ReduceSender <T>), confBuilder);
}
}
else if (OperatorSpec is ScatterOperatorSpec)
{
ScatterOperatorSpec scatterSpec = OperatorSpec as ScatterOperatorSpec;
if (taskId.Equals(scatterSpec.SenderId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ScatterSender <T> > .Class);
SetMessageType(typeof(ScatterSender <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ScatterReceiver <T> > .Class);
SetMessageType(typeof(ScatterReceiver <T>), confBuilder);
}
}
else
{
throw new NotSupportedException("Spec type not supported");
}
return(Configurations.Merge(confBuilder.Build(), OperatorSpec.Configiration));
}
public void TestReferenceType()
{
AReferenceClass o = (AReferenceClass)TangFactory.GetTang().NewInjector().GetInstance(typeof(IAInterface));
}
/// <summary>
/// Adds broadcast and reduce operators to the default communication group
/// </summary>
private void AddGroupCommunicationOperators()
{
var reduceFunctionConfig = _configurationManager.ReduceFunctionConfiguration;
var mapOutputPipelineDataConverterConfig = _configurationManager.MapOutputPipelineDataConverterConfiguration;
var mapInputPipelineDataConverterConfig = _configurationManager.MapInputPipelineDataConverterConfiguration;
try
{
TangFactory.GetTang()
.NewInjector(mapInputPipelineDataConverterConfig)
.GetInstance <IPipelineDataConverter <TMapInput> >();
mapInputPipelineDataConverterConfig =
TangFactory.GetTang()
.NewConfigurationBuilder(mapInputPipelineDataConverterConfig)
.BindImplementation(
GenericType <IPipelineDataConverter <MapInputWithControlMessage <TMapInput> > > .Class,
GenericType <MapInputwithControlMessagePipelineDataConverter <TMapInput> > .Class)
.Build();
}
catch (Exception)
{
mapInputPipelineDataConverterConfig = TangFactory.GetTang()
.NewConfigurationBuilder()
.BindImplementation(
GenericType <IPipelineDataConverter <MapInputWithControlMessage <TMapInput> > > .Class,
GenericType <DefaultPipelineDataConverter <MapInputWithControlMessage <TMapInput> > > .Class)
.Build();
}
try
{
TangFactory.GetTang()
.NewInjector(mapOutputPipelineDataConverterConfig)
.GetInstance <IPipelineDataConverter <TMapOutput> >();
}
catch (Exception)
{
mapOutputPipelineDataConverterConfig =
TangFactory.GetTang()
.NewConfigurationBuilder()
.BindImplementation(GenericType <IPipelineDataConverter <TMapOutput> > .Class,
GenericType <DefaultPipelineDataConverter <TMapOutput> > .Class)
.Build();
}
_commGroup =
_groupCommDriver.DefaultGroup
.AddBroadcast <MapInputWithControlMessage <TMapInput> >(
IMRUConstants.BroadcastOperatorName,
IMRUConstants.UpdateTaskName,
TopologyTypes.Tree,
mapInputPipelineDataConverterConfig)
.AddReduce <TMapOutput>(
IMRUConstants.ReduceOperatorName,
IMRUConstants.UpdateTaskName,
TopologyTypes.Tree,
reduceFunctionConfig,
mapOutputPipelineDataConverterConfig)
.Build();
}
protected ConfigurationBuilderImpl()
{
this.ClassHierarchy = TangFactory.GetTang().GetDefaultClassHierarchy();
}
public void TestSetup()
{
tang = TangFactory.GetTang();
}
public async Task TestCommonStreamingCodecs()
{
IInjector injector = TangFactory.GetTang().NewInjector();
IStreamingCodec <int> intCodec = injector.GetInstance <IntStreamingCodec>();
IStreamingCodec <double> doubleCodec = injector.GetInstance <DoubleStreamingCodec>();
IStreamingCodec <float> floatCodec = injector.GetInstance <FloatStreamingCodec>();
IStreamingCodec <int[]> intArrCodec = injector.GetInstance <IntArrayStreamingCodec>();
IStreamingCodec <double[]> doubleArrCodec = injector.GetInstance <DoubleArrayStreamingCodec>();
IStreamingCodec <float[]> floatArrCodec = injector.GetInstance <FloatArrayStreamingCodec>();
IStreamingCodec <string> stringCodec = injector.GetInstance <StringStreamingCodec>();
CancellationToken token = new CancellationToken();
int obj = 5;
int[] intArr = { 1, 2 };
double[] doubleArr = { 1, 2 };
float[] floatArr = { 1, 2 };
string stringObj = "hello";
var stream = new MemoryStream();
IDataWriter writer = new StreamDataWriter(stream);
intCodec.Write(obj, writer);
await intCodec.WriteAsync(obj + 1, writer, token);
doubleCodec.Write(obj + 2, writer);
await doubleCodec.WriteAsync(obj + 3, writer, token);
floatCodec.Write(obj + 4, writer);
await floatCodec.WriteAsync(obj + 5, writer, token);
intArrCodec.Write(intArr, writer);
await intArrCodec.WriteAsync(intArr, writer, token);
doubleArrCodec.Write(doubleArr, writer);
await doubleArrCodec.WriteAsync(doubleArr, writer, token);
floatArrCodec.Write(floatArr, writer);
await floatArrCodec.WriteAsync(floatArr, writer, token);
stringCodec.Write(stringObj, writer);
await stringCodec.WriteAsync(stringObj, writer, token);
stream.Position = 0;
IDataReader reader = new StreamDataReader(stream);
int res1 = intCodec.Read(reader);
int res2 = await intCodec.ReadAsync(reader, token);
double res3 = doubleCodec.Read(reader);
double res4 = await doubleCodec.ReadAsync(reader, token);
float res5 = floatCodec.Read(reader);
float res6 = await floatCodec.ReadAsync(reader, token);
int[] resArr1 = intArrCodec.Read(reader);
int[] resArr2 = await intArrCodec.ReadAsync(reader, token);
double[] resArr3 = doubleArrCodec.Read(reader);
double[] resArr4 = await doubleArrCodec.ReadAsync(reader, token);
float[] resArr5 = floatArrCodec.Read(reader);
float[] resArr6 = await floatArrCodec.ReadAsync(reader, token);
string resArr7 = stringCodec.Read(reader);
string resArr8 = await stringCodec.ReadAsync(reader, token);
Assert.AreEqual(obj, res1);
Assert.AreEqual(obj + 1, res2);
Assert.AreEqual(obj + 2, res3);
Assert.AreEqual(obj + 3, res4);
Assert.AreEqual(obj + 4, res5);
Assert.AreEqual(obj + 5, res6);
Assert.AreEqual(stringObj, resArr7);
Assert.AreEqual(stringObj, resArr8);
for (int i = 0; i < intArr.Length; i++)
{
Assert.AreEqual(intArr[i], resArr1[i]);
Assert.AreEqual(intArr[i], resArr2[i]);
}
for (int i = 0; i < doubleArr.Length; i++)
{
Assert.AreEqual(doubleArr[i], resArr3[i]);
Assert.AreEqual(doubleArr[i], resArr4[i]);
}
for (int i = 0; i < floatArr.Length; i++)
{
Assert.AreEqual(floatArr[i], resArr5[i]);
Assert.AreEqual(floatArr[i], resArr6[i]);
}
}
public void TestDefaultImplementaion()
{
ClassWithDefaultConstructor impl = (ClassWithDefaultConstructor)TangFactory.GetTang().NewInjector().GetInstance(typeof(ClassWithDefaultConstructor));
Assert.IsNotNull(impl);
}
/// <summary>
/// Gets the task configuration for the operator topology.
/// </summary>
/// <param name="taskId">The task identifier</param>
/// <returns>The task configuration</returns>
public IConfiguration GetTaskConfiguration(string taskId)
{
if (taskId == null)
{
throw new ArgumentException("TaskId is null when GetTaskConfiguration");
}
TaskNode selfTaskNode = GetTaskNode(taskId);
if (selfTaskNode == null)
{
throw new ArgumentException("Task has not been added to the topology");
}
string parentId;
TaskNode parent = selfTaskNode.Parent;
if (parent == null)
{
parentId = selfTaskNode.TaskId;
}
else
{
parentId = parent.TaskId;
}
// add parentid, if no parent, add itself
ICsConfigurationBuilder confBuilder = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <GroupCommConfigurationOptions.TopologyRootTaskId, string>(
GenericType <GroupCommConfigurationOptions.TopologyRootTaskId> .Class,
parentId);
// add all its children
foreach (TaskNode childNode in selfTaskNode.GetChildren())
{
confBuilder.BindSetEntry <GroupCommConfigurationOptions.TopologyChildTaskIds, string>(
GenericType <GroupCommConfigurationOptions.TopologyChildTaskIds> .Class,
childNode.TaskId);
}
if (OperatorSpec is BroadcastOperatorSpec)
{
var broadcastSpec = OperatorSpec as BroadcastOperatorSpec;
if (taskId.Equals(broadcastSpec.SenderId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <BroadcastSender <T> > .Class);
SetMessageType(typeof(BroadcastSender <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <BroadcastReceiver <T> > .Class);
SetMessageType(typeof(BroadcastReceiver <T>), confBuilder);
}
}
else if (OperatorSpec is ReduceOperatorSpec)
{
var reduceSpec = OperatorSpec as ReduceOperatorSpec;
if (taskId.Equals(reduceSpec.ReceiverId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ReduceReceiver <T> > .Class);
SetMessageType(typeof(ReduceReceiver <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ReduceSender <T> > .Class);
SetMessageType(typeof(ReduceSender <T>), confBuilder);
}
}
else if (OperatorSpec is ScatterOperatorSpec)
{
ScatterOperatorSpec scatterSpec = OperatorSpec as ScatterOperatorSpec;
if (taskId.Equals(scatterSpec.SenderId))
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ScatterSender <T> > .Class);
SetMessageType(typeof(ScatterSender <T>), confBuilder);
}
else
{
confBuilder.BindImplementation(GenericType <IGroupCommOperator <T> > .Class, GenericType <ScatterReceiver <T> > .Class);
SetMessageType(typeof(ScatterReceiver <T>), confBuilder);
}
}
else
{
throw new NotSupportedException("Spec type not supported");
}
return(Configurations.Merge(confBuilder.Build(), OperatorSpec.Configuration));
}
public async Task TestApplicationSubmissionAndQuery()
{
var client = TangFactory.GetTang().NewInjector().GetInstance <IYarnRMClient>();
var newApplication = await client.CreateNewApplicationAsync();
Assert.IsNotNull(newApplication);
Assert.IsFalse(string.IsNullOrEmpty(newApplication.ApplicationId));
Assert.IsTrue(newApplication.MaximumResourceCapability.MemoryMB > 0);
Assert.IsTrue(newApplication.MaximumResourceCapability.VCores > 0);
string applicationName = "REEFTEST_APPLICATION_" + Guid.NewGuid();
Console.WriteLine(applicationName);
const string anyApplicationType = "REEFTest";
var submitApplicationRequest = new SubmitApplication
{
ApplicationId = newApplication.ApplicationId,
AmResource = new Resouce
{
MemoryMB = 500,
VCores = 1
},
ApplicationType = anyApplicationType,
ApplicationName = applicationName,
KeepContainersAcrossApplicationAttempts = false,
MaxAppAttempts = 1,
Priority = 1,
UnmanagedAM = false,
AmContainerSpec = new AmContainerSpec
{
Commands = new Commands
{
Command = @"DONTCARE"
},
LocalResources = new LocalResources
{
Entries = new List <YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue> >
{
new YARN.RestClient.DataModel.KeyValuePair <string, LocalResourcesValue>
{
Key = "APPLICATIONWILLFAILBUTWEDONTCAREHERE",
Value = new LocalResourcesValue
{
Resource = "Foo",
Type = ResourceType.FILE,
Visibility = Visibility.APPLICATION
}
}
}
}
}
};
var application = await client.SubmitApplicationAsync(submitApplicationRequest);
Assert.IsNotNull(application);
Assert.AreEqual(newApplication.ApplicationId, application.Id);
Assert.AreEqual(applicationName, application.Name);
Assert.AreEqual(anyApplicationType, application.ApplicationType);
var getApplicationResult = client.GetApplicationAsync(newApplication.ApplicationId).GetAwaiter().GetResult();
Assert.IsNotNull(getApplicationResult);
Assert.AreEqual(newApplication.ApplicationId, getApplicationResult.Id);
Assert.AreEqual(applicationName, getApplicationResult.Name);
Assert.AreEqual(anyApplicationType, getApplicationResult.ApplicationType);
}
