/// <exception cref="System.IO.IOException"/>
private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app)
{
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2
= new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
(), 2);
nmContext.GetContainers()[container1.GetContainerId()] = container1;
nmContext.GetContainers()[container2.GetContainerId()] = container2;
app.GetContainers()[container1.GetContainerId()] = container1;
app.GetContainers()[container2.GetContainerId()] = container2;
Dictionary <string, string> hash = new Dictionary <string, string>();
hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString
();
hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString
();
return(hash);
}
public virtual RMContext MockRMContext(int n, long time)
{
IList <RMApp> apps = NewRMApps(n, time, RMAppState.Finished);
ConcurrentMap <ApplicationId, RMApp> map = Maps.NewConcurrentMap();
foreach (RMApp app in apps)
{
map[app.GetApplicationId()] = app;
}
Dispatcher rmDispatcher = new AsyncDispatcher();
ContainerAllocationExpirer containerAllocationExpirer = new ContainerAllocationExpirer
(rmDispatcher);
AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(rmDispatcher);
AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(rmDispatcher);
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContext context = new _RMContextImpl_121(map, rmDispatcher, containerAllocationExpirer
, amLivelinessMonitor, amFinishingMonitor, null, null, null, null, null);
((RMContextImpl)context).SetStateStore(Org.Mockito.Mockito.Mock <RMStateStore>());
metricsPublisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher>();
context.SetSystemMetricsPublisher(metricsPublisher);
context.SetRMApplicationHistoryWriter(writer);
return(context);
}
public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
// for UT purpose
protected internal virtual void StartDispatcher()
{
// start dispatcher
AsyncDispatcher asyncDispatcher = (AsyncDispatcher)dispatcher;
asyncDispatcher.Start();
}
private static void Initialize()
{
if (_instance == null)
{
_instance = new GameObject("Dispatcher").AddComponent <AsyncDispatcher>();
DontDestroyOnLoad(_instance.gameObject);
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled
, YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled);
InitAndStartRecoveryStore(conf);
NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager
(conf, nmStore);
NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore
);
RecoverTokens(nmTokenSecretManager, containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor
>(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf
);
try
{
exec.Init();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to initialize container executor", e);
}
DeletionService del = CreateDeletionService(exec);
AddService(del);
// NodeManager level dispatcher
this.dispatcher = new AsyncDispatcher();
nodeHealthChecker = new NodeHealthCheckerService();
AddService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.GetDiskHandler();
this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager,
nmStore);
nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker
);
NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor();
AddService(nodeResourceMonitor);
containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater,
this.aclsManager, dirsHandler);
AddService(containerManager);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor
(), this.aclsManager, dirsHandler);
AddService(webServer);
((NodeManager.NMContext)context).SetWebServer(webServer);
dispatcher.Register(typeof(ContainerManagerEventType), containerManager);
dispatcher.Register(typeof(NodeManagerEventType), this);
AddService(dispatcher);
DefaultMetricsSystem.Initialize("NodeManager");
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
AddService(nodeStatusUpdater);
base.ServiceInit(conf);
}
// for UT purpose
protected internal virtual void InitDispatcher(Configuration conf)
{
// create async handler
dispatcher = new AsyncDispatcher();
AsyncDispatcher asyncDispatcher = (AsyncDispatcher)dispatcher;
asyncDispatcher.Init(conf);
asyncDispatcher.SetDrainEventsOnStop();
}
public MultiThreadedDispatcher(int num)
: base(typeof(SystemMetricsPublisher.MultiThreadedDispatcher).FullName)
{
for (int i = 0; i < num; ++i)
{
AsyncDispatcher dispatcher = CreateDispatcher();
dispatchers.AddItem(dispatcher);
AddIfService(dispatcher);
}
}
public MultiThreadedDispatcher(int num)
: base(typeof(RMApplicationHistoryWriter.MultiThreadedDispatcher).FullName)
{
for (int i = 0; i < num; ++i)
{
AsyncDispatcher dispatcher = CreateDispatcher();
dispatchers.AddItem(dispatcher);
AddIfService(dispatcher);
}
}
public ContainersMonitorImpl(ContainerExecutor exec, AsyncDispatcher dispatcher,
Context context)
: base("containers-monitor")
{
this.containerExecutor = exec;
this.eventDispatcher = dispatcher;
this.context = context;
this.containersToBeAdded = new Dictionary <ContainerId, ContainersMonitorImpl.ProcessTreeInfo
>();
this.containersToBeRemoved = new AList <ContainerId>();
this.monitoringThread = new ContainersMonitorImpl.MonitoringThread(this);
}
public virtual void TestSchedulerEventDispatcherForPreemptionEvents()
{
AsyncDispatcher rmDispatcher = new AsyncDispatcher();
CapacityScheduler sched = Org.Mockito.Mockito.Spy(new CapacityScheduler());
YarnConfiguration conf = new YarnConfiguration();
ResourceManager.SchedulerEventDispatcher schedulerDispatcher = new ResourceManager.SchedulerEventDispatcher
(sched);
rmDispatcher.Register(typeof(SchedulerEventType), schedulerDispatcher);
rmDispatcher.Init(conf);
rmDispatcher.Start();
schedulerDispatcher.Init(conf);
schedulerDispatcher.Start();
try
{
ApplicationAttemptId appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
RMContainer container = Org.Mockito.Mockito.Mock <RMContainer>();
ContainerPreemptEvent event1 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.DropReservation);
rmDispatcher.GetEventHandler().Handle(event1);
ContainerPreemptEvent event2 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.KillContainer);
rmDispatcher.GetEventHandler().Handle(event2);
ContainerPreemptEvent event3 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.PreemptContainer);
rmDispatcher.GetEventHandler().Handle(event3);
// Wait for events to be processed by scheduler dispatcher.
Sharpen.Thread.Sleep(1000);
Org.Mockito.Mockito.Verify(sched, Org.Mockito.Mockito.Times(3)).Handle(Matchers.Any
<SchedulerEvent>());
Org.Mockito.Mockito.Verify(sched).DropContainerReservation(container);
Org.Mockito.Mockito.Verify(sched).PreemptContainer(appAttemptId, container);
Org.Mockito.Mockito.Verify(sched).KillContainer(container);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
finally
{
schedulerDispatcher.Stop();
rmDispatcher.Stop();
}
}
static void Main(string[] args)
{
const int numOfCalls = 100000;
///***** PURE C# *****/
var handler = new SampleHandler();
Time <Message>("Pure C# - Shared handler", numOfCalls, handler.HandleAsync);
Time <Message>("Pure C# - New handler", numOfCalls, m => new SampleHandler().HandleAsync(m));
Time <MyRequest>("Pure C# Request - Shared handler", numOfCalls, handler.HandleAsync);
Time <MyRequest>("Pure C# Request - New handler", numOfCalls, m => new SampleHandler().HandleAsync(m));
/***** ROUTEMEISTER *****/
var sampleAssembly = typeof(Program).GetTypeInfo().Assembly;
var routeFactory = new MessageRouteFactory();
var routes = routeFactory.Create(sampleAssembly, typeof(IMyHandlerOf <>));
var reqRoutes = routeFactory.Create(sampleAssembly, typeof(IMyRequestHandlerOf <,>));
var sharedHandlerRouter = new SequentialAsyncRouter((t, e) => handler, routes);
var newHandlerRouter = new SequentialAsyncRouter((t, e) => new SampleHandler(), routes);
var asyncDispatcherSharedHandler = new AsyncDispatcher((t, e) => handler, reqRoutes);
var asyncDispatcherNewHandler = new AsyncDispatcher((t, e) => new SampleHandler(), reqRoutes);
Time <Message>("SequentialAsyncRouter - Shared handler", numOfCalls, sharedHandlerRouter.RouteAsync);
Time <Message>("SequentialAsyncRouter - New handler", numOfCalls, newHandlerRouter.RouteAsync);
Time <MyRequest>("AsyncDispatcher.Send - Shared handler", numOfCalls, asyncDispatcherSharedHandler.SendAsync);
Time <MyRequest>("AsyncDispatcher.Send - New handler", numOfCalls, asyncDispatcherNewHandler.SendAsync);
Time <MyRequest>("AsyncDispatcher.Publish - Shared handler", numOfCalls, asyncDispatcherSharedHandler.PublishAsync);
Time <MyRequest>("AsyncDispatcher.Publish - New handler", numOfCalls, asyncDispatcherNewHandler.PublishAsync);
Time <MyRequest>("AsyncDispatcher.Request - Shared handler", numOfCalls, asyncDispatcherSharedHandler.RequestAsync);
Time <MyRequest>("AsyncDispatcher.Request - New handler", numOfCalls, asyncDispatcherNewHandler.RequestAsync);
var messageType = typeof(Message);
var route = routes.GetRoute(messageType);
var routeAction = route.Actions.Single();
Time <Message>("Manual Route - Shared handler", numOfCalls, m => (Task)routeAction.Invoke(handler, m));
Time <Message>("Manual Route - New handler", numOfCalls, m => (Task)routeAction.Invoke(new SampleHandler(), m));
}
protected override void Load(ContainerBuilder builder)
{
//MessageRoutes
builder.Register(ctx =>
{
var factory = new MessageRouteFactory();
return(factory.Create(
SampleAssembly.Item,
typeof(IAsyncMessageHandler <>)));
})
.SingleInstance();
//MiddlewareEnabledAsyncRouter sample
builder.Register(ctx =>
{
var routes = ctx.Resolve <MessageRoutes>();
var parentScope = ctx.Resolve <ILifetimeScope>();
var router = new MiddlewareEnabledAsyncRouter(
(type, envelope) => envelope.GetScope().Resolve(type),
routes);
router.Use(next => envelope =>
{
using (var childScope = parentScope.BeginLifetimeScope())
{
envelope.SetScope(childScope);
return(next(envelope));
}
});
return(router);
})
.SingleInstance();
//SequentialAsyncRouter sample
builder.Register(ctx =>
{
var routes = ctx.Resolve <MessageRoutes>();
var parentScope = ctx.Resolve <ILifetimeScope>();
var router = new SequentialAsyncRouter(
(type, envelope) => envelope.GetScope().Resolve(type),
routes)
{
OnBeforeRouting = envelope => envelope.SetScope(parentScope.BeginLifetimeScope()),
OnAfterRouted = envelope => envelope.GetScope()?.Dispose()
};
return(router);
})
.SingleInstance();
//AsyncDispatcher sample
builder.Register(ctx =>
{
var routes = ctx.Resolve <MessageRoutes>();
var parentScope = ctx.Resolve <ILifetimeScope>();
var router = new AsyncDispatcher(
(type, envelope) => envelope.GetScope().Resolve(type),
routes)
{
OnBeforeRouting = envelope => envelope.SetScope(parentScope.BeginLifetimeScope()),
OnAfterRouted = envelope => envelope.GetScope()?.Dispose()
};
return(router);
})
.SingleInstance();
builder.RegisterType <SomeDependency>();
builder.RegisterType <ConcreteHandler>();
}
// for UT purpose
protected internal virtual void StopDispatcher()
{
AsyncDispatcher asyncDispatcher = (AsyncDispatcher)dispatcher;
asyncDispatcher.Stop();
}
public Task <object> CallAsync(Dictionary <string, object> header, MethodContext methodContext, Func <object, Task> callback, CancellationToken token,
Stream stream, params object[] pureArgs)
{
if (callback != null)
{
_callbackDispatcher = new AsyncDispatcher();
}
var action = methodContext.InstanceMethod.MethodInfo.ToActionInfo();
var tcs = new TaskCompletionSource <object>();
var t = Task.Run(async() =>
{
_convert.ResultStream += (s, e) => { SetStreamResult(tcs, e.Value); };
_convert.Result += (s, e) => { SetResult(tcs, e.Value); };
if (callback != null)
{
_convert.CallbackAsync += async(s, e) =>
{
try
{
await _callbackDispatcher.InvokeAsync(() => callback(e.Value)).Unwrap();
}
catch (Exception ex)
{
_logger.LogWarning(ex, "client callback");
throw;
}
}
}
;
_convert.Fault += (s, e) => SetFault(tcs, e.Value);
try
{
//Send cmd
var postStream = methodContext.ContractMethod.IsMQPost ? stream.StreamToBytes() : null;
var p = new OnceCallParam(header, action, postStream != null || stream != null,
postStream, stream.GetLength(), pureArgs);
if (token.IsCancellationRequested)
{
SetCancel(tcs);
return;
}
//sendCmd
var sendStreamNext = await _convert.SendCmdAsync(p, methodContext, stream, methodContext.ContractMethod.IsMQPost, token);
//cancel token
_reg = token.Register(async() =>
{
try
{
await _convert.SendCancelAsync();
}
catch (Exception e)
{
SetFault(tcs, e);
}
});
if (!sendStreamNext || stream == null)
{
return;
}
//timeout
#pragma warning disable 4014
Task.Delay(_timeoutInterval, _timeOutCts.Token).ContinueWith(i =>
#pragma warning restore 4014
{
SetFault(tcs, new TimeoutException($"Service is not response over {_timeoutInterval} ms, time out."));
}, _timeOutCts.Token);
//send stream
await Helper.SendStreamAsync(_convert.SendBufferAsync, _convert.SendBufferEndAsync, stream, token, OnSendRequestStreamStarted);
OnSendRequestStreamFinished();
}
catch (Exception e)
{
SetFault(tcs, e);
}
}, token);
if (t.IsCanceled)
{
SetCancel(tcs);
}
return(tcs.Task);
}
//Old method
#if false
private async void BuildStreamlines()
{
//Delete all existing streamlines
await AsyncDispatcher.RunOnMainThread(() => {
for (int i = 0; i < transform.childCount; i++)
{
Destroy(transform.GetChild(i).gameObject);
}
return(null);
}).NoSync();
/** One line object per segment method
* Usefull for setting colors easily, but very uneficient performance-wise
* //Loop through all injection point
* for (float y = 0.1f; y <= GridManager.Size; y += GridManager.Spacing) {
* for (float z = 0.1f; z <= GridManager.Size; z += GridManager.Spacing) {
* //Init list
* var startPosition = new Vector3(transform.position.x, y, z);
*
* //Get speed at currentPoint
* Vector3 currentSpeed;
* while ((currentSpeed = DataBase.GetSpeedAtPoint(startPosition)) != Vector3.zero) {
* //Move to next point by going into the speed direction by a defined distance
* var endPosition = startPosition + currentSpeed.normalized * StreamLineIncDistance;
*
* //Create a new line renderer
* var line = Instantiate(Line, Vector3.zero, Quaternion.identity, transform);
* var lineRenderer = line.GetComponent<LineRenderer>();
*
* //Apply points
* lineRenderer.positionCount = 2;
* lineRenderer.SetPositions(new []{startPosition, endPosition});
*
* //Move to next segment
* startPosition = endPosition;
* }
* }
* }
*
* return;
*/
/** Old code
* //Loop through all injection point
* for (float y = 0.1f; y <= TrajectoriesManager.Size; y += TrajectoriesManager.Spacing) {
* for (float z = 0.1f; z <= TrajectoriesManager.Size; z += TrajectoriesManager.Spacing) {
* //Build stream line points
* var positions = BuildStreamLine(new Vector3(transform.position.x, y, z), out var speedList);
* if (positions == null) continue;
*
* //Create a new line renderer
* var line = Instantiate(Line, Vector3.zero, Quaternion.identity, transform);
* var lineRenderer = line.GetComponent<LineRenderer>();
*
* //Apply points
* lineRenderer.positionCount = positions.Length;
* lineRenderer.SetPositions(positions);
*
* //
* //const int textLength = 1024;
* var texture = new Texture2D(speedList.Count, 1);
* var maxSpeed = speedList.Max();
* texture.SetPixels32(speedList.Select(s => (Color32)Color.Lerp(Color.blue, Color.red, s / maxSpeed)).ToArray());
* texture.Apply();
*
* lineRenderer.material.mainTexture = texture;
* }
* }*/
foreach (var trajectory in TrajectoriesManager.Instance.Trajectories)
{
LineRenderer lineRenderer = (LineRenderer)await AsyncDispatcher.RunOnMainThread(() => {
//Create a new line renderer
var line = Instantiate(Line, Vector3.zero, Quaternion.identity, transform);
lineRenderer = line.GetComponent <LineRenderer>();
//Apply points
lineRenderer.positionCount = trajectory.Points.Length;
lineRenderer.SetPositions(trajectory.Points);
return(lineRenderer);
}).NoSync();
//-- Calculate colors --
/** Obsolete
* //Build distance array
* var distances = new float[trajectory.Points.Length - 1];
* for (int i = 0; i < distances.Length; i++)
* distances[i] = Vector3.Distance(trajectory.Points[i + 1], trajectory.Points[i]);
*/
var distances = trajectory.Distances;
int textureSize = Mathf.CeilToInt(distances.Length / 10f);
var distanceIncrement = distances.Sum() / (textureSize - 1);
var colorValues = new float[textureSize];
int currentDistanceIndex = 0;
var currentDistance = distances[currentDistanceIndex];
for (int i = 0; i < colorValues.Length; i++)
{
while (currentDistance < distanceIncrement * i && currentDistanceIndex + 1 < distances.Length) //Index condition needed because of float imprecision.
{
currentDistance += distances[++currentDistanceIndex];
}
colorValues[i] = distances[currentDistanceIndex];
}
//Apply colors
var texture = new Texture2D(textureSize, 1);
texture.SetPixels32(colorValues.Select(s => (Color32)Color.Lerp(Color.blue, Color.red, s / (3 * TrajectoriesManager.Instance.TrajectoriesAverageDistance))).ToArray());
texture.Apply();
lineRenderer.material.mainTexture = texture;
//TODO this method render black line when app is launched. Work fine after change de resolution (rebuild).
//lineRenderer.GetPropertyBlock(_materialPropertyBlock);
//_materialPropertyBlock.SetTexture("_MainTex", texture);
//lineRenderer.SetPropertyBlock(_materialPropertyBlock);
}
}
public virtual void TestSuccessfulContainerLaunch()
{
FileContext localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
YarnConfiguration conf = new YarnConfiguration();
Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService());
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.Create();
NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher
, healthChecker, metrics);
// Don't start any updating thread.
DummyContainerManager containerManager = new DummyContainerManager(context, exec,
del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler);
nodeStatusUpdater.Init(conf);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Start();
containerManager.Init(conf);
containerManager.Start();
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0);
string user = "******";
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context.
GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Running);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cID);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Complete);
containerManager.Stop();
}
// this has to be at least as much as map slot requirement
// This is a huge kluge. The real implementations have a decent approach
private void CoreTestEstimator(TaskRuntimeEstimator testedEstimator, int expectedSpeculations
)
{
estimator = testedEstimator;
clock = new TestRuntimeEstimators.MockClock();
dispatcher = new AsyncDispatcher();
myJob = null;
slotsInUse.Set(0);
completedMaps.Set(0);
completedReduces.Set(0);
successfulSpeculations.Set(0);
taskTimeSavedBySpeculation.Set(0);
clock.AdvanceTime(1000);
Configuration conf = new Configuration();
myAppContext = new TestRuntimeEstimators.MyAppContext(this, MapTasks, ReduceTasks
);
myJob = myAppContext.GetAllJobs().Values.GetEnumerator().Next();
estimator.Contextualize(conf, myAppContext);
conf.SetLong(MRJobConfig.SpeculativeRetryAfterNoSpeculate, 500L);
conf.SetLong(MRJobConfig.SpeculativeRetryAfterSpeculate, 5000L);
conf.SetDouble(MRJobConfig.SpeculativecapRunningTasks, 0.1);
conf.SetDouble(MRJobConfig.SpeculativecapTotalTasks, 0.001);
conf.SetInt(MRJobConfig.SpeculativeMinimumAllowedTasks, 5);
speculator = new DefaultSpeculator(conf, myAppContext, estimator, clock);
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_RETRY_AFTER_NO_SPECULATE value"
, 500L, speculator.GetSoonestRetryAfterNoSpeculate());
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_RETRY_AFTER_SPECULATE value",
5000L, speculator.GetSoonestRetryAfterSpeculate());
NUnit.Framework.Assert.AreEqual(speculator.GetProportionRunningTasksSpeculatable(
), 0.1, 0.00001);
NUnit.Framework.Assert.AreEqual(speculator.GetProportionTotalTasksSpeculatable(),
0.001, 0.00001);
NUnit.Framework.Assert.AreEqual("wrong SPECULATIVE_MINIMUM_ALLOWED_TASKS value",
5, speculator.GetMinimumAllowedSpeculativeTasks());
dispatcher.Register(typeof(Speculator.EventType), speculator);
dispatcher.Register(typeof(TaskEventType), new TestRuntimeEstimators.SpeculationRequestEventHandler
(this));
dispatcher.Init(conf);
dispatcher.Start();
speculator.Init(conf);
speculator.Start();
// Now that the plumbing is hooked up, we do the following:
// do until all tasks are finished, ...
// 1: If we have spare capacity, assign as many map tasks as we can, then
// assign as many reduce tasks as we can. Note that an odd reduce
// task might be started while there are still map tasks, because
// map tasks take 3 slots and reduce tasks 2 slots.
// 2: Send a speculation event for every task attempt that's running
// note that new attempts might get started by the speculator
// discover undone tasks
int undoneMaps = MapTasks;
int undoneReduces = ReduceTasks;
// build a task sequence where all the maps precede any of the reduces
IList <Task> allTasksSequence = new List <Task>();
Sharpen.Collections.AddAll(allTasksSequence, myJob.GetTasks(TaskType.Map).Values);
Sharpen.Collections.AddAll(allTasksSequence, myJob.GetTasks(TaskType.Reduce).Values
);
while (undoneMaps + undoneReduces > 0)
{
undoneMaps = 0;
undoneReduces = 0;
// start all attempts which are new but for which there is enough slots
foreach (Task task in allTasksSequence)
{
if (!task.IsFinished())
{
if (task.GetType() == TaskType.Map)
{
++undoneMaps;
}
else
{
++undoneReduces;
}
}
foreach (TaskAttempt attempt in task.GetAttempts().Values)
{
if (attempt.GetState() == TaskAttemptState.New && InitialNumberFreeSlots - slotsInUse
.Get() >= TaskTypeSlots(task.GetType()))
{
TestRuntimeEstimators.MyTaskAttemptImpl attemptImpl = (TestRuntimeEstimators.MyTaskAttemptImpl
)attempt;
SpeculatorEvent @event = new SpeculatorEvent(attempt.GetID(), false, clock.GetTime
());
speculator.Handle(@event);
attemptImpl.StartUp();
}
else
{
// If a task attempt is in progress we should send the news to
// the Speculator.
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
status.id = attempt.GetID();
status.progress = attempt.GetProgress();
status.stateString = attempt.GetState().ToString();
status.taskState = attempt.GetState();
SpeculatorEvent @event = new SpeculatorEvent(status, clock.GetTime());
speculator.Handle(@event);
}
}
}
long startTime = Runtime.CurrentTimeMillis();
// drain the speculator event queue
while (!speculator.EventQueueEmpty())
{
Sharpen.Thread.Yield();
if (Runtime.CurrentTimeMillis() > startTime + 130000)
{
return;
}
}
clock.AdvanceTime(1000L);
if (clock.GetTime() % 10000L == 0L)
{
speculator.ScanForSpeculations();
}
}
NUnit.Framework.Assert.AreEqual("We got the wrong number of successful speculations."
, expectedSpeculations, successfulSpeculations.Get());
}
public virtual void TestCommitWindow()
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
AsyncDispatcher dispatcher = new AsyncDispatcher();
dispatcher.Init(conf);
dispatcher.Start();
TestCommitterEventHandler.TestingJobEventHandler jeh = new TestCommitterEventHandler.TestingJobEventHandler
();
dispatcher.Register(typeof(JobEventType), jeh);
SystemClock clock = new SystemClock();
AppContext appContext = Org.Mockito.Mockito.Mock <AppContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
Org.Mockito.Mockito.When(appContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(appContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(appContext.GetEventHandler()).ThenReturn(dispatcher.GetEventHandler
());
Org.Mockito.Mockito.When(appContext.GetClock()).ThenReturn(clock);
OutputCommitter committer = Org.Mockito.Mockito.Mock <OutputCommitter>();
TestCommitterEventHandler.TestingRMHeartbeatHandler rmhh = new TestCommitterEventHandler.TestingRMHeartbeatHandler
();
CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh
);
ceh.Init(conf);
ceh.Start();
// verify trying to commit when RM heartbeats are stale does not commit
ceh.Handle(new CommitterJobCommitEvent(null, null));
long timeToWaitMs = 5000;
while (rmhh.GetNumCallbacks() != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not register a heartbeat callback"
, 1, rmhh.GetNumCallbacks());
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Never()).CommitJob(Matchers.Any
<JobContext>());
NUnit.Framework.Assert.AreEqual("committer should not have committed", 0, jeh.numCommitCompletedEvents
);
// set a fresh heartbeat and verify commit completes
rmhh.SetLastHeartbeatTime(clock.GetTime());
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not complete commit after RM hearbeat"
, 1, jeh.numCommitCompletedEvents);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(1)).CommitJob(Matchers.Any
<JobContext>());
//Clean up so we can try to commit again (Don't do this at home)
Cleanup();
// try to commit again and verify it goes through since the heartbeat
// is still fresh
ceh.Handle(new CommitterJobCommitEvent(null, null));
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not commit", 2, jeh.numCommitCompletedEvents
);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(2)).CommitJob(Matchers.Any
<JobContext>());
ceh.Stop();
dispatcher.Stop();
}