public virtual void TestJobTokenRpc()
{
TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>();
Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion
(Matchers.AnyString(), Matchers.AnyLong());
Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof(
TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT
).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt
());
JobTokenSecretManager sm = new JobTokenSecretManager();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).
SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string jobId = current.GetUserName();
JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId));
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(tokenId, sm);
sm.AddTokenForJob(jobId, token);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service address for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_110(addr, server));
}
public virtual void TestSucceedAndFailedCopyMap <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(2);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_251();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
MapHost host1 = new MapHost("host1", null);
TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0)
, TaskType.Map, 0), 0);
TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0
), TaskType.Map, 1), 1);
// handle output fetch failure for failedAttemptID, part I
scheduler.HostFailed(host1.GetHostName());
// handle output fetch succeed for succeedAttemptID
long bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
// handle output fetch failure for failedAttemptID, part II
// for MAPREDUCE-6361: verify no NPE exception get thrown out
scheduler.CopyFailed(failedAttemptID, host1, true, false);
}
public EventFetcher(TaskAttemptID reduce, TaskUmbilicalProtocol umbilical, ShuffleScheduler
<K, V> scheduler, ExceptionReporter reporter, int maxEventsToFetch)
{
SetName("EventFetcher for fetching Map Completion Events");
SetDaemon(true);
this.reduce = reduce;
this.umbilical = umbilical;
this.scheduler = scheduler;
exceptionReporter = reporter;
this.maxEventsToFetch = maxEventsToFetch;
}
public virtual void TestConsumerApi()
{
JobConf jobConf = new JobConf();
ShuffleConsumerPlugin <K, V> shuffleConsumerPlugin = new TestShufflePlugin.TestShuffleConsumerPlugin
<K, V>();
//mock creation
ReduceTask mockReduceTask = Org.Mockito.Mockito.Mock <ReduceTask>();
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = jobConf.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
try
{
string[] dirs = jobConf.GetLocalDirs();
// verify that these APIs are available through super class handler
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, jobConf, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
shuffleConsumerPlugin.Init(context);
shuffleConsumerPlugin.Run();
shuffleConsumerPlugin.Close();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("Threw exception:" + e, false);
}
// verify that these APIs are available for 3rd party plugins
mockReduceTask.GetTaskID();
mockReduceTask.GetJobID();
mockReduceTask.GetNumMaps();
mockReduceTask.GetPartition();
mockReporter.Progress();
}
//Used for status updates
public virtual void Init(ShuffleConsumerPlugin.Context context)
{
this.context = context;
this.reduceId = context.GetReduceId();
this.jobConf = context.GetJobConf();
this.umbilical = context.GetUmbilical();
this.reporter = context.GetReporter();
this.metrics = new ShuffleClientMetrics(reduceId, jobConf);
this.copyPhase = context.GetCopyPhase();
this.taskStatus = context.GetStatus();
this.reduceTask = context.GetReduceTask();
this.localMapFiles = context.GetLocalMapFiles();
scheduler = new ShuffleSchedulerImpl <K, V>(jobConf, taskStatus, reduceId, this, copyPhase
, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes(), context.GetFailedShuffleCounter
());
merger = CreateMergeManager(context);
}
/// <exception cref="System.Exception"/>
public object Run()
{
TaskUmbilicalProtocol proxy = null;
try
{
proxy = (TaskUmbilicalProtocol)RPC.GetProxy <TaskUmbilicalProtocol>(TaskUmbilicalProtocol
.versionID, addr, TestUmbilicalProtocolWithJobToken.conf);
proxy.Ping(null);
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
return(null);
}
public virtual void TestConsecutiveFetch()
{
int MaxEventsToFetch = 100;
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 1);
TaskUmbilicalProtocol umbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>
();
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.AnyInt(), Matchers.AnyInt(), Matchers.Any <TaskAttemptID>())).ThenReturn
(GetMockedCompletionEventsUpdate(0, 0));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid))).ThenReturn(GetMockedCompletionEventsUpdate
(0, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)))
.ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid
))).ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch * 2, 3));
ShuffleScheduler <string, string> scheduler = Org.Mockito.Mockito.Mock <ShuffleScheduler
>();
ExceptionReporter reporter = Org.Mockito.Mockito.Mock <ExceptionReporter>();
TestEventFetcher.EventFetcherForTest <string, string> ef = new TestEventFetcher.EventFetcherForTest
<string, string>(tid, umbilical, scheduler, reporter, MaxEventsToFetch);
ef.GetMapCompletionEvents();
Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Never()).ReportException
(Matchers.Any <Exception>());
InOrder inOrder = Org.Mockito.Mockito.InOrder(umbilical);
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(MaxEventsToFetch
* 2 + 3)).Resolve(Matchers.Any <TaskCompletionEvent>());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.TypeLoadException"/>
/// <exception cref="System.Exception"/>
public override void Run(JobConf job, TaskUmbilicalProtocol umbilical)
{
return;
}
public EventFetcherForTest(TaskAttemptID reduce, TaskUmbilicalProtocol umbilical,
ShuffleScheduler <K, V> scheduler, ExceptionReporter reporter, int maxEventsToFetch
)
: base(reduce, umbilical, scheduler, reporter, maxEventsToFetch)
{
}
public virtual void TestAggregatedTransferRate <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(10);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_115();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
TaskAttemptID attemptID0 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 0), 0);
//adding the 1st interval, 40MB from 60s to 100s
long bytes = (long)40 * 1024 * 1024;
scheduler.CopySucceeded(attemptID0, new MapHost(null, null), bytes, 60000, 100000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(1, 1, 1), progress.ToString());
TaskAttemptID attemptID1 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 1), 1);
//adding the 2nd interval before the 1st interval, 50MB from 0s to 50s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID1, new MapHost(null, null), bytes, 0, 50000, output
);
NUnit.Framework.Assert.AreEqual(CopyMessage(2, 1, 1), progress.ToString());
TaskAttemptID attemptID2 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 2), 2);
//adding the 3rd interval overlapping with the 1st and the 2nd interval
//110MB from 25s to 80s
bytes = (long)110 * 1024 * 1024;
scheduler.CopySucceeded(attemptID2, new MapHost(null, null), bytes, 25000, 80000,
output);
NUnit.Framework.Assert.AreEqual(CopyMessage(3, 2, 2), progress.ToString());
TaskAttemptID attemptID3 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 3), 3);
//adding the 4th interval just after the 2nd interval, 100MB from 100s to 300s
bytes = (long)100 * 1024 * 1024;
scheduler.CopySucceeded(attemptID3, new MapHost(null, null), bytes, 100000, 300000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(4, 0.5, 1), progress.ToString());
TaskAttemptID attemptID4 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 4), 4);
//adding the 5th interval between after 4th, 50MB from 350s to 400s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID4, new MapHost(null, null), bytes, 350000, 400000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(5, 1, 1), progress.ToString());
TaskAttemptID attemptID5 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 5), 5);
//adding the 6th interval between after 5th, 50MB from 450s to 500s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID5, new MapHost(null, null), bytes, 450000, 500000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(6, 1, 1), progress.ToString());
TaskAttemptID attemptID6 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 6), 6);
//adding the 7th interval between after 5th and 6th interval, 20MB from 320s to 340s
bytes = (long)20 * 1024 * 1024;
scheduler.CopySucceeded(attemptID6, new MapHost(null, null), bytes, 320000, 340000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(7, 1, 1), progress.ToString());
TaskAttemptID attemptID7 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 7), 7);
//adding the 8th interval overlapping with 4th, 5th, and 7th 30MB from 290s to 350s
bytes = (long)30 * 1024 * 1024;
scheduler.CopySucceeded(attemptID7, new MapHost(null, null), bytes, 290000, 350000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(8, 0.5, 1), progress.ToString());
TaskAttemptID attemptID8 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 8), 8);
//adding the 9th interval overlapping with 5th and 6th, 50MB from 400s to 450s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID8, new MapHost(null, null), bytes, 400000, 450000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(9, 1, 1), progress.ToString());
TaskAttemptID attemptID9 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 9), 9);
//adding the 10th interval overlapping with all intervals, 500MB from 0s to 500s
bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(attemptID9, new MapHost(null, null), bytes, 0, 500000, output
);
NUnit.Framework.Assert.AreEqual(CopyMessage(10, 1, 2), progress.ToString());
}
