internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics , ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id) { /* Default read timeout (in milliseconds) */ // Initiative value is 0, which means it hasn't retried yet. this.jobConf = job; this.reporter = reporter; this.scheduler = scheduler; this.merger = merger; this.metrics = metrics; this.exceptionReporter = exceptionReporter; this.id = id; this.reduce = reduceId.GetTaskID().GetId(); this.shuffleSecretKey = shuffleKey; ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString ()); wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength .ToString()); badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString ()); wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap .ToString()); connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection .ToString()); wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce .ToString()); this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout ); this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout); this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig .DefaultShuffleFetchRetryIntervalMs); this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout ); bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled , YarnConfiguration.DefaultNmRecoveryEnabled); this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault ); SetName("fetcher#" + id); SetDaemon(true); lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher)) { sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault ); if (sslShuffle && sslFactory == null) { sslFactory = new SSLFactory(SSLFactory.Mode.Client, job); try { sslFactory.Init(); } catch (Exception ex) { sslFactory.Destroy(); throw new RuntimeException(ex); } } } }
public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler , MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey) : this(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey , ++nextId) { }
public virtual void Setup() { // mocked generics Log.Info(">>>> " + name.GetMethodName()); job = new JobConf(); job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false); jobWithRetry = new JobConf(); jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true); id = TaskAttemptID.ForName("attempt_0_1_r_1_1"); ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>(); mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>(); r = Org.Mockito.Mockito.Mock <Reporter>(); metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>(); except = Org.Mockito.Mockito.Mock <ExceptionReporter>(); key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 }); connection = Org.Mockito.Mockito.Mock <HttpURLConnection>(); allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>(); Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString()) ).ThenReturn(allErrs); AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1); maps.AddItem(map1ID); maps.AddItem(map2ID); Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps); }
public FakeFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V > scheduler, MergeManagerImpl <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey jobTokenSecret, HttpURLConnection connection, int id) : base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, jobTokenSecret , id) { this.connection = connection; }
public virtual void TestSucceedAndFailedCopyMap <K, V>() { JobConf job = new JobConf(); job.SetNumMapTasks(2); //mock creation TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol >(); Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>(); FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>(); Type combinerClass = job.GetCombinerClass(); Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector <K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>(); // needed for mock with generic TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>(); LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator >(); CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec >(); Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>(); TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>(); Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>(); MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task >(); MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>(); ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K , V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator , mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus , mockProgress, mockProgress, mockTask, mockMapOutputFile, null); TaskStatus status = new _TaskStatus_251(); Progress progress = new Progress(); ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status , null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes (), context.GetFailedShuffleCounter()); MapHost host1 = new MapHost("host1", null); TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0) , TaskType.Map, 0), 0); TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0 ), TaskType.Map, 1), 1); // handle output fetch failure for failedAttemptID, part I scheduler.HostFailed(host1.GetHostName()); // handle output fetch succeed for succeedAttemptID long bytes = (long)500 * 1024 * 1024; scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output); // handle output fetch failure for failedAttemptID, part II // for MAPREDUCE-6361: verify no NPE exception get thrown out scheduler.CopyFailed(failedAttemptID, host1, true, false); }
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles) : base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey ) { this.job = job; this.localMapFiles = localMapFiles; SetName("localfetcher#" + id); SetDaemon(true); }
//Used for status updates public virtual void Init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.GetReduceId(); this.jobConf = context.GetJobConf(); this.umbilical = context.GetUmbilical(); this.reporter = context.GetReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.GetCopyPhase(); this.taskStatus = context.GetStatus(); this.reduceTask = context.GetReduceTask(); this.localMapFiles = context.GetLocalMapFiles(); scheduler = new ShuffleSchedulerImpl <K, V>(jobConf, taskStatus, reduceId, this, copyPhase , context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes(), context.GetFailedShuffleCounter ()); merger = CreateMergeManager(context); }
/// <exception cref="System.Exception"/> public virtual void TestCopyFromHostWithRetry() { InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>( ); ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>(); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry , id, ss, mm, r, metrics, except, key, connection, true); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(immo); long retryTime = Time.MonotonicNow(); Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong() , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>()); // Emulate host down for 3 seconds. underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any <TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean ()); }
public virtual void TestTipFailed() { JobConf job = new JobConf(); job.SetNumMapTasks(2); TaskStatus status = new _TaskStatus_49(); Progress progress = new Progress(); TaskAttemptID reduceId = new TaskAttemptID("314159", 0, TaskType.Reduce, 0, 0); ShuffleSchedulerImpl scheduler = new ShuffleSchedulerImpl(job, status, reduceId, null, progress, null, null, null); JobID jobId = new JobID(); TaskID taskId1 = new TaskID(jobId, TaskType.Reduce, 1); scheduler.TipFailed(taskId1); NUnit.Framework.Assert.AreEqual("Progress should be 0.5", 0.5f, progress.GetProgress (), 0.0f); NUnit.Framework.Assert.IsFalse(scheduler.WaitUntilDone(1)); TaskID taskId0 = new TaskID(jobId, TaskType.Reduce, 0); scheduler.TipFailed(taskId0); NUnit.Framework.Assert.AreEqual("Progress should be 1.0", 1.0f, progress.GetProgress (), 0.0f); NUnit.Framework.Assert.IsTrue(scheduler.WaitUntilDone(1)); }
public virtual void TestAggregatedTransferRate <K, V>() { JobConf job = new JobConf(); job.SetNumMapTasks(10); //mock creation TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol >(); Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>(); FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>(); Type combinerClass = job.GetCombinerClass(); Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector <K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>(); // needed for mock with generic TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>(); LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator >(); CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec >(); Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>(); TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>(); Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>(); MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task >(); MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>(); ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K , V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator , mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus , mockProgress, mockProgress, mockTask, mockMapOutputFile, null); TaskStatus status = new _TaskStatus_115(); Progress progress = new Progress(); ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status , null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes (), context.GetFailedShuffleCounter()); TaskAttemptID attemptID0 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 0), 0); //adding the 1st interval, 40MB from 60s to 100s long bytes = (long)40 * 1024 * 1024; scheduler.CopySucceeded(attemptID0, new MapHost(null, null), bytes, 60000, 100000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(1, 1, 1), progress.ToString()); TaskAttemptID attemptID1 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 1), 1); //adding the 2nd interval before the 1st interval, 50MB from 0s to 50s bytes = (long)50 * 1024 * 1024; scheduler.CopySucceeded(attemptID1, new MapHost(null, null), bytes, 0, 50000, output ); NUnit.Framework.Assert.AreEqual(CopyMessage(2, 1, 1), progress.ToString()); TaskAttemptID attemptID2 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 2), 2); //adding the 3rd interval overlapping with the 1st and the 2nd interval //110MB from 25s to 80s bytes = (long)110 * 1024 * 1024; scheduler.CopySucceeded(attemptID2, new MapHost(null, null), bytes, 25000, 80000, output); NUnit.Framework.Assert.AreEqual(CopyMessage(3, 2, 2), progress.ToString()); TaskAttemptID attemptID3 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 3), 3); //adding the 4th interval just after the 2nd interval, 100MB from 100s to 300s bytes = (long)100 * 1024 * 1024; scheduler.CopySucceeded(attemptID3, new MapHost(null, null), bytes, 100000, 300000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(4, 0.5, 1), progress.ToString()); TaskAttemptID attemptID4 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 4), 4); //adding the 5th interval between after 4th, 50MB from 350s to 400s bytes = (long)50 * 1024 * 1024; scheduler.CopySucceeded(attemptID4, new MapHost(null, null), bytes, 350000, 400000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(5, 1, 1), progress.ToString()); TaskAttemptID attemptID5 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 5), 5); //adding the 6th interval between after 5th, 50MB from 450s to 500s bytes = (long)50 * 1024 * 1024; scheduler.CopySucceeded(attemptID5, new MapHost(null, null), bytes, 450000, 500000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(6, 1, 1), progress.ToString()); TaskAttemptID attemptID6 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 6), 6); //adding the 7th interval between after 5th and 6th interval, 20MB from 320s to 340s bytes = (long)20 * 1024 * 1024; scheduler.CopySucceeded(attemptID6, new MapHost(null, null), bytes, 320000, 340000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(7, 1, 1), progress.ToString()); TaskAttemptID attemptID7 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 7), 7); //adding the 8th interval overlapping with 4th, 5th, and 7th 30MB from 290s to 350s bytes = (long)30 * 1024 * 1024; scheduler.CopySucceeded(attemptID7, new MapHost(null, null), bytes, 290000, 350000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(8, 0.5, 1), progress.ToString()); TaskAttemptID attemptID8 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 8), 8); //adding the 9th interval overlapping with 5th and 6th, 50MB from 400s to 450s bytes = (long)50 * 1024 * 1024; scheduler.CopySucceeded(attemptID8, new MapHost(null, null), bytes, 400000, 450000 , output); NUnit.Framework.Assert.AreEqual(CopyMessage(9, 1, 1), progress.ToString()); TaskAttemptID attemptID9 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType .Map, 9), 9); //adding the 10th interval overlapping with all intervals, 500MB from 0s to 500s bytes = (long)500 * 1024 * 1024; scheduler.CopySucceeded(attemptID9, new MapHost(null, null), bytes, 0, 500000, output ); NUnit.Framework.Assert.AreEqual(CopyMessage(10, 1, 2), progress.ToString()); }
public Referee(ShuffleSchedulerImpl <K, V> _enclosing) { this._enclosing = _enclosing; this.SetName("ShufflePenaltyReferee"); this.SetDaemon(true); }