public virtual void ObsoleteMapOutput(TaskAttemptID mapId) { lock (this) { obsoleteMaps.AddItem(mapId); } }
public virtual void PutBackKnownMapOutput(MapHost host, TaskAttemptID mapId) { lock (this) { host.AddKnownMap(mapId); } }
public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId , ExceptionReporter reporter, Progress progress, Counters.Counter shuffledMapsCounter , Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter) { referee = new ShuffleSchedulerImpl.Referee(this); totalMaps = job.GetNumMapTasks(); abortFailureLimit = Math.Max(30, totalMaps / 10); copyTimeTracker = new ShuffleSchedulerImpl.CopyTimeTracker(); remainingMaps = totalMaps; finishedMaps = new bool[remainingMaps]; this.reporter = reporter; this.status = status; this.reduceId = reduceId; this.progress = progress; this.shuffledMapsCounter = shuffledMapsCounter; this.reduceShuffleBytes = reduceShuffleBytes; this.failedShuffleCounter = failedShuffleCounter; this.startTime = Time.MonotonicNow(); lastProgressTime = startTime; referee.Start(); this.maxFailedUniqueFetches = Math.Min(totalMaps, 5); this.maxFetchFailuresBeforeReporting = job.GetInt(MRJobConfig.ShuffleFetchFailures , ReportFailureLimit); this.reportReadErrorImmediately = job.GetBoolean(MRJobConfig.ShuffleNotifyReaderror , true); this.maxDelay = job.GetLong(MRJobConfig.MaxShuffleFetchRetryDelay, MRJobConfig.DefaultMaxShuffleFetchRetryDelay ); this.maxHostFailures = job.GetInt(MRJobConfig.MaxShuffleFetchHostFailures, MRJobConfig .DefaultMaxShuffleFetchHostFailures); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Reduce(Text key, IEnumerable <IntWritable> values, Reducer.Context context) { // Make one reducer slower for speculative execution TaskAttemptID taid = context.GetTaskAttemptID(); long sleepTime = 100; Configuration conf = context.GetConfiguration(); bool test_speculate_reduce = conf.GetBoolean(MRJobConfig.ReduceSpeculative, false ); // IF TESTING REDUCE SPECULATIVE EXECUTION: // Make the "*_r_000000_0" attempt take much longer than the others. // When speculative execution is enabled, this should cause the attempt // to be killed and restarted. At that point, the attempt ID will be // "*_r_000000_1", so sleepTime will still remain 100ms. if ((taid.GetTaskType() == TaskType.Reduce) && test_speculate_reduce && (taid.GetTaskID ().GetId() == 0) && (taid.GetId() == 0)) { sleepTime = 10000; } try { Sharpen.Thread.Sleep(sleepTime); } catch (Exception) { } // Ignore context.Write(key, new IntWritable(0)); }
public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime , string trackerName, int httpPort, int shufflePort, string locality, string avataar ) : this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort, ConverterUtils .ToContainerId("container_-1_-1_-1_-1"), locality, avataar) { }
public virtual void Setup() { // mocked generics Log.Info(">>>> " + name.GetMethodName()); job = new JobConf(); job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false); jobWithRetry = new JobConf(); jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true); id = TaskAttemptID.ForName("attempt_0_1_r_1_1"); ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>(); mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>(); r = Org.Mockito.Mockito.Mock <Reporter>(); metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>(); except = Org.Mockito.Mockito.Mock <ExceptionReporter>(); key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 }); connection = Org.Mockito.Mockito.Mock <HttpURLConnection>(); allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>(); Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString()) ).ThenReturn(allErrs); AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1); maps.AddItem(map1ID); maps.AddItem(map2ID); Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestFailingMapper() { Log.Info("\n\n\nStarting testFailingMapper()."); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } Job job = RunFailingMapperJob(); TaskID taskID = new TaskID(job.GetJobID(), TaskType.Map, 0); TaskAttemptID aId = new TaskAttemptID(taskID, 0); System.Console.Out.WriteLine("Diagnostics for " + aId + " :"); foreach (string diag in job.GetTaskDiagnostics(aId)) { System.Console.Out.WriteLine(diag); } aId = new TaskAttemptID(taskID, 1); System.Console.Out.WriteLine("Diagnostics for " + aId + " :"); foreach (string diag_1 in job.GetTaskDiagnostics(aId)) { System.Console.Out.WriteLine(diag_1); } TaskCompletionEvent[] events = job.GetTaskCompletionEvents(0, 2); NUnit.Framework.Assert.AreEqual(TaskCompletionEvent.Status.Failed, events[0].GetStatus ()); NUnit.Framework.Assert.AreEqual(TaskCompletionEvent.Status.Tipfailed, events[1].GetStatus ()); NUnit.Framework.Assert.AreEqual(JobStatus.State.Failed, job.GetJobState()); VerifyFailingMapperCounters(job); }
public ReduceAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, string taskStatus , long shuffleFinishTime, long sortFinishTime, long finishTime, string hostname, string state, Counters counters) : this(id, taskType, taskStatus, shuffleFinishTime, sortFinishTime, finishTime, hostname , -1, string.Empty, state, counters, null) { }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override RecordReader CreateRecordReader(InputSplit split, TaskAttemptContext taskContext) { try { if (!rrCstrMap.Contains(ident)) { throw new IOException("No RecordReader for " + ident); } Configuration conf = GetConf(taskContext.GetConfiguration()); TaskAttemptContext context = new TaskAttemptContextImpl(conf, TaskAttemptID.ForName (conf.Get(MRJobConfig.TaskAttemptId)), new Parser.WrappedStatusReporter(taskContext )); return(rrCstrMap[ident].NewInstance(id, inf.CreateRecordReader(split, context), cmpcl )); } catch (MemberAccessException e) { throw new IOException(e); } catch (InstantiationException e) { throw new IOException(e); } catch (TargetInvocationException e) { throw new IOException(e); } }
public TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID id, TaskType taskType , string status, long finishTime, string hostname, int port, string rackName, string error, int[][] allSplits) : this(id, taskType, status, finishTime, hostname, port, rackName, error, EmptyCounters , null) { }
public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taskId, StatusReporter reporter) : base(conf, taskId.GetJobID()) { this.taskId = taskId; this.reporter = reporter; }
/// <exception cref="System.IO.IOException"/> public virtual void CleanUpPartialOutputForTask(TaskAttemptContext context) { // we double check this is never invoked from a non-preemptable subclass. // This should never happen, since the invoking codes is checking it too, // but it is safer to double check. Errors handling this would produce // inconsistent output. if (!this.GetType().IsAnnotationPresent(typeof(Checkpointable))) { throw new InvalidOperationException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable class" ); } FileSystem fs = FsFor(GetTaskAttemptPath(context), context.GetConfiguration()); Log.Info("cleanUpPartialOutputForTask: removing everything belonging to " + context .GetTaskAttemptID().GetTaskID() + " in: " + GetCommittedTaskPath(context).GetParent ()); TaskAttemptID taid = context.GetTaskAttemptID(); TaskID tid = taid.GetTaskID(); Path pCommit = GetCommittedTaskPath(context).GetParent(); // remove any committed output for (int i = 0; i < taid.GetId(); ++i) { TaskAttemptID oldId = new TaskAttemptID(tid, i); Path pTask = new Path(pCommit, oldId.ToString()); if (fs.Exists(pTask) && !fs.Delete(pTask, true)) { throw new IOException("Failed to delete " + pTask); } } }
public MapOutput(TaskAttemptID mapId, long size, bool primaryMapOutput) { this.id = Id.IncrementAndGet(); this.mapId = mapId; this.size = size; this.primaryMapOutput = primaryMapOutput; }
public virtual void TestTaskStartTimes() { TaskId taskId = Org.Mockito.Mockito.Mock <TaskId>(); JobHistoryParser.TaskInfo taskInfo = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskInfo >(); IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> taskAttempts = new SortedDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo>(); TaskAttemptID id = new TaskAttemptID("0", 0, TaskType.Map, 0, 0); JobHistoryParser.TaskAttemptInfo info = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskAttemptInfo >(); Org.Mockito.Mockito.When(info.GetAttemptId()).ThenReturn(id); Org.Mockito.Mockito.When(info.GetStartTime()).ThenReturn(10l); taskAttempts[id] = info; id = new TaskAttemptID("1", 0, TaskType.Map, 1, 1); info = Org.Mockito.Mockito.Mock <JobHistoryParser.TaskAttemptInfo>(); Org.Mockito.Mockito.When(info.GetAttemptId()).ThenReturn(id); Org.Mockito.Mockito.When(info.GetStartTime()).ThenReturn(20l); taskAttempts[id] = info; Org.Mockito.Mockito.When(taskInfo.GetAllTaskAttempts()).ThenReturn(taskAttempts); CompletedTask task = new CompletedTask(taskId, taskInfo); TaskReport report = task.GetReport(); // Make sure the startTime returned by report is the lesser of the // attempy launch times NUnit.Framework.Assert.IsTrue(report.GetStartTime() == 10); }
public MapAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, string taskStatus , long mapFinishTime, long finishTime, string hostname, string state, Counters counters ) : this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, -1, string.Empty , state, counters, null) { }
/// <exception cref="System.Exception"/> /// <exception cref="System.IO.IOException"/> public ReduceContextImpl(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator input, Counter inputKeyCounter, Counter inputValueCounter, RecordWriter <KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter, RawComparator <KEYIN> comparator, Type keyClass, Type valueClass) : base(conf, taskid, output, committer, reporter) { iterable = new ReduceContextImpl.ValueIterable(this); // current key // current value // first value in key // more w/ this key // more in file this.input = input; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; this.comparator = comparator; this.serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.GetDeserializer(keyClass); this.keyDeserializer.Open(buffer); this.valueDeserializer = serializationFactory.GetDeserializer(valueClass); this.valueDeserializer.Open(buffer); hasMore = input.Next(); this.keyClass = keyClass; this.valueClass = valueClass; this.conf = conf; this.taskid = taskid; }
/// <summary> /// Tests the class loader set by /// <see cref="Org.Apache.Hadoop.Conf.Configuration.SetClassLoader(Sharpen.ClassLoader) /// "/> /// is inherited by any /// <see cref="WrappedRecordReader{K, U}"/> /// s created by /// <see cref="CompositeRecordReader{K, V, X}"/> /// </summary> /// <exception cref="System.Exception"/> public virtual void TestClassLoader() { Configuration conf = new Configuration(); TestWrappedRRClassloader.Fake_ClassLoader classLoader = new TestWrappedRRClassloader.Fake_ClassLoader (); conf.SetClassLoader(classLoader); NUnit.Framework.Assert.IsTrue(conf.GetClassLoader() is TestWrappedRRClassloader.Fake_ClassLoader ); FileSystem fs = FileSystem.Get(conf); Path testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified (fs); Path @base = new Path(testdir, "/empty"); Path[] src = new Path[] { new Path(@base, "i0"), new Path("i1"), new Path("i2") }; conf.Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose("outer", typeof( TestWrappedRRClassloader.IF_ClassLoaderChecker), src)); CompositeInputFormat <NullWritable> inputFormat = new CompositeInputFormat <NullWritable >(); // create dummy TaskAttemptID TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0); conf.Set(MRJobConfig.TaskAttemptId, tid.ToString()); inputFormat.CreateRecordReader(inputFormat.GetSplits(Job.GetInstance(conf))[0], new TaskAttemptContextImpl(conf, tid)); }
public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, RecordWriter <KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter) : base(conf, taskid, reporter) { this.output = output; this.committer = committer; }
/* * (non-Javadoc) * * @see * org.apache.hadoop.mapred.MapReduceBase#configure(org.apache.hadoop.mapred * .JobConf) */ public override void Configure(JobConf conf) { // MapReduceBase try { config = new ConfigExtractor(conf); ConfigExtractor.DumpOptions(config); filesystem = config.GetBaseDirectory().GetFileSystem(conf); } catch (Exception e) { Log.Error("Unable to setup slive " + StringUtils.StringifyException(e)); throw new RuntimeException("Unable to setup slive configuration", e); } if (conf.Get(MRJobConfig.TaskAttemptId) != null) { this.taskId = TaskAttemptID.ForName(conf.Get(MRJobConfig.TaskAttemptId)).GetTaskID ().GetId(); } else { // So that branch-1/0.20 can run this same code as well this.taskId = TaskAttemptID.ForName(conf.Get("mapred.task.id")).GetTaskID().GetId (); } }
/// <summary>Do some basic verification on the input received -- Being defensive</summary> /// <param name="compressedLength"/> /// <param name="decompressedLength"/> /// <param name="forReduce"/> /// <param name="remaining"/> /// <param name="mapId"/> /// <returns>true/false, based on if the verification succeeded or not</returns> private bool VerifySanity(long compressedLength, long decompressedLength, int forReduce , ICollection <TaskAttemptID> remaining, TaskAttemptID mapId) { if (compressedLength < 0 || decompressedLength < 0) { wrongLengthErrs.Increment(1); Log.Warn(GetName() + " invalid lengths in map output header: id: " + mapId + " len: " + compressedLength + ", decomp len: " + decompressedLength); return(false); } if (forReduce != reduce) { wrongReduceErrs.Increment(1); Log.Warn(GetName() + " data for the wrong reduce map: " + mapId + " len: " + compressedLength + " decomp len: " + decompressedLength + " for reduce " + forReduce); return(false); } // Sanity check if (!remaining.Contains(mapId)) { wrongMapErrs.Increment(1); Log.Warn("Invalid map-output! Received output for " + mapId); return(false); } return(true); }
internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics , ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id) { /* Default read timeout (in milliseconds) */ // Initiative value is 0, which means it hasn't retried yet. this.jobConf = job; this.reporter = reporter; this.scheduler = scheduler; this.merger = merger; this.metrics = metrics; this.exceptionReporter = exceptionReporter; this.id = id; this.reduce = reduceId.GetTaskID().GetId(); this.shuffleSecretKey = shuffleKey; ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString ()); wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength .ToString()); badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString ()); wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap .ToString()); connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection .ToString()); wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce .ToString()); this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout ); this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout); this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig .DefaultShuffleFetchRetryIntervalMs); this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout ); bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled , YarnConfiguration.DefaultNmRecoveryEnabled); this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault ); SetName("fetcher#" + id); SetDaemon(true); lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher)) { sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault ); if (sslShuffle && sslFactory == null) { sslFactory = new SSLFactory(SSLFactory.Mode.Client, job); try { sslFactory.Init(); } catch (Exception ex) { sslFactory.Destroy(); throw new RuntimeException(ex); } } } }
/// <exception cref="System.IO.IOException"/> public override void Merge(IList <InMemoryMapOutput <K, V> > inputs) { if (inputs == null || inputs.Count == 0) { return; } TaskAttemptID dummyMapId = inputs[0].GetMapId(); IList <Merger.Segment <K, V> > inMemorySegments = new AList <Merger.Segment <K, V> >(); long mergeOutputSize = this._enclosing.CreateInMemorySegments(inputs, inMemorySegments , 0); int noInMemorySegments = inMemorySegments.Count; InMemoryMapOutput <K, V> mergedMapOutputs = this._enclosing.UnconditionalReserve(dummyMapId , mergeOutputSize, false); IFile.Writer <K, V> writer = new InMemoryWriter <K, V>(mergedMapOutputs.GetArrayStream ()); MergeManagerImpl.Log.Info("Initiating Memory-to-Memory merge with " + noInMemorySegments + " segments of total-size: " + mergeOutputSize); RawKeyValueIterator rIter = Merger.Merge(this._enclosing.jobConf, this._enclosing .rfs, (Type)this._enclosing.jobConf.GetMapOutputKeyClass(), (Type)this._enclosing .jobConf.GetMapOutputValueClass(), inMemorySegments, inMemorySegments.Count, new Path(this._enclosing.reduceId.ToString()), (RawComparator <K>) this._enclosing.jobConf .GetOutputKeyComparator(), this._enclosing.reporter, null, null, null); Merger.WriteFile(rIter, writer, this._enclosing.reporter, this._enclosing.jobConf ); writer.Close(); MergeManagerImpl.Log.Info(this._enclosing.reduceId + " Memory-to-Memory merge of the " + noInMemorySegments + " files in-memory complete."); // Note the output of the merge this._enclosing.CloseInMemoryMergedFile(mergedMapOutputs); }
public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler , MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey) : this(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey , ++nextId) { }
public MapContextImpl(Configuration conf, TaskAttemptID taskid, RecordReader <KEYIN , VALUEIN> reader, RecordWriter <KEYOUT, VALUEOUT> writer, OutputCommitter committer , StatusReporter reporter, InputSplit split) : base(conf, taskid, writer, committer, reporter) { this.reader = reader; this.split = split; }
/// <exception cref="System.IO.IOException"/> public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl <K, V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher , bool primaryMapOutput) : this(mapId, reduceId, merger, size, conf, mapOutputFile, fetcher, primaryMapOutput , FileSystem.GetLocal(conf).GetRaw(), mapOutputFile.GetInputFileForWrite(mapId.GetTaskID (), size)) { }
/// <exception cref="System.Exception"/> private string ReadStdOut(JobConf conf) { TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig .TaskAttemptId))); FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout); return(ReadFile(stdOut)); }
public FakeFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V > scheduler, MergeManagerImpl <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey jobTokenSecret, HttpURLConnection connection, int id) : base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, jobTokenSecret , id) { this.connection = connection; }
public virtual void CommitTask(TaskAttemptContext context, Path taskAttemptPath) { TaskAttemptID attemptId = context.GetTaskAttemptID(); if (HasOutputPath()) { context.Progress(); if (taskAttemptPath == null) { taskAttemptPath = GetTaskAttemptPath(context); } FileSystem fs = taskAttemptPath.GetFileSystem(context.GetConfiguration()); FileStatus taskAttemptDirStatus; try { taskAttemptDirStatus = fs.GetFileStatus(taskAttemptPath); } catch (FileNotFoundException) { taskAttemptDirStatus = null; } if (taskAttemptDirStatus != null) { if (algorithmVersion == 1) { Path committedTaskPath = GetCommittedTaskPath(context); if (fs.Exists(committedTaskPath)) { if (!fs.Delete(committedTaskPath, true)) { throw new IOException("Could not delete " + committedTaskPath); } } if (!fs.Rename(taskAttemptPath, committedTaskPath)) { throw new IOException("Could not rename " + taskAttemptPath + " to " + committedTaskPath ); } Log.Info("Saved output of task '" + attemptId + "' to " + committedTaskPath); } else { // directly merge everything from taskAttemptPath to output directory MergePaths(fs, taskAttemptDirStatus, outputPath); Log.Info("Saved output of task '" + attemptId + "' to " + outputPath); } } else { Log.Warn("No Output found for " + attemptId); } } else { Log.Warn("Output Path is null in commitTask()"); } }
public virtual void TestSucceedAndFailedCopyMap <K, V>() { JobConf job = new JobConf(); job.SetNumMapTasks(2); //mock creation TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol >(); Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>(); FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>(); Type combinerClass = job.GetCombinerClass(); Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector <K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>(); // needed for mock with generic TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>(); LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator >(); CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec >(); Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>(); TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>(); Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>(); MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task >(); MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>(); ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K , V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator , mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus , mockProgress, mockProgress, mockTask, mockMapOutputFile, null); TaskStatus status = new _TaskStatus_251(); Progress progress = new Progress(); ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status , null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes (), context.GetFailedShuffleCounter()); MapHost host1 = new MapHost("host1", null); TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0) , TaskType.Map, 0), 0); TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0 ), TaskType.Map, 1), 1); // handle output fetch failure for failedAttemptID, part I scheduler.HostFailed(host1.GetHostName()); // handle output fetch succeed for succeedAttemptID long bytes = (long)500 * 1024 * 1024; scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output); // handle output fetch failure for failedAttemptID, part II // for MAPREDUCE-6361: verify no NPE exception get thrown out scheduler.CopyFailed(failedAttemptID, host1, true, false); }
/// <summary>Unconditional Reserve is used by the Memory-to-Memory thread</summary> /// <returns/> private InMemoryMapOutput <K, V> UnconditionalReserve(TaskAttemptID mapId, long requestedSize , bool primaryMapOutput) { lock (this) { usedMemory += requestedSize; return(new InMemoryMapOutput <K, V>(jobConf, mapId, this, (int)requestedSize, codec , primaryMapOutput)); } }