public CombineOutputCollector(TestPipeApplication _enclosing, Counters.Counter outCounter
, Progressable progressable)
{
this._enclosing = _enclosing;
this.outCounter = outCounter;
this.progressable = progressable;
}
/// <exception cref="System.IO.IOException"/>
private void CombineAndSpill(RawKeyValueIterator kvIter, Counters.Counter inCounter
)
{
JobConf job = jobConf;
Reducer combiner = ReflectionUtils.NewInstance(combinerClass, job);
Type keyClass = (Type)job.GetMapOutputKeyClass();
Type valClass = (Type)job.GetMapOutputValueClass();
RawComparator <K> comparator = (RawComparator <K>)job.GetCombinerKeyGroupingComparator
();
try
{
Task.CombineValuesIterator values = new Task.CombineValuesIterator(kvIter, comparator
, keyClass, valClass, job, Reporter.Null, inCounter);
while (values.More())
{
combiner.Reduce(values.GetKey(), values, combineCollector, Reporter.Null);
values.NextKey();
}
}
finally
{
combiner.Close();
}
}
internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V>
scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics
, ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id)
{
/* Default read timeout (in milliseconds) */
// Initiative value is 0, which means it hasn't retried yet.
this.jobConf = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
this.metrics = metrics;
this.exceptionReporter = exceptionReporter;
this.id = id;
this.reduce = reduceId.GetTaskID().GetId();
this.shuffleSecretKey = shuffleKey;
ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString
());
wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength
.ToString());
badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString
());
wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap
.ToString());
connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection
.ToString());
wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce
.ToString());
this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout
);
this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout);
this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig
.DefaultShuffleFetchRetryIntervalMs);
this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout
);
bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled
, YarnConfiguration.DefaultNmRecoveryEnabled);
this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault
);
SetName("fetcher#" + id);
SetDaemon(true);
lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher))
{
sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
);
if (sslShuffle && sslFactory == null)
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, job);
try
{
sslFactory.Init();
}
catch (Exception ex)
{
sslFactory.Destroy();
throw new RuntimeException(ex);
}
}
}
}
public TimeCounter(string name, string notes, Counters counters)
{
int id = counters.FindId(name, notes);
_counter = counters.GetCounter(id);
_ticks = 0;
}
public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId
, ExceptionReporter reporter, Progress progress, Counters.Counter shuffledMapsCounter
, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter)
{
referee = new ShuffleSchedulerImpl.Referee(this);
totalMaps = job.GetNumMapTasks();
abortFailureLimit = Math.Max(30, totalMaps / 10);
copyTimeTracker = new ShuffleSchedulerImpl.CopyTimeTracker();
remainingMaps = totalMaps;
finishedMaps = new bool[remainingMaps];
this.reporter = reporter;
this.status = status;
this.reduceId = reduceId;
this.progress = progress;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.startTime = Time.MonotonicNow();
lastProgressTime = startTime;
referee.Start();
this.maxFailedUniqueFetches = Math.Min(totalMaps, 5);
this.maxFetchFailuresBeforeReporting = job.GetInt(MRJobConfig.ShuffleFetchFailures
, ReportFailureLimit);
this.reportReadErrorImmediately = job.GetBoolean(MRJobConfig.ShuffleNotifyReaderror
, true);
this.maxDelay = job.GetLong(MRJobConfig.MaxShuffleFetchRetryDelay, MRJobConfig.DefaultMaxShuffleFetchRetryDelay
);
this.maxHostFailures = job.GetInt(MRJobConfig.MaxShuffleFetchHostFailures, MRJobConfig
.DefaultMaxShuffleFetchHostFailures);
}
public virtual void Setup()
{
// mocked generics
Log.Info(">>>> " + name.GetMethodName());
job = new JobConf();
job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false);
jobWithRetry = new JobConf();
jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true);
id = TaskAttemptID.ForName("attempt_0_1_r_1_1");
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>();
r = Org.Mockito.Mockito.Mock <Reporter>();
metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>();
except = Org.Mockito.Mockito.Mock <ExceptionReporter>();
key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 });
connection = Org.Mockito.Mockito.Mock <HttpURLConnection>();
allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>();
Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString())
).ThenReturn(allErrs);
AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1);
maps.AddItem(map1ID);
maps.AddItem(map2ID);
Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps);
}
public virtual void TestSucceedAndFailedCopyMap <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(2);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_251();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
MapHost host1 = new MapHost("host1", null);
TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0)
, TaskType.Map, 0), 0);
TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0
), TaskType.Map, 1), 1);
// handle output fetch failure for failedAttemptID, part I
scheduler.HostFailed(host1.GetHostName());
// handle output fetch succeed for succeedAttemptID
long bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
// handle output fetch failure for failedAttemptID, part II
// for MAPREDUCE-6361: verify no NPE exception get thrown out
scheduler.CopyFailed(failedAttemptID, host1, true, false);
}
public virtual void TestConsumerApi()
{
JobConf jobConf = new JobConf();
ShuffleConsumerPlugin <K, V> shuffleConsumerPlugin = new TestShufflePlugin.TestShuffleConsumerPlugin
<K, V>();
//mock creation
ReduceTask mockReduceTask = Org.Mockito.Mockito.Mock <ReduceTask>();
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = jobConf.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
try
{
string[] dirs = jobConf.GetLocalDirs();
// verify that these APIs are available through super class handler
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, jobConf, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
shuffleConsumerPlugin.Init(context);
shuffleConsumerPlugin.Run();
shuffleConsumerPlugin.Close();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("Threw exception:" + e, false);
}
// verify that these APIs are available for 3rd party plugins
mockReduceTask.GetTaskID();
mockReduceTask.GetJobID();
mockReduceTask.GetNumMaps();
mockReduceTask.GetPartition();
mockReporter.Progress();
}
/// <exception cref="System.IO.IOException"/>
public virtual void IncrementCounter(int id, long amount)
{
if (id < registeredCounters.Count)
{
Counters.Counter counter = registeredCounters[id];
counter.Increment(amount);
}
else
{
throw new IOException("Invalid counter with id: " + id);
}
}
public override Counters.Counter GetCounter(string group, string name)
{
Counters.Counter counter = null;
if (this.counters != null)
{
counter = this.counters.FindCounter(group, name);
if (counter == null)
{
Counters.Group grp = this.counters.AddGroup(group, group);
counter = grp.AddCounter(name, name, 10);
}
}
return(counter);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestMergeShouldReturnProperProgress(IList <Merger.Segment <Text
, Text> > segments)
{
Path tmpDir = new Path("localpath");
Type keyClass = (Type)jobConf.GetMapOutputKeyClass();
Type valueClass = (Type)jobConf.GetMapOutputValueClass();
RawComparator <Text> comparator = jobConf.GetOutputKeyComparator();
Counters.Counter readsCounter = new Counters.Counter();
Counters.Counter writesCounter = new Counters.Counter();
Progress mergePhase = new Progress();
RawKeyValueIterator mergeQueue = Merger.Merge(conf, fs, keyClass, valueClass, segments
, 2, tmpDir, comparator, GetReporter(), readsCounter, writesCounter, mergePhase);
float epsilon = 0.00001f;
// Reading 6 keys total, 3 each in 2 segments, so each key read moves the
// progress forward 1/6th of the way. Initially the first keys from each
// segment have been read as part of the merge setup, so progress = 2/6.
NUnit.Framework.Assert.AreEqual(2 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
// The first next() returns one of the keys already read during merge setup
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(2 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
// Subsequent next() calls should read one key and move progress
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(3 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(4 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
// At this point we've exhausted all of the keys in one segment
// so getting the next key will return the already cached key from the
// other segment
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(4 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
// Subsequent next() calls should read one key and move progress
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(5 / 6.0f, mergeQueue.GetProgress().Get(), epsilon
);
NUnit.Framework.Assert.IsTrue(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(1.0f, mergeQueue.GetProgress().Get(), epsilon);
// Now there should be no more input
NUnit.Framework.Assert.IsFalse(mergeQueue.Next());
NUnit.Framework.Assert.AreEqual(1.0f, mergeQueue.GetProgress().Get(), epsilon);
NUnit.Framework.Assert.IsTrue(mergeQueue.GetKey() == null);
NUnit.Framework.Assert.AreEqual(0, mergeQueue.GetValue().GetData().Length);
}
public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS
, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec codec
, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector, Counters.Counter
spilledRecordsCounter, Counters.Counter reduceCombineInputCounter, Counters.Counter
mergedMapOutputsCounter, ExceptionReporter exceptionReporter, Progress mergePhase
, MapOutputFile mapOutputFile)
{
/* Maximum percentage of the in-memory limit that a single shuffle can
* consume*/
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localDirAllocator = localDirAllocator;
this.exceptionReporter = exceptionReporter;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.spilledRecordsCounter = spilledRecordsCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.mapOutputFile = mapOutputFile;
this.mapOutputFile.SetConf(jobConf);
this.localFS = localFS;
this.rfs = ((LocalFileSystem)localFS).GetRaw();
float maxInMemCopyUse = jobConf.GetFloat(MRJobConfig.ShuffleInputBufferPercent, MRJobConfig
.DefaultShuffleInputBufferPercent);
if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleInputBufferPercent
+ ": " + maxInMemCopyUse);
}
// Allow unit tests to fix Runtime memory
this.memoryLimit = (long)(jobConf.GetLong(MRJobConfig.ReduceMemoryTotalBytes, Runtime
.GetRuntime().MaxMemory()) * maxInMemCopyUse);
this.ioSortFactor = jobConf.GetInt(MRJobConfig.IoSortFactor, 100);
float singleShuffleMemoryLimitPercent = jobConf.GetFloat(MRJobConfig.ShuffleMemoryLimitPercent
, DefaultShuffleMemoryLimitPercent);
if (singleShuffleMemoryLimitPercent <= 0.0f || singleShuffleMemoryLimitPercent >
1.0f)
{
throw new ArgumentException("Invalid value for " + MRJobConfig.ShuffleMemoryLimitPercent
+ ": " + singleShuffleMemoryLimitPercent);
}
usedMemory = 0L;
commitMemory = 0L;
this.maxSingleShuffleLimit = (long)(memoryLimit * singleShuffleMemoryLimitPercent
);
this.memToMemMergeOutputsThreshold = jobConf.GetInt(MRJobConfig.ReduceMemtomemThreshold
, ioSortFactor);
this.mergeThreshold = (long)(this.memoryLimit * jobConf.GetFloat(MRJobConfig.ShuffleMergePercent
, 0.90f));
Log.Info("MergerManager: memoryLimit=" + memoryLimit + ", " + "maxSingleShuffleLimit="
+ maxSingleShuffleLimit + ", " + "mergeThreshold=" + mergeThreshold + ", " + "ioSortFactor="
+ ioSortFactor + ", " + "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold
);
if (this.maxSingleShuffleLimit >= this.mergeThreshold)
{
throw new RuntimeException("Invalid configuration: " + "maxSingleShuffleLimit should be less than mergeThreshold "
+ "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit + "mergeThreshold: " +
this.mergeThreshold);
}
bool allowMemToMemMerge = jobConf.GetBoolean(MRJobConfig.ReduceMemtomemEnabled, false
);
if (allowMemToMemMerge)
{
this.memToMemMerger = new MergeManagerImpl.IntermediateMemoryToMemoryMerger(this,
this, memToMemMergeOutputsThreshold);
this.memToMemMerger.Start();
}
else
{
this.memToMemMerger = null;
}
this.inMemoryMerger = CreateInMemoryMerger();
this.inMemoryMerger.Start();
this.onDiskMerger = new MergeManagerImpl.OnDiskMerger(this, this);
this.onDiskMerger.Start();
this.mergePhase = mergePhase;
}
public TimeCounter(int counterId, Counters counters)
{
_counter = counters.GetCounter(counterId);
_ticks = 0;
}
public TimeCounter(int counterId, Counters counters)
{
_counter = counters.GetCounter(counterId);
_ticks = 0;
}
public virtual void TestAggregatedTransferRate <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(10);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_115();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
TaskAttemptID attemptID0 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 0), 0);
//adding the 1st interval, 40MB from 60s to 100s
long bytes = (long)40 * 1024 * 1024;
scheduler.CopySucceeded(attemptID0, new MapHost(null, null), bytes, 60000, 100000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(1, 1, 1), progress.ToString());
TaskAttemptID attemptID1 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 1), 1);
//adding the 2nd interval before the 1st interval, 50MB from 0s to 50s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID1, new MapHost(null, null), bytes, 0, 50000, output
);
NUnit.Framework.Assert.AreEqual(CopyMessage(2, 1, 1), progress.ToString());
TaskAttemptID attemptID2 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 2), 2);
//adding the 3rd interval overlapping with the 1st and the 2nd interval
//110MB from 25s to 80s
bytes = (long)110 * 1024 * 1024;
scheduler.CopySucceeded(attemptID2, new MapHost(null, null), bytes, 25000, 80000,
output);
NUnit.Framework.Assert.AreEqual(CopyMessage(3, 2, 2), progress.ToString());
TaskAttemptID attemptID3 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 3), 3);
//adding the 4th interval just after the 2nd interval, 100MB from 100s to 300s
bytes = (long)100 * 1024 * 1024;
scheduler.CopySucceeded(attemptID3, new MapHost(null, null), bytes, 100000, 300000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(4, 0.5, 1), progress.ToString());
TaskAttemptID attemptID4 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 4), 4);
//adding the 5th interval between after 4th, 50MB from 350s to 400s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID4, new MapHost(null, null), bytes, 350000, 400000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(5, 1, 1), progress.ToString());
TaskAttemptID attemptID5 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 5), 5);
//adding the 6th interval between after 5th, 50MB from 450s to 500s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID5, new MapHost(null, null), bytes, 450000, 500000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(6, 1, 1), progress.ToString());
TaskAttemptID attemptID6 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 6), 6);
//adding the 7th interval between after 5th and 6th interval, 20MB from 320s to 340s
bytes = (long)20 * 1024 * 1024;
scheduler.CopySucceeded(attemptID6, new MapHost(null, null), bytes, 320000, 340000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(7, 1, 1), progress.ToString());
TaskAttemptID attemptID7 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 7), 7);
//adding the 8th interval overlapping with 4th, 5th, and 7th 30MB from 290s to 350s
bytes = (long)30 * 1024 * 1024;
scheduler.CopySucceeded(attemptID7, new MapHost(null, null), bytes, 290000, 350000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(8, 0.5, 1), progress.ToString());
TaskAttemptID attemptID8 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 8), 8);
//adding the 9th interval overlapping with 5th and 6th, 50MB from 400s to 450s
bytes = (long)50 * 1024 * 1024;
scheduler.CopySucceeded(attemptID8, new MapHost(null, null), bytes, 400000, 450000
, output);
NUnit.Framework.Assert.AreEqual(CopyMessage(9, 1, 1), progress.ToString());
TaskAttemptID attemptID9 = new TaskAttemptID(new TaskID(new JobID("test", 0), TaskType
.Map, 9), 9);
//adding the 10th interval overlapping with all intervals, 500MB from 0s to 500s
bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(attemptID9, new MapHost(null, null), bytes, 0, 500000, output
);
NUnit.Framework.Assert.AreEqual(CopyMessage(10, 1, 2), progress.ToString());
}
/// <exception cref="System.IO.IOException"/>
public virtual void RegisterCounter(int id, string group, string name)
{
Counters.Counter counter = reporter.GetCounter(group, name);
registeredCounters[id] = counter;
}
public FakeCollector(TestPipeApplication _enclosing, Counters.Counter outCounter,
Progressable progressable)
: base(_enclosing)
{
this._enclosing = _enclosing;
}
public TimeCounter(string name, string notes, Counters counters)
{
int id = counters.FindId(name, notes);
_counter = counters.GetCounter(id);
_ticks = 0;
}