/// <exception cref="System.IO.IOException"/>
private long CreateInMemorySegments(IList <InMemoryMapOutput <K, V> > inMemoryMapOutputs
, IList <Merger.Segment <K, V> > inMemorySegments, long leaveBytes)
{
long totalSize = 0L;
// We could use fullSize could come from the RamManager, but files can be
// closed but not yet present in inMemoryMapOutputs
long fullSize = 0L;
foreach (InMemoryMapOutput <K, V> mo in inMemoryMapOutputs)
{
fullSize += mo.GetMemory().Length;
}
while (fullSize > leaveBytes)
{
InMemoryMapOutput <K, V> mo_1 = inMemoryMapOutputs.Remove(0);
byte[] data = mo_1.GetMemory();
long size = data.Length;
totalSize += size;
fullSize -= size;
IFile.Reader <K, V> reader = new InMemoryReader <K, V>(this, mo_1.GetMapId(), data,
0, (int)size, jobConf);
inMemorySegments.AddItem(new Merger.Segment <K, V>(reader, true, (mo_1.IsPrimaryMapOutput
() ? mergedMapOutputsCounter : null)));
}
return(totalSize);
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostCompressFailure()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
, true, false);
}
/// <exception cref="System.IO.IOException"/>
public override void Merge(IList <InMemoryMapOutput <K, V> > inputs)
{
if (inputs == null || inputs.Count == 0)
{
return;
}
TaskAttemptID dummyMapId = inputs[0].GetMapId();
IList <Merger.Segment <K, V> > inMemorySegments = new AList <Merger.Segment <K, V> >();
long mergeOutputSize = this._enclosing.CreateInMemorySegments(inputs, inMemorySegments
, 0);
int noInMemorySegments = inMemorySegments.Count;
InMemoryMapOutput <K, V> mergedMapOutputs = this._enclosing.UnconditionalReserve(dummyMapId
, mergeOutputSize, false);
IFile.Writer <K, V> writer = new InMemoryWriter <K, V>(mergedMapOutputs.GetArrayStream
());
MergeManagerImpl.Log.Info("Initiating Memory-to-Memory merge with " + noInMemorySegments
+ " segments of total-size: " + mergeOutputSize);
RawKeyValueIterator rIter = Merger.Merge(this._enclosing.jobConf, this._enclosing
.rfs, (Type)this._enclosing.jobConf.GetMapOutputKeyClass(), (Type)this._enclosing
.jobConf.GetMapOutputValueClass(), inMemorySegments, inMemorySegments.Count, new
Path(this._enclosing.reduceId.ToString()), (RawComparator <K>) this._enclosing.jobConf
.GetOutputKeyComparator(), this._enclosing.reporter, null, null, null);
Merger.WriteFile(rIter, writer, this._enclosing.reporter, this._enclosing.jobConf
);
writer.Close();
MergeManagerImpl.Log.Info(this._enclosing.reduceId + " Memory-to-Memory merge of the "
+ noInMemorySegments + " files in-memory complete.");
// Note the output of the merge
this._enclosing.CloseInMemoryMergedFile(mergedMapOutputs);
}
public virtual void CloseInMemoryFile(InMemoryMapOutput <K, V> mapOutput)
{
lock (this)
{
inMemoryMapOutputs.AddItem(mapOutput);
Log.Info("closeInMemoryFile -> map-output of size: " + mapOutput.GetSize() + ", inMemoryMapOutputs.size() -> "
+ inMemoryMapOutputs.Count + ", commitMemory -> " + commitMemory + ", usedMemory ->"
+ usedMemory);
commitMemory += mapOutput.GetSize();
// Can hang if mergeThreshold is really low.
if (commitMemory >= mergeThreshold)
{
Log.Info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold="
+ mergeThreshold + ". Current usedMemory=" + usedMemory);
Sharpen.Collections.AddAll(inMemoryMapOutputs, inMemoryMergedMapOutputs);
inMemoryMergedMapOutputs.Clear();
inMemoryMerger.StartMerge(inMemoryMapOutputs);
commitMemory = 0L;
}
// Reset commitMemory.
if (memToMemMerger != null)
{
if (inMemoryMapOutputs.Count >= memToMemMergeOutputsThreshold)
{
memToMemMerger.StartMerge(inMemoryMapOutputs);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetryUnreserve()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
// Verify that unreserve occurs if an exception happens after shuffle
// buffer is reserved.
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(immo).Abort();
}
public virtual void CloseInMemoryMergedFile(InMemoryMapOutput <K, V> mapOutput)
{
lock (this)
{
inMemoryMergedMapOutputs.AddItem(mapOutput);
Log.Info("closeInMemoryMergedFile -> size: " + mapOutput.GetSize() + ", inMemoryMergedMapOutputs.size() -> "
+ inMemoryMergedMapOutputs.Count);
}
}
/// <exception cref="System.IO.IOException"/>
private void FillOutput(InMemoryMapOutput <Text, Text> output)
{
BoundedByteArrayOutputStream stream = output.GetArrayStream();
int count = stream.GetLimit();
for (int i = 0; i < count; ++i)
{
stream.Write(i);
}
}
public virtual void TestCopyFromHostExtraBytes()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bout);
IFileOutputStream ios = new IFileOutputStream(dos);
header.Write(dos);
ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
ios.Finish();
ShuffleHeader header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
IFileOutputStream ios2 = new IFileOutputStream(dos);
header2.Write(dos);
ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
ios2.Finish();
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
// 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
, mm, 8, null, true);
InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
, mm, 10, null, true);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut2);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(allErrs).Increment(1);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
/// <exception cref="System.Exception"/>
public virtual void TestInterruptInMemory()
{
int Fetcher = 2;
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput
<Text, Text>(job, id, mm, 100, null, true));
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
(bout.ToByteArray()));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection, Fetcher);
underTest.Start();
// wait for read in inputstream
@in.WaitForFetcher();
underTest.ShutDown();
underTest.Join();
// rely on test timeout to kill if stuck
NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
Org.Mockito.Mockito.Verify(immo).Abort();
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetry()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection, true);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
long retryTime = Time.MonotonicNow();
Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
// Emulate host down for 3 seconds.
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
<TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
());
}
public virtual void TestInMemoryAndOnDiskMerger()
{
JobID jobId = new JobID("a", 0);
TaskAttemptID reduceId1 = new TaskAttemptID(new TaskID(jobId, TaskType.Reduce, 0)
, 0);
TaskAttemptID mapId1 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 1), 0);
TaskAttemptID mapId2 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 2), 0);
LocalDirAllocator lda = new LocalDirAllocator(MRConfig.LocalDir);
MergeManagerImpl <Text, Text> mergeManager = new MergeManagerImpl <Text, Text>(reduceId1
, jobConf, fs, lda, Reporter.Null, null, null, null, null, null, null, null, new
Progress(), new MROutputFiles());
// write map outputs
IDictionary <string, string> map1 = new SortedDictionary <string, string>();
map1["apple"] = "disgusting";
map1["carrot"] = "delicious";
IDictionary <string, string> map2 = new SortedDictionary <string, string>();
map1["banana"] = "pretty good";
byte[] mapOutputBytes1 = WriteMapOutput(conf, map1);
byte[] mapOutputBytes2 = WriteMapOutput(conf, map2);
InMemoryMapOutput <Text, Text> mapOutput1 = new InMemoryMapOutput <Text, Text>(conf
, mapId1, mergeManager, mapOutputBytes1.Length, null, true);
InMemoryMapOutput <Text, Text> mapOutput2 = new InMemoryMapOutput <Text, Text>(conf
, mapId2, mergeManager, mapOutputBytes2.Length, null, true);
System.Array.Copy(mapOutputBytes1, 0, mapOutput1.GetMemory(), 0, mapOutputBytes1.
Length);
System.Array.Copy(mapOutputBytes2, 0, mapOutput2.GetMemory(), 0, mapOutputBytes2.
Length);
// create merger and run merge
MergeThread <InMemoryMapOutput <Text, Text>, Text, Text> inMemoryMerger = mergeManager
.CreateInMemoryMerger();
IList <InMemoryMapOutput <Text, Text> > mapOutputs1 = new AList <InMemoryMapOutput <Text
, Text> >();
mapOutputs1.AddItem(mapOutput1);
mapOutputs1.AddItem(mapOutput2);
inMemoryMerger.Merge(mapOutputs1);
NUnit.Framework.Assert.AreEqual(1, mergeManager.onDiskMapOutputs.Count);
TaskAttemptID reduceId2 = new TaskAttemptID(new TaskID(jobId, TaskType.Reduce, 3)
, 0);
TaskAttemptID mapId3 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 4), 0);
TaskAttemptID mapId4 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 5), 0);
// write map outputs
IDictionary <string, string> map3 = new SortedDictionary <string, string>();
map3["apple"] = "awesome";
map3["carrot"] = "amazing";
IDictionary <string, string> map4 = new SortedDictionary <string, string>();
map4["banana"] = "bla";
byte[] mapOutputBytes3 = WriteMapOutput(conf, map3);
byte[] mapOutputBytes4 = WriteMapOutput(conf, map4);
InMemoryMapOutput <Text, Text> mapOutput3 = new InMemoryMapOutput <Text, Text>(conf
, mapId3, mergeManager, mapOutputBytes3.Length, null, true);
InMemoryMapOutput <Text, Text> mapOutput4 = new InMemoryMapOutput <Text, Text>(conf
, mapId4, mergeManager, mapOutputBytes4.Length, null, true);
System.Array.Copy(mapOutputBytes3, 0, mapOutput3.GetMemory(), 0, mapOutputBytes3.
Length);
System.Array.Copy(mapOutputBytes4, 0, mapOutput4.GetMemory(), 0, mapOutputBytes4.
Length);
// // create merger and run merge
MergeThread <InMemoryMapOutput <Text, Text>, Text, Text> inMemoryMerger2 = mergeManager
.CreateInMemoryMerger();
IList <InMemoryMapOutput <Text, Text> > mapOutputs2 = new AList <InMemoryMapOutput <Text
, Text> >();
mapOutputs2.AddItem(mapOutput3);
mapOutputs2.AddItem(mapOutput4);
inMemoryMerger2.Merge(mapOutputs2);
NUnit.Framework.Assert.AreEqual(2, mergeManager.onDiskMapOutputs.Count);
IList <MergeManagerImpl.CompressAwarePath> paths = new AList <MergeManagerImpl.CompressAwarePath
>();
IEnumerator <MergeManagerImpl.CompressAwarePath> iterator = mergeManager.onDiskMapOutputs
.GetEnumerator();
IList <string> keys = new AList <string>();
IList <string> values = new AList <string>();
while (iterator.HasNext())
{
MergeManagerImpl.CompressAwarePath next = iterator.Next();
ReadOnDiskMapOutput(conf, fs, next, keys, values);
paths.AddItem(next);
}
NUnit.Framework.Assert.AreEqual(keys, Arrays.AsList("apple", "banana", "carrot",
"apple", "banana", "carrot"));
NUnit.Framework.Assert.AreEqual(values, Arrays.AsList("awesome", "bla", "amazing"
, "disgusting", "pretty good", "delicious"));
mergeManager.Close();
mergeManager = new MergeManagerImpl <Text, Text>(reduceId2, jobConf, fs, lda, Reporter
.Null, null, null, null, null, null, null, null, new Progress(), new MROutputFiles
());
MergeThread <MergeManagerImpl.CompressAwarePath, Text, Text> onDiskMerger = mergeManager
.CreateOnDiskMerger();
onDiskMerger.Merge(paths);
NUnit.Framework.Assert.AreEqual(1, mergeManager.onDiskMapOutputs.Count);
keys = new AList <string>();
values = new AList <string>();
ReadOnDiskMapOutput(conf, fs, mergeManager.onDiskMapOutputs.GetEnumerator().Next(
), keys, values);
NUnit.Framework.Assert.AreEqual(keys, Arrays.AsList("apple", "apple", "banana", "banana"
, "carrot", "carrot"));
NUnit.Framework.Assert.AreEqual(values, Arrays.AsList("awesome", "disgusting", "pretty good"
, "bla", "amazing", "delicious"));
mergeManager.Close();
NUnit.Framework.Assert.AreEqual(0, mergeManager.inMemoryMapOutputs.Count);
NUnit.Framework.Assert.AreEqual(0, mergeManager.inMemoryMergedMapOutputs.Count);
NUnit.Framework.Assert.AreEqual(0, mergeManager.onDiskMapOutputs.Count);
}
/// <exception cref="System.Exception"/>
public virtual void TestMemoryMerge()
{
int TotalMemBytes = 10000;
int OutputSize = 7950;
JobConf conf = new JobConf();
conf.SetFloat(MRJobConfig.ShuffleInputBufferPercent, 1.0f);
conf.SetLong(MRJobConfig.ReduceMemoryTotalBytes, TotalMemBytes);
conf.SetFloat(MRJobConfig.ShuffleMemoryLimitPercent, 0.8f);
conf.SetFloat(MRJobConfig.ShuffleMergePercent, 0.9f);
TestMergeManager.TestExceptionReporter reporter = new TestMergeManager.TestExceptionReporter
();
CyclicBarrier mergeStart = new CyclicBarrier(2);
CyclicBarrier mergeComplete = new CyclicBarrier(2);
TestMergeManager.StubbedMergeManager mgr = new TestMergeManager.StubbedMergeManager
(conf, reporter, mergeStart, mergeComplete);
// reserve enough map output to cause a merge when it is committed
MapOutput <Text, Text> out1 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
));
InMemoryMapOutput <Text, Text> mout1 = (InMemoryMapOutput <Text, Text>)out1;
FillOutput(mout1);
MapOutput <Text, Text> out2 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
));
InMemoryMapOutput <Text, Text> mout2 = (InMemoryMapOutput <Text, Text>)out2;
FillOutput(mout2);
// next reservation should be a WAIT
MapOutput <Text, Text> out3 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
// trigger the first merge and wait for merge thread to start merging
// and free enough output to reserve more
mout1.Commit();
mout2.Commit();
mergeStart.Await();
NUnit.Framework.Assert.AreEqual(1, mgr.GetNumMerges());
// reserve enough map output to cause another merge when committed
out1 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
));
mout1 = (InMemoryMapOutput <Text, Text>)out1;
FillOutput(mout1);
out2 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
));
mout2 = (InMemoryMapOutput <Text, Text>)out2;
FillOutput(mout2);
// next reservation should be null
out3 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
// commit output *before* merge thread completes
mout1.Commit();
mout2.Commit();
// allow the first merge to complete
mergeComplete.Await();
// start the second merge and verify
mergeStart.Await();
NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
// trigger the end of the second merge
mergeComplete.Await();
NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
NUnit.Framework.Assert.AreEqual("exception reporter invoked", 0, reporter.GetNumExceptions
());
}