예제 #1
0
        /// <exception cref="System.IO.IOException"/>
        private long CreateInMemorySegments(IList <InMemoryMapOutput <K, V> > inMemoryMapOutputs
                                            , IList <Merger.Segment <K, V> > inMemorySegments, long leaveBytes)
        {
            long totalSize = 0L;
            // We could use fullSize could come from the RamManager, but files can be
            // closed but not yet present in inMemoryMapOutputs
            long fullSize = 0L;

            foreach (InMemoryMapOutput <K, V> mo in inMemoryMapOutputs)
            {
                fullSize += mo.GetMemory().Length;
            }
            while (fullSize > leaveBytes)
            {
                InMemoryMapOutput <K, V> mo_1 = inMemoryMapOutputs.Remove(0);
                byte[] data = mo_1.GetMemory();
                long   size = data.Length;
                totalSize += size;
                fullSize  -= size;
                IFile.Reader <K, V> reader = new InMemoryReader <K, V>(this, mo_1.GetMapId(), data,
                                                                       0, (int)size, jobConf);
                inMemorySegments.AddItem(new Merger.Segment <K, V>(reader, true, (mo_1.IsPrimaryMapOutput
                                                                                      () ? mergedMapOutputsCounter : null)));
            }
            return(totalSize);
        }
예제 #2
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostCompressFailure()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
                                                                                    MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
                                                                                Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
                                                                      , encHash);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
                                                                                    , true, false);
        }
예제 #3
0
            /// <exception cref="System.IO.IOException"/>
            public override void Merge(IList <InMemoryMapOutput <K, V> > inputs)
            {
                if (inputs == null || inputs.Count == 0)
                {
                    return;
                }
                TaskAttemptID dummyMapId = inputs[0].GetMapId();
                IList <Merger.Segment <K, V> > inMemorySegments = new AList <Merger.Segment <K, V> >();
                long mergeOutputSize = this._enclosing.CreateInMemorySegments(inputs, inMemorySegments
                                                                              , 0);
                int noInMemorySegments = inMemorySegments.Count;
                InMemoryMapOutput <K, V> mergedMapOutputs = this._enclosing.UnconditionalReserve(dummyMapId
                                                                                                 , mergeOutputSize, false);

                IFile.Writer <K, V> writer = new InMemoryWriter <K, V>(mergedMapOutputs.GetArrayStream
                                                                           ());
                MergeManagerImpl.Log.Info("Initiating Memory-to-Memory merge with " + noInMemorySegments
                                          + " segments of total-size: " + mergeOutputSize);
                RawKeyValueIterator rIter = Merger.Merge(this._enclosing.jobConf, this._enclosing
                                                         .rfs, (Type)this._enclosing.jobConf.GetMapOutputKeyClass(), (Type)this._enclosing
                                                         .jobConf.GetMapOutputValueClass(), inMemorySegments, inMemorySegments.Count, new
                                                         Path(this._enclosing.reduceId.ToString()), (RawComparator <K>) this._enclosing.jobConf
                                                         .GetOutputKeyComparator(), this._enclosing.reporter, null, null, null);

                Merger.WriteFile(rIter, writer, this._enclosing.reporter, this._enclosing.jobConf
                                 );
                writer.Close();
                MergeManagerImpl.Log.Info(this._enclosing.reduceId + " Memory-to-Memory merge of the "
                                          + noInMemorySegments + " files in-memory complete.");
                // Note the output of the merge
                this._enclosing.CloseInMemoryMergedFile(mergedMapOutputs);
            }
예제 #4
0
 public virtual void CloseInMemoryFile(InMemoryMapOutput <K, V> mapOutput)
 {
     lock (this)
     {
         inMemoryMapOutputs.AddItem(mapOutput);
         Log.Info("closeInMemoryFile -> map-output of size: " + mapOutput.GetSize() + ", inMemoryMapOutputs.size() -> "
                  + inMemoryMapOutputs.Count + ", commitMemory -> " + commitMemory + ", usedMemory ->"
                  + usedMemory);
         commitMemory += mapOutput.GetSize();
         // Can hang if mergeThreshold is really low.
         if (commitMemory >= mergeThreshold)
         {
             Log.Info("Starting inMemoryMerger's merge since commitMemory=" + commitMemory + " > mergeThreshold="
                      + mergeThreshold + ". Current usedMemory=" + usedMemory);
             Sharpen.Collections.AddAll(inMemoryMapOutputs, inMemoryMergedMapOutputs);
             inMemoryMergedMapOutputs.Clear();
             inMemoryMerger.StartMerge(inMemoryMapOutputs);
             commitMemory = 0L;
         }
         // Reset commitMemory.
         if (memToMemMerger != null)
         {
             if (inMemoryMapOutputs.Count >= memToMemMergeOutputsThreshold)
             {
                 memToMemMerger.StartMerge(inMemoryMapOutputs);
             }
         }
     }
 }
예제 #5
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostWithRetryUnreserve()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
                                                                                      , id, ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            // Verify that unreserve occurs if an exception happens after shuffle
            // buffer is reserved.
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
                                                                                            <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
                                                                                            , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(immo).Abort();
        }
예제 #6
0
 public virtual void CloseInMemoryMergedFile(InMemoryMapOutput <K, V> mapOutput)
 {
     lock (this)
     {
         inMemoryMergedMapOutputs.AddItem(mapOutput);
         Log.Info("closeInMemoryMergedFile -> size: " + mapOutput.GetSize() + ", inMemoryMergedMapOutputs.size() -> "
                  + inMemoryMergedMapOutputs.Count);
     }
 }
예제 #7
0
        /// <exception cref="System.IO.IOException"/>
        private void FillOutput(InMemoryMapOutput <Text, Text> output)
        {
            BoundedByteArrayOutputStream stream = output.GetArrayStream();
            int count = stream.GetLimit();

            for (int i = 0; i < count; ++i)
            {
                stream.Write(i);
            }
        }
예제 #8
0
        public virtual void TestCopyFromHostExtraBytes()
        {
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();
            DataOutputStream      dos    = new DataOutputStream(bout);
            IFileOutputStream     ios    = new IFileOutputStream(dos);

            header.Write(dos);
            ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
            ios.Finish();
            ShuffleHeader     header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
            IFileOutputStream ios2    = new IFileOutputStream(dos);

            header2.Write(dos);
            ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
            ios2.Finish();
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            // 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
            InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
                                                                                       , mm, 8, null, true);
            InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
                                                                                        , mm, 10, null, true);

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut2);
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(allErrs).Increment(1);
            Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
                                                                                   , true, false);
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map1ID));
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map2ID));
        }
예제 #9
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInterruptInMemory()
        {
            int Fetcher = 2;
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput
                                                                          <Text, Text>(job, id, mm, 100, null, true));

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
            Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
                                                                                    (bout.ToByteArray()));
            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect();
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection, Fetcher);

            underTest.Start();
            // wait for read in inputstream
            @in.WaitForFetcher();
            underTest.ShutDown();
            underTest.Join();
            // rely on test timeout to kill if stuck
            NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
            Org.Mockito.Mockito.Verify(immo).Abort();
        }
예제 #10
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostWithRetry()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );

            ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
                                                                                      , id, ss, mm, r, metrics, except, key, connection, true);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            long retryTime = Time.MonotonicNow();

            Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
                                                                                        <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
                                                                                        , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            // Emulate host down for 3 seconds.
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
                                                                                   <TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
                                                                                       ());
        }
예제 #11
0
        public virtual void TestInMemoryAndOnDiskMerger()
        {
            JobID         jobId     = new JobID("a", 0);
            TaskAttemptID reduceId1 = new TaskAttemptID(new TaskID(jobId, TaskType.Reduce, 0)
                                                        , 0);
            TaskAttemptID                 mapId1       = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 1), 0);
            TaskAttemptID                 mapId2       = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 2), 0);
            LocalDirAllocator             lda          = new LocalDirAllocator(MRConfig.LocalDir);
            MergeManagerImpl <Text, Text> mergeManager = new MergeManagerImpl <Text, Text>(reduceId1
                                                                                           , jobConf, fs, lda, Reporter.Null, null, null, null, null, null, null, null, new
                                                                                           Progress(), new MROutputFiles());
            // write map outputs
            IDictionary <string, string> map1 = new SortedDictionary <string, string>();

            map1["apple"]  = "disgusting";
            map1["carrot"] = "delicious";
            IDictionary <string, string> map2 = new SortedDictionary <string, string>();

            map1["banana"] = "pretty good";
            byte[] mapOutputBytes1 = WriteMapOutput(conf, map1);
            byte[] mapOutputBytes2 = WriteMapOutput(conf, map2);
            InMemoryMapOutput <Text, Text> mapOutput1 = new InMemoryMapOutput <Text, Text>(conf
                                                                                           , mapId1, mergeManager, mapOutputBytes1.Length, null, true);
            InMemoryMapOutput <Text, Text> mapOutput2 = new InMemoryMapOutput <Text, Text>(conf
                                                                                           , mapId2, mergeManager, mapOutputBytes2.Length, null, true);

            System.Array.Copy(mapOutputBytes1, 0, mapOutput1.GetMemory(), 0, mapOutputBytes1.
                              Length);
            System.Array.Copy(mapOutputBytes2, 0, mapOutput2.GetMemory(), 0, mapOutputBytes2.
                              Length);
            // create merger and run merge
            MergeThread <InMemoryMapOutput <Text, Text>, Text, Text> inMemoryMerger = mergeManager
                                                                                      .CreateInMemoryMerger();
            IList <InMemoryMapOutput <Text, Text> > mapOutputs1 = new AList <InMemoryMapOutput <Text
                                                                                                , Text> >();

            mapOutputs1.AddItem(mapOutput1);
            mapOutputs1.AddItem(mapOutput2);
            inMemoryMerger.Merge(mapOutputs1);
            NUnit.Framework.Assert.AreEqual(1, mergeManager.onDiskMapOutputs.Count);
            TaskAttemptID reduceId2 = new TaskAttemptID(new TaskID(jobId, TaskType.Reduce, 3)
                                                        , 0);
            TaskAttemptID mapId3 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 4), 0);
            TaskAttemptID mapId4 = new TaskAttemptID(new TaskID(jobId, TaskType.Map, 5), 0);
            // write map outputs
            IDictionary <string, string> map3 = new SortedDictionary <string, string>();

            map3["apple"]  = "awesome";
            map3["carrot"] = "amazing";
            IDictionary <string, string> map4 = new SortedDictionary <string, string>();

            map4["banana"] = "bla";
            byte[] mapOutputBytes3 = WriteMapOutput(conf, map3);
            byte[] mapOutputBytes4 = WriteMapOutput(conf, map4);
            InMemoryMapOutput <Text, Text> mapOutput3 = new InMemoryMapOutput <Text, Text>(conf
                                                                                           , mapId3, mergeManager, mapOutputBytes3.Length, null, true);
            InMemoryMapOutput <Text, Text> mapOutput4 = new InMemoryMapOutput <Text, Text>(conf
                                                                                           , mapId4, mergeManager, mapOutputBytes4.Length, null, true);

            System.Array.Copy(mapOutputBytes3, 0, mapOutput3.GetMemory(), 0, mapOutputBytes3.
                              Length);
            System.Array.Copy(mapOutputBytes4, 0, mapOutput4.GetMemory(), 0, mapOutputBytes4.
                              Length);
            //    // create merger and run merge
            MergeThread <InMemoryMapOutput <Text, Text>, Text, Text> inMemoryMerger2 = mergeManager
                                                                                       .CreateInMemoryMerger();
            IList <InMemoryMapOutput <Text, Text> > mapOutputs2 = new AList <InMemoryMapOutput <Text
                                                                                                , Text> >();

            mapOutputs2.AddItem(mapOutput3);
            mapOutputs2.AddItem(mapOutput4);
            inMemoryMerger2.Merge(mapOutputs2);
            NUnit.Framework.Assert.AreEqual(2, mergeManager.onDiskMapOutputs.Count);
            IList <MergeManagerImpl.CompressAwarePath> paths = new AList <MergeManagerImpl.CompressAwarePath
                                                                          >();
            IEnumerator <MergeManagerImpl.CompressAwarePath> iterator = mergeManager.onDiskMapOutputs
                                                                        .GetEnumerator();
            IList <string> keys   = new AList <string>();
            IList <string> values = new AList <string>();

            while (iterator.HasNext())
            {
                MergeManagerImpl.CompressAwarePath next = iterator.Next();
                ReadOnDiskMapOutput(conf, fs, next, keys, values);
                paths.AddItem(next);
            }
            NUnit.Framework.Assert.AreEqual(keys, Arrays.AsList("apple", "banana", "carrot",
                                                                "apple", "banana", "carrot"));
            NUnit.Framework.Assert.AreEqual(values, Arrays.AsList("awesome", "bla", "amazing"
                                                                  , "disgusting", "pretty good", "delicious"));
            mergeManager.Close();
            mergeManager = new MergeManagerImpl <Text, Text>(reduceId2, jobConf, fs, lda, Reporter
                                                             .Null, null, null, null, null, null, null, null, new Progress(), new MROutputFiles
                                                                 ());
            MergeThread <MergeManagerImpl.CompressAwarePath, Text, Text> onDiskMerger = mergeManager
                                                                                        .CreateOnDiskMerger();

            onDiskMerger.Merge(paths);
            NUnit.Framework.Assert.AreEqual(1, mergeManager.onDiskMapOutputs.Count);
            keys   = new AList <string>();
            values = new AList <string>();
            ReadOnDiskMapOutput(conf, fs, mergeManager.onDiskMapOutputs.GetEnumerator().Next(
                                    ), keys, values);
            NUnit.Framework.Assert.AreEqual(keys, Arrays.AsList("apple", "apple", "banana", "banana"
                                                                , "carrot", "carrot"));
            NUnit.Framework.Assert.AreEqual(values, Arrays.AsList("awesome", "disgusting", "pretty good"
                                                                  , "bla", "amazing", "delicious"));
            mergeManager.Close();
            NUnit.Framework.Assert.AreEqual(0, mergeManager.inMemoryMapOutputs.Count);
            NUnit.Framework.Assert.AreEqual(0, mergeManager.inMemoryMergedMapOutputs.Count);
            NUnit.Framework.Assert.AreEqual(0, mergeManager.onDiskMapOutputs.Count);
        }
예제 #12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestMemoryMerge()
        {
            int     TotalMemBytes = 10000;
            int     OutputSize    = 7950;
            JobConf conf          = new JobConf();

            conf.SetFloat(MRJobConfig.ShuffleInputBufferPercent, 1.0f);
            conf.SetLong(MRJobConfig.ReduceMemoryTotalBytes, TotalMemBytes);
            conf.SetFloat(MRJobConfig.ShuffleMemoryLimitPercent, 0.8f);
            conf.SetFloat(MRJobConfig.ShuffleMergePercent, 0.9f);
            TestMergeManager.TestExceptionReporter reporter = new TestMergeManager.TestExceptionReporter
                                                                  ();
            CyclicBarrier mergeStart    = new CyclicBarrier(2);
            CyclicBarrier mergeComplete = new CyclicBarrier(2);

            TestMergeManager.StubbedMergeManager mgr = new TestMergeManager.StubbedMergeManager
                                                           (conf, reporter, mergeStart, mergeComplete);
            // reserve enough map output to cause a merge when it is committed
            MapOutput <Text, Text> out1 = mgr.Reserve(null, OutputSize, 0);

            NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
                                                                       ));
            InMemoryMapOutput <Text, Text> mout1 = (InMemoryMapOutput <Text, Text>)out1;

            FillOutput(mout1);
            MapOutput <Text, Text> out2 = mgr.Reserve(null, OutputSize, 0);

            NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
                                                                       ));
            InMemoryMapOutput <Text, Text> mout2 = (InMemoryMapOutput <Text, Text>)out2;

            FillOutput(mout2);
            // next reservation should be a WAIT
            MapOutput <Text, Text> out3 = mgr.Reserve(null, OutputSize, 0);

            NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
            // trigger the first merge and wait for merge thread to start merging
            // and free enough output to reserve more
            mout1.Commit();
            mout2.Commit();
            mergeStart.Await();
            NUnit.Framework.Assert.AreEqual(1, mgr.GetNumMerges());
            // reserve enough map output to cause another merge when committed
            out1 = mgr.Reserve(null, OutputSize, 0);
            NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
                                                                       ));
            mout1 = (InMemoryMapOutput <Text, Text>)out1;
            FillOutput(mout1);
            out2 = mgr.Reserve(null, OutputSize, 0);
            NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
                                                                       ));
            mout2 = (InMemoryMapOutput <Text, Text>)out2;
            FillOutput(mout2);
            // next reservation should be null
            out3 = mgr.Reserve(null, OutputSize, 0);
            NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
            // commit output *before* merge thread completes
            mout1.Commit();
            mout2.Commit();
            // allow the first merge to complete
            mergeComplete.Await();
            // start the second merge and verify
            mergeStart.Await();
            NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
            // trigger the end of the second merge
            mergeComplete.Await();
            NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
            NUnit.Framework.Assert.AreEqual("exception reporter invoked", 0, reporter.GetNumExceptions
                                                ());
        }