Beispiel #1
0
 /// <summary>
 /// Test case that stops a writer after finalizing a block but
 /// before calling completeFile, recovers a file from another writer,
 /// starts writing from that writer, and then has the old lease holder
 /// call completeFile
 /// </summary>
 /// <exception cref="System.Exception"/>
 public virtual void TestCompleteOtherLeaseHoldersFile()
 {
     cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build();
     try
     {
         cluster.WaitActive();
         NamenodeProtocols preSpyNN = cluster.GetNameNodeRpc();
         NamenodeProtocols spyNN    = Org.Mockito.Mockito.Spy(preSpyNN);
         // Delay completeFile
         GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
         Org.Mockito.Mockito.DoAnswer(delayer).When(spyNN).Complete(Matchers.AnyString(),
                                                                    Matchers.AnyString(), (ExtendedBlock)Matchers.AnyObject(), Matchers.AnyLong());
         DFSClient client = new DFSClient(null, spyNN, conf, null);
         file1 = new Path("/testCompleteOtherLease");
         OutputStream stm = client.Create("/testCompleteOtherLease", true);
         // write 1/2 block
         AppendTestUtil.Write(stm, 0, 4096);
         AtomicReference <Exception> err = new AtomicReference <Exception>();
         Sharpen.Thread t = new _Thread_242(stm, err);
         t.Start();
         Log.Info("Waiting for close to get to latch...");
         delayer.WaitForCall();
         // At this point, the block is finalized on the DNs, but the file
         // has not been completed in the NN.
         // Lose the leases
         Log.Info("Killing lease checker");
         client.GetLeaseRenewer().InterruptAndJoin();
         FileSystem fs1 = cluster.GetFileSystem();
         FileSystem fs2 = AppendTestUtil.CreateHdfsWithDifferentUsername(fs1.GetConf());
         Log.Info("Recovering file");
         RecoverFile(fs2);
         Log.Info("Opening file for append from new fs");
         FSDataOutputStream appenderStream = fs2.Append(file1);
         Log.Info("Writing some data from new appender");
         AppendTestUtil.Write(appenderStream, 0, 4096);
         Log.Info("Telling old close to proceed.");
         delayer.Proceed();
         Log.Info("Waiting for close to finish.");
         t.Join();
         Log.Info("Close finished.");
         // We expect that close will get a "Lease mismatch"
         // error.
         Exception thrownByClose = err.Get();
         NUnit.Framework.Assert.IsNotNull(thrownByClose);
         NUnit.Framework.Assert.IsTrue(thrownByClose is IOException);
         if (!thrownByClose.Message.Contains("Lease mismatch"))
         {
             throw thrownByClose;
         }
         // The appender should be able to close properly
         appenderStream.Close();
     }
     finally
     {
         cluster.Shutdown();
     }
 }
Beispiel #2
0
        public virtual void TestFactory()
        {
            string[] groups            = new string[] { "supergroup" };
            UserGroupInformation[] ugi = new UserGroupInformation[3];
            for (int i = 0; i < ugi.Length; i++)
            {
                ugi[i] = UserGroupInformation.CreateUserForTesting("user" + i, groups);
            }
            Org.Mockito.Mockito.DoReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission
                                                                ((short)777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, unchecked (
                                                                (byte)0))).When(mcp).GetFileInfo(Matchers.AnyString());
            Org.Mockito.Mockito.DoReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission
                                                                ((short)777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, unchecked (
                                                                (byte)0))).When(mcp).Create(Matchers.AnyString(), (FsPermission)Matchers.AnyObject
                                                                                                (), Matchers.AnyString(), (EnumSetWritable <CreateFlag>)Matchers.AnyObject(), Matchers.AnyBoolean
                                                                                                (), Matchers.AnyShort(), Matchers.AnyLong(), (CryptoProtocolVersion[])Matchers.AnyObject
                                                                                                ());
            Configuration      conf = new Configuration();
            DFSClient          c1   = CreateDFSClientAs(ugi[0], conf);
            FSDataOutputStream out1 = CreateFsOut(c1, "/out1");
            DFSClient          c2   = CreateDFSClientAs(ugi[0], conf);
            FSDataOutputStream out2 = CreateFsOut(c2, "/out2");

            NUnit.Framework.Assert.AreEqual(c1.GetLeaseRenewer(), c2.GetLeaseRenewer());
            DFSClient          c3   = CreateDFSClientAs(ugi[1], conf);
            FSDataOutputStream out3 = CreateFsOut(c3, "/out3");

            NUnit.Framework.Assert.IsTrue(c1.GetLeaseRenewer() != c3.GetLeaseRenewer());
            DFSClient          c4   = CreateDFSClientAs(ugi[1], conf);
            FSDataOutputStream out4 = CreateFsOut(c4, "/out4");

            NUnit.Framework.Assert.AreEqual(c3.GetLeaseRenewer(), c4.GetLeaseRenewer());
            DFSClient          c5   = CreateDFSClientAs(ugi[2], conf);
            FSDataOutputStream out5 = CreateFsOut(c5, "/out5");

            NUnit.Framework.Assert.IsTrue(c1.GetLeaseRenewer() != c5.GetLeaseRenewer());
            NUnit.Framework.Assert.IsTrue(c3.GetLeaseRenewer() != c5.GetLeaseRenewer());
        }
Beispiel #3
0
        public virtual void TestLeaseAbort()
        {
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();

            try
            {
                cluster.WaitActive();
                NamenodeProtocols  preSpyNN = cluster.GetNameNodeRpc();
                NamenodeProtocols  spyNN    = Org.Mockito.Mockito.Spy(preSpyNN);
                DFSClient          dfs      = new DFSClient(null, spyNN, conf, null);
                byte[]             buf      = new byte[1024];
                FSDataOutputStream c_out    = CreateFsOut(dfs, dirString + "c");
                c_out.Write(buf, 0, 1024);
                c_out.Close();
                DFSInputStream     c_in  = dfs.Open(dirString + "c");
                FSDataOutputStream d_out = CreateFsOut(dfs, dirString + "d");
                // stub the renew method.
                Org.Mockito.Mockito.DoThrow(new RemoteException(typeof(SecretManager.InvalidToken
                                                                       ).FullName, "Your token is worthless")).When(spyNN).RenewLease(Matchers.AnyString
                                                                                                                                          ());
                // We don't need to wait the lease renewer thread to act.
                // call renewLease() manually.
                // make it look like the soft limit has been exceeded.
                LeaseRenewer originalRenewer = dfs.GetLeaseRenewer();
                dfs.lastLeaseRenewal = Time.MonotonicNow() - HdfsConstants.LeaseSoftlimitPeriod -
                                       1000;
                try
                {
                    dfs.RenewLease();
                }
                catch (IOException)
                {
                }
                // Things should continue to work it passes hard limit without
                // renewing.
                try
                {
                    d_out.Write(buf, 0, 1024);
                    Log.Info("Write worked beyond the soft limit as expected.");
                }
                catch (IOException)
                {
                    NUnit.Framework.Assert.Fail("Write failed.");
                }
                // make it look like the hard limit has been exceeded.
                dfs.lastLeaseRenewal = Time.MonotonicNow() - HdfsConstants.LeaseHardlimitPeriod -
                                       1000;
                dfs.RenewLease();
                // this should not work.
                try
                {
                    d_out.Write(buf, 0, 1024);
                    d_out.Close();
                    NUnit.Framework.Assert.Fail("Write did not fail even after the fatal lease renewal failure"
                                                );
                }
                catch (IOException e)
                {
                    Log.Info("Write failed as expected. ", e);
                }
                // If aborted, the renewer should be empty. (no reference to clients)
                Sharpen.Thread.Sleep(1000);
                NUnit.Framework.Assert.IsTrue(originalRenewer.IsEmpty());
                // unstub
                Org.Mockito.Mockito.DoNothing().When(spyNN).RenewLease(Matchers.AnyString());
                // existing input streams should work
                try
                {
                    int num = c_in.Read(buf, 0, 1);
                    if (num != 1)
                    {
                        NUnit.Framework.Assert.Fail("Failed to read 1 byte");
                    }
                    c_in.Close();
                }
                catch (IOException e)
                {
                    Log.Error("Read failed with ", e);
                    NUnit.Framework.Assert.Fail("Read after lease renewal failure failed");
                }
                // new file writes should work.
                try
                {
                    c_out = CreateFsOut(dfs, dirString + "c");
                    c_out.Write(buf, 0, 1024);
                    c_out.Close();
                }
                catch (IOException e)
                {
                    Log.Error("Write failed with ", e);
                    NUnit.Framework.Assert.Fail("Write failed");
                }
            }
            finally
            {
                cluster.Shutdown();
            }
        }