Esempio n. 1
0
        /// <exception cref="System.IO.IOException"/>
        private int DownloadImage(NNStorage storage, NamenodeProtocol proxy)
        {
            // Load the newly formatted image, using all of the directories
            // (including shared edits)
            long    imageTxId = proxy.GetMostRecentCheckpointTxId();
            long    curTxId   = proxy.GetTransactionID();
            FSImage image     = new FSImage(conf);

            try
            {
                image.GetStorage().SetStorageInfo(storage);
                image.InitEditLog(HdfsServerConstants.StartupOption.Regular);
                System.Diagnostics.Debug.Assert(image.GetEditLog().IsOpenForRead(), "Expected edit log to be open for read"
                                                );
                // Ensure that we have enough edits already in the shared directory to
                // start up from the last checkpoint on the active.
                if (!skipSharedEditsCheck && !CheckLogsAvailableForRead(image, imageTxId, curTxId
                                                                        ))
                {
                    return(ErrCodeLogsUnavailable);
                }
                image.GetStorage().WriteTransactionIdFileToStorage(curTxId);
                // Download that checkpoint into our storage directories.
                MD5Hash hash = TransferFsImage.DownloadImageToStorage(otherHttpAddr, imageTxId, storage
                                                                      , true);
                image.SaveDigestAndRenameCheckpointImage(NNStorage.NameNodeFile.Image, imageTxId,
                                                         hash);
            }
            catch (IOException ioe)
            {
                image.Close();
                throw;
            }
            return(0);
        }
Esempio n. 2
0
        /// <exception cref="System.IO.IOException"/>
        public KeyManager(string blockpoolID, NamenodeProtocol namenode, bool encryptDataTransfer
                          , Configuration conf)
        {
            this.namenode            = namenode;
            this.encryptDataTransfer = encryptDataTransfer;
            ExportedBlockKeys keys = namenode.GetBlockKeys();

            this.isBlockTokenEnabled = keys.IsBlockTokenEnabled();
            if (isBlockTokenEnabled)
            {
                long updateInterval = keys.GetKeyUpdateInterval();
                long tokenLifetime  = keys.GetTokenLifetime();
                Log.Info("Block token params received from NN: update interval=" + StringUtils.FormatTime
                             (updateInterval) + ", token lifetime=" + StringUtils.FormatTime(tokenLifetime));
                string encryptionAlgorithm = conf.Get(DFSConfigKeys.DfsDataEncryptionAlgorithmKey
                                                      );
                this.blockTokenSecretManager = new BlockTokenSecretManager(updateInterval, tokenLifetime
                                                                           , blockpoolID, encryptionAlgorithm);
                this.blockTokenSecretManager.AddKeys(keys);
                // sync block keys with NN more frequently than NN updates its block keys
                this.blockKeyUpdater = new KeyManager.BlockKeyUpdater(this, updateInterval / 4);
                this.shouldRun       = true;
            }
            else
            {
                this.blockTokenSecretManager = null;
                this.blockKeyUpdater         = null;
            }
        }
Esempio n. 3
0
        /// <exception cref="System.IO.IOException"/>
        public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths
                                 , Configuration conf, int maxNotChangedIterations)
        {
            this.nameNodeUri = nameNodeUri;
            this.idPath      = idPath;
            this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new
                                                                                            Path("/")) : targetPaths;
            this.maxNotChangedIterations = maxNotChangedIterations;
            this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri).
                            GetProxy();
            this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth
                                                                       ).GetProxy();
            this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf);
            NamespaceInfo namespaceinfo = namenode.VersionRequest();

            this.blockpoolID = namespaceinfo.GetBlockPoolID();
            FsServerDefaults defaults = fs.GetServerDefaults(new Path("/"));

            this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer
                                                 (), conf);
            // if it is for test, we do not create the id file
            @out = CheckAndMarkRunning();
            if (@out == null)
            {
                // Exit if there is another one running.
                throw new IOException("Another " + name + " is running.");
            }
        }
Esempio n. 4
0
 /// <exception cref="System.IO.IOException"/>
 private NamenodeProtocol GetActiveNodeProxy()
 {
     if (cachedActiveProxy == null)
     {
         int rpcTimeout = conf.GetInt(DFSConfigKeys.DfsHaLogrollRpcTimeoutKey, DFSConfigKeys
                                      .DfsHaLogrollRpcTimeoutDefault);
         NamenodeProtocolPB proxy = RPC.WaitForProxy <NamenodeProtocolPB>(RPC.GetProtocolVersion
                                                                              (typeof(NamenodeProtocolPB)), activeAddr, conf, rpcTimeout, long.MaxValue);
         cachedActiveProxy = new NamenodeProtocolTranslatorPB(proxy);
     }
     System.Diagnostics.Debug.Assert(cachedActiveProxy != null);
     return(cachedActiveProxy);
 }
Esempio n. 5
0
        public virtual void TestNamenodeProtocol()
        {
            NamenodeProtocol np = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
                                                                                      , UserGroupInformation.GetCurrentUser(), true).GetProxy();
            bool exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
                                                          .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "rollEditLog"
                                                          );

            NUnit.Framework.Assert.IsTrue(exists);
            exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
                                                     .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "bogusMethod"
                                                     );
            NUnit.Framework.Assert.IsFalse(exists);
        }
Esempio n. 6
0
        /// <exception cref="System.IO.IOException"/>
        private void GetBlocksWithException(NamenodeProtocol namenode, DatanodeInfo datanode
                                            , long size)
        {
            bool getException = false;

            try
            {
                namenode.GetBlocks(DFSTestUtil.GetLocalDatanodeInfo(), 2);
            }
            catch (RemoteException e)
            {
                getException = true;
                NUnit.Framework.Assert.IsTrue(e.GetClassName().Contains("HadoopIllegalArgumentException"
                                                                        ));
            }
            NUnit.Framework.Assert.IsTrue(getException);
        }
Esempio n. 7
0
 public override void Stop()
 {
     // NameNode
     if (checkpointManager != null)
     {
         // Prevent from starting a new checkpoint.
         // Checkpoints that has already been started may proceed until
         // the error reporting to the name-node is complete.
         // Checkpoint manager should not be interrupted yet because it will
         // close storage file channels and the checkpoint may fail with
         // ClosedByInterruptException.
         checkpointManager.shouldRun = false;
     }
     if (namenode != null && GetRegistration() != null)
     {
         // Exclude this node from the list of backup streams on the name-node
         try
         {
             namenode.ErrorReport(GetRegistration(), NamenodeProtocol.Fatal, "Shutting down.");
         }
         catch (IOException e)
         {
             Log.Error("Failed to report to name-node.", e);
         }
     }
     // Stop the RPC client
     if (namenode != null)
     {
         RPC.StopProxy(namenode);
     }
     namenode = null;
     // Stop the checkpoint manager
     if (checkpointManager != null)
     {
         checkpointManager.Interrupt();
         checkpointManager = null;
     }
     // Abort current log segment - otherwise the NN shutdown code
     // will close it gracefully, which is incorrect.
     GetFSImage().GetEditLog().AbortCurrentLogSegment();
     // Stop name-node threads
     base.Stop();
 }
Esempio n. 8
0
        // TODO: move to a common with DataNode util class
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Sharpen.SocketTimeoutException"/>
        private static NamespaceInfo Handshake(NamenodeProtocol namenode)
        {
            NamespaceInfo nsInfo;

            nsInfo = namenode.VersionRequest();
            // throws SocketTimeoutException
            string errorMsg = null;

            // verify build version
            if (!nsInfo.GetBuildVersion().Equals(Storage.GetBuildVersion()))
            {
                errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.GetBuildVersion
                               () + "; backup node BV = " + Storage.GetBuildVersion();
                Log.Error(errorMsg);
                throw new IOException(errorMsg);
            }
            System.Diagnostics.Debug.Assert(HdfsConstants.NamenodeLayoutVersion == nsInfo.GetLayoutVersion
                                                (), "Active and backup node layout versions must be the same. Expected: " + HdfsConstants
                                            .NamenodeLayoutVersion + " actual " + nsInfo.GetLayoutVersion());
            return(nsInfo);
        }
Esempio n. 9
0
        /// <exception cref="System.IO.IOException"/>
        private NamespaceInfo Handshake(Configuration conf)
        {
            // connect to name node
            IPEndPoint nnAddress = NameNode.GetServiceAddress(conf, true);

            this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
                                                                                , UserGroupInformation.GetCurrentUser(), true).GetProxy();
            this.nnRpcAddress  = NetUtils.GetHostPortString(nnAddress);
            this.nnHttpAddress = DFSUtil.GetInfoServer(nnAddress, conf, DFSUtil.GetHttpClientScheme
                                                           (conf)).ToURL();
            // get version and id info from the name-node
            NamespaceInfo nsInfo = null;

            while (!IsStopRequested())
            {
                try
                {
                    nsInfo = Handshake(namenode);
                    break;
                }
                catch (SocketTimeoutException e)
                {
                    // name-node is busy
                    Log.Info("Problem connecting to server: " + nnAddress);
                    try
                    {
                        Sharpen.Thread.Sleep(1000);
                    }
                    catch (Exception)
                    {
                        Log.Warn("Encountered exception ", e);
                    }
                }
            }
            return(nsInfo);
        }
Esempio n. 10
0
        /// <exception cref="System.IO.IOException"/>
        private int DoRun()
        {
            NamenodeProtocol proxy = CreateNNProtocolProxy();
            NamespaceInfo    nsInfo;
            bool             isUpgradeFinalized;

            try
            {
                nsInfo             = proxy.VersionRequest();
                isUpgradeFinalized = proxy.IsUpgradeFinalized();
            }
            catch (IOException ioe)
            {
                Log.Fatal("Unable to fetch namespace information from active NN at " + otherIpcAddr
                          + ": " + ioe.Message);
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("Full exception trace", ioe);
                }
                return(ErrCodeFailedConnect);
            }
            if (!CheckLayoutVersion(nsInfo))
            {
                Log.Fatal("Layout version on remote node (" + nsInfo.GetLayoutVersion() + ") does not match "
                          + "this node's layout version (" + HdfsConstants.NamenodeLayoutVersion + ")");
                return(ErrCodeInvalidVersion);
            }
            System.Console.Out.WriteLine("=====================================================\n"
                                         + "About to bootstrap Standby ID " + nnId + " from:\n" + "           Nameservice ID: "
                                         + nsId + "\n" + "        Other Namenode ID: " + otherNNId + "\n" + "  Other NN's HTTP address: "
                                         + otherHttpAddr + "\n" + "  Other NN's IPC  address: " + otherIpcAddr + "\n" +
                                         "             Namespace ID: " + nsInfo.GetNamespaceID() + "\n" + "            Block pool ID: "
                                         + nsInfo.GetBlockPoolID() + "\n" + "               Cluster ID: " + nsInfo.GetClusterID
                                             () + "\n" + "           Layout version: " + nsInfo.GetLayoutVersion() + "\n" + "       isUpgradeFinalized: "
                                         + isUpgradeFinalized + "\n" + "====================================================="
                                         );
            NNStorage storage = new NNStorage(conf, dirsToFormat, editUrisToFormat);

            if (!isUpgradeFinalized)
            {
                // the remote NameNode is in upgrade state, this NameNode should also
                // create the previous directory. First prepare the upgrade and rename
                // the current dir to previous.tmp.
                Log.Info("The active NameNode is in Upgrade. " + "Prepare the upgrade for the standby NameNode as well."
                         );
                if (!DoPreUpgrade(storage, nsInfo))
                {
                    return(ErrCodeAlreadyFormatted);
                }
            }
            else
            {
                if (!Format(storage, nsInfo))
                {
                    // prompt the user to format storage
                    return(ErrCodeAlreadyFormatted);
                }
            }
            // download the fsimage from active namenode
            int download = DownloadImage(storage, proxy);

            if (download != 0)
            {
                return(download);
            }
            // finish the upgrade: rename previous.tmp to previous
            if (!isUpgradeFinalized)
            {
                DoUpgrade(storage);
            }
            return(0);
        }
Esempio n. 11
0
 public NamenodeProtocolServerSideTranslatorPB(NamenodeProtocol impl)
 {
     this.impl = impl;
 }
Esempio n. 12
0
        public virtual void TestGetBlocks()
        {
            Configuration Conf = new HdfsConfiguration();
            short         ReplicationFactor = (short)2;
            int           DefaultBlockSize  = 1024;
            Random        r = new Random();

            Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor
                                                                                   ).Build();

            try
            {
                cluster.WaitActive();
                // create a file with two blocks
                FileSystem         fs   = cluster.GetFileSystem();
                FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor);
                byte[]             data = new byte[1024];
                long fileLen            = 2 * DefaultBlockSize;
                long bytesToWrite       = fileLen;
                while (bytesToWrite > 0)
                {
                    r.NextBytes(data);
                    int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite;
                    @out.Write(data, 0, bytesToWriteNext);
                    bytesToWrite -= bytesToWriteNext;
                }
                @out.Close();
                // get blocks & data nodes
                IList <LocatedBlock> locatedBlocks;
                DatanodeInfo[]       dataNodes = null;
                bool notWritten;
                do
                {
                    DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf);
                    locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen)
                                    .GetLocatedBlocks();
                    NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count);
                    notWritten = false;
                    for (int i = 0; i < 2; i++)
                    {
                        dataNodes = locatedBlocks[i].GetLocations();
                        if (dataNodes.Length != ReplicationFactor)
                        {
                            notWritten = true;
                            try
                            {
                                Sharpen.Thread.Sleep(10);
                            }
                            catch (Exception)
                            {
                            }
                            break;
                        }
                    }
                }while (notWritten);
                // get RPC client to namenode
                IPEndPoint       addr     = new IPEndPoint("localhost", cluster.GetNameNodePort());
                NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode
                                                                                           .GetUri(addr)).GetProxy();
                // get blocks of size fileLen from dataNodes[0]
                BlocksWithLocations.BlockWithLocations[] locs;
                locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2);
                // get blocks of size BlockSize from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 1 from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 0 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], 0);
                // get blocks of size -1 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], -1);
                // get blocks of size BlockSize from a non-existent datanode
                DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4");
                GetBlocksWithException(namenode, info, 2);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Esempio n. 13
0
        /// <summary>Initialize SecondaryNameNode.</summary>
        /// <exception cref="System.IO.IOException"/>
        private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
                                )
        {
            IPEndPoint infoSocAddr     = GetHttpAddress(conf);
            string     infoBindAddress = infoSocAddr.GetHostName();

            UserGroupInformation.SetConfiguration(conf);
            if (UserGroupInformation.IsSecurityEnabled())
            {
                SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys
                                   .DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress);
            }
            // initiate Java VM metrics
            DefaultMetricsSystem.Initialize("SecondaryNameNode");
            JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey
                                                            ), DefaultMetricsSystem.Instance());
            // Create connection to the namenode.
            shouldRun     = true;
            nameNodeAddr  = NameNode.GetServiceAddress(conf, true);
            this.conf     = conf;
            this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr
                                                                                , UserGroupInformation.GetCurrentUser(), true).GetProxy();
            // initialize checkpoint directories
            fsName              = GetInfoServer();
            checkpointDirs      = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
            checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"
                                                                 );
            checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs
                                                                      );
            checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat());
            checkpointImage.DeleteTempEdits();
            namesystem = new FSNamesystem(conf, checkpointImage, true);
            // Disable quota checks
            namesystem.dir.DisableQuotaChecks();
            // Initialize other scheduling parameters from the configuration
            checkpointConf = new CheckpointConf(conf);
            IPEndPoint httpAddr        = infoSocAddr;
            string     httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey
                                                         , DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault);
            IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);

            HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
                                                                               , httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey
                                                                               , DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey);
            nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo"
                                                     , this);
            infoServer = builder.Build();
            infoServer.SetAttribute("secondary.name.node", this);
            infoServer.SetAttribute("name.system.image", checkpointImage);
            infoServer.SetAttribute(JspHelper.CurrentConf, conf);
            infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet
                                                                                         ), true);
            infoServer.Start();
            Log.Info("Web server init done");
            HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
            int connIdx = 0;

            if (policy.IsHttpEnabled())
            {
                IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++);
                conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString
                             (httpAddress));
            }
            if (policy.IsHttpsEnabled())
            {
                IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx);
                conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString
                             (httpsAddress));
            }
            legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey);
            Log.Info("Checkpoint Period   :" + checkpointConf.GetPeriod() + " secs " + "(" +
                     checkpointConf.GetPeriod() / 60 + " min)");
            Log.Info("Log Size Trigger    :" + checkpointConf.GetTxnCount() + " txns");
        }
Esempio n. 14
0
 internal virtual void SetNameNode(NamenodeProtocol namenode)
 {
     this.namenode = namenode;
 }