Esempio n. 1
0
        /// <exception cref="System.IO.IOException"/>
        internal static void DoMerge(CheckpointSignature sig, RemoteEditLogManifest manifest
                                     , bool loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
        {
            NNStorage dstStorage = dstImage.GetStorage();

            dstStorage.SetStorageInfo(sig);
            if (loadImage)
            {
                FilePath file = dstStorage.FindImageFile(NNStorage.NameNodeFile.Image, sig.mostRecentCheckpointTxId
                                                         );
                if (file == null)
                {
                    throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId
                                          + " even though it should have " + "just been downloaded");
                }
                dstNamesystem.WriteLock();
                try
                {
                    dstImage.ReloadFromImageFile(file, dstNamesystem);
                }
                finally
                {
                    dstNamesystem.WriteUnlock();
                }
                dstNamesystem.ImageLoadComplete();
            }
            // error simulation code for junit test
            CheckpointFaultInjector.GetInstance().DuringMerge();
            Checkpointer.RollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
            // The following has the side effect of purging old fsimages/edit logs.
            dstImage.SaveFSImageInAllDirs(dstNamesystem, dstImage.GetLastAppliedTxId());
            dstStorage.WriteAll();
        }
Esempio n. 2
0
        /// <summary>Test to verify the timeout of Image upload</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestImageUploadTimeout()
        {
            Configuration conf        = new HdfsConfiguration();
            NNStorage     mockStorage = Org.Mockito.Mockito.Mock <NNStorage>();
            HttpServer2   testServer  = HttpServerFunctionalTest.CreateServer("hdfs");

            try
            {
                testServer.AddServlet("ImageTransfer", ImageServlet.PathSpec, typeof(TestTransferFsImage.TestImageTransferServlet
                                                                                     ));
                testServer.Start();
                Uri serverURL = HttpServerFunctionalTest.GetServerURL(testServer);
                // set the timeout here, otherwise it will take default.
                TransferFsImage.timeout = 2000;
                FilePath tmpDir = new FilePath(new FileSystemTestHelper().GetTestRootDir());
                tmpDir.Mkdirs();
                FilePath         mockImageFile = FilePath.CreateTempFile("image", string.Empty, tmpDir);
                FileOutputStream imageFile     = new FileOutputStream(mockImageFile);
                imageFile.Write(Sharpen.Runtime.GetBytesForString("data"));
                imageFile.Close();
                Org.Mockito.Mockito.When(mockStorage.FindImageFile(Org.Mockito.Mockito.Any <NNStorage.NameNodeFile
                                                                                            >(), Org.Mockito.Mockito.AnyLong())).ThenReturn(mockImageFile);
                Org.Mockito.Mockito.When(mockStorage.ToColonSeparatedString()).ThenReturn("storage:info:string"
                                                                                          );
                try
                {
                    TransferFsImage.UploadImageFromStorage(serverURL, conf, mockStorage, NNStorage.NameNodeFile
                                                           .Image, 1L);
                    NUnit.Framework.Assert.Fail("TransferImage Should fail with timeout");
                }
                catch (SocketTimeoutException e)
                {
                    NUnit.Framework.Assert.AreEqual("Upload should timeout", "Read timed out", e.Message
                                                    );
                }
            }
            finally
            {
                testServer.Stop();
            }
        }
Esempio n. 3
0
        /// <summary>Create a new checkpoint</summary>
        /// <exception cref="System.IO.IOException"/>
        internal virtual void DoCheckpoint()
        {
            BackupImage bnImage   = GetFSImage();
            NNStorage   bnStorage = bnImage.GetStorage();
            long        startTime = Time.MonotonicNow();

            bnImage.FreezeNamespaceAtNextRoll();
            NamenodeCommand cmd = GetRemoteNamenodeProxy().StartCheckpoint(backupNode.GetRegistration
                                                                               ());
            CheckpointCommand cpCmd = null;

            switch (cmd.GetAction())
            {
            case NamenodeProtocol.ActShutdown:
            {
                Shutdown();
                throw new IOException("Name-node " + backupNode.nnRpcAddress + " requested shutdown."
                                      );
            }

            case NamenodeProtocol.ActCheckpoint:
            {
                cpCmd = (CheckpointCommand)cmd;
                break;
            }

            default:
            {
                throw new IOException("Unsupported NamenodeCommand: " + cmd.GetAction());
            }
            }
            bnImage.WaitUntilNamespaceFrozen();
            CheckpointSignature sig = cpCmd.GetSignature();

            // Make sure we're talking to the same NN!
            sig.ValidateStorageInfo(bnImage);
            long lastApplied = bnImage.GetLastAppliedTxId();

            Log.Debug("Doing checkpoint. Last applied: " + lastApplied);
            RemoteEditLogManifest manifest = GetRemoteNamenodeProxy().GetEditLogManifest(bnImage
                                                                                         .GetLastAppliedTxId() + 1);
            bool needReloadImage = false;

            if (!manifest.GetLogs().IsEmpty())
            {
                RemoteEditLog firstRemoteLog = manifest.GetLogs()[0];
                // we don't have enough logs to roll forward using only logs. Need
                // to download and load the image.
                if (firstRemoteLog.GetStartTxId() > lastApplied + 1)
                {
                    Log.Info("Unable to roll forward using only logs. Downloading " + "image with txid "
                             + sig.mostRecentCheckpointTxId);
                    MD5Hash downloadedHash = TransferFsImage.DownloadImageToStorage(backupNode.nnHttpAddress
                                                                                    , sig.mostRecentCheckpointTxId, bnStorage, true);
                    bnImage.SaveDigestAndRenameCheckpointImage(NNStorage.NameNodeFile.Image, sig.mostRecentCheckpointTxId
                                                               , downloadedHash);
                    lastApplied     = sig.mostRecentCheckpointTxId;
                    needReloadImage = true;
                }
                if (firstRemoteLog.GetStartTxId() > lastApplied + 1)
                {
                    throw new IOException("No logs to roll forward from " + lastApplied);
                }
                // get edits files
                foreach (RemoteEditLog log in manifest.GetLogs())
                {
                    TransferFsImage.DownloadEditsToStorage(backupNode.nnHttpAddress, log, bnStorage);
                }
                if (needReloadImage)
                {
                    Log.Info("Loading image with txid " + sig.mostRecentCheckpointTxId);
                    FilePath file = bnStorage.FindImageFile(NNStorage.NameNodeFile.Image, sig.mostRecentCheckpointTxId
                                                            );
                    bnImage.ReloadFromImageFile(file, backupNode.GetNamesystem());
                }
                RollForwardByApplyingLogs(manifest, bnImage, backupNode.GetNamesystem());
            }
            long txid = bnImage.GetLastAppliedTxId();

            backupNode.namesystem.WriteLock();
            try
            {
                backupNode.namesystem.SetImageLoaded();
                if (backupNode.namesystem.GetBlocksTotal() > 0)
                {
                    backupNode.namesystem.SetBlockTotal();
                }
                bnImage.SaveFSImageInAllDirs(backupNode.GetNamesystem(), txid);
                bnStorage.WriteAll();
            }
            finally
            {
                backupNode.namesystem.WriteUnlock();
            }
            if (cpCmd.NeedToReturnImage())
            {
                TransferFsImage.UploadImageFromStorage(backupNode.nnHttpAddress, conf, bnStorage,
                                                       NNStorage.NameNodeFile.Image, txid);
            }
            GetRemoteNamenodeProxy().EndCheckpoint(backupNode.GetRegistration(), sig);
            if (backupNode.GetRole() == HdfsServerConstants.NamenodeRole.Backup)
            {
                bnImage.ConvergeJournalSpool();
            }
            backupNode.SetRegistration();
            // keep registration up to date
            long imageSize = bnImage.GetStorage().GetFsImageName(txid).Length();

            Log.Info("Checkpoint completed in " + (Time.MonotonicNow() - startTime) / 1000 +
                     " seconds." + " New Image Size: " + imageSize);
        }
Esempio n. 4
0
        /*
         * Uploads the imagefile using HTTP PUT method
         */
        /// <exception cref="System.IO.IOException"/>
        private static void UploadImage(Uri url, Configuration conf, NNStorage storage, NNStorage.NameNodeFile
                                        nnf, long txId, Canceler canceler)
        {
            FilePath imageFile = storage.FindImageFile(nnf, txId);

            if (imageFile == null)
            {
                throw new IOException("Could not find image with txid " + txId);
            }
            HttpURLConnection connection = null;

            try
            {
                URIBuilder uriBuilder = new URIBuilder(url.ToURI());
                // write all params for image upload request as query itself.
                // Request body contains the image to be uploaded.
                IDictionary <string, string> @params = ImageServlet.GetParamsForPutImage(storage,
                                                                                         txId, imageFile.Length(), nnf);
                foreach (KeyValuePair <string, string> entry in @params)
                {
                    uriBuilder.AddParameter(entry.Key, entry.Value);
                }
                Uri urlWithParams = uriBuilder.Build().ToURL();
                connection = (HttpURLConnection)connectionFactory.OpenConnection(urlWithParams, UserGroupInformation
                                                                                 .IsSecurityEnabled());
                // Set the request to PUT
                connection.SetRequestMethod("PUT");
                connection.SetDoOutput(true);
                int chunkSize = conf.GetInt(DFSConfigKeys.DfsImageTransferChunksizeKey, DFSConfigKeys
                                            .DfsImageTransferChunksizeDefault);
                if (imageFile.Length() > chunkSize)
                {
                    // using chunked streaming mode to support upload of 2GB+ files and to
                    // avoid internal buffering.
                    // this mode should be used only if more than chunkSize data is present
                    // to upload. otherwise upload may not happen sometimes.
                    connection.SetChunkedStreamingMode(chunkSize);
                }
                SetTimeout(connection);
                // set headers for verification
                ImageServlet.SetVerificationHeadersForPut(connection, imageFile);
                // Write the file to output stream.
                WriteFileToPutRequest(conf, connection, imageFile, canceler);
                int responseCode = connection.GetResponseCode();
                if (responseCode != HttpURLConnection.HttpOk)
                {
                    throw new TransferFsImage.HttpPutFailedException(string.Format("Image uploading failed, status: %d, url: %s, message: %s"
                                                                                   , responseCode, urlWithParams, connection.GetResponseMessage()), responseCode);
                }
            }
            catch (AuthenticationException e)
            {
                throw new IOException(e);
            }
            catch (URISyntaxException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (connection != null)
                {
                    connection.Disconnect();
                }
            }
        }