/// <exception cref="System.IO.IOException"/> public static void DownloadMostRecentImageToDirectory(Uri infoServer, FilePath dir ) { string fileId = ImageServlet.GetParamStringForMostRecentImage(); GetFileClient(infoServer, fileId, Lists.NewArrayList(dir), null, false); }
/// <exception cref="System.IO.IOException"/> internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage dstStorage) { System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: " + log); string fileid = ImageServlet.GetParamStringForLog(log, dstStorage); string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log .GetEndTxId()); IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, finalFileName); System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets."); foreach (FilePath f in finalFiles) { if (f.Exists() && FileUtil.CanRead(f)) { Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at " + f); return; } else { if (Log.IsDebugEnabled()) { Log.Debug("Dest file: " + f); } } } long milliTime = Time.MonotonicNow(); string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log. GetEndTxId(), milliTime); IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName ); GetFileClient(fsName, fileid, tmpFiles, dstStorage, false); Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length () + " bytes."); CheckpointFaultInjector.GetInstance().BeforeEditsRename(); foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType .Edits)) { FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId (), milliTime); FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(), log.GetEndTxId()); if (Log.IsDebugEnabled()) { Log.Debug("Renaming " + tmpFile + " to " + finalizedFile); } bool success = tmpFile.RenameTo(finalizedFile); if (!success) { Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile); } } }
public virtual void TestIsValidRequestor() { Configuration conf = new HdfsConfiguration(); KerberosName.SetRules("RULE:[1:$1]\nRULE:[2:$1]"); // Set up generic HA configs. conf.Set(DFSConfigKeys.DfsNameservices, "ns1"); conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2" ); // Set up NN1 HA configs. conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1" ), "host1:1234"); conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1" , "nn1"), "hdfs/[email protected]"); // Set up NN2 HA configs. conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2" ), "host2:1234"); conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1" , "nn2"), "hdfs/[email protected]"); // Initialize this conf object as though we're running on NN1. NameNode.InitializeGenericKeys(conf, "ns1", "nn1"); AccessControlList acls = Org.Mockito.Mockito.Mock <AccessControlList>(); Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.Any <UserGroupInformation >())).ThenReturn(false); ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>(); Org.Mockito.Mockito.When(context.GetAttribute(HttpServer2.AdminsAcl)).ThenReturn( acls); // Make sure that NN2 is considered a valid fsimage/edits requestor. NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]" , conf)); // Mark atm as an admin. Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.ArgThat(new _ArgumentMatcher_76 ()))).ThenReturn(true); // Make sure that NN2 is still considered a valid requestor. NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]" , conf)); // Make sure an admin is considered a valid requestor. NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "*****@*****.**" , conf)); // Make sure other users are *not* considered valid requestors. NUnit.Framework.Assert.IsFalse(ImageServlet.IsValidRequestor(context, "*****@*****.**" , conf)); }
/// <exception cref="System.IO.IOException"/> public static MD5Hash DownloadImageToStorage(Uri fsName, long imageTxId, Storage dstStorage, bool needDigest) { string fileid = ImageServlet.GetParamStringForImage(null, imageTxId, dstStorage); string fileName = NNStorage.GetCheckpointImageFileName(imageTxId); IList <FilePath> dstFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Image, fileName ); if (dstFiles.IsEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash hash = GetFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); Log.Info("Downloaded file " + dstFiles[0].GetName() + " size " + dstFiles[0].Length () + " bytes."); return(hash); }
/// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="System.IO.IOException"/> private static void WriteFileToPutRequest(Configuration conf, HttpURLConnection connection , FilePath imageFile, Canceler canceler) { connection.SetRequestProperty(ContentType, "application/octet-stream"); connection.SetRequestProperty(ContentTransferEncoding, "binary"); OutputStream output = connection.GetOutputStream(); FileInputStream input = new FileInputStream(imageFile); try { CopyFileToStream(output, imageFile, input, ImageServlet.GetThrottler(conf), canceler ); } finally { IOUtils.CloseStream(input); IOUtils.CloseStream(output); } }
/// <exception cref="System.IO.IOException"/> private void ServeFile(FilePath file) { FileInputStream fis = new FileInputStream(file); try { ImageServlet.SetVerificationHeadersForGet(response, file); ImageServlet.SetFileNameHeaders(response, file); if (!file.Exists()) { throw new FileNotFoundException(file.ToString()); } TransferFsImage.CopyFileToStream(response.GetOutputStream(), file, fis, ImageServlet .GetThrottler(conf)); } finally { IOUtils.CloseStream(fis); } }
/// <exception cref="System.Exception"/> public Void Run() { long txid = parsedParams.GetTxId(); NNStorage.NameNodeFile nnf = parsedParams.GetNameNodeFile(); if (!nnImage.AddToCheckpointing(txid)) { response.SendError(HttpServletResponse.ScConflict, "Either current namenode is checkpointing or another" + " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID " + txid); return(null); } try { if (nnImage.GetStorage().FindImageFile(nnf, txid) != null) { response.SendError(HttpServletResponse.ScConflict, "Either current namenode has checkpointed or " + "another checkpointer already uploaded an " + "checkpoint for txid " + txid); return(null); } InputStream stream = request.GetInputStream(); try { long start = Time.MonotonicNow(); MD5Hash downloadImageDigest = TransferFsImage.HandleUploadImageRequest(request, txid , nnImage.GetStorage(), stream, parsedParams.GetFileSize(), ImageServlet.GetThrottler (conf)); nnImage.SaveDigestAndRenameCheckpointImage(nnf, txid, downloadImageDigest); if (metrics != null) { long elapsed = Time.MonotonicNow() - start; metrics.AddPutImage(elapsed); } nnImage.PurgeOldStorage(nnf); } finally { stream.Close(); } } finally { nnImage.RemoveFromCheckpointing(txid); } return(null); }
/* * Uploads the imagefile using HTTP PUT method */ /// <exception cref="System.IO.IOException"/> private static void UploadImage(Uri url, Configuration conf, NNStorage storage, NNStorage.NameNodeFile nnf, long txId, Canceler canceler) { FilePath imageFile = storage.FindImageFile(nnf, txId); if (imageFile == null) { throw new IOException("Could not find image with txid " + txId); } HttpURLConnection connection = null; try { URIBuilder uriBuilder = new URIBuilder(url.ToURI()); // write all params for image upload request as query itself. // Request body contains the image to be uploaded. IDictionary <string, string> @params = ImageServlet.GetParamsForPutImage(storage, txId, imageFile.Length(), nnf); foreach (KeyValuePair <string, string> entry in @params) { uriBuilder.AddParameter(entry.Key, entry.Value); } Uri urlWithParams = uriBuilder.Build().ToURL(); connection = (HttpURLConnection)connectionFactory.OpenConnection(urlWithParams, UserGroupInformation .IsSecurityEnabled()); // Set the request to PUT connection.SetRequestMethod("PUT"); connection.SetDoOutput(true); int chunkSize = conf.GetInt(DFSConfigKeys.DfsImageTransferChunksizeKey, DFSConfigKeys .DfsImageTransferChunksizeDefault); if (imageFile.Length() > chunkSize) { // using chunked streaming mode to support upload of 2GB+ files and to // avoid internal buffering. // this mode should be used only if more than chunkSize data is present // to upload. otherwise upload may not happen sometimes. connection.SetChunkedStreamingMode(chunkSize); } SetTimeout(connection); // set headers for verification ImageServlet.SetVerificationHeadersForPut(connection, imageFile); // Write the file to output stream. WriteFileToPutRequest(conf, connection, imageFile, canceler); int responseCode = connection.GetResponseCode(); if (responseCode != HttpURLConnection.HttpOk) { throw new TransferFsImage.HttpPutFailedException(string.Format("Image uploading failed, status: %d, url: %s, message: %s" , responseCode, urlWithParams, connection.GetResponseMessage()), responseCode); } } catch (AuthenticationException e) { throw new IOException(e); } catch (URISyntaxException e) { throw new IOException(e); } finally { if (connection != null) { connection.Disconnect(); } } }