/// <summary>Requests that the NameNode download an image from this node.</summary> /// <remarks> /// Requests that the NameNode download an image from this node. Allows for /// optional external cancelation. /// </remarks> /// <param name="fsName">the http address for the remote NN</param> /// <param name="conf">Configuration</param> /// <param name="storage">the storage directory to transfer the image from</param> /// <param name="nnf">the NameNodeFile type of the image</param> /// <param name="txid">the transaction ID of the image to be uploaded</param> /// <param name="canceler">optional canceler to check for abort of upload</param> /// <exception cref="System.IO.IOException">if there is an I/O error or cancellation</exception> public static void UploadImageFromStorage(Uri fsName, Configuration conf, NNStorage storage, NNStorage.NameNodeFile nnf, long txid, Canceler canceler) { Uri url = new Uri(fsName, ImageServlet.PathSpec); long startTime = Time.MonotonicNow(); try { UploadImage(url, conf, storage, nnf, txid, canceler); } catch (TransferFsImage.HttpPutFailedException e) { if (e.GetResponseCode() == HttpServletResponse.ScConflict) { // this is OK - this means that a previous attempt to upload // this checkpoint succeeded even though we thought it failed. Log.Info("Image upload with txid " + txid + " conflicted with a previous image upload to the " + "same NameNode. Continuing...", e); return; } else { throw; } } double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001 ); Log.Info("Uploaded image with txid " + txid + " to namenode at " + fsName + " in " + xferSec + " seconds"); }
public _Callable_204(StandbyCheckpointer _enclosing, NNStorage.NameNodeFile imageType , long txid) { this._enclosing = _enclosing; this.imageType = imageType; this.txid = txid; }
internal static string GetParamStringForImage(NNStorage.NameNodeFile nnf, long txid , StorageInfo remoteStorageInfo) { string imageType = nnf == null ? string.Empty : "&" + ImageFileType + "=" + nnf.ToString (); return("getimage=1&" + TxidParam + "=" + txid + imageType + "&" + StorageinfoParam + "=" + remoteStorageInfo.ToColonSeparatedString()); }
/// <param name="request">the object from which this servlet reads the url contents</param> /// <param name="response">the object into which this servlet writes the url contents /// </param> /// <exception cref="System.IO.IOException">if the request is bad</exception> public GetImageParams(HttpServletRequest request, HttpServletResponse response) { IDictionary <string, string[]> pmap = request.GetParameterMap(); isGetImage = isGetEdit = fetchLatest = false; foreach (KeyValuePair <string, string[]> entry in pmap) { string key = entry.Key; string[] val = entry.Value; if (key.Equals("getimage")) { isGetImage = true; try { txId = ServletUtil.ParseLongParam(request, TxidParam); string imageType = ServletUtil.GetParameter(request, ImageFileType); nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf (imageType); } catch (FormatException nfe) { if (request.GetParameter(TxidParam).Equals(LatestFsimageValue)) { fetchLatest = true; } else { throw; } } } else { if (key.Equals("getedit")) { isGetEdit = true; startTxId = ServletUtil.ParseLongParam(request, StartTxidParam); endTxId = ServletUtil.ParseLongParam(request, EndTxidParam); } else { if (key.Equals(StorageinfoParam)) { storageInfoString = val[0]; } } } } int numGets = (isGetImage ? 1 : 0) + (isGetEdit ? 1 : 0); if ((numGets > 1) || (numGets == 0)) { throw new IOException("Illegal parameters to TransferFsImage"); } }
/// <exception cref="System.IO.IOException"/> internal virtual void PurgeOldStorage(NNStorage.NameNodeFile nnf) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (EnumSet.Of(nnf)); storage.InspectStorageDirs(inspector); long minImageTxId = GetImageTxIdToRetain(inspector); PurgeCheckpointsOlderThan(inspector, minImageTxId); if (nnf == NNStorage.NameNodeFile.ImageRollback) { // do not purge edits for IMAGE_ROLLBACK. return; } // If fsimage_N is the image we want to keep, then we need to keep // all txns > N. We can remove anything < N+1, since fsimage_N // reflects the state up to and including N. However, we also // provide a "cushion" of older txns that we keep, which is // handy for HA, where a remote node may not have as many // new images. // // First, determine the target number of extra transactions to retain based // on the configured amount. long minimumRequiredTxId = minImageTxId + 1; long purgeLogsFrom = Math.Max(0, minimumRequiredTxId - numExtraEditsToRetain); AList <EditLogInputStream> editLogs = new AList <EditLogInputStream>(); purgeableLogs.SelectInputStreams(editLogs, purgeLogsFrom, false); editLogs.Sort(new _IComparer_138()); // Remove from consideration any edit logs that are in fact required. while (editLogs.Count > 0 && editLogs[editLogs.Count - 1].GetFirstTxId() >= minimumRequiredTxId ) { editLogs.Remove(editLogs.Count - 1); } // Next, adjust the number of transactions to retain if doing so would mean // keeping too many segments around. while (editLogs.Count > maxExtraEditsSegmentsToRetain) { purgeLogsFrom = editLogs[0].GetLastTxId() + 1; editLogs.Remove(0); } // Finally, ensure that we're not trying to purge any transactions that we // actually need. if (purgeLogsFrom > minimumRequiredTxId) { throw new Exception("Should not purge more edits than required to " + "restore: " + purgeLogsFrom + " should be <= " + minimumRequiredTxId); } purgeableLogs.PurgeLogsOlderThan(purgeLogsFrom); }
/// <exception cref="System.IO.IOException"/> internal virtual void PurgeCheckpoinsAfter(NNStorage.NameNodeFile nnf, long fromTxId ) { FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector (EnumSet.Of(nnf)); storage.InspectStorageDirs(inspector); foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages()) { if (image.GetCheckpointTxId() > fromTxId) { purger.PurgeImage(image); } } }
/// <exception cref="System.Exception"/> public Void Run() { long txid = parsedParams.GetTxId(); NNStorage.NameNodeFile nnf = parsedParams.GetNameNodeFile(); if (!nnImage.AddToCheckpointing(txid)) { response.SendError(HttpServletResponse.ScConflict, "Either current namenode is checkpointing or another" + " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID " + txid); return(null); } try { if (nnImage.GetStorage().FindImageFile(nnf, txid) != null) { response.SendError(HttpServletResponse.ScConflict, "Either current namenode has checkpointed or " + "another checkpointer already uploaded an " + "checkpoint for txid " + txid); return(null); } InputStream stream = request.GetInputStream(); try { long start = Time.MonotonicNow(); MD5Hash downloadImageDigest = TransferFsImage.HandleUploadImageRequest(request, txid , nnImage.GetStorage(), stream, parsedParams.GetFileSize(), ImageServlet.GetThrottler (conf)); nnImage.SaveDigestAndRenameCheckpointImage(nnf, txid, downloadImageDigest); if (metrics != null) { long elapsed = Time.MonotonicNow() - start; metrics.AddPutImage(elapsed); } nnImage.PurgeOldStorage(nnf); } finally { stream.Close(); } } finally { nnImage.RemoveFromCheckpointing(txid); } return(null); }
/// <exception cref="System.IO.IOException"/> public PutImageParams(HttpServletRequest request, HttpServletResponse response, Configuration conf) { /* * Params required to handle put image request */ txId = ServletUtil.ParseLongParam(request, TxidParam); storageInfoString = ServletUtil.GetParameter(request, StorageinfoParam); fileSize = ServletUtil.ParseLongParam(request, TransferFsImage.FileLength); string imageType = ServletUtil.GetParameter(request, ImageFileType); nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf (imageType); if (fileSize == 0 || txId == -1 || storageInfoString == null || storageInfoString .IsEmpty()) { throw new IOException("Illegal parameters to TransferFsImage"); } }
/// <exception cref="System.IO.IOException"/> internal virtual void PurgeCheckpoints(NNStorage.NameNodeFile nnf) { PurgeCheckpoinsAfter(nnf, -1); }
/// <summary>Set the required parameters for uploading image</summary> /// <param name="httpMethod">instance of method to set the parameters</param> /// <param name="storage">colon separated storageInfo string</param> /// <param name="txid">txid of the image</param> /// <param name="imageFileSize">size of the imagefile to be uploaded</param> /// <param name="nnf">NameNodeFile Type</param> /// <returns>Returns map of parameters to be used with PUT request.</returns> internal static IDictionary <string, string> GetParamsForPutImage(Storage storage, long txid, long imageFileSize, NNStorage.NameNodeFile nnf) { IDictionary <string, string> @params = new Dictionary <string, string>(); @params[TxidParam] = System.Convert.ToString(txid); @params[StorageinfoParam] = storage.ToColonSeparatedString(); // setting the length of the file to be uploaded in separate property as // Content-Length only supports up to 2GB @params[TransferFsImage.FileLength] = System.Convert.ToString(imageFileSize); @params[ImageFileType] = nnf.ToString(); return(@params); }
/* * Uploads the imagefile using HTTP PUT method */ /// <exception cref="System.IO.IOException"/> private static void UploadImage(Uri url, Configuration conf, NNStorage storage, NNStorage.NameNodeFile nnf, long txId, Canceler canceler) { FilePath imageFile = storage.FindImageFile(nnf, txId); if (imageFile == null) { throw new IOException("Could not find image with txid " + txId); } HttpURLConnection connection = null; try { URIBuilder uriBuilder = new URIBuilder(url.ToURI()); // write all params for image upload request as query itself. // Request body contains the image to be uploaded. IDictionary <string, string> @params = ImageServlet.GetParamsForPutImage(storage, txId, imageFile.Length(), nnf); foreach (KeyValuePair <string, string> entry in @params) { uriBuilder.AddParameter(entry.Key, entry.Value); } Uri urlWithParams = uriBuilder.Build().ToURL(); connection = (HttpURLConnection)connectionFactory.OpenConnection(urlWithParams, UserGroupInformation .IsSecurityEnabled()); // Set the request to PUT connection.SetRequestMethod("PUT"); connection.SetDoOutput(true); int chunkSize = conf.GetInt(DFSConfigKeys.DfsImageTransferChunksizeKey, DFSConfigKeys .DfsImageTransferChunksizeDefault); if (imageFile.Length() > chunkSize) { // using chunked streaming mode to support upload of 2GB+ files and to // avoid internal buffering. // this mode should be used only if more than chunkSize data is present // to upload. otherwise upload may not happen sometimes. connection.SetChunkedStreamingMode(chunkSize); } SetTimeout(connection); // set headers for verification ImageServlet.SetVerificationHeadersForPut(connection, imageFile); // Write the file to output stream. WriteFileToPutRequest(conf, connection, imageFile, canceler); int responseCode = connection.GetResponseCode(); if (responseCode != HttpURLConnection.HttpOk) { throw new TransferFsImage.HttpPutFailedException(string.Format("Image uploading failed, status: %d, url: %s, message: %s" , responseCode, urlWithParams, connection.GetResponseMessage()), responseCode); } } catch (AuthenticationException e) { throw new IOException(e); } catch (URISyntaxException e) { throw new IOException(e); } finally { if (connection != null) { connection.Disconnect(); } } }
/// <summary>Requests that the NameNode download an image from this node.</summary> /// <param name="fsName">the http address for the remote NN</param> /// <param name="conf">Configuration</param> /// <param name="storage">the storage directory to transfer the image from</param> /// <param name="nnf">the NameNodeFile type of the image</param> /// <param name="txid">the transaction ID of the image to be uploaded</param> /// <exception cref="System.IO.IOException">if there is an I/O error</exception> public static void UploadImageFromStorage(Uri fsName, Configuration conf, NNStorage storage, NNStorage.NameNodeFile nnf, long txid) { UploadImageFromStorage(fsName, conf, storage, nnf, txid, null); }