/// <summary>Requests that the NameNode download an image from this node.</summary>
/// <remarks>
/// Requests that the NameNode download an image from this node. Allows for
/// optional external cancelation.
/// </remarks>
/// <param name="fsName">the http address for the remote NN</param>
/// <param name="conf">Configuration</param>
/// <param name="storage">the storage directory to transfer the image from</param>
/// <param name="nnf">the NameNodeFile type of the image</param>
/// <param name="txid">the transaction ID of the image to be uploaded</param>
/// <param name="canceler">optional canceler to check for abort of upload</param>
/// <exception cref="System.IO.IOException">if there is an I/O error or cancellation</exception>
public static void UploadImageFromStorage(Uri fsName, Configuration conf, NNStorage
storage, NNStorage.NameNodeFile nnf, long txid, Canceler canceler)
{
Uri url = new Uri(fsName, ImageServlet.PathSpec);
long startTime = Time.MonotonicNow();
try
{
UploadImage(url, conf, storage, nnf, txid, canceler);
}
catch (TransferFsImage.HttpPutFailedException e)
{
if (e.GetResponseCode() == HttpServletResponse.ScConflict)
{
// this is OK - this means that a previous attempt to upload
// this checkpoint succeeded even though we thought it failed.
Log.Info("Image upload with txid " + txid + " conflicted with a previous image upload to the "
+ "same NameNode. Continuing...", e);
return;
}
else
{
throw;
}
}
double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001
);
Log.Info("Uploaded image with txid " + txid + " to namenode at " + fsName + " in "
+ xferSec + " seconds");
}
public _Callable_204(StandbyCheckpointer _enclosing, NNStorage.NameNodeFile imageType
, long txid)
{
this._enclosing = _enclosing;
this.imageType = imageType;
this.txid = txid;
}
internal static string GetParamStringForImage(NNStorage.NameNodeFile nnf, long txid
, StorageInfo remoteStorageInfo)
{
string imageType = nnf == null ? string.Empty : "&" + ImageFileType + "=" + nnf.ToString
();
return("getimage=1&" + TxidParam + "=" + txid + imageType + "&" + StorageinfoParam
+ "=" + remoteStorageInfo.ToColonSeparatedString());
}
/// <param name="request">the object from which this servlet reads the url contents</param>
/// <param name="response">the object into which this servlet writes the url contents
/// </param>
/// <exception cref="System.IO.IOException">if the request is bad</exception>
public GetImageParams(HttpServletRequest request, HttpServletResponse response)
{
IDictionary <string, string[]> pmap = request.GetParameterMap();
isGetImage = isGetEdit = fetchLatest = false;
foreach (KeyValuePair <string, string[]> entry in pmap)
{
string key = entry.Key;
string[] val = entry.Value;
if (key.Equals("getimage"))
{
isGetImage = true;
try
{
txId = ServletUtil.ParseLongParam(request, TxidParam);
string imageType = ServletUtil.GetParameter(request, ImageFileType);
nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf
(imageType);
}
catch (FormatException nfe)
{
if (request.GetParameter(TxidParam).Equals(LatestFsimageValue))
{
fetchLatest = true;
}
else
{
throw;
}
}
}
else
{
if (key.Equals("getedit"))
{
isGetEdit = true;
startTxId = ServletUtil.ParseLongParam(request, StartTxidParam);
endTxId = ServletUtil.ParseLongParam(request, EndTxidParam);
}
else
{
if (key.Equals(StorageinfoParam))
{
storageInfoString = val[0];
}
}
}
}
int numGets = (isGetImage ? 1 : 0) + (isGetEdit ? 1 : 0);
if ((numGets > 1) || (numGets == 0))
{
throw new IOException("Illegal parameters to TransferFsImage");
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void PurgeOldStorage(NNStorage.NameNodeFile nnf)
{
FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
(EnumSet.Of(nnf));
storage.InspectStorageDirs(inspector);
long minImageTxId = GetImageTxIdToRetain(inspector);
PurgeCheckpointsOlderThan(inspector, minImageTxId);
if (nnf == NNStorage.NameNodeFile.ImageRollback)
{
// do not purge edits for IMAGE_ROLLBACK.
return;
}
// If fsimage_N is the image we want to keep, then we need to keep
// all txns > N. We can remove anything < N+1, since fsimage_N
// reflects the state up to and including N. However, we also
// provide a "cushion" of older txns that we keep, which is
// handy for HA, where a remote node may not have as many
// new images.
//
// First, determine the target number of extra transactions to retain based
// on the configured amount.
long minimumRequiredTxId = minImageTxId + 1;
long purgeLogsFrom = Math.Max(0, minimumRequiredTxId - numExtraEditsToRetain);
AList <EditLogInputStream> editLogs = new AList <EditLogInputStream>();
purgeableLogs.SelectInputStreams(editLogs, purgeLogsFrom, false);
editLogs.Sort(new _IComparer_138());
// Remove from consideration any edit logs that are in fact required.
while (editLogs.Count > 0 && editLogs[editLogs.Count - 1].GetFirstTxId() >= minimumRequiredTxId
)
{
editLogs.Remove(editLogs.Count - 1);
}
// Next, adjust the number of transactions to retain if doing so would mean
// keeping too many segments around.
while (editLogs.Count > maxExtraEditsSegmentsToRetain)
{
purgeLogsFrom = editLogs[0].GetLastTxId() + 1;
editLogs.Remove(0);
}
// Finally, ensure that we're not trying to purge any transactions that we
// actually need.
if (purgeLogsFrom > minimumRequiredTxId)
{
throw new Exception("Should not purge more edits than required to " + "restore: "
+ purgeLogsFrom + " should be <= " + minimumRequiredTxId);
}
purgeableLogs.PurgeLogsOlderThan(purgeLogsFrom);
}
/// <exception cref="System.IO.IOException"/>
internal virtual void PurgeCheckpoinsAfter(NNStorage.NameNodeFile nnf, long fromTxId
)
{
FSImageTransactionalStorageInspector inspector = new FSImageTransactionalStorageInspector
(EnumSet.Of(nnf));
storage.InspectStorageDirs(inspector);
foreach (FSImageStorageInspector.FSImageFile image in inspector.GetFoundImages())
{
if (image.GetCheckpointTxId() > fromTxId)
{
purger.PurgeImage(image);
}
}
}
/// <exception cref="System.Exception"/>
public Void Run()
{
long txid = parsedParams.GetTxId();
NNStorage.NameNodeFile nnf = parsedParams.GetNameNodeFile();
if (!nnImage.AddToCheckpointing(txid))
{
response.SendError(HttpServletResponse.ScConflict, "Either current namenode is checkpointing or another"
+ " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID "
+ txid);
return(null);
}
try
{
if (nnImage.GetStorage().FindImageFile(nnf, txid) != null)
{
response.SendError(HttpServletResponse.ScConflict, "Either current namenode has checkpointed or "
+ "another checkpointer already uploaded an " + "checkpoint for txid " + txid);
return(null);
}
InputStream stream = request.GetInputStream();
try
{
long start = Time.MonotonicNow();
MD5Hash downloadImageDigest = TransferFsImage.HandleUploadImageRequest(request, txid
, nnImage.GetStorage(), stream, parsedParams.GetFileSize(), ImageServlet.GetThrottler
(conf));
nnImage.SaveDigestAndRenameCheckpointImage(nnf, txid, downloadImageDigest);
if (metrics != null)
{
long elapsed = Time.MonotonicNow() - start;
metrics.AddPutImage(elapsed);
}
nnImage.PurgeOldStorage(nnf);
}
finally
{
stream.Close();
}
}
finally
{
nnImage.RemoveFromCheckpointing(txid);
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
public PutImageParams(HttpServletRequest request, HttpServletResponse response, Configuration
conf)
{
/*
* Params required to handle put image request
*/
txId = ServletUtil.ParseLongParam(request, TxidParam);
storageInfoString = ServletUtil.GetParameter(request, StorageinfoParam);
fileSize = ServletUtil.ParseLongParam(request, TransferFsImage.FileLength);
string imageType = ServletUtil.GetParameter(request, ImageFileType);
nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf
(imageType);
if (fileSize == 0 || txId == -1 || storageInfoString == null || storageInfoString
.IsEmpty())
{
throw new IOException("Illegal parameters to TransferFsImage");
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void PurgeCheckpoints(NNStorage.NameNodeFile nnf)
{
PurgeCheckpoinsAfter(nnf, -1);
}
/// <summary>Set the required parameters for uploading image</summary>
/// <param name="httpMethod">instance of method to set the parameters</param>
/// <param name="storage">colon separated storageInfo string</param>
/// <param name="txid">txid of the image</param>
/// <param name="imageFileSize">size of the imagefile to be uploaded</param>
/// <param name="nnf">NameNodeFile Type</param>
/// <returns>Returns map of parameters to be used with PUT request.</returns>
internal static IDictionary <string, string> GetParamsForPutImage(Storage storage,
long txid, long imageFileSize, NNStorage.NameNodeFile nnf)
{
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[TxidParam] = System.Convert.ToString(txid);
@params[StorageinfoParam] = storage.ToColonSeparatedString();
// setting the length of the file to be uploaded in separate property as
// Content-Length only supports up to 2GB
@params[TransferFsImage.FileLength] = System.Convert.ToString(imageFileSize);
@params[ImageFileType] = nnf.ToString();
return(@params);
}
/*
* Uploads the imagefile using HTTP PUT method
*/
/// <exception cref="System.IO.IOException"/>
private static void UploadImage(Uri url, Configuration conf, NNStorage storage, NNStorage.NameNodeFile
nnf, long txId, Canceler canceler)
{
FilePath imageFile = storage.FindImageFile(nnf, txId);
if (imageFile == null)
{
throw new IOException("Could not find image with txid " + txId);
}
HttpURLConnection connection = null;
try
{
URIBuilder uriBuilder = new URIBuilder(url.ToURI());
// write all params for image upload request as query itself.
// Request body contains the image to be uploaded.
IDictionary <string, string> @params = ImageServlet.GetParamsForPutImage(storage,
txId, imageFile.Length(), nnf);
foreach (KeyValuePair <string, string> entry in @params)
{
uriBuilder.AddParameter(entry.Key, entry.Value);
}
Uri urlWithParams = uriBuilder.Build().ToURL();
connection = (HttpURLConnection)connectionFactory.OpenConnection(urlWithParams, UserGroupInformation
.IsSecurityEnabled());
// Set the request to PUT
connection.SetRequestMethod("PUT");
connection.SetDoOutput(true);
int chunkSize = conf.GetInt(DFSConfigKeys.DfsImageTransferChunksizeKey, DFSConfigKeys
.DfsImageTransferChunksizeDefault);
if (imageFile.Length() > chunkSize)
{
// using chunked streaming mode to support upload of 2GB+ files and to
// avoid internal buffering.
// this mode should be used only if more than chunkSize data is present
// to upload. otherwise upload may not happen sometimes.
connection.SetChunkedStreamingMode(chunkSize);
}
SetTimeout(connection);
// set headers for verification
ImageServlet.SetVerificationHeadersForPut(connection, imageFile);
// Write the file to output stream.
WriteFileToPutRequest(conf, connection, imageFile, canceler);
int responseCode = connection.GetResponseCode();
if (responseCode != HttpURLConnection.HttpOk)
{
throw new TransferFsImage.HttpPutFailedException(string.Format("Image uploading failed, status: %d, url: %s, message: %s"
, responseCode, urlWithParams, connection.GetResponseMessage()), responseCode);
}
}
catch (AuthenticationException e)
{
throw new IOException(e);
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
finally
{
if (connection != null)
{
connection.Disconnect();
}
}
}
/// <summary>Requests that the NameNode download an image from this node.</summary>
/// <param name="fsName">the http address for the remote NN</param>
/// <param name="conf">Configuration</param>
/// <param name="storage">the storage directory to transfer the image from</param>
/// <param name="nnf">the NameNodeFile type of the image</param>
/// <param name="txid">the transaction ID of the image to be uploaded</param>
/// <exception cref="System.IO.IOException">if there is an I/O error</exception>
public static void UploadImageFromStorage(Uri fsName, Configuration conf, NNStorage
storage, NNStorage.NameNodeFile nnf, long txid)
{
UploadImageFromStorage(fsName, conf, storage, nnf, txid, null);
}
