/// <exception cref="System.IO.IOException"/>
public static void DownloadMostRecentImageToDirectory(Uri infoServer, FilePath dir
)
{
string fileId = ImageServlet.GetParamStringForMostRecentImage();
GetFileClient(infoServer, fileId, Lists.NewArrayList(dir), null, false);
}
/// <exception cref="System.IO.IOException"/>
internal static void DownloadEditsToStorage(Uri fsName, RemoteEditLog log, NNStorage
dstStorage)
{
System.Diagnostics.Debug.Assert(log.GetStartTxId() > 0 && log.GetEndTxId() > 0, "bad log: "
+ log);
string fileid = ImageServlet.GetParamStringForLog(log, dstStorage);
string finalFileName = NNStorage.GetFinalizedEditsFileName(log.GetStartTxId(), log
.GetEndTxId());
IList <FilePath> finalFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits,
finalFileName);
System.Diagnostics.Debug.Assert(!finalFiles.IsEmpty(), "No checkpoint targets.");
foreach (FilePath f in finalFiles)
{
if (f.Exists() && FileUtil.CanRead(f))
{
Log.Info("Skipping download of remote edit log " + log + " since it already is stored locally at "
+ f);
return;
}
else
{
if (Log.IsDebugEnabled())
{
Log.Debug("Dest file: " + f);
}
}
}
long milliTime = Time.MonotonicNow();
string tmpFileName = NNStorage.GetTemporaryEditsFileName(log.GetStartTxId(), log.
GetEndTxId(), milliTime);
IList <FilePath> tmpFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Edits, tmpFileName
);
GetFileClient(fsName, fileid, tmpFiles, dstStorage, false);
Log.Info("Downloaded file " + tmpFiles[0].GetName() + " size " + finalFiles[0].Length
() + " bytes.");
CheckpointFaultInjector.GetInstance().BeforeEditsRename();
foreach (Storage.StorageDirectory sd in dstStorage.DirIterable(NNStorage.NameNodeDirType
.Edits))
{
FilePath tmpFile = NNStorage.GetTemporaryEditsFile(sd, log.GetStartTxId(), log.GetEndTxId
(), milliTime);
FilePath finalizedFile = NNStorage.GetFinalizedEditsFile(sd, log.GetStartTxId(),
log.GetEndTxId());
if (Log.IsDebugEnabled())
{
Log.Debug("Renaming " + tmpFile + " to " + finalizedFile);
}
bool success = tmpFile.RenameTo(finalizedFile);
if (!success)
{
Log.Warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile);
}
}
}
public virtual void TestIsValidRequestor()
{
Configuration conf = new HdfsConfiguration();
KerberosName.SetRules("RULE:[1:$1]\nRULE:[2:$1]");
// Set up generic HA configs.
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
// Set up NN1 HA configs.
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), "host1:1234");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
, "nn1"), "hdfs/[email protected]");
// Set up NN2 HA configs.
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
), "host2:1234");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
, "nn2"), "hdfs/[email protected]");
// Initialize this conf object as though we're running on NN1.
NameNode.InitializeGenericKeys(conf, "ns1", "nn1");
AccessControlList acls = Org.Mockito.Mockito.Mock <AccessControlList>();
Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.Any <UserGroupInformation
>())).ThenReturn(false);
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
Org.Mockito.Mockito.When(context.GetAttribute(HttpServer2.AdminsAcl)).ThenReturn(
acls);
// Make sure that NN2 is considered a valid fsimage/edits requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
, conf));
// Mark atm as an admin.
Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.ArgThat(new _ArgumentMatcher_76
()))).ThenReturn(true);
// Make sure that NN2 is still considered a valid requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
, conf));
// Make sure an admin is considered a valid requestor.
NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "*****@*****.**"
, conf));
// Make sure other users are *not* considered valid requestors.
NUnit.Framework.Assert.IsFalse(ImageServlet.IsValidRequestor(context, "*****@*****.**"
, conf));
}
/// <exception cref="System.IO.IOException"/>
public static MD5Hash DownloadImageToStorage(Uri fsName, long imageTxId, Storage
dstStorage, bool needDigest)
{
string fileid = ImageServlet.GetParamStringForImage(null, imageTxId, dstStorage);
string fileName = NNStorage.GetCheckpointImageFileName(imageTxId);
IList <FilePath> dstFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Image, fileName
);
if (dstFiles.IsEmpty())
{
throw new IOException("No targets in destination storage!");
}
MD5Hash hash = GetFileClient(fsName, fileid, dstFiles, dstStorage, needDigest);
Log.Info("Downloaded file " + dstFiles[0].GetName() + " size " + dstFiles[0].Length
() + " bytes.");
return(hash);
}
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="System.IO.IOException"/>
private static void WriteFileToPutRequest(Configuration conf, HttpURLConnection connection
, FilePath imageFile, Canceler canceler)
{
connection.SetRequestProperty(ContentType, "application/octet-stream");
connection.SetRequestProperty(ContentTransferEncoding, "binary");
OutputStream output = connection.GetOutputStream();
FileInputStream input = new FileInputStream(imageFile);
try
{
CopyFileToStream(output, imageFile, input, ImageServlet.GetThrottler(conf), canceler
);
}
finally
{
IOUtils.CloseStream(input);
IOUtils.CloseStream(output);
}
}
/// <exception cref="System.IO.IOException"/>
private void ServeFile(FilePath file)
{
FileInputStream fis = new FileInputStream(file);
try
{
ImageServlet.SetVerificationHeadersForGet(response, file);
ImageServlet.SetFileNameHeaders(response, file);
if (!file.Exists())
{
throw new FileNotFoundException(file.ToString());
}
TransferFsImage.CopyFileToStream(response.GetOutputStream(), file, fis, ImageServlet
.GetThrottler(conf));
}
finally
{
IOUtils.CloseStream(fis);
}
}
/// <exception cref="System.Exception"/>
public Void Run()
{
long txid = parsedParams.GetTxId();
NNStorage.NameNodeFile nnf = parsedParams.GetNameNodeFile();
if (!nnImage.AddToCheckpointing(txid))
{
response.SendError(HttpServletResponse.ScConflict, "Either current namenode is checkpointing or another"
+ " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID "
+ txid);
return(null);
}
try
{
if (nnImage.GetStorage().FindImageFile(nnf, txid) != null)
{
response.SendError(HttpServletResponse.ScConflict, "Either current namenode has checkpointed or "
+ "another checkpointer already uploaded an " + "checkpoint for txid " + txid);
return(null);
}
InputStream stream = request.GetInputStream();
try
{
long start = Time.MonotonicNow();
MD5Hash downloadImageDigest = TransferFsImage.HandleUploadImageRequest(request, txid
, nnImage.GetStorage(), stream, parsedParams.GetFileSize(), ImageServlet.GetThrottler
(conf));
nnImage.SaveDigestAndRenameCheckpointImage(nnf, txid, downloadImageDigest);
if (metrics != null)
{
long elapsed = Time.MonotonicNow() - start;
metrics.AddPutImage(elapsed);
}
nnImage.PurgeOldStorage(nnf);
}
finally
{
stream.Close();
}
}
finally
{
nnImage.RemoveFromCheckpointing(txid);
}
return(null);
}
/*
* Uploads the imagefile using HTTP PUT method
*/
/// <exception cref="System.IO.IOException"/>
private static void UploadImage(Uri url, Configuration conf, NNStorage storage, NNStorage.NameNodeFile
nnf, long txId, Canceler canceler)
{
FilePath imageFile = storage.FindImageFile(nnf, txId);
if (imageFile == null)
{
throw new IOException("Could not find image with txid " + txId);
}
HttpURLConnection connection = null;
try
{
URIBuilder uriBuilder = new URIBuilder(url.ToURI());
// write all params for image upload request as query itself.
// Request body contains the image to be uploaded.
IDictionary <string, string> @params = ImageServlet.GetParamsForPutImage(storage,
txId, imageFile.Length(), nnf);
foreach (KeyValuePair <string, string> entry in @params)
{
uriBuilder.AddParameter(entry.Key, entry.Value);
}
Uri urlWithParams = uriBuilder.Build().ToURL();
connection = (HttpURLConnection)connectionFactory.OpenConnection(urlWithParams, UserGroupInformation
.IsSecurityEnabled());
// Set the request to PUT
connection.SetRequestMethod("PUT");
connection.SetDoOutput(true);
int chunkSize = conf.GetInt(DFSConfigKeys.DfsImageTransferChunksizeKey, DFSConfigKeys
.DfsImageTransferChunksizeDefault);
if (imageFile.Length() > chunkSize)
{
// using chunked streaming mode to support upload of 2GB+ files and to
// avoid internal buffering.
// this mode should be used only if more than chunkSize data is present
// to upload. otherwise upload may not happen sometimes.
connection.SetChunkedStreamingMode(chunkSize);
}
SetTimeout(connection);
// set headers for verification
ImageServlet.SetVerificationHeadersForPut(connection, imageFile);
// Write the file to output stream.
WriteFileToPutRequest(conf, connection, imageFile, canceler);
int responseCode = connection.GetResponseCode();
if (responseCode != HttpURLConnection.HttpOk)
{
throw new TransferFsImage.HttpPutFailedException(string.Format("Image uploading failed, status: %d, url: %s, message: %s"
, responseCode, urlWithParams, connection.GetResponseMessage()), responseCode);
}
}
catch (AuthenticationException e)
{
throw new IOException(e);
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
finally
{
if (connection != null)
{
connection.Disconnect();
}
}
}