/// <summary>verify that edits log and fsimage are in different directories and of a correct size
/// </summary>
private void VerifyDifferentDirs(FSImage img, long expectedImgSize, long expectedEditsSize
)
{
Storage.StorageDirectory sd = null;
for (IEnumerator <Storage.StorageDirectory> it = img.GetStorage().DirIterator(); it
.HasNext();)
{
sd = it.Next();
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Image))
{
img.GetStorage();
FilePath imf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Image, 0);
Log.Info("--image file " + imf.GetAbsolutePath() + "; len = " + imf.Length() + "; expected = "
+ expectedImgSize);
NUnit.Framework.Assert.AreEqual(expectedImgSize, imf.Length());
}
else
{
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Edits))
{
img.GetStorage();
FilePath edf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Edits, 0);
Log.Info("-- edits file " + edf.GetAbsolutePath() + "; len = " + edf.Length() + "; expected = "
+ expectedEditsSize);
NUnit.Framework.Assert.AreEqual(expectedEditsSize, edf.Length());
}
else
{
NUnit.Framework.Assert.Fail("Image/Edits directories are not different");
}
}
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual bool DoRecovery()
{
Log.Debug("Performing recovery in " + latestNameSD + " and " + latestEditsSD);
bool needToSave = false;
FilePath curFile = NNStorage.GetStorageFile(latestNameSD, NNStorage.NameNodeFile.
Image);
FilePath ckptFile = NNStorage.GetStorageFile(latestNameSD, NNStorage.NameNodeFile
.ImageNew);
//
// If we were in the midst of a checkpoint
//
if (ckptFile.Exists())
{
needToSave = true;
if (NNStorage.GetStorageFile(latestEditsSD, NNStorage.NameNodeFile.EditsNew).Exists
())
{
//
// checkpointing migth have uploaded a new
// merged image, but we discard it here because we are
// not sure whether the entire merged image was uploaded
// before the namenode crashed.
//
if (!ckptFile.Delete())
{
throw new IOException("Unable to delete " + ckptFile);
}
}
else
{
//
// checkpointing was in progress when the namenode
// shutdown. The fsimage.ckpt was created and the edits.new
// file was moved to edits. We complete that checkpoint by
// moving fsimage.new to fsimage. There is no need to
// update the fstime file here. renameTo fails on Windows
// if the destination file already exists.
//
if (!ckptFile.RenameTo(curFile))
{
if (!curFile.Delete())
{
Log.Warn("Unable to delete dir " + curFile + " before rename");
}
if (!ckptFile.RenameTo(curFile))
{
throw new IOException("Unable to rename " + ckptFile + " to " + curFile);
}
}
}
}
return(needToSave);
}
/// <returns>
/// a list with the paths to EDITS and EDITS_NEW (if it exists)
/// in a given storage directory.
/// </returns>
internal static IList <FilePath> GetEditsInStorageDir(Storage.StorageDirectory sd)
{
AList <FilePath> files = new AList <FilePath>();
FilePath edits = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Edits);
System.Diagnostics.Debug.Assert(edits.Exists(), "Expected edits file at " + edits
);
files.AddItem(edits);
FilePath editsNew = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.EditsNew);
if (editsNew.Exists())
{
files.AddItem(editsNew);
}
return(files);
}
/* Flag if there is at least one storage dir that doesn't contain the newest
* fstime */
/* Flag set false if there are any "previous" directories found */
// Track the name and edits dir with the latest times
/// <exception cref="System.IO.IOException"/>
internal override void InspectDirectory(Storage.StorageDirectory sd)
{
// Was the file just formatted?
if (!sd.GetVersionFile().Exists())
{
hasOutOfDateStorageDirs = true;
return;
}
bool imageExists = false;
bool editsExists = false;
// Determine if sd is image, edits or both
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Image))
{
imageExists = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Image).Exists();
imageDirs.AddItem(sd.GetRoot().GetCanonicalPath());
}
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Edits))
{
editsExists = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Edits).Exists();
editsDirs.AddItem(sd.GetRoot().GetCanonicalPath());
}
long checkpointTime = ReadCheckpointTime(sd);
checkpointTimes.AddItem(checkpointTime);
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Image) && (latestNameCheckpointTime
< checkpointTime) && imageExists)
{
latestNameCheckpointTime = checkpointTime;
latestNameSD = sd;
}
if (sd.GetStorageDirType().IsOfType(NNStorage.NameNodeDirType.Edits) && (latestEditsCheckpointTime
< checkpointTime) && editsExists)
{
latestEditsCheckpointTime = checkpointTime;
latestEditsSD = sd;
}
// check that we have a valid, non-default checkpointTime
if (checkpointTime <= 0L)
{
hasOutOfDateStorageDirs = true;
}
// set finalized flag
isUpgradeFinalized = isUpgradeFinalized && !sd.GetPreviousDir().Exists();
}
/// <exception cref="System.IO.IOException"/>
internal override IList <FSImageStorageInspector.FSImageFile> GetLatestImages()
{
// We should have at least one image and one edits dirs
if (latestNameSD == null)
{
throw new IOException("Image file is not found in " + imageDirs);
}
if (latestEditsSD == null)
{
throw new IOException("Edits file is not found in " + editsDirs);
}
// Make sure we are loading image and edits from same checkpoint
if (latestNameCheckpointTime > latestEditsCheckpointTime && latestNameSD != latestEditsSD &&
latestNameSD.GetStorageDirType() == NNStorage.NameNodeDirType.Image && latestEditsSD
.GetStorageDirType() == NNStorage.NameNodeDirType.Edits)
{
// This is a rare failure when NN has image-only and edits-only
// storage directories, and fails right after saving images,
// in some of the storage directories, but before purging edits.
// See -NOTE- in saveNamespace().
Log.Error("This is a rare failure scenario!!!");
Log.Error("Image checkpoint time " + latestNameCheckpointTime + " > edits checkpoint time "
+ latestEditsCheckpointTime);
Log.Error("Name-node will treat the image as the latest state of " + "the namespace. Old edits will be discarded."
);
}
else
{
if (latestNameCheckpointTime != latestEditsCheckpointTime)
{
throw new IOException("Inconsistent storage detected, " + "image and edits checkpoint times do not match. "
+ "image checkpoint time = " + latestNameCheckpointTime + "edits checkpoint time = "
+ latestEditsCheckpointTime);
}
}
needToSaveAfterRecovery = DoRecovery();
FSImageStorageInspector.FSImageFile file = new FSImageStorageInspector.FSImageFile
(latestNameSD, NNStorage.GetStorageFile(latestNameSD, NNStorage.NameNodeFile.Image
), HdfsConstants.InvalidTxid);
List <FSImageStorageInspector.FSImageFile> ret = new List <FSImageStorageInspector.FSImageFile
>();
ret.AddItem(file);
return(ret);
}
/// <summary>Determine the checkpoint time of the specified StorageDirectory</summary>
/// <param name="sd">StorageDirectory to check</param>
/// <returns>If file exists and can be read, last checkpoint time. If not, 0L.</returns>
/// <exception cref="System.IO.IOException">On errors processing file pointed to by sd
/// </exception>
internal static long ReadCheckpointTime(Storage.StorageDirectory sd)
{
FilePath timeFile = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Time);
long timeStamp = 0L;
if (timeFile.Exists() && FileUtil.CanRead(timeFile))
{
DataInputStream @in = new DataInputStream(new FileInputStream(timeFile));
try
{
timeStamp = @in.ReadLong();
@in.Close();
@in = null;
}
finally
{
IOUtils.Cleanup(Log, @in);
}
}
return(timeStamp);
}
public virtual void TestSNNStartup()
{
//setUpConfig();
Log.Info("--starting SecondNN startup test");
// different name dirs
config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI
(new FilePath(hdfsDir, "name")).ToString());
config.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI
(new FilePath(hdfsDir, "name")).ToString());
// same checkpoint dirs
config.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI
(new FilePath(hdfsDir, "chkpt_edits")).ToString());
config.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, Org.Apache.Hadoop.Hdfs.Server.Common.Util.FileAsURI
(new FilePath(hdfsDir, "chkpt")).ToString());
Log.Info("--starting NN ");
MiniDFSCluster cluster = null;
SecondaryNameNode sn = null;
NameNode nn = null;
try
{
cluster = new MiniDFSCluster.Builder(config).ManageDataDfsDirs(false).ManageNameDfsDirs
(false).Build();
cluster.WaitActive();
nn = cluster.GetNameNode();
NUnit.Framework.Assert.IsNotNull(nn);
// start secondary node
Log.Info("--starting SecondNN");
sn = new SecondaryNameNode(config);
NUnit.Framework.Assert.IsNotNull(sn);
Log.Info("--doing checkpoint");
sn.DoCheckpoint();
// this shouldn't fail
Log.Info("--done checkpoint");
// now verify that image and edits are created in the different directories
FSImage image = nn.GetFSImage();
Storage.StorageDirectory sd = image.GetStorage().GetStorageDir(0);
//only one
NUnit.Framework.Assert.AreEqual(sd.GetStorageDirType(), NNStorage.NameNodeDirType
.ImageAndEdits);
image.GetStorage();
FilePath imf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Image, 0);
image.GetStorage();
FilePath edf = NNStorage.GetStorageFile(sd, NNStorage.NameNodeFile.Edits, 0);
Log.Info("--image file " + imf.GetAbsolutePath() + "; len = " + imf.Length());
Log.Info("--edits file " + edf.GetAbsolutePath() + "; len = " + edf.Length());
FSImage chkpImage = sn.GetFSImage();
VerifyDifferentDirs(chkpImage, imf.Length(), edf.Length());
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(StringUtils.StringifyException(e));
System.Console.Error.WriteLine("checkpoint failed");
throw;
}
finally
{
if (sn != null)
{
sn.Shutdown();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
