/// <summary>
/// Sets up the storage directories for a block pool under
/// <see cref="DFSConfigKeys.DfsDatanodeDataDirKey"/>
/// . For each element
/// in
/// <see cref="DFSConfigKeys.DfsDatanodeDataDirKey"/>
/// , the subdirectories
/// represented by the first four elements of the <code>state</code> array
/// will be created and populated.
/// See
/// <see cref="UpgradeUtilities.CreateBlockPoolStorageDirs(string[], string, string)"
/// />
/// </summary>
/// <param name="bpid">block pool Id</param>
/// <param name="state">
/// a row from the testCases table which indicates which directories
/// to setup for the node
/// </param>
/// <returns>file paths representing block pool storage directories</returns>
/// <exception cref="System.Exception"/>
internal virtual string[] CreateBlockPoolStorageState(string bpid, bool[] state)
{
string[] baseDirs = conf.GetStrings(DFSConfigKeys.DfsDatanodeDataDirKey);
UpgradeUtilities.CreateEmptyDirs(baseDirs);
UpgradeUtilities.CreateDataNodeStorageDirs(baseDirs, "current");
// After copying the storage directories from master datanode, empty
// the block pool storage directories
string[] bpDirs = UpgradeUtilities.CreateEmptyBPDirs(baseDirs, bpid);
if (state[CurrentExists])
{
// current
UpgradeUtilities.CreateBlockPoolStorageDirs(baseDirs, "current", bpid);
}
if (state[PreviousExists])
{
// previous
UpgradeUtilities.CreateBlockPoolStorageDirs(baseDirs, "previous", bpid);
}
if (state[PreviousTmpExists])
{
// previous.tmp
UpgradeUtilities.CreateBlockPoolStorageDirs(baseDirs, "previous.tmp", bpid);
}
if (state[RemovedTmpExists])
{
// removed.tmp
UpgradeUtilities.CreateBlockPoolStorageDirs(baseDirs, "removed.tmp", bpid);
}
return(bpDirs);
}
/// <summary>
/// Sets up the storage directories for a datanode under
/// <see cref="DFSConfigKeys.DfsDatanodeDataDirKey"/>
/// . For each element in
/// <see cref="DFSConfigKeys.DfsDatanodeDataDirKey"/>
/// , the subdirectories
/// represented by the first four elements of the <code>state</code> array
/// will be created and populated.
/// See
/// <see cref="UpgradeUtilities.CreateDataNodeStorageDirs(string[], string)"/>
/// </summary>
/// <param name="state">
/// a row from the testCases table which indicates which directories
/// to setup for the node
/// </param>
/// <returns>file paths representing datanode storage directories</returns>
/// <exception cref="System.Exception"/>
internal virtual string[] CreateDataNodeStorageState(bool[] state)
{
string[] baseDirs = conf.GetStrings(DFSConfigKeys.DfsDatanodeDataDirKey);
UpgradeUtilities.CreateEmptyDirs(baseDirs);
if (state[CurrentExists])
{
// current
UpgradeUtilities.CreateDataNodeStorageDirs(baseDirs, "current");
}
if (state[PreviousExists])
{
// previous
UpgradeUtilities.CreateDataNodeStorageDirs(baseDirs, "previous");
}
if (state[PreviousTmpExists])
{
// previous.tmp
UpgradeUtilities.CreateDataNodeStorageDirs(baseDirs, "previous.tmp");
}
if (state[RemovedTmpExists])
{
// removed.tmp
UpgradeUtilities.CreateDataNodeStorageDirs(baseDirs, "removed.tmp");
}
return(baseDirs);
}
public virtual void TestUpgrade4()
{
int numDirs = 4;
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
conf.SetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion, false);
conf = UpgradeUtilities.InitializeStorageStateConf(numDirs, conf);
string[] nameNodeDirs = conf.GetStrings(DFSConfigKeys.DfsNamenodeNameDirKey);
Log("NameNode upgrade with one bad storage dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
try
{
// assert("storage dir has been prepared for failure before reaching this point");
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade, typeof(IOException
), Sharpen.Pattern.Compile("failed in 1 storage"));
}
finally
{
// assert("storage dir shall be returned to normal state before exiting");
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
}
}
public virtual void TestRollback()
{
FilePath[] baseDirs;
UpgradeUtilities.Initialize();
StorageInfo storageInfo = null;
for (int numDirs = 1; numDirs <= 2; numDirs++)
{
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
conf = UpgradeUtilities.InitializeStorageStateConf(numDirs, conf);
string[] nameNodeDirs = conf.GetStrings(DFSConfigKeys.DfsNamenodeNameDirKey);
string[] dataNodeDirs = conf.GetStrings(DFSConfigKeys.DfsDatanodeDataDirKey);
Log("Normal NameNode rollback", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
NameNode.DoRollback(conf, false);
CheckResult(HdfsServerConstants.NodeType.NameNode, nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("Normal DataNode rollback", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
NameNode.DoRollback(conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageDataDfsDirs
(false).ManageNameDfsDirs(false).DnStartupOption(HdfsServerConstants.StartupOption
.Rollback).Build();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "previous");
cluster.StartDataNodes(conf, 1, false, HdfsServerConstants.StartupOption.Rollback
, null);
CheckResult(HdfsServerConstants.NodeType.DataNode, dataNodeDirs);
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("Normal BlockPool rollback", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
NameNode.DoRollback(conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageDataDfsDirs
(false).ManageNameDfsDirs(false).DnStartupOption(HdfsServerConstants.StartupOption
.Rollback).Build();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
UpgradeUtilities.CreateBlockPoolStorageDirs(dataNodeDirs, "current", UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
// Create a previous snapshot for the blockpool
UpgradeUtilities.CreateBlockPoolStorageDirs(dataNodeDirs, "previous", UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
// Put newer layout version in current.
storageInfo = new StorageInfo(HdfsConstants.DatanodeLayoutVersion - 1, UpgradeUtilities
.GetCurrentNamespaceID(cluster), UpgradeUtilities.GetCurrentClusterID(cluster),
UpgradeUtilities.GetCurrentFsscTime(cluster), HdfsServerConstants.NodeType.DataNode
);
// Overwrite VERSION file in the current directory of
// volume directories and block pool slice directories
// with a layout version from future.
FilePath[] dataCurrentDirs = new FilePath[dataNodeDirs.Length];
for (int i = 0; i < dataNodeDirs.Length; i++)
{
dataCurrentDirs[i] = new FilePath((new Path(dataNodeDirs[i] + "/current")).ToString
());
}
UpgradeUtilities.CreateDataNodeVersionFile(dataCurrentDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
cluster.StartDataNodes(conf, 1, false, HdfsServerConstants.StartupOption.Rollback
, null);
NUnit.Framework.Assert.IsTrue(cluster.IsDataNodeUp());
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("NameNode rollback without existing previous dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
StartNameNodeShouldFail("None of the storage directories contain previous fs state"
);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("DataNode rollback without existing previous dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageDataDfsDirs
(false).ManageNameDfsDirs(false).StartupOption(HdfsServerConstants.StartupOption
.Upgrade).Build();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
cluster.StartDataNodes(conf, 1, false, HdfsServerConstants.StartupOption.Rollback
, null);
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("DataNode rollback with future stored layout version in previous", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
NameNode.DoRollback(conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageDataDfsDirs
(false).ManageNameDfsDirs(false).DnStartupOption(HdfsServerConstants.StartupOption
.Rollback).Build();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "previous");
storageInfo = new StorageInfo(int.MinValue, UpgradeUtilities.GetCurrentNamespaceID
(cluster), UpgradeUtilities.GetCurrentClusterID(cluster), UpgradeUtilities.GetCurrentFsscTime
(cluster), HdfsServerConstants.NodeType.DataNode);
UpgradeUtilities.CreateDataNodeVersionFile(baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartBlockPoolShouldFail(HdfsServerConstants.StartupOption.Rollback, cluster.GetNamesystem
().GetBlockPoolId());
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("DataNode rollback with newer fsscTime in previous", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
NameNode.DoRollback(conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(false).ManageDataDfsDirs
(false).ManageNameDfsDirs(false).DnStartupOption(HdfsServerConstants.StartupOption
.Rollback).Build();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "previous");
storageInfo = new StorageInfo(HdfsConstants.DatanodeLayoutVersion, UpgradeUtilities
.GetCurrentNamespaceID(cluster), UpgradeUtilities.GetCurrentClusterID(cluster),
long.MaxValue, HdfsServerConstants.NodeType.DataNode);
UpgradeUtilities.CreateDataNodeVersionFile(baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartBlockPoolShouldFail(HdfsServerConstants.StartupOption.Rollback, cluster.GetNamesystem
().GetBlockPoolId());
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("NameNode rollback with no edits file", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
DeleteMatchingFiles(baseDirs, "edits.*");
StartNameNodeShouldFail("Gap in transactions");
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode rollback with no image file", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
DeleteMatchingFiles(baseDirs, "fsimage_.*");
StartNameNodeShouldFail("No valid image files found");
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode rollback with corrupt version file", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
foreach (FilePath f in baseDirs)
{
UpgradeUtilities.CorruptFile(new FilePath(f, "VERSION"), Sharpen.Runtime.GetBytesForString
("layoutVersion", Charsets.Utf8), Sharpen.Runtime.GetBytesForString("xxxxxxxxxxxxx"
, Charsets.Utf8));
}
StartNameNodeShouldFail("file VERSION has layoutVersion missing");
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode rollback with old layout version in previous", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
storageInfo = new StorageInfo(1, UpgradeUtilities.GetCurrentNamespaceID(null), UpgradeUtilities
.GetCurrentClusterID(null), UpgradeUtilities.GetCurrentFsscTime(null), HdfsServerConstants.NodeType
.NameNode);
UpgradeUtilities.CreateNameNodeVersionFile(conf, baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartNameNodeShouldFail("Cannot rollback to storage version 1 using this version"
);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
}
}
public virtual void TestFinalize()
{
UpgradeUtilities.Initialize();
for (int numDirs = 1; numDirs <= 2; numDirs++)
{
/* This test requires that "current" directory not change after
* the upgrade. Actually it is ok for those contents to change.
* For now disabling block verification so that the contents are
* not changed.
* Disable duplicate replica deletion as the test intentionally
* mirrors the contents of storage directories.
*/
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
conf.SetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion, false);
conf = UpgradeUtilities.InitializeStorageStateConf(numDirs, conf);
string[] nameNodeDirs = conf.GetStrings(DFSConfigKeys.DfsNamenodeNameDirKey);
string[] dataNodeDirs = conf.GetStrings(DFSConfigKeys.DfsDatanodeDataDirKey);
Log("Finalize NN & DN with existing previous dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "previous");
cluster = new MiniDFSCluster.Builder(conf).Format(false).ManageDataDfsDirs(false)
.ManageNameDfsDirs(false).StartupOption(HdfsServerConstants.StartupOption.Regular
).Build();
cluster.FinalizeCluster(conf);
cluster.TriggerBlockReports();
// 1 second should be enough for asynchronous DN finalize
Sharpen.Thread.Sleep(1000);
CheckResult(nameNodeDirs, dataNodeDirs, null);
Log("Finalize NN & DN without existing previous dir", numDirs);
cluster.FinalizeCluster(conf);
cluster.TriggerBlockReports();
// 1 second should be enough for asynchronous DN finalize
Sharpen.Thread.Sleep(1000);
CheckResult(nameNodeDirs, dataNodeDirs, null);
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("Finalize NN & BP with existing previous dir", numDirs);
string bpid = UpgradeUtilities.GetCurrentBlockPoolID(cluster);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
UpgradeUtilities.CreateBlockPoolStorageDirs(dataNodeDirs, "current", bpid);
UpgradeUtilities.CreateBlockPoolStorageDirs(dataNodeDirs, "previous", bpid);
cluster = new MiniDFSCluster.Builder(conf).Format(false).ManageDataDfsDirs(false)
.ManageNameDfsDirs(false).StartupOption(HdfsServerConstants.StartupOption.Regular
).Build();
cluster.FinalizeCluster(conf);
cluster.TriggerBlockReports();
// 1 second should be enough for asynchronous BP finalize
Sharpen.Thread.Sleep(1000);
CheckResult(nameNodeDirs, dataNodeDirs, bpid);
Log("Finalize NN & BP without existing previous dir", numDirs);
cluster.FinalizeCluster(conf);
cluster.TriggerBlockReports();
// 1 second should be enough for asynchronous BP finalize
Sharpen.Thread.Sleep(1000);
CheckResult(nameNodeDirs, dataNodeDirs, bpid);
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
}
}
/// <summary>
/// This test attempts to upgrade the NameNode and DataNode under
/// a number of valid and invalid conditions.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpgrade()
{
FilePath[] baseDirs;
StorageInfo storageInfo = null;
for (int numDirs = 1; numDirs <= 2; numDirs++)
{
conf = new HdfsConfiguration();
conf = UpgradeUtilities.InitializeStorageStateConf(numDirs, conf);
string[] nameNodeDirs = conf.GetStrings(DFSConfigKeys.DfsNamenodeNameDirKey);
string[] dataNodeDirs = conf.GetStrings(DFSConfigKeys.DfsDatanodeDataDirKey);
conf.SetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion, false);
Log("Normal NameNode upgrade", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
// make sure that rolling upgrade cannot be started
try
{
DistributedFileSystem dfs = cluster.GetFileSystem();
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare);
NUnit.Framework.Assert.Fail();
}
catch (RemoteException re)
{
NUnit.Framework.Assert.AreEqual(typeof(InconsistentFSStateException).FullName, re
.GetClassName());
Log.Info("The exception is expected.", re);
}
CheckNameNode(nameNodeDirs, ExpectedTxid);
if (numDirs > 1)
{
TestParallelImageWrite.CheckImages(cluster.GetNamesystem(), numDirs);
}
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("Normal DataNode upgrade", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
cluster.StartDataNodes(conf, 1, false, HdfsServerConstants.StartupOption.Regular,
null);
CheckDataNode(dataNodeDirs, UpgradeUtilities.GetCurrentBlockPoolID(null));
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("NameNode upgrade with existing previous dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "previous");
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("DataNode upgrade with existing previous dir", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "previous");
cluster.StartDataNodes(conf, 1, false, HdfsServerConstants.StartupOption.Regular,
null);
CheckDataNode(dataNodeDirs, UpgradeUtilities.GetCurrentBlockPoolID(null));
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("DataNode upgrade with future stored layout version in current", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
baseDirs = UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
storageInfo = new StorageInfo(int.MinValue, UpgradeUtilities.GetCurrentNamespaceID
(cluster), UpgradeUtilities.GetCurrentClusterID(cluster), UpgradeUtilities.GetCurrentFsscTime
(cluster), HdfsServerConstants.NodeType.DataNode);
UpgradeUtilities.CreateDataNodeVersionFile(baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartBlockPoolShouldFail(HdfsServerConstants.StartupOption.Regular, UpgradeUtilities
.GetCurrentBlockPoolID(null));
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("DataNode upgrade with newer fsscTime in current", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
baseDirs = UpgradeUtilities.CreateDataNodeStorageDirs(dataNodeDirs, "current");
storageInfo = new StorageInfo(HdfsConstants.DatanodeLayoutVersion, UpgradeUtilities
.GetCurrentNamespaceID(cluster), UpgradeUtilities.GetCurrentClusterID(cluster),
long.MaxValue, HdfsServerConstants.NodeType.DataNode);
UpgradeUtilities.CreateDataNodeVersionFile(baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
// Ensure corresponding block pool failed to initialized
StartBlockPoolShouldFail(HdfsServerConstants.StartupOption.Regular, UpgradeUtilities
.GetCurrentBlockPoolID(null));
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
UpgradeUtilities.CreateEmptyDirs(dataNodeDirs);
Log("NameNode upgrade with no edits file", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
DeleteStorageFilesWithPrefix(nameNodeDirs, "edits_");
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode upgrade with no image file", numDirs);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
DeleteStorageFilesWithPrefix(nameNodeDirs, "fsimage_");
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode upgrade with corrupt version file", numDirs);
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
foreach (FilePath f in baseDirs)
{
UpgradeUtilities.CorruptFile(new FilePath(f, "VERSION"), Sharpen.Runtime.GetBytesForString
("layoutVersion", Charsets.Utf8), Sharpen.Runtime.GetBytesForString("xxxxxxxxxxxxx"
, Charsets.Utf8));
}
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode upgrade with old layout version in current", numDirs);
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
storageInfo = new StorageInfo(Storage.LastUpgradableLayoutVersion + 1, UpgradeUtilities
.GetCurrentNamespaceID(null), UpgradeUtilities.GetCurrentClusterID(null), UpgradeUtilities
.GetCurrentFsscTime(null), HdfsServerConstants.NodeType.NameNode);
UpgradeUtilities.CreateNameNodeVersionFile(conf, baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
Log("NameNode upgrade with future layout version in current", numDirs);
baseDirs = UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
storageInfo = new StorageInfo(int.MinValue, UpgradeUtilities.GetCurrentNamespaceID
(null), UpgradeUtilities.GetCurrentClusterID(null), UpgradeUtilities.GetCurrentFsscTime
(null), HdfsServerConstants.NodeType.NameNode);
UpgradeUtilities.CreateNameNodeVersionFile(conf, baseDirs, storageInfo, UpgradeUtilities
.GetCurrentBlockPoolID(cluster));
StartNameNodeShouldFail(HdfsServerConstants.StartupOption.Upgrade);
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
}
// end numDir loop
// One more check: normal NN upgrade with 4 directories, concurrent write
int numDirs_1 = 4;
{
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
conf.SetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion, false);
conf = UpgradeUtilities.InitializeStorageStateConf(numDirs_1, conf);
string[] nameNodeDirs = conf.GetStrings(DFSConfigKeys.DfsNamenodeNameDirKey);
Log("Normal NameNode upgrade", numDirs_1);
UpgradeUtilities.CreateNameNodeStorageDirs(nameNodeDirs, "current");
cluster = CreateCluster();
// make sure that rolling upgrade cannot be started
try
{
DistributedFileSystem dfs = cluster.GetFileSystem();
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare);
NUnit.Framework.Assert.Fail();
}
catch (RemoteException re)
{
NUnit.Framework.Assert.AreEqual(typeof(InconsistentFSStateException).FullName, re
.GetClassName());
Log.Info("The exception is expected.", re);
}
CheckNameNode(nameNodeDirs, ExpectedTxid);
TestParallelImageWrite.CheckImages(cluster.GetNamesystem(), numDirs_1);
cluster.Shutdown();
UpgradeUtilities.CreateEmptyDirs(nameNodeDirs);
}
}
