/// <exception cref="System.IO.IOException"/>
public static void CreateBlockPoolVersionFile(FilePath bpDir, StorageInfo version
, string bpid)
{
// Create block pool version files
if (DataNodeLayoutVersion.Supports(LayoutVersion.Feature.Federation, version.layoutVersion
))
{
FilePath bpCurDir = new FilePath(bpDir, Storage.StorageDirCurrent);
BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version, bpid);
FilePath versionFile = new FilePath(bpCurDir, "VERSION");
Storage.StorageDirectory sd = new Storage.StorageDirectory(bpDir);
bpStorage.WriteProperties(versionFile, sd);
}
}
/// <summary>
/// Verify that the current directory exists and that the previous directory
/// does not exist.
/// </summary>
/// <remarks>
/// Verify that the current directory exists and that the previous directory
/// does not exist. Verify that current hasn't been modified by comparing
/// the checksum of all it's containing files with their original checksum.
/// </remarks>
/// <exception cref="System.Exception"/>
internal static void CheckResult(string[] nameNodeDirs, string[] dataNodeDirs, string
bpid)
{
IList <FilePath> dirs = Lists.NewArrayList();
for (int i = 0; i < nameNodeDirs.Length; i++)
{
FilePath curDir = new FilePath(nameNodeDirs[i], "current");
dirs.AddItem(curDir);
FSImageTestUtil.AssertReasonableNameCurrentDir(curDir);
}
FSImageTestUtil.AssertParallelFilesAreIdentical(dirs, Sharpen.Collections.EmptySet
<string>());
FilePath[] dnCurDirs = new FilePath[dataNodeDirs.Length];
for (int i_1 = 0; i_1 < dataNodeDirs.Length; i_1++)
{
dnCurDirs[i_1] = new FilePath(dataNodeDirs[i_1], "current");
NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType
.DataNode, dnCurDirs[i_1], false), UpgradeUtilities.ChecksumMasterDataNodeContents
());
}
for (int i_2 = 0; i_2 < nameNodeDirs.Length; i_2++)
{
NUnit.Framework.Assert.IsFalse(new FilePath(nameNodeDirs[i_2], "previous").IsDirectory
());
}
if (bpid == null)
{
for (int i_3 = 0; i_3 < dataNodeDirs.Length; i_3++)
{
NUnit.Framework.Assert.IsFalse(new FilePath(dataNodeDirs[i_3], "previous").IsDirectory
());
}
}
else
{
for (int i_3 = 0; i_3 < dataNodeDirs.Length; i_3++)
{
FilePath bpRoot = BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDirs[i_3]);
NUnit.Framework.Assert.IsFalse(new FilePath(bpRoot, "previous").IsDirectory());
FilePath bpCurFinalizeDir = new FilePath(bpRoot, "current/" + DataStorage.StorageDirFinalized
);
NUnit.Framework.Assert.AreEqual(UpgradeUtilities.ChecksumContents(HdfsServerConstants.NodeType
.DataNode, bpCurFinalizeDir, true), UpgradeUtilities.ChecksumMasterBlockPoolFinalizedContents
());
}
}
}
/// <summary>
/// Create a <code>version</code> file for datanode inside the specified parent
/// directory.
/// </summary>
/// <remarks>
/// Create a <code>version</code> file for datanode inside the specified parent
/// directory. If such a file already exists, it will be overwritten.
/// The given version string will be written to the file as the layout
/// version. None of the parameters may be null.
/// </remarks>
/// <param name="parent">directory where namenode VERSION file is stored</param>
/// <param name="version">StorageInfo to create VERSION file from</param>
/// <param name="bpid">Block pool Id</param>
/// <param name="bpidToWrite">Block pool Id to write into the version file</param>
/// <exception cref="System.IO.IOException"/>
public static void CreateDataNodeVersionFile(FilePath[] parent, StorageInfo version
, string bpid, string bpidToWrite)
{
DataStorage storage = new DataStorage(version);
storage.SetDatanodeUuid("FixedDatanodeUuid");
FilePath[] versionFiles = new FilePath[parent.Length];
for (int i = 0; i < parent.Length; i++)
{
FilePath versionFile = new FilePath(parent[i], "VERSION");
Storage.StorageDirectory sd = new Storage.StorageDirectory(parent[i].GetParentFile
());
storage.CreateStorageID(sd, false);
storage.WriteProperties(versionFile, sd);
versionFiles[i] = versionFile;
FilePath bpDir = BlockPoolSliceStorage.GetBpRoot(bpid, parent[i]);
CreateBlockPoolVersionFile(bpDir, version, bpidToWrite);
}
}
// Root scratch directory on local filesystem
// The singleton master storage directory for Namenode
// A checksum of the contents in namenodeStorage directory
// The namespaceId of the namenodeStorage directory
// The clusterId of the namenodeStorage directory
// The blockpoolId of the namenodeStorage directory
// The fsscTime of the namenodeStorage directory
// The singleton master storage directory for Datanode
// A checksum of the contents in datanodeStorage directory
// A checksum of the contents in blockpool storage directory
// A checksum of the contents in blockpool finalize storage directory
// A checksum of the contents in blockpool rbw storage directory
/// <summary>Initialize the data structures used by this class.</summary>
/// <remarks>
/// Initialize the data structures used by this class.
/// IMPORTANT NOTE: This method must be called once before calling
/// any other public method on this class.
/// <p>
/// Creates a singleton master populated storage
/// directory for a Namenode (contains edits, fsimage,
/// version, and time files) and a Datanode (contains version and
/// block files). This can be a lengthy operation.
/// </remarks>
/// <exception cref="System.Exception"/>
public static void Initialize()
{
CreateEmptyDirs(new string[] { TestRootDir.ToString() });
Configuration config = new HdfsConfiguration();
config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, namenodeStorage.ToString());
config.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, namenodeStorage.ToString());
config.Set(DFSConfigKeys.DfsDatanodeDataDirKey, datanodeStorage.ToString());
MiniDFSCluster cluster = null;
string bpid = null;
try
{
// format data-node
CreateEmptyDirs(new string[] { datanodeStorage.ToString() });
// format and start NameNode and start DataNode
DFSTestUtil.FormatNameNode(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).StartupOption(HdfsServerConstants.StartupOption
.Regular).Format(false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).Build(
);
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
namenodeStorageNamespaceID = namenode.VersionRequest().GetNamespaceID();
namenodeStorageFsscTime = namenode.VersionRequest().GetCTime();
namenodeStorageClusterID = namenode.VersionRequest().GetClusterID();
namenodeStorageBlockPoolID = namenode.VersionRequest().GetBlockPoolID();
FileSystem fs = FileSystem.Get(config);
Path baseDir = new Path("/TestUpgrade");
fs.Mkdirs(baseDir);
// write some files
int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
for (int i = 0; i < bufferSize; i++)
{
buffer[i] = unchecked ((byte)((byte)('0') + i % 50));
}
WriteFile(fs, new Path(baseDir, "file1"), buffer, bufferSize);
WriteFile(fs, new Path(baseDir, "file2"), buffer, bufferSize);
// save image
namenode.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false);
namenode.SaveNamespace();
namenode.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false);
// write more files
WriteFile(fs, new Path(baseDir, "file3"), buffer, bufferSize);
WriteFile(fs, new Path(baseDir, "file4"), buffer, bufferSize);
bpid = cluster.GetNamesystem(0).GetBlockPoolId();
}
finally
{
// shutdown
if (cluster != null)
{
cluster.Shutdown();
}
FileUtil.FullyDelete(new FilePath(namenodeStorage, "in_use.lock"));
FileUtil.FullyDelete(new FilePath(datanodeStorage, "in_use.lock"));
}
namenodeStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.NameNode,
new FilePath(namenodeStorage, "current"), false);
FilePath dnCurDir = new FilePath(datanodeStorage, "current");
datanodeStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode,
dnCurDir, false);
FilePath bpCurDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir),
"current");
blockPoolStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode
, bpCurDir, false);
FilePath bpCurFinalizeDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir
), "current/" + DataStorage.StorageDirFinalized);
blockPoolFinalizedStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType
.DataNode, bpCurFinalizeDir, true);
FilePath bpCurRbwDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir
), "current/" + DataStorage.StorageDirRbw);
blockPoolRbwStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode
, bpCurRbwDir, false);
}
