/// <summary>Upgrade to any release after 0.22 (0.22 included) release e.g.</summary>
/// <remarks>
/// Upgrade to any release after 0.22 (0.22 included) release e.g. 0.22 => 0.23
/// Upgrade procedure is as follows:
/// <ol>
/// <li>If <SD>/current/<bpid>/previous exists then delete it</li>
/// <li>Rename <SD>/current/<bpid>/current to
/// <SD>/current/bpid/current/previous.tmp</li>
/// <li>Create new <SD>current/<bpid>/current directory</li>
/// <ol>
/// <li>Hard links for block files are created from previous.tmp to current</li>
/// <li>Save new version file in current directory</li>
/// </ol>
/// <li>Rename previous.tmp to previous</li> </ol>
/// </remarks>
/// <param name="bpSd">storage directory <SD>/current/<bpid></param>
/// <param name="nsInfo">Namespace Info from the namenode</param>
/// <exception cref="System.IO.IOException">on error</exception>
internal virtual void DoUpgrade(DataNode datanode, Storage.StorageDirectory bpSd,
NamespaceInfo nsInfo)
{
// Upgrading is applicable only to release with federation or after
if (!DataNodeLayoutVersion.Supports(LayoutVersion.Feature.Federation, layoutVersion
))
{
return;
}
Log.Info("Upgrading block pool storage directory " + bpSd.GetRoot() + ".\n old LV = "
+ this.GetLayoutVersion() + "; old CTime = " + this.GetCTime() + ".\n new LV = "
+ HdfsConstants.DatanodeLayoutVersion + "; new CTime = " + nsInfo.GetCTime());
// get <SD>/previous directory
string dnRoot = GetDataNodeStorageRoot(bpSd.GetRoot().GetCanonicalPath());
Storage.StorageDirectory dnSdStorage = new Storage.StorageDirectory(new FilePath(
dnRoot));
FilePath dnPrevDir = dnSdStorage.GetPreviousDir();
// If <SD>/previous directory exists delete it
if (dnPrevDir.Exists())
{
DeleteDir(dnPrevDir);
}
FilePath bpCurDir = bpSd.GetCurrentDir();
FilePath bpPrevDir = bpSd.GetPreviousDir();
System.Diagnostics.Debug.Assert(bpCurDir.Exists(), "BP level current directory must exist."
);
CleanupDetachDir(new FilePath(bpCurDir, DataStorage.StorageDirDetached));
// 1. Delete <SD>/current/<bpid>/previous dir before upgrading
if (bpPrevDir.Exists())
{
DeleteDir(bpPrevDir);
}
FilePath bpTmpDir = bpSd.GetPreviousTmp();
System.Diagnostics.Debug.Assert(!bpTmpDir.Exists(), "previous.tmp directory must not exist."
);
// 2. Rename <SD>/current/<bpid>/current to
// <SD>/current/<bpid>/previous.tmp
Rename(bpCurDir, bpTmpDir);
// 3. Create new <SD>/current with block files hardlinks and VERSION
LinkAllBlocks(datanode, bpTmpDir, bpCurDir);
this.layoutVersion = HdfsConstants.DatanodeLayoutVersion;
System.Diagnostics.Debug.Assert(this.namespaceID == nsInfo.GetNamespaceID(), "Data-node and name-node layout versions must be the same."
);
this.cTime = nsInfo.GetCTime();
WriteProperties(bpSd);
// 4.rename <SD>/current/<bpid>/previous.tmp to
// <SD>/current/<bpid>/previous
Rename(bpTmpDir, bpPrevDir);
Log.Info("Upgrade of block pool " + blockpoolID + " at " + bpSd.GetRoot() + " is complete"
);
}
/// <summary>
/// Perform any steps that must succeed across all storage dirs/JournalManagers
/// involved in an upgrade before proceeding onto the actual upgrade stage.
/// </summary>
/// <remarks>
/// Perform any steps that must succeed across all storage dirs/JournalManagers
/// involved in an upgrade before proceeding onto the actual upgrade stage. If
/// a call to any JM's or local storage dir's doPreUpgrade method fails, then
/// doUpgrade will not be called for any JM. The existing current dir is
/// renamed to previous.tmp, and then a new, empty current dir is created.
/// </remarks>
/// <param name="conf">
/// configuration for creating
/// <see cref="EditLogFileOutputStream"/>
/// </param>
/// <param name="sd">the storage directory to perform the pre-upgrade procedure.</param>
/// <exception cref="System.IO.IOException">in the event of error</exception>
internal static void DoPreUpgrade(Configuration conf, Storage.StorageDirectory sd
)
{
Log.Info("Starting upgrade of storage directory " + sd.GetRoot());
// rename current to tmp
RenameCurToTmp(sd);
FilePath curDir = sd.GetCurrentDir();
FilePath tmpDir = sd.GetPreviousTmp();
IList <string> fileNameList = IOUtils.ListDirectory(tmpDir, new _FilenameFilter_121
(tmpDir));
foreach (string s in fileNameList)
{
FilePath prevFile = new FilePath(tmpDir, s);
FilePath newFile = new FilePath(curDir, prevFile.GetName());
Files.CreateLink(newFile.ToPath(), prevFile.ToPath());
}
}
/// <summary>
/// Rename the existing current dir to previous.tmp, and create a new empty
/// current dir.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public static void RenameCurToTmp(Storage.StorageDirectory sd)
{
FilePath curDir = sd.GetCurrentDir();
FilePath prevDir = sd.GetPreviousDir();
FilePath tmpDir = sd.GetPreviousTmp();
Preconditions.CheckState(curDir.Exists(), "Current directory must exist for preupgrade."
);
Preconditions.CheckState(!prevDir.Exists(), "Previous directory must not exist for preupgrade."
);
Preconditions.CheckState(!tmpDir.Exists(), "Previous.tmp directory must not exist for preupgrade."
+ "Consider restarting for recovery.");
// rename current to tmp
NNStorage.Rename(curDir, tmpDir);
if (!curDir.Mkdir())
{
throw new IOException("Cannot create directory " + curDir);
}
}
/// <summary>Perform the upgrade of the storage dir to the given storage info.</summary>
/// <remarks>
/// Perform the upgrade of the storage dir to the given storage info. The new
/// storage info is written into the current directory, and the previous.tmp
/// directory is renamed to previous.
/// </remarks>
/// <param name="sd">the storage directory to upgrade</param>
/// <param name="storage">info about the new upgraded versions.</param>
/// <exception cref="System.IO.IOException">in the event of error</exception>
public static void DoUpgrade(Storage.StorageDirectory sd, Storage storage)
{
Log.Info("Performing upgrade of storage directory " + sd.GetRoot());
try
{
// Write the version file, since saveFsImage only makes the
// fsimage_<txid>, and the directory is otherwise empty.
storage.WriteProperties(sd);
FilePath prevDir = sd.GetPreviousDir();
FilePath tmpDir = sd.GetPreviousTmp();
Preconditions.CheckState(!prevDir.Exists(), "previous directory must not exist for upgrade."
);
Preconditions.CheckState(tmpDir.Exists(), "previous.tmp directory must exist for upgrade."
);
// rename tmp to previous
NNStorage.Rename(tmpDir, prevDir);
}
catch (IOException ioe)
{
Log.Error("Unable to rename temp to previous for " + sd.GetRoot(), ioe);
throw;
}
}