/// <exception cref="System.IO.IOException"/>
protected internal override void Initialize(Configuration conf)
{
// NameNode
// Trash is disabled in BackupNameNode,
// but should be turned back on if it ever becomes active.
conf.SetLong(CommonConfigurationKeys.FsTrashIntervalKey, CommonConfigurationKeys.
FsTrashIntervalDefault);
NamespaceInfo nsInfo = Handshake(conf);
base.Initialize(conf);
namesystem.SetBlockPoolId(nsInfo.GetBlockPoolID());
if (false == namesystem.IsInSafeMode())
{
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
}
// Backup node should never do lease recovery,
// therefore lease hard limit should never expire.
namesystem.leaseManager.SetLeasePeriod(HdfsConstants.LeaseSoftlimitPeriod, long.MaxValue
);
// register with the active name-node
RegisterWith(nsInfo);
// Checkpoint daemon should start after the rpc server started
RunCheckpointDaemon(conf);
IPEndPoint addr = GetHttpAddress();
if (addr != null)
{
conf.Set(BnHttpAddressNameKey, NetUtils.GetHostPortString(GetHttpAddress()));
}
}
/// <summary>Format a block pool slice storage.</summary>
/// <param name="dnCurDir">DataStorage current directory</param>
/// <param name="nsInfo">the name space info</param>
/// <exception cref="System.IO.IOException">Signals that an I/O exception has occurred.
/// </exception>
internal virtual void Format(FilePath dnCurDir, NamespaceInfo nsInfo)
{
FilePath curBpDir = GetBpRoot(nsInfo.GetBlockPoolID(), dnCurDir);
Storage.StorageDirectory bpSdir = new Storage.StorageDirectory(curBpDir);
Format(bpSdir, nsInfo);
}
/// <summary>Analyze and load storage directories.</summary>
/// <remarks>
/// Analyze and load storage directories. Recover from previous transitions if
/// required.
/// The block pool storages are either all analyzed or none of them is loaded.
/// Therefore, a failure on loading any block pool storage results a faulty
/// data volume.
/// </remarks>
/// <param name="datanode">Datanode to which this storage belongs to</param>
/// <param name="nsInfo">namespace information</param>
/// <param name="dataDirs">storage directories of block pool</param>
/// <param name="startOpt">startup option</param>
/// <returns>an array of loaded block pool directories.</returns>
/// <exception cref="System.IO.IOException">on error</exception>
internal virtual IList <Storage.StorageDirectory> LoadBpStorageDirectories(DataNode
datanode, NamespaceInfo nsInfo, ICollection <FilePath> dataDirs, HdfsServerConstants.StartupOption
startOpt)
{
IList <Storage.StorageDirectory> succeedDirs = Lists.NewArrayList();
try
{
foreach (FilePath dataDir in dataDirs)
{
if (ContainsStorageDir(dataDir))
{
throw new IOException("BlockPoolSliceStorage.recoverTransitionRead: " + "attempt to load an used block storage: "
+ dataDir);
}
Storage.StorageDirectory sd = LoadStorageDirectory(datanode, nsInfo, dataDir, startOpt
);
succeedDirs.AddItem(sd);
}
}
catch (IOException e)
{
Log.Warn("Failed to analyze storage directories for block pool " + nsInfo.GetBlockPoolID
(), e);
throw;
}
return(succeedDirs);
}
/// <exception cref="System.IO.IOException"/>
public NameNodeConnector(string name, URI nameNodeUri, Path idPath, IList <Path> targetPaths
, Configuration conf, int maxNotChangedIterations)
{
this.nameNodeUri = nameNodeUri;
this.idPath = idPath;
this.targetPaths = targetPaths == null || targetPaths.IsEmpty() ? Arrays.AsList(new
Path("/")) : targetPaths;
this.maxNotChangedIterations = maxNotChangedIterations;
this.namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(conf, nameNodeUri).
GetProxy();
this.client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, nameNodeUri, fallbackToSimpleAuth
).GetProxy();
this.fs = (DistributedFileSystem)FileSystem.Get(nameNodeUri, conf);
NamespaceInfo namespaceinfo = namenode.VersionRequest();
this.blockpoolID = namespaceinfo.GetBlockPoolID();
FsServerDefaults defaults = fs.GetServerDefaults(new Path("/"));
this.keyManager = new KeyManager(blockpoolID, namenode, defaults.GetEncryptDataTransfer
(), conf);
// if it is for test, we do not create the id file
@out = CheckAndMarkRunning();
if (@out == null)
{
// Exit if there is another one running.
throw new IOException("Another " + name + " is running.");
}
}
/// <summary>This is called when using bootstrapStandby for HA upgrade.</summary>
/// <remarks>
/// This is called when using bootstrapStandby for HA upgrade. The SBN should
/// also create previous directory so that later when it starts, it understands
/// that the cluster is in the upgrade state. This function renames the old
/// current directory to previous.tmp.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private bool DoPreUpgrade(NNStorage storage, NamespaceInfo nsInfo)
{
bool isFormatted = false;
IDictionary <Storage.StorageDirectory, Storage.StorageState> dataDirStates = new Dictionary
<Storage.StorageDirectory, Storage.StorageState>();
try
{
isFormatted = FSImage.RecoverStorageDirs(HdfsServerConstants.StartupOption.Upgrade
, storage, dataDirStates);
if (dataDirStates.Values.Contains(Storage.StorageState.NotFormatted))
{
// recoverStorageDirs returns true if there is a formatted directory
isFormatted = false;
System.Console.Error.WriteLine("The original storage directory is not formatted."
);
}
}
catch (InconsistentFSStateException e)
{
// if the storage is in a bad state,
Log.Warn("The storage directory is in an inconsistent state", e);
}
finally
{
storage.UnlockAll();
}
// if there is InconsistentFSStateException or the storage is not formatted,
// format the storage. Although this format is done through the new
// software, since in HA setup the SBN is rolled back through
// "-bootstrapStandby", we should still be fine.
if (!isFormatted && !Format(storage, nsInfo))
{
return(false);
}
// make sure there is no previous directory
FSImage.CheckUpgrade(storage);
// Do preUpgrade for each directory
for (IEnumerator <Storage.StorageDirectory> it = storage.DirIterator(false); it.HasNext
();)
{
Storage.StorageDirectory sd = it.Next();
try
{
NNUpgradeUtil.RenameCurToTmp(sd);
}
catch (IOException e)
{
Log.Error("Failed to move aside pre-upgrade storage " + "in image directory " + sd
.GetRoot(), e);
throw;
}
}
storage.SetStorageInfo(nsInfo);
storage.SetBlockPoolID(nsInfo.GetBlockPoolID());
return(true);
}
/// <summary>Analyze storage directories.</summary>
/// <remarks>
/// Analyze storage directories. Recover from previous transitions if required.
/// The block pool storages are either all analyzed or none of them is loaded.
/// Therefore, a failure on loading any block pool storage results a faulty
/// data volume.
/// </remarks>
/// <param name="datanode">Datanode to which this storage belongs to</param>
/// <param name="nsInfo">namespace information</param>
/// <param name="dataDirs">storage directories of block pool</param>
/// <param name="startOpt">startup option</param>
/// <exception cref="System.IO.IOException">on error</exception>
internal virtual void RecoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo
, ICollection <FilePath> dataDirs, HdfsServerConstants.StartupOption startOpt)
{
Log.Info("Analyzing storage directories for bpid " + nsInfo.GetBlockPoolID());
foreach (Storage.StorageDirectory sd in LoadBpStorageDirectories(datanode, nsInfo
, dataDirs, startOpt))
{
AddStorageDir(sd);
}
}
public virtual void TestConvertNamespaceInfo()
{
NamespaceInfo info = new NamespaceInfo(37, "clusterID", "bpID", 2300);
HdfsProtos.NamespaceInfoProto proto = PBHelper.Convert(info);
NamespaceInfo info2 = PBHelper.Convert(proto);
Compare(info, info2);
//Compare the StorageInfo
NUnit.Framework.Assert.AreEqual(info.GetBlockPoolID(), info2.GetBlockPoolID());
NUnit.Framework.Assert.AreEqual(info.GetBuildVersion(), info2.GetBuildVersion());
}
/// <summary>Format a block pool slice storage.</summary>
/// <param name="bpSdir">the block pool storage</param>
/// <param name="nsInfo">the name space info</param>
/// <exception cref="System.IO.IOException">Signals that an I/O exception has occurred.
/// </exception>
private void Format(Storage.StorageDirectory bpSdir, NamespaceInfo nsInfo)
{
Log.Info("Formatting block pool " + blockpoolID + " directory " + bpSdir.GetCurrentDir
());
bpSdir.ClearDirectory();
// create directory
this.layoutVersion = HdfsConstants.DatanodeLayoutVersion;
this.cTime = nsInfo.GetCTime();
this.namespaceID = nsInfo.GetNamespaceID();
this.blockpoolID = nsInfo.GetBlockPoolID();
WriteProperties(bpSdir);
}
internal virtual string GetBlockPoolId()
{
ReadLock();
try
{
if (bpNSInfo != null)
{
return(bpNSInfo.GetBlockPoolID());
}
else
{
Log.Warn("Block pool ID needed, but service not yet registered with NN", new Exception
("trace"));
return(null);
}
}
finally
{
ReadUnlock();
}
}
/// <summary>Called by the BPServiceActors when they handshake to a NN.</summary>
/// <remarks>
/// Called by the BPServiceActors when they handshake to a NN.
/// If this is the first NN connection, this sets the namespace info
/// for this BPOfferService. If it's a connection to a new NN, it
/// verifies that this namespace matches (eg to prevent a misconfiguration
/// where a StandbyNode from a different cluster is specified)
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal virtual void VerifyAndSetNamespaceInfo(NamespaceInfo nsInfo)
{
WriteLock();
try
{
if (this.bpNSInfo == null)
{
this.bpNSInfo = nsInfo;
bool success = false;
// Now that we know the namespace ID, etc, we can pass this to the DN.
// The DN can now initialize its local storage if we are the
// first BP to handshake, etc.
try
{
dn.InitBlockPool(this);
success = true;
}
finally
{
if (!success)
{
// The datanode failed to initialize the BP. We need to reset
// the namespace info so that other BPService actors still have
// a chance to set it, and re-initialize the datanode.
this.bpNSInfo = null;
}
}
}
else
{
CheckNSEquality(bpNSInfo.GetBlockPoolID(), nsInfo.GetBlockPoolID(), "Blockpool ID"
);
CheckNSEquality(bpNSInfo.GetNamespaceID(), nsInfo.GetNamespaceID(), "Namespace ID"
);
CheckNSEquality(bpNSInfo.GetClusterID(), nsInfo.GetClusterID(), "Cluster ID");
}
}
finally
{
WriteUnlock();
}
}
/// <exception cref="System.IO.IOException"/>
private void CheckEnv()
{
lock (this)
{
if (!initialized)
{
try
{
Stat versionStat = zkc.Exists(versionPath, false);
if (versionStat == null)
{
throw new IOException("Environment not initialized. " + "Have you forgotten to format?"
);
}
byte[] d = zkc.GetData(versionPath, false, versionStat);
BKJournalProtos.VersionProto.Builder builder = BKJournalProtos.VersionProto.NewBuilder
();
TextFormat.Merge(new string(d, Charsets.Utf8), builder);
if (!builder.IsInitialized())
{
throw new IOException("Invalid/Incomplete data in znode");
}
BKJournalProtos.VersionProto vp = ((BKJournalProtos.VersionProto)builder.Build());
// There's only one version at the moment
System.Diagnostics.Debug.Assert(vp.GetLayoutVersion() == BkjmLayoutVersion);
NamespaceInfo readns = PBHelper.Convert(vp.GetNamespaceInfo());
if (nsInfo.GetNamespaceID() != readns.GetNamespaceID() || !nsInfo.clusterID.Equals
(readns.GetClusterID()) || !nsInfo.GetBlockPoolID().Equals(readns.GetBlockPoolID
()))
{
string err = string.Format("Environment mismatch. Running process %s" + ", stored in ZK %s"
, nsInfo, readns);
Log.Error(err);
throw new IOException(err);
}
ci.Init();
initialized = true;
}
catch (KeeperException ke)
{
throw new IOException("Cannot access ZooKeeper", ke);
}
catch (Exception ie)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException("Interrupted while checking environment", ie);
}
}
}
}
/// <summary>
/// Analyze whether a transition of the BP state is required and
/// perform it if necessary.
/// </summary>
/// <remarks>
/// Analyze whether a transition of the BP state is required and
/// perform it if necessary.
/// <br />
/// Rollback if previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime.
/// Upgrade if this.LV > LAYOUT_VERSION || this.cTime < namenode.cTime Regular
/// startup if this.LV = LAYOUT_VERSION && this.cTime = namenode.cTime
/// </remarks>
/// <param name="sd">storage directory <SD>/current/<bpid></param>
/// <param name="nsInfo">namespace info</param>
/// <param name="startOpt">startup option</param>
/// <exception cref="System.IO.IOException"/>
private void DoTransition(DataNode datanode, Storage.StorageDirectory sd, NamespaceInfo
nsInfo, HdfsServerConstants.StartupOption startOpt)
{
if (startOpt == HdfsServerConstants.StartupOption.Rollback && sd.GetPreviousDir()
.Exists())
{
Preconditions.CheckState(!GetTrashRootDir(sd).Exists(), sd.GetPreviousDir() + " and "
+ GetTrashRootDir(sd) + " should not " + " both be present.");
DoRollback(sd, nsInfo);
}
else
{
// rollback if applicable
if (startOpt == HdfsServerConstants.StartupOption.Rollback && !sd.GetPreviousDir(
).Exists())
{
// Restore all the files in the trash. The restored files are retained
// during rolling upgrade rollback. They are deleted during rolling
// upgrade downgrade.
int restored = RestoreBlockFilesFromTrash(GetTrashRootDir(sd));
Log.Info("Restored " + restored + " block files from trash.");
}
}
ReadProperties(sd);
CheckVersionUpgradable(this.layoutVersion);
System.Diagnostics.Debug.Assert(this.layoutVersion >= HdfsConstants.DatanodeLayoutVersion
, "Future version is not allowed");
if (GetNamespaceID() != nsInfo.GetNamespaceID())
{
throw new IOException("Incompatible namespaceIDs in " + sd.GetRoot().GetCanonicalPath
() + ": namenode namespaceID = " + nsInfo.GetNamespaceID() + "; datanode namespaceID = "
+ GetNamespaceID());
}
if (!blockpoolID.Equals(nsInfo.GetBlockPoolID()))
{
throw new IOException("Incompatible blockpoolIDs in " + sd.GetRoot().GetCanonicalPath
() + ": namenode blockpoolID = " + nsInfo.GetBlockPoolID() + "; datanode blockpoolID = "
+ blockpoolID);
}
if (this.layoutVersion == HdfsConstants.DatanodeLayoutVersion && this.cTime == nsInfo
.GetCTime())
{
return;
}
// regular startup
if (this.layoutVersion > HdfsConstants.DatanodeLayoutVersion)
{
int restored = RestoreBlockFilesFromTrash(GetTrashRootDir(sd));
Log.Info("Restored " + restored + " block files from trash " + "before the layout upgrade. These blocks will be moved to "
+ "the previous directory during the upgrade");
}
if (this.layoutVersion > HdfsConstants.DatanodeLayoutVersion || this.cTime < nsInfo
.GetCTime())
{
DoUpgrade(datanode, sd, nsInfo);
// upgrade
return;
}
// layoutVersion == LAYOUT_VERSION && this.cTime > nsInfo.cTime
// must shutdown
throw new IOException("Datanode state: LV = " + this.GetLayoutVersion() + " CTime = "
+ this.GetCTime() + " is newer than the namespace state: LV = " + nsInfo.GetLayoutVersion
() + " CTime = " + nsInfo.GetCTime());
}
/// <summary>Load one storage directory.</summary>
/// <remarks>Load one storage directory. Recover from previous transitions if required.
/// </remarks>
/// <param name="datanode">datanode instance</param>
/// <param name="nsInfo">namespace information</param>
/// <param name="dataDir">the root path of the storage directory</param>
/// <param name="startOpt">startup option</param>
/// <returns>the StorageDirectory successfully loaded.</returns>
/// <exception cref="System.IO.IOException"/>
private Storage.StorageDirectory LoadStorageDirectory(DataNode datanode, NamespaceInfo
nsInfo, FilePath dataDir, HdfsServerConstants.StartupOption startOpt)
{
Storage.StorageDirectory sd = new Storage.StorageDirectory(dataDir, null, true);
try
{
Storage.StorageState curState = sd.AnalyzeStorage(startOpt, this);
switch (curState)
{
case Storage.StorageState.Normal:
{
// sd is locked but not opened
break;
}
case Storage.StorageState.NonExistent:
{
Log.Info("Block pool storage directory " + dataDir + " does not exist");
throw new IOException("Storage directory " + dataDir + " does not exist");
}
case Storage.StorageState.NotFormatted:
{
// format
Log.Info("Block pool storage directory " + dataDir + " is not formatted for " + nsInfo
.GetBlockPoolID());
Log.Info("Formatting ...");
Format(sd, nsInfo);
break;
}
default:
{
// recovery part is common
sd.DoRecover(curState);
break;
}
}
// 2. Do transitions
// Each storage directory is treated individually.
// During startup some of them can upgrade or roll back
// while others could be up-to-date for the regular startup.
DoTransition(datanode, sd, nsInfo, startOpt);
if (GetCTime() != nsInfo.GetCTime())
{
throw new IOException("Data-node and name-node CTimes must be the same.");
}
// 3. Update successfully loaded storage.
SetServiceLayoutVersion(GetServiceLayoutVersion());
WriteProperties(sd);
return(sd);
}
catch (IOException ioe)
{
sd.Unlock();
throw;
}
}
/// <summary>Register this backup node with the active name-node.</summary>
/// <param name="nsInfo">namespace information</param>
/// <exception cref="System.IO.IOException"/>
private void RegisterWith(NamespaceInfo nsInfo)
{
BackupImage bnImage = (BackupImage)GetFSImage();
NNStorage storage = bnImage.GetStorage();
// verify namespaceID
if (storage.GetNamespaceID() == 0)
{
// new backup storage
storage.SetStorageInfo(nsInfo);
storage.SetBlockPoolID(nsInfo.GetBlockPoolID());
storage.SetClusterID(nsInfo.GetClusterID());
}
else
{
nsInfo.ValidateStorage(storage);
}
bnImage.InitEditLog(HdfsServerConstants.StartupOption.Regular);
SetRegistration();
NamenodeRegistration nnReg = null;
while (!IsStopRequested())
{
try
{
nnReg = namenode.RegisterSubordinateNamenode(GetRegistration());
break;
}
catch (SocketTimeoutException e)
{
// name-node is busy
Log.Info("Problem connecting to name-node: " + nnRpcAddress);
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
Log.Warn("Encountered exception ", e);
}
}
}
string msg = null;
if (nnReg == null)
{
// consider as a rejection
msg = "Registration rejected by " + nnRpcAddress;
}
else
{
if (!nnReg.IsRole(HdfsServerConstants.NamenodeRole.Namenode))
{
msg = "Name-node " + nnRpcAddress + " is not active";
}
}
if (msg != null)
{
msg += ". Shutting down.";
Log.Error(msg);
throw new IOException(msg);
}
// stop the node
nnRpcAddress = nnReg.GetAddress();
}
