/// <exception cref="System.Exception"/>
private void SetUpHaCluster(bool security)
{
conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.HadoopSecurityAuthorization, security);
cluster = new MiniQJMHACluster.Builder(conf).Build();
SetHAConf(conf, cluster.GetDfsCluster().GetNameNode(0).GetHostAndPort(), cluster.
GetDfsCluster().GetNameNode(1).GetHostAndPort());
cluster.GetDfsCluster().GetNameNode(0).GetHostAndPort();
admin = new DFSAdmin();
admin.SetConf(conf);
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
originOut = System.Console.Out;
originErr = System.Console.Error;
Runtime.SetOut(new TextWriter(@out));
Runtime.SetErr(new TextWriter(err));
}
public virtual void SetUpCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointCheckPeriodKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeNumCheckpointsRetainedKey, 10);
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
HAUtil.SetAllowStandbyReads(conf, true);
if (clusterType == TestFailureToReadEdits.TestType.SharedDirHa)
{
MiniDFSNNTopology topology = MiniQJMHACluster.CreateDefaultTopology(10000);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).CheckExitOnShutdown
(false).Build();
}
else
{
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(0).CheckExitOnShutdown(false);
miniQjmHaCluster = builder.Build();
cluster = miniQjmHaCluster.GetDfsCluster();
}
cluster.WaitActive();
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
}
public virtual void TestFinalizeWithJournalNodes()
{
MiniQJMHACluster qjCluster = null;
FileSystem fs = null;
try
{
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(0);
qjCluster = builder.Build();
MiniDFSCluster cluster = qjCluster.GetDfsCluster();
// No upgrade is in progress at the moment.
CheckJnPreviousDirExistence(qjCluster, false);
CheckClusterPreviousDirExistence(cluster, false);
AssertCTimesEqual(cluster);
// Transition NN0 to active and do some FS ops.
cluster.TransitionToActive(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1")));
long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster);
// Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
// flag.
cluster.ShutdownNameNode(1);
cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade
);
cluster.RestartNameNode(0, false);
NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade <= GetCommittedTxnIdValue(qjCluster
));
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2")));
CheckNnPreviousDirExistence(cluster, 0, true);
CheckNnPreviousDirExistence(cluster, 1, false);
CheckJnPreviousDirExistence(qjCluster, true);
// Now bootstrap the standby with the upgraded info.
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
cluster.RestartNameNode(1);
long cidDuringUpgrade = GetCommittedTxnIdValue(qjCluster);
NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidBeforeUpgrade);
RunFinalizeCommand(cluster);
NUnit.Framework.Assert.AreEqual(cidDuringUpgrade, GetCommittedTxnIdValue(qjCluster
));
CheckClusterPreviousDirExistence(cluster, false);
CheckJnPreviousDirExistence(qjCluster, false);
AssertCTimesEqual(cluster);
}
finally
{
if (fs != null)
{
fs.Close();
}
if (qjCluster != null)
{
qjCluster.Shutdown();
}
}
}
/// <summary>Tests all FsEditLogOps that are converted to inotify events.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/>
public virtual void TestBasic()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
// so that we can get an atime change
conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 1);
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(2);
MiniQJMHACluster cluster = builder.Build();
try
{
cluster.GetDfsCluster().WaitActive();
cluster.GetDfsCluster().TransitionToActive(0);
DFSClient client = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress
(), conf);
FileSystem fs = cluster.GetDfsCluster().GetFileSystem(0);
DFSTestUtil.CreateFile(fs, new Path("/file"), BlockSize, (short)1, 0L);
DFSTestUtil.CreateFile(fs, new Path("/file3"), BlockSize, (short)1, 0L);
DFSTestUtil.CreateFile(fs, new Path("/file5"), BlockSize, (short)1, 0L);
DFSInotifyEventInputStream eis = client.GetInotifyEventStream();
client.Rename("/file", "/file4", null);
// RenameOp -> RenameEvent
client.Rename("/file4", "/file2");
// RenameOldOp -> RenameEvent
// DeleteOp, AddOp -> UnlinkEvent, CreateEvent
OutputStream os = client.Create("/file2", true, (short)2, BlockSize);
os.Write(new byte[BlockSize]);
os.Close();
// CloseOp -> CloseEvent
// AddOp -> AppendEvent
os = client.Append("/file2", BlockSize, EnumSet.Of(CreateFlag.Append), null, null
);
os.Write(new byte[BlockSize]);
os.Close();
// CloseOp -> CloseEvent
Sharpen.Thread.Sleep(10);
// so that the atime will get updated on the next line
client.Open("/file2").Read(new byte[1]);
// TimesOp -> MetadataUpdateEvent
// SetReplicationOp -> MetadataUpdateEvent
client.SetReplication("/file2", (short)1);
// ConcatDeleteOp -> AppendEvent, UnlinkEvent, CloseEvent
client.Concat("/file2", new string[] { "/file3" });
client.Delete("/file2", false);
// DeleteOp -> UnlinkEvent
client.Mkdirs("/dir", null, false);
// MkdirOp -> CreateEvent
// SetPermissionsOp -> MetadataUpdateEvent
client.SetPermission("/dir", FsPermission.ValueOf("-rw-rw-rw-"));
// SetOwnerOp -> MetadataUpdateEvent
client.SetOwner("/dir", "username", "groupname");
client.CreateSymlink("/dir", "/dir2", false);
// SymlinkOp -> CreateEvent
client.SetXAttr("/file5", "user.field", Sharpen.Runtime.GetBytesForString("value"
), EnumSet.Of(XAttrSetFlag.Create));
// SetXAttrOp -> MetadataUpdateEvent
// RemoveXAttrOp -> MetadataUpdateEvent
client.RemoveXAttr("/file5", "user.field");
// SetAclOp -> MetadataUpdateEvent
client.SetAcl("/file5", AclEntry.ParseAclSpec("user::rwx,user:foo:rw-,group::r--,other::---"
, true));
client.RemoveAcl("/file5");
// SetAclOp -> MetadataUpdateEvent
client.Rename("/file5", "/dir");
// RenameOldOp -> RenameEvent
EventBatch batch = null;
// RenameOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
long txid = batch.GetTxid();
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.AreEqual("/file4", re.GetDstPath());
NUnit.Framework.Assert.AreEqual("/file", re.GetSrcPath());
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
long eventsBehind = eis.GetTxidsBehindEstimate();
// RenameOldOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re2 = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(re2.GetDstPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(re2.GetSrcPath().Equals("/file4"));
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
// AddOp with overwrite
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce.GetiNodeType() == Event.CreateEvent.INodeType.File
);
NUnit.Framework.Assert.IsTrue(ce.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce.GetReplication() > 0);
NUnit.Framework.Assert.IsTrue(ce.GetSymlinkTarget() == null);
NUnit.Framework.Assert.IsTrue(ce.GetOverwrite());
NUnit.Framework.Assert.AreEqual(BlockSize, ce.GetDefaultBlockSize());
// CloseOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Close);
Event.CloseEvent ce2 = (Event.CloseEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce2.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce2.GetFileSize() > 0);
NUnit.Framework.Assert.IsTrue(ce2.GetTimestamp() > 0);
// AppendOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Append);
Event.AppendEvent append2 = (Event.AppendEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.AreEqual("/file2", append2.GetPath());
NUnit.Framework.Assert.IsFalse(append2.ToNewBlock());
// CloseOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Close);
NUnit.Framework.Assert.IsTrue(((Event.CloseEvent)batch.GetEvents()[0]).GetPath().
Equals("/file2"));
// TimesOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(mue.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Times);
// SetReplicationOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue2.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(mue2.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Replication);
NUnit.Framework.Assert.IsTrue(mue2.GetReplication() == 1);
// ConcatDeleteOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(3, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Append);
NUnit.Framework.Assert.IsTrue(((Event.AppendEvent)batch.GetEvents()[0]).GetPath()
.Equals("/file2"));
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[1].GetEventType() == Event.EventType
.Unlink);
Event.UnlinkEvent ue2 = (Event.UnlinkEvent)batch.GetEvents()[1];
NUnit.Framework.Assert.IsTrue(ue2.GetPath().Equals("/file3"));
NUnit.Framework.Assert.IsTrue(ue2.GetTimestamp() > 0);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[2].GetEventType() == Event.EventType
.Close);
Event.CloseEvent ce3 = (Event.CloseEvent)batch.GetEvents()[2];
NUnit.Framework.Assert.IsTrue(ce3.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce3.GetTimestamp() > 0);
// DeleteOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Unlink);
Event.UnlinkEvent ue = (Event.UnlinkEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ue.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ue.GetTimestamp() > 0);
// MkdirOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce4 = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce4.GetiNodeType() == Event.CreateEvent.INodeType.Directory
);
NUnit.Framework.Assert.IsTrue(ce4.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(ce4.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce4.GetReplication() == 0);
NUnit.Framework.Assert.IsTrue(ce4.GetSymlinkTarget() == null);
// SetPermissionsOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue3.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(mue3.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Perms);
NUnit.Framework.Assert.IsTrue(mue3.GetPerms().ToString().Contains("rw-rw-rw-"));
// SetOwnerOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue4.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(mue4.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Owner);
NUnit.Framework.Assert.IsTrue(mue4.GetOwnerName().Equals("username"));
NUnit.Framework.Assert.IsTrue(mue4.GetGroupName().Equals("groupname"));
// SymlinkOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce5 = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce5.GetiNodeType() == Event.CreateEvent.INodeType.Symlink
);
NUnit.Framework.Assert.IsTrue(ce5.GetPath().Equals("/dir2"));
NUnit.Framework.Assert.IsTrue(ce5.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce5.GetReplication() == 0);
NUnit.Framework.Assert.IsTrue(ce5.GetSymlinkTarget().Equals("/dir"));
// SetXAttrOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue5.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue5.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Xattrs);
NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs().Count == 1);
NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs()[0].GetName().Contains("field"));
NUnit.Framework.Assert.IsTrue(!mue5.IsxAttrsRemoved());
// RemoveXAttrOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue6.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue6.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Xattrs);
NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs().Count == 1);
NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs()[0].GetName().Contains("field"));
NUnit.Framework.Assert.IsTrue(mue6.IsxAttrsRemoved());
// SetAclOp (1)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue7.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue7.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Acls);
NUnit.Framework.Assert.IsTrue(mue7.GetAcls().Contains(AclEntry.ParseAclEntry("user::rwx"
, true)));
// SetAclOp (2)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue8.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue8.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Acls);
NUnit.Framework.Assert.IsTrue(mue8.GetAcls() == null);
// RenameOp (2)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re3 = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(re3.GetDstPath().Equals("/dir/file5"));
NUnit.Framework.Assert.IsTrue(re3.GetSrcPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
// Returns null when there are no further events
NUnit.Framework.Assert.IsTrue(eis.Poll() == null);
// make sure the estimate hasn't changed since the above assertion
// tells us that we are fully caught up to the current namesystem state
// and we should not have been behind at all when eventsBehind was set
// either, since there were few enough events that they should have all
// been read to the client during the first poll() call
NUnit.Framework.Assert.IsTrue(eis.GetTxidsBehindEstimate() == eventsBehind);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestRollbackWithJournalNodes()
{
MiniQJMHACluster qjCluster = null;
FileSystem fs = null;
try
{
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(0);
qjCluster = builder.Build();
MiniDFSCluster cluster = qjCluster.GetDfsCluster();
// No upgrade is in progress at the moment.
CheckClusterPreviousDirExistence(cluster, false);
AssertCTimesEqual(cluster);
CheckJnPreviousDirExistence(qjCluster, false);
// Transition NN0 to active and do some FS ops.
cluster.TransitionToActive(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1")));
long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster);
// Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
// flag.
cluster.ShutdownNameNode(1);
cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade
);
cluster.RestartNameNode(0, false);
CheckNnPreviousDirExistence(cluster, 0, true);
CheckNnPreviousDirExistence(cluster, 1, false);
CheckJnPreviousDirExistence(qjCluster, true);
// NN0 should come up in the active state when given the -upgrade option,
// so no need to transition it to active.
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2")));
long cidDuringUpgrade = GetCommittedTxnIdValue(qjCluster);
NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidBeforeUpgrade);
// Now bootstrap the standby with the upgraded info.
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
cluster.RestartNameNode(1);
CheckNnPreviousDirExistence(cluster, 0, true);
CheckNnPreviousDirExistence(cluster, 1, true);
CheckJnPreviousDirExistence(qjCluster, true);
AssertCTimesEqual(cluster);
// Shut down the NNs, but deliberately leave the JNs up and running.
ICollection <URI> nn1NameDirs = cluster.GetNameDirs(0);
cluster.Shutdown();
conf.SetStrings(DFSConfigKeys.DfsNamenodeNameDirKey, Joiner.On(",").Join(nn1NameDirs
));
NameNode.DoRollback(conf, false);
long cidAfterRollback = GetCommittedTxnIdValue(qjCluster);
NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade < cidAfterRollback);
// make sure the committedTxnId has been reset correctly after rollback
NUnit.Framework.Assert.IsTrue(cidDuringUpgrade > cidAfterRollback);
// The rollback operation should have rolled back the first NN's local
// dirs, and the shared dir, but not the other NN's dirs. Those have to be
// done by bootstrapping the standby.
CheckNnPreviousDirExistence(cluster, 0, false);
CheckJnPreviousDirExistence(qjCluster, false);
}
finally
{
if (fs != null)
{
fs.Close();
}
if (qjCluster != null)
{
qjCluster.Shutdown();
}
}
}
public virtual void TestUpgradeWithJournalNodes()
{
MiniQJMHACluster qjCluster = null;
FileSystem fs = null;
try
{
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(0);
qjCluster = builder.Build();
MiniDFSCluster cluster = qjCluster.GetDfsCluster();
// No upgrade is in progress at the moment.
CheckJnPreviousDirExistence(qjCluster, false);
CheckClusterPreviousDirExistence(cluster, false);
AssertCTimesEqual(cluster);
// Transition NN0 to active and do some FS ops.
cluster.TransitionToActive(0);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo1")));
// get the value of the committedTxnId in journal nodes
long cidBeforeUpgrade = GetCommittedTxnIdValue(qjCluster);
// Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
// flag.
cluster.ShutdownNameNode(1);
cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Upgrade
);
cluster.RestartNameNode(0, false);
CheckNnPreviousDirExistence(cluster, 0, true);
CheckNnPreviousDirExistence(cluster, 1, false);
CheckJnPreviousDirExistence(qjCluster, true);
NUnit.Framework.Assert.IsTrue(cidBeforeUpgrade <= GetCommittedTxnIdValue(qjCluster
));
// NN0 should come up in the active state when given the -upgrade option,
// so no need to transition it to active.
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo2")));
// Restart NN0 without the -upgrade flag, to make sure that works.
cluster.GetNameNodeInfos()[0].SetStartOpt(HdfsServerConstants.StartupOption.Regular
);
cluster.RestartNameNode(0, false);
// Make sure we can still do FS ops after upgrading.
cluster.TransitionToActive(0);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo3")));
NUnit.Framework.Assert.IsTrue(GetCommittedTxnIdValue(qjCluster) > cidBeforeUpgrade
);
// Now bootstrap the standby with the upgraded info.
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(0, rc);
// Now restart NN1 and make sure that we can do ops against that as well.
cluster.RestartNameNode(1);
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path("/foo4")));
AssertCTimesEqual(cluster);
}
finally
{
if (fs != null)
{
fs.Close();
}
if (qjCluster != null)
{
qjCluster.Shutdown();
}
}
}