public virtual void Setup()
{
// start 1 journal node
jCluster = new MiniJournalCluster.Builder(new Configuration()).Format(true).NumJournalNodes
(NumJn).Build();
jn = jCluster.GetJournalNode(0);
}
/// <exception cref="System.IO.IOException"/>
private void CheckRecovery(MiniJournalCluster cluster, long segmentTxId, long expectedEndTxId
)
{
int numFinalized = 0;
for (int i = 0; i < cluster.GetNumNodes(); i++)
{
FilePath logDir = cluster.GetCurrentDir(i, QJMTestUtil.Jid);
FileJournalManager.EditLogFile elf = FileJournalManager.GetLogFile(logDir, segmentTxId
);
if (elf == null)
{
continue;
}
if (!elf.IsInProgress())
{
numFinalized++;
if (elf.GetLastTxId() != expectedEndTxId)
{
NUnit.Framework.Assert.Fail("File " + elf + " finalized to wrong txid, expected "
+ expectedEndTxId);
}
}
}
if (numFinalized < cluster.GetQuorumSize())
{
NUnit.Framework.Assert.Fail("Did not find a quorum of finalized logs starting at "
+ segmentTxId);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
private static QuorumJournalManager CreateRandomFaultyQJM(MiniJournalCluster cluster
, Random seedGenerator)
{
AsyncLogger.Factory spyFactory = new _Factory_502(seedGenerator);
return(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil.Jid
), QJMTestUtil.FakeNsinfo, spyFactory));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
private static QuorumJournalManager CreateInjectableQJM(MiniJournalCluster cluster
)
{
AsyncLogger.Factory spyFactory = new _Factory_487();
return(new QuorumJournalManager(conf, cluster.GetQuorumJournalURI(QJMTestUtil.Jid
), QJMTestUtil.FakeNsinfo, spyFactory));
}
private static Configuration SetConf(Configuration conf, FilePath dir, MiniJournalCluster
mjc)
{
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, dir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
conf.SetLong(DFSConfigKeys.DfsNamenodeCheckpointTxnsKey, 0L);
return(conf);
}
public virtual void TestRollbackWithQJM()
{
Configuration conf = new HdfsConfiguration();
MiniJournalCluster mjc = null;
MiniDFSCluster cluster = null;
Path foo = new Path("/foo");
Path bar = new Path("/bar");
try
{
mjc = new MiniJournalCluster.Builder(conf).NumJournalNodes(NumJournalNodes).Build
();
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI(JournalId)
.ToString());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
DFSAdmin dfsadmin = new DFSAdmin(conf);
dfs.Mkdirs(foo);
// start rolling upgrade
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(new string[] { "-rollingUpgrade",
"prepare" }));
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// create new directory
dfs.Mkdirs(bar);
dfs.Close();
// rollback
cluster.RestartNameNode("-rollingUpgrade", "rollback");
// make sure /foo is still there, but /bar is not
dfs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(dfs.Exists(foo));
NUnit.Framework.Assert.IsFalse(dfs.Exists(bar));
// check storage in JNs
for (int i = 0; i < NumJournalNodes; i++)
{
FilePath dir = mjc.GetCurrentDir(0, JournalId);
// segments:(startSegment, mkdir, endSegment), (startSegment, upgrade
// marker, mkdir, endSegment)
CheckJNStorage(dir, 4, 7);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (mjc != null)
{
mjc.Shutdown();
}
}
}
public virtual void Setup()
{
conf = new Configuration();
// Don't retry connections - it just slows down the tests.
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, 0);
cluster = new MiniJournalCluster.Builder(conf).Build();
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
qjm.Format(QJMTestUtil.FakeNsinfo);
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.AreEqual(1, qjm.GetLoggerSetForTests().GetEpoch());
}
/// <summary>
/// Run a simple workload of becoming the active writer and writing
/// two log segments: 1-3 and 4-6.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private static int DoWorkload(MiniJournalCluster cluster, QuorumJournalManager qjm
)
{
int lastAcked = 0;
try
{
qjm.RecoverUnfinalizedSegments();
QJMTestUtil.WriteSegment(cluster, qjm, 1, 3, true);
lastAcked = 3;
QJMTestUtil.WriteSegment(cluster, qjm, 4, 3, true);
lastAcked = 6;
}
catch (QuorumException qe)
{
Log.Info("Failed to write at txid " + lastAcked, qe);
}
return(lastAcked);
}
public virtual void Setup()
{
Configuration conf = new Configuration();
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).Build();
cluster = miniQjmHaCluster.GetDfsCluster();
jCluster = miniQjmHaCluster.GetJournalCluster();
// make nn0 active
cluster.TransitionToActive(0);
// do sth to generate in-progress edit log data
DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs
(cluster, conf);
dfs.Mkdirs(new Path("/test2"));
dfs.Close();
}
public virtual void TestJournalNodeMXBean()
{
// we have not formatted the journals yet, and the journal status in jmx
// should be empty since journal objects are created lazily
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=JournalNode,name=JournalNodeInfo"
);
// getJournalsStatus
string journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
NUnit.Framework.Assert.IsFalse(journalStatus.Contains(Nameservice));
// format the journal ns1
NamespaceInfo FakeNsinfo = new NamespaceInfo(12345, "mycluster", "my-bp", 0L);
jn.GetOrCreateJournal(Nameservice).Format(FakeNsinfo);
// check again after format
// getJournalsStatus
journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
IDictionary <string, IDictionary <string, string> > jMap = new Dictionary <string, IDictionary
<string, string> >();
IDictionary <string, string> infoMap = new Dictionary <string, string>();
infoMap["Formatted"] = "true";
jMap[Nameservice] = infoMap;
NUnit.Framework.Assert.AreEqual(JSON.ToString(jMap), journalStatus);
// restart journal node without formatting
jCluster = new MiniJournalCluster.Builder(new Configuration()).Format(false).NumJournalNodes
(NumJn).Build();
jn = jCluster.GetJournalNode(0);
// re-check
journalStatus = (string)mbs.GetAttribute(mxbeanName, "JournalsStatus");
NUnit.Framework.Assert.AreEqual(jn.GetJournalsStatus(), journalStatus);
jMap = new Dictionary <string, IDictionary <string, string> >();
infoMap = new Dictionary <string, string>();
infoMap["Formatted"] = "true";
jMap[Nameservice] = infoMap;
NUnit.Framework.Assert.AreEqual(JSON.ToString(jMap), journalStatus);
}
private long WriteSegmentUntilCrash(MiniJournalCluster cluster, QuorumJournalManager
qjm, long txid, int numTxns, Holder <Exception> thrown)
{
long firstTxId = txid;
long lastAcked = txid - 1;
try
{
EditLogOutputStream stm = qjm.StartLogSegment(txid, NameNodeLayoutVersion.CurrentLayoutVersion
);
for (int i = 0; i < numTxns; i++)
{
QJMTestUtil.WriteTxns(stm, txid++, 1);
lastAcked++;
}
stm.Close();
qjm.FinalizeLogSegment(firstTxId, lastAcked);
}
catch (Exception t)
{
thrown.held = t;
}
return(lastAcked);
}
