public virtual void TestPipelineRecoveryOnRestartFailure()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientDatanodeRestartTimeoutKey, "5");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol3.dat");
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)2, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.GetDataNodes()[0];
string dnAddr1 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args1 = new string[] { "-shutdownDatanode", dnAddr1, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args1));
Sharpen.Thread.Sleep(4000);
// This should succeed without restarting the node. The restart will
// expire and regular pipeline recovery will kick in.
@out.Close();
// At this point there is only one node in the cluster.
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
dn = cluster.GetDataNodes()[1];
string dnAddr2 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args2 = new string[] { "-shutdownDatanode", dnAddr2, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args2));
Sharpen.Thread.Sleep(4000);
try
{
// close should fail
@out.Close();
System.Diagnostics.Debug.Assert(false);
}
catch (IOException)
{
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestPipelineRecoveryOnOOB()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientDatanodeRestartTimeoutKey, "15");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 1;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)1, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.GetDataNodes()[0];
string dnAddr = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args1 = new string[] { "-shutdownDatanode", dnAddr, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args1));
// Wait long enough to receive an OOB ack before closing the file.
Sharpen.Thread.Sleep(4000);
// Retart the datanode
cluster.RestartDataNode(0, true);
// The following forces a data packet and end of block packets to be sent.
@out.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestUpdatePipelineAfterDelete()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path("/test-file");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
DFSOutputStream @out = null;
try
{
// Create a file and make sure a block is allocated for it.
@out = (DFSOutputStream)(fs.Create(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
// Create a snapshot that includes the file.
SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1");
// Grab the block info of this file for later use.
FSDataInputStream @in = null;
ExtendedBlock oldBlock = null;
try
{
@in = fs.Open(file);
oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// Allocate a new block ID/gen stamp so we can simulate pipeline
// recovery.
string clientName = ((DistributedFileSystem)fs).GetClient().GetClientName();
LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName
);
ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId
(), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp());
// Delete the file from the present FS. It will still exist the
// previously-created snapshot. This will log an OP_DELETE for the
// file in question.
fs.Delete(file, true);
// Simulate a pipeline recovery, wherein a new block is allocated
// for the existing block, resulting in an OP_UPDATE_BLOCKS being
// logged for the file in question.
try
{
namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations
(), newLocatedBlock.GetStorageIDs());
}
catch (IOException ioe)
{
// normal
GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction"
, ioe);
}
// Make sure the NN can restart with the edit logs as we have them now.
cluster.RestartNameNode(true);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestGetNewStamp()
{
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test getNewStampAndToken on a finalized block
try
{
namenode.UpdateBlockForPipeline(firstBlock, string.Empty);
NUnit.Framework.Assert.Fail("Can not get a new GS from a finalized block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is not under Construction"));
}
// test getNewStampAndToken on a non-existent block
try
{
long newBlockId = firstBlock.GetBlockId() + 1;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
namenode.UpdateBlockForPipeline(newBlock, string.Empty);
NUnit.Framework.Assert.Fail("Cannot get a new GS from a non-existent block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("does not exist"));
}
/* Test RBW replicas */
// change first block to a RBW
DFSOutputStream @out = null;
try
{
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = null;
try
{
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// test non-lease holder
DFSClient dfs = ((DistributedFileSystem)fileSys).dfs;
try
{
namenode.UpdateBlockForPipeline(firstBlock, "test" + dfs.clientName);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a non lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test null lease holder
try
{
namenode.UpdateBlockForPipeline(firstBlock, null);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a null lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test getNewStampAndToken on a rbw block
namenode.UpdateBlockForPipeline(firstBlock, dfs.clientName);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}