// create a bunch of files. Write to them and then verify.
public override void Run()
{
System.Console.Out.WriteLine("Workload starting ");
for (int i = 0; i < numberOfFiles; i++)
{
Path filename = new Path(id + "." + i);
try
{
System.Console.Out.WriteLine("Workload processing file " + filename);
FSDataOutputStream stm = CreateFile(fs, filename, replication);
DFSOutputStream dfstream = (DFSOutputStream)(stm.GetWrappedStream());
dfstream.SetArtificialSlowdown(1000);
WriteFile(stm, myseed);
stm.Close();
CheckFile(fs, filename, replication, numBlocks, fileSize, myseed);
}
catch (Exception e)
{
System.Console.Out.WriteLine("Workload exception " + e);
NUnit.Framework.Assert.IsTrue(e.ToString(), false);
}
// increment the stamp to indicate that another file is done.
lock (this)
{
stamp++;
}
}
}
public virtual void TestManyDfsClientsWhereSomeNotOpen()
{
// First DFSClient has no files open so doesn't renew leases.
DFSClient mockClient1 = CreateMockClient();
Org.Mockito.Mockito.DoReturn(false).When(mockClient1).RenewLease();
NUnit.Framework.Assert.AreSame(renewer, LeaseRenewer.GetInstance(FakeAuthority, FakeUgiA
, mockClient1));
// Set up a file so that we start renewing our lease.
DFSOutputStream mockStream1 = Org.Mockito.Mockito.Mock <DFSOutputStream>();
long fileId = 456L;
renewer.Put(fileId, mockStream1, mockClient1);
// Second DFSClient does renew lease
DFSClient mockClient2 = CreateMockClient();
Org.Mockito.Mockito.DoReturn(true).When(mockClient2).RenewLease();
NUnit.Framework.Assert.AreSame(renewer, LeaseRenewer.GetInstance(FakeAuthority, FakeUgiA
, mockClient2));
// Set up a file so that we start renewing our lease.
DFSOutputStream mockStream2 = Org.Mockito.Mockito.Mock <DFSOutputStream>();
renewer.Put(fileId, mockStream2, mockClient2);
// Wait for lease to get renewed
GenericTestUtils.WaitFor(new _Supplier_156(mockClient1, mockClient2), 100, 10000);
// should not throw!
renewer.CloseFile(fileId, mockClient1);
renewer.CloseFile(fileId, mockClient2);
}
public virtual void TestThreadName()
{
DFSOutputStream mockStream = Org.Mockito.Mockito.Mock <DFSOutputStream>();
long fileId = 789L;
NUnit.Framework.Assert.IsFalse("Renewer not initially running", renewer.IsRunning
());
// Pretend to open a file
renewer.Put(fileId, mockStream, MockDfsclient);
NUnit.Framework.Assert.IsTrue("Renewer should have started running", renewer.IsRunning
());
// Check the thread name is reasonable
string threadName = renewer.GetDaemonName();
NUnit.Framework.Assert.AreEqual("LeaseRenewer:myuser@hdfs://nn1/", threadName);
// Pretend to close the file
renewer.CloseFile(fileId, MockDfsclient);
renewer.SetEmptyTime(Time.MonotonicNow());
// Should stop the renewer running within a few seconds
long failTime = Time.MonotonicNow() + 5000;
while (renewer.IsRunning() && Time.MonotonicNow() < failTime)
{
Sharpen.Thread.Sleep(50);
}
NUnit.Framework.Assert.IsFalse(renewer.IsRunning());
}
public virtual void TestRenewal()
{
// Keep track of how many times the lease gets renewed
AtomicInteger leaseRenewalCount = new AtomicInteger();
Org.Mockito.Mockito.DoAnswer(new _Answer_99(leaseRenewalCount)).When(MockDfsclient
).RenewLease();
// Set up a file so that we start renewing our lease.
DFSOutputStream mockStream = Org.Mockito.Mockito.Mock <DFSOutputStream>();
long fileId = 123L;
renewer.Put(fileId, mockStream, MockDfsclient);
// Wait for lease to get renewed
long failTime = Time.MonotonicNow() + 5000;
while (Time.MonotonicNow() < failTime && leaseRenewalCount.Get() == 0)
{
Sharpen.Thread.Sleep(50);
}
if (leaseRenewalCount.Get() == 0)
{
NUnit.Framework.Assert.Fail("Did not renew lease at all!");
}
renewer.CloseFile(fileId, MockDfsclient);
}
public virtual void TestCloseTwice()
{
DistributedFileSystem fs = cluster.GetFileSystem();
FSDataOutputStream os = fs.Create(new Path("/test"));
DFSOutputStream dos = (DFSOutputStream)Whitebox.GetInternalState(os, "wrappedStream"
);
AtomicReference <IOException> ex = (AtomicReference <IOException>)Whitebox.GetInternalState
(dos, "lastException");
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
IOException dummy = new IOException("dummy");
ex.Set(dummy);
try
{
dos.Close();
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(e, dummy);
}
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
DFSOutputStream dfsos = dfs.PrimitiveCreate(GetUriPath(f), absolutePermission, createFlag
, createParent, replication, blockSize, progress, bufferSize, checksumOpt);
return(dfs.CreateWrappedOutputStream(dfsos, statistics, dfsos.GetInitialLen()));
}
public virtual void TestPipelineRecoveryOnRestartFailure()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientDatanodeRestartTimeoutKey, "5");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol3.dat");
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)2, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.GetDataNodes()[0];
string dnAddr1 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args1 = new string[] { "-shutdownDatanode", dnAddr1, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args1));
Sharpen.Thread.Sleep(4000);
// This should succeed without restarting the node. The restart will
// expire and regular pipeline recovery will kick in.
@out.Close();
// At this point there is only one node in the cluster.
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
dn = cluster.GetDataNodes()[1];
string dnAddr2 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args2 = new string[] { "-shutdownDatanode", dnAddr2, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args2));
Sharpen.Thread.Sleep(4000);
try
{
// close should fail
@out.Close();
System.Diagnostics.Debug.Assert(false);
}
catch (IOException)
{
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestPacketTransmissionDelay()
{
// Make the first datanode to not relay heartbeat packet.
DataNodeFaultInjector dnFaultInjector = new _DataNodeFaultInjector_171();
DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.Get();
DataNodeFaultInjector.Set(dnFaultInjector);
// Setting the timeout to be 3 seconds. Normally heartbeat packet
// would be sent every 1.5 seconds if there is no data traffic.
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientSocketTimeoutKey, "3000");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("noheartbeat.dat"), (short)2);
@out.Write(unchecked ((int)(0x31)));
@out.Hflush();
DFSOutputStream dfsOut = (DFSOutputStream)@out.GetWrappedStream();
// original pipeline
DatanodeInfo[] orgNodes = dfsOut.GetPipeline();
// Cause the second datanode to timeout on reading packet
Sharpen.Thread.Sleep(3500);
@out.Write(unchecked ((int)(0x32)));
@out.Hflush();
// new pipeline
DatanodeInfo[] newNodes = dfsOut.GetPipeline();
@out.Close();
bool contains = false;
for (int i = 0; i < newNodes.Length; i++)
{
if (orgNodes[0].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
throw new IOException("The first datanode should have been replaced.");
}
if (orgNodes[1].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
contains = true;
}
}
NUnit.Framework.Assert.IsTrue(contains);
}
finally
{
DataNodeFaultInjector.Set(oldDnInjector);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
internal virtual void Put(long inodeId, DFSOutputStream @out, DFSClient dfsc)
{
lock (this)
{
if (dfsc.IsClientRunning())
{
if (!IsRunning() || IsRenewerExpired())
{
//start a new deamon with a new id.
int id = ++currentId;
daemon = new Daemon(new _Runnable_296(this, id));
daemon.Start();
}
dfsc.PutFileBeingWritten(inodeId, @out);
emptyTime = long.MaxValue;
}
}
}
public virtual void TestComputePacketChunkSize()
{
DistributedFileSystem fs = cluster.GetFileSystem();
FSDataOutputStream os = fs.Create(new Path("/test"));
DFSOutputStream dos = (DFSOutputStream)Whitebox.GetInternalState(os, "wrappedStream"
);
int packetSize = 64 * 1024;
int bytesPerChecksum = 512;
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(dos.GetType(), "computePacketChunkSize"
, typeof(int), typeof(int));
method.Invoke(dos, packetSize, bytesPerChecksum);
FieldInfo field = Sharpen.Runtime.GetDeclaredField(dos.GetType(), "packetSize");
NUnit.Framework.Assert.IsTrue((int)field.GetValue(dos) + 33 < packetSize);
// If PKT_MAX_HEADER_LEN is 257, actual packet size come to over 64KB
// without a fix on HDFS-7308.
NUnit.Framework.Assert.IsTrue((int)field.GetValue(dos) + 257 < packetSize);
}
/// <exception cref="System.Exception"/>
public virtual void TestDatanodePeersXceiver()
{
// Test DatanodeXceiver has correct peer-dataxceiver pairs for sending OOB message
try
{
StartCluster();
// Create files in DFS.
string testFile1 = "/" + GenericTestUtils.GetMethodName() + ".01.dat";
string testFile2 = "/" + GenericTestUtils.GetMethodName() + ".02.dat";
string testFile3 = "/" + GenericTestUtils.GetMethodName() + ".03.dat";
DFSClient client1 = new DFSClient(NameNode.GetAddress(conf), conf);
DFSClient client2 = new DFSClient(NameNode.GetAddress(conf), conf);
DFSClient client3 = new DFSClient(NameNode.GetAddress(conf), conf);
DFSOutputStream s1 = (DFSOutputStream)client1.Create(testFile1, true);
DFSOutputStream s2 = (DFSOutputStream)client2.Create(testFile2, true);
DFSOutputStream s3 = (DFSOutputStream)client3.Create(testFile3, true);
byte[] toWrite = new byte[1024 * 1024 * 8];
Random rb = new Random(1111);
rb.NextBytes(toWrite);
s1.Write(toWrite, 0, 1024 * 1024 * 8);
s1.Flush();
s2.Write(toWrite, 0, 1024 * 1024 * 8);
s2.Flush();
s3.Write(toWrite, 0, 1024 * 1024 * 8);
s3.Flush();
NUnit.Framework.Assert.IsTrue(dn0.GetXferServer().GetNumPeersXceiver() == dn0.GetXferServer
().GetNumPeersXceiver());
s1.Close();
s2.Close();
s3.Close();
NUnit.Framework.Assert.IsTrue(dn0.GetXferServer().GetNumPeersXceiver() == dn0.GetXferServer
().GetNumPeersXceiver());
client1.Close();
client2.Close();
client3.Close();
}
finally
{
ShutdownCluster();
}
}
public virtual void TestPipelineRecoveryOnOOB()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientDatanodeRestartTimeoutKey, "15");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 1;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)1, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.GetDataNodes()[0];
string dnAddr = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args1 = new string[] { "-shutdownDatanode", dnAddr, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args1));
// Wait long enough to receive an OOB ack before closing the file.
Sharpen.Thread.Sleep(4000);
// Retart the datanode
cluster.RestartDataNode(0, true);
// The following forces a data packet and end of block packets to be sent.
@out.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public HdfsDataOutputStream(DFSOutputStream @out, FileSystem.Statistics stats, long
startPosition)
: base(@out, stats, startPosition)
{
}
public virtual void TestGetNewStamp()
{
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test getNewStampAndToken on a finalized block
try
{
namenode.UpdateBlockForPipeline(firstBlock, string.Empty);
NUnit.Framework.Assert.Fail("Can not get a new GS from a finalized block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is not under Construction"));
}
// test getNewStampAndToken on a non-existent block
try
{
long newBlockId = firstBlock.GetBlockId() + 1;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
namenode.UpdateBlockForPipeline(newBlock, string.Empty);
NUnit.Framework.Assert.Fail("Cannot get a new GS from a non-existent block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("does not exist"));
}
/* Test RBW replicas */
// change first block to a RBW
DFSOutputStream @out = null;
try
{
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = null;
try
{
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// test non-lease holder
DFSClient dfs = ((DistributedFileSystem)fileSys).dfs;
try
{
namenode.UpdateBlockForPipeline(firstBlock, "test" + dfs.clientName);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a non lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test null lease holder
try
{
namenode.UpdateBlockForPipeline(firstBlock, null);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a null lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test getNewStampAndToken on a rbw block
namenode.UpdateBlockForPipeline(firstBlock, dfs.clientName);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestUpdatePipelineAfterDelete()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path("/test-file");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
DFSOutputStream @out = null;
try
{
// Create a file and make sure a block is allocated for it.
@out = (DFSOutputStream)(fs.Create(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
// Create a snapshot that includes the file.
SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1");
// Grab the block info of this file for later use.
FSDataInputStream @in = null;
ExtendedBlock oldBlock = null;
try
{
@in = fs.Open(file);
oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// Allocate a new block ID/gen stamp so we can simulate pipeline
// recovery.
string clientName = ((DistributedFileSystem)fs).GetClient().GetClientName();
LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName
);
ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId
(), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp());
// Delete the file from the present FS. It will still exist the
// previously-created snapshot. This will log an OP_DELETE for the
// file in question.
fs.Delete(file, true);
// Simulate a pipeline recovery, wherein a new block is allocated
// for the existing block, resulting in an OP_UPDATE_BLOCKS being
// logged for the file in question.
try
{
namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations
(), newLocatedBlock.GetStorageIDs());
}
catch (IOException ioe)
{
// normal
GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction"
, ioe);
}
// Make sure the NN can restart with the edit logs as we have them now.
cluster.RestartNameNode(true);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestRoundTripAckMetric()
{
int datanodeCount = 2;
int interval = 1;
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey, string.Empty + interval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(datanodeCount
).Build();
try
{
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
// Open a file and get the head of the pipeline
Path testFile = new Path("/testRoundTripAckMetric.txt");
FSDataOutputStream fsout = fs.Create(testFile, (short)datanodeCount);
DFSOutputStream dout = (DFSOutputStream)fsout.GetWrappedStream();
// Slow down the writes to catch the write pipeline
dout.SetChunksPerPacket(5);
dout.SetArtificialSlowdown(3000);
fsout.Write(new byte[10000]);
DatanodeInfo[] pipeline = null;
int count = 0;
while (pipeline == null && count < 5)
{
pipeline = dout.GetPipeline();
System.Console.Out.WriteLine("Waiting for pipeline to be created.");
Sharpen.Thread.Sleep(1000);
count++;
}
// Get the head node that should be receiving downstream acks
DatanodeInfo headInfo = pipeline[0];
DataNode headNode = null;
foreach (DataNode datanode in cluster.GetDataNodes())
{
if (datanode.GetDatanodeId().Equals(headInfo))
{
headNode = datanode;
break;
}
}
NUnit.Framework.Assert.IsNotNull("Could not find the head of the datanode write pipeline"
, headNode);
// Close the file and wait for the metrics to rollover
Sharpen.Thread.Sleep((interval + 1) * 1000);
// Check the ack was received
MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(headNode.GetMetrics().
Name());
NUnit.Framework.Assert.IsTrue("Expected non-zero number of acks", MetricsAsserts.GetLongCounter
("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0);
MetricsAsserts.AssertQuantileGauges("PacketAckRoundTripTimeNanos" + interval + "s"
, dnMetrics);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public HdfsDataOutputStream(DFSOutputStream @out, FileSystem.Statistics stats)
: this(@out, stats, 0L)
{
}
public static long GetFileId(DFSOutputStream @out)
{
return(@out.GetFileId());
}
public virtual void TestXceiverCount()
{
Configuration conf = new HdfsConfiguration();
// retry one time, if close fails
conf.SetInt(DFSConfigKeys.DfsClientBlockWriteLocatefollowingblockRetriesKey, 1);
MiniDFSCluster cluster = null;
int nodes = 8;
int fileCount = 5;
short fileRepl = 3;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nodes).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager();
IList <DataNode> datanodes = cluster.GetDataNodes();
DistributedFileSystem fs = cluster.GetFileSystem();
// trigger heartbeats in case not already sent
TriggerHeartbeats(datanodes);
// check that all nodes are live and in service
int expectedTotalLoad = nodes;
// xceiver server adds 1 to load
int expectedInServiceNodes = nodes;
int expectedInServiceLoad = nodes;
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// shutdown half the nodes and force a heartbeat check to ensure
// counts are accurate
for (int i = 0; i < nodes / 2; i++)
{
DataNode dn = datanodes[i];
DatanodeDescriptor dnd = dnm.GetDatanode(dn.GetDatanodeId());
dn.Shutdown();
DFSTestUtil.SetDatanodeDead(dnd);
BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager());
//Verify decommission of dead node won't impact nodesInService metrics.
dnm.GetDecomManager().StartDecommission(dnd);
expectedInServiceNodes--;
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, namesystem.GetNumLiveDataNodes
());
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
//Verify recommission of dead node won't impact nodesInService metrics.
dnm.GetDecomManager().StopDecommission(dnd);
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
}
// restart the nodes to verify that counts are correct after
// node re-registration
cluster.RestartDataNodes();
cluster.WaitActive();
datanodes = cluster.GetDataNodes();
expectedInServiceNodes = nodes;
NUnit.Framework.Assert.AreEqual(nodes, datanodes.Count);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// create streams and hsync to force datastreamers to start
DFSOutputStream[] streams = new DFSOutputStream[fileCount];
for (int i_1 = 0; i_1 < fileCount; i_1++)
{
streams[i_1] = (DFSOutputStream)fs.Create(new Path("/f" + i_1), fileRepl).GetWrappedStream
();
streams[i_1].Write(Sharpen.Runtime.GetBytesForString("1"));
streams[i_1].Hsync();
// the load for writers is 2 because both the write xceiver & packet
// responder threads are counted in the load
expectedTotalLoad += 2 * fileRepl;
expectedInServiceLoad += 2 * fileRepl;
}
// force nodes to send load update
TriggerHeartbeats(datanodes);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
// decomm a few nodes, substract their load from the expected load,
// trigger heartbeat to force load update
for (int i_2 = 0; i_2 < fileRepl; i_2++)
{
expectedInServiceNodes--;
DatanodeDescriptor dnd = dnm.GetDatanode(datanodes[i_2].GetDatanodeId());
expectedInServiceLoad -= dnd.GetXceiverCount();
dnm.GetDecomManager().StartDecommission(dnd);
DataNodeTestUtils.TriggerHeartbeat(datanodes[i_2]);
Sharpen.Thread.Sleep(100);
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
}
// check expected load while closing each stream. recalc expected
// load based on whether the nodes in the pipeline are decomm
for (int i_3 = 0; i_3 < fileCount; i_3++)
{
int decomm = 0;
foreach (DatanodeInfo dni in streams[i_3].GetPipeline())
{
DatanodeDescriptor dnd = dnm.GetDatanode(dni);
expectedTotalLoad -= 2;
if (dnd.IsDecommissionInProgress() || dnd.IsDecommissioned())
{
decomm++;
}
else
{
expectedInServiceLoad -= 2;
}
}
try
{
streams[i_3].Close();
}
catch (IOException ioe)
{
// nodes will go decommissioned even if there's a UC block whose
// other locations are decommissioned too. we'll ignore that
// bug for now
if (decomm < fileRepl)
{
throw;
}
}
TriggerHeartbeats(datanodes);
// verify node count and loads
CheckClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes,
expectedInServiceLoad);
}
// shutdown each node, verify node counts based on decomm state
for (int i_4 = 0; i_4 < nodes; i_4++)
{
DataNode dn = datanodes[i_4];
dn.Shutdown();
// force it to appear dead so live count decreases
DatanodeDescriptor dnDesc = dnm.GetDatanode(dn.GetDatanodeId());
DFSTestUtil.SetDatanodeDead(dnDesc);
BlockManagerTestUtil.CheckHeartbeat(namesystem.GetBlockManager());
NUnit.Framework.Assert.AreEqual(nodes - 1 - i_4, namesystem.GetNumLiveDataNodes()
);
// first few nodes are already out of service
if (i_4 >= fileRepl)
{
expectedInServiceNodes--;
}
NUnit.Framework.Assert.AreEqual(expectedInServiceNodes, GetNumDNInService(namesystem
));
// live nodes always report load of 1. no nodes is load 0
double expectedXceiverAvg = (i_4 == nodes - 1) ? 0.0 : 1.0;
NUnit.Framework.Assert.AreEqual((double)expectedXceiverAvg, GetInServiceXceiverAverage
(namesystem), Epsilon);
}
// final sanity check
CheckClusterHealth(0, namesystem, 0.0, 0, 0.0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Write to one file, then kill one datanode in the pipeline and then
/// close the file.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void SimpleTest(int datanodeToKill)
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 2000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 2);
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000);
int myMaxNodes = 5;
System.Console.Out.WriteLine("SimpleTest starting with DataNode to Kill " + datanodeToKill
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(myMaxNodes
).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
short repl = 3;
Path filename = new Path("simpletest.dat");
try
{
// create a file and write one block of data
System.Console.Out.WriteLine("SimpleTest creating file " + filename);
FSDataOutputStream stm = CreateFile(fs, filename, repl);
DFSOutputStream dfstream = (DFSOutputStream)(stm.GetWrappedStream());
// these are test settings
dfstream.SetChunksPerPacket(5);
dfstream.SetArtificialSlowdown(3000);
long myseed = AppendTestUtil.NextLong();
byte[] buffer = AppendTestUtil.RandomBytes(myseed, fileSize);
int mid = fileSize / 4;
stm.Write(buffer, 0, mid);
DatanodeInfo[] targets = dfstream.GetPipeline();
int count = 5;
while (count-- > 0 && targets == null)
{
try
{
System.Console.Out.WriteLine("SimpleTest: Waiting for pipeline to be created.");
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
targets = dfstream.GetPipeline();
}
if (targets == null)
{
int victim = AppendTestUtil.NextInt(myMaxNodes);
System.Console.Out.WriteLine("SimpleTest stopping datanode random " + victim);
cluster.StopDataNode(victim);
}
else
{
int victim = datanodeToKill;
System.Console.Out.WriteLine("SimpleTest stopping datanode " + targets[victim]);
cluster.StopDataNode(targets[victim].GetXferAddr());
}
System.Console.Out.WriteLine("SimpleTest stopping datanode complete");
// write some more data to file, close and verify
stm.Write(buffer, mid, fileSize - mid);
stm.Close();
CheckFile(fs, filename, repl, numBlocks, fileSize, myseed);
}
catch (Exception e)
{
System.Console.Out.WriteLine("Simple Workload exception " + e);
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.IsTrue(e.ToString(), false);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
