/// <summary>
/// Test the case that remove a data volume on a particular DataNode when the
/// volume is actively being written.
/// </summary>
/// <param name="dataNodeIdx">the index of the DataNode to remove a volume.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.BrokenBarrierException"/>
private void TestRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
{
// Starts DFS cluster with 3 DataNodes to form a pipeline.
StartDFSCluster(1, 3);
short Replication = 3;
DataNode dn = cluster.GetDataNodes()[dataNodeIdx];
FileSystem fs = cluster.GetFileSystem();
Path testFile = new Path("/test");
long lastTimeDiskErrorCheck = dn.GetLastDiskErrorCheck();
FSDataOutputStream @out = fs.Create(testFile, Replication);
Random rb = new Random(0);
byte[] writeBuf = new byte[BlockSize / 2];
// half of the block.
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
// Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
// BlockReceiver releases volume reference before finalizeBlock(), the blocks
// on the volume will be removed, and finalizeBlock() throws IOE.
FsDatasetSpi <FsVolumeSpi> data = dn.data;
dn.data = Org.Mockito.Mockito.Spy(data);
Org.Mockito.Mockito.DoAnswer(new _Answer_599(data)).When(dn.data).FinalizeBlock(Matchers.Any
<ExtendedBlock>());
// Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
// the block is not removed, since the volume reference should not
// be released at this point.
CyclicBarrier barrier = new CyclicBarrier(2);
IList <string> oldDirs = GetDataDirs(dn);
string newDirs = oldDirs[1];
// Remove the first volume.
IList <Exception> exceptions = new AList <Exception>();
Sharpen.Thread reconfigThread = new _Thread_616(barrier, dn, newDirs, exceptions);
reconfigThread.Start();
barrier.Await();
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
@out.Close();
reconfigThread.Join();
// Verify the file has sufficient replications.
DFSTestUtil.WaitReplication(fs, testFile, Replication);
// Read the content back
byte[] content = DFSTestUtil.ReadFileBuffer(fs, testFile);
NUnit.Framework.Assert.AreEqual(BlockSize, content.Length);
// If an IOException thrown from BlockReceiver#run, it triggers
// DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
// to see whether there is IOException in BlockReceiver#run().
NUnit.Framework.Assert.AreEqual(lastTimeDiskErrorCheck, dn.GetLastDiskErrorCheck(
));
if (!exceptions.IsEmpty())
{
throw new IOException(exceptions[0].InnerException);
}
}
public virtual void TestPacketTransmissionDelay()
{
// Make the first datanode to not relay heartbeat packet.
DataNodeFaultInjector dnFaultInjector = new _DataNodeFaultInjector_171();
DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.Get();
DataNodeFaultInjector.Set(dnFaultInjector);
// Setting the timeout to be 3 seconds. Normally heartbeat packet
// would be sent every 1.5 seconds if there is no data traffic.
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientSocketTimeoutKey, "3000");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("noheartbeat.dat"), (short)2);
@out.Write(unchecked ((int)(0x31)));
@out.Hflush();
DFSOutputStream dfsOut = (DFSOutputStream)@out.GetWrappedStream();
// original pipeline
DatanodeInfo[] orgNodes = dfsOut.GetPipeline();
// Cause the second datanode to timeout on reading packet
Sharpen.Thread.Sleep(3500);
@out.Write(unchecked ((int)(0x32)));
@out.Hflush();
// new pipeline
DatanodeInfo[] newNodes = dfsOut.GetPipeline();
@out.Close();
bool contains = false;
for (int i = 0; i < newNodes.Length; i++)
{
if (orgNodes[0].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
throw new IOException("The first datanode should have been replaced.");
}
if (orgNodes[1].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
contains = true;
}
}
NUnit.Framework.Assert.IsTrue(contains);
}
finally
{
DataNodeFaultInjector.Set(oldDnInjector);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestHSyncOperation(bool testWithAppend)
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSync/foo");
int len = 1 << 16;
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null);
if (testWithAppend)
{
// re-open the file with append call
@out.Close();
@out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.SyncBlock), 4096, null
);
}
@out.Hflush();
// hflush does not sync
CheckSyncMetric(cluster, 0);
@out.Hsync();
// hsync on empty file does nothing
CheckSyncMetric(cluster, 0);
@out.Write(1);
CheckSyncMetric(cluster, 0);
@out.Hsync();
CheckSyncMetric(cluster, 1);
// avoiding repeated hsyncs is a potential future optimization
@out.Hsync();
CheckSyncMetric(cluster, 2);
@out.Hflush();
// hflush still does not sync
CheckSyncMetric(cluster, 2);
@out.Close();
// close is sync'ing
CheckSyncMetric(cluster, 3);
// same with a file created with out SYNC_BLOCK
@out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite), 4096, (short)1, len, null);
@out.Hsync();
CheckSyncMetric(cluster, 3);
@out.Write(1);
CheckSyncMetric(cluster, 3);
@out.Hsync();
CheckSyncMetric(cluster, 4);
// repeated hsyncs
@out.Hsync();
CheckSyncMetric(cluster, 5);
@out.Close();
// close does not sync (not opened with SYNC_BLOCK)
CheckSyncMetric(cluster, 5);
cluster.Shutdown();
}
public virtual void TestSimpleFlush()
{
Configuration conf = new HdfsConfiguration();
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// create a new file.
Path file1 = new Path("/simpleFlush.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file simpleFlush.dat");
// write to file
int mid = AppendTestUtil.FileSize / 2;
stm.Write(fileContents, 0, mid);
stm.Hflush();
System.Console.Out.WriteLine("Wrote and Flushed first part of file.");
// write the remainder of the file
stm.Write(fileContents, mid, AppendTestUtil.FileSize - mid);
System.Console.Out.WriteLine("Written second part of file");
stm.Hflush();
stm.Hflush();
System.Console.Out.WriteLine("Wrote and Flushed second part of file.");
// verify that full blocks are sane
CheckFile(fs, file1, 1);
stm.Close();
System.Console.Out.WriteLine("Closed file.");
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <summary>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// </summary>
/// <remarks>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// This prevents the second instance from running because it can not
/// creates the file while the first one is running.
/// This method checks if there is any running instance. If no, mark yes.
/// Note that this is an atomic operation.
/// </remarks>
/// <returns>
/// null if there is a running instance;
/// otherwise, the output stream to the newly created file.
/// </returns>
/// <exception cref="System.IO.IOException"/>
private OutputStream CheckAndMarkRunning()
{
try
{
if (fs.Exists(idPath))
{
// try appending to it so that it will fail fast if another balancer is
// running.
IOUtils.CloseStream(fs.Append(idPath));
fs.Delete(idPath, true);
}
FSDataOutputStream fsout = fs.Create(idPath, false);
// mark balancer idPath to be deleted during filesystem closure
fs.DeleteOnExit(idPath);
if (write2IdFile)
{
fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName());
fsout.Hflush();
}
return(fsout);
}
catch (RemoteException e)
{
if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName()))
{
return(null);
}
else
{
throw;
}
}
}
public virtual void TestAbandonBlock()
{
string src = FileNamePrefix + "foo";
// Start writing a file but do not close it
FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L);
for (int i = 0; i < 1024; i++)
{
fout.Write(123);
}
fout.Hflush();
long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId();
// Now abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue
);
int orginalNumBlocks = blocks.LocatedBlockCount();
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// call abandonBlock again to make sure the operation is idempotent
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// And close the file
fout.Close();
// Close cluster and check the block has been abandoned after restart
cluster.RestartNameNode();
blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue);
NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks
, blocks.LocatedBlockCount() + 1);
}
public virtual void TestBlockReportsWhileFileBeingWritten()
{
FSDataOutputStream @out = fs.Create(TestFilePath);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Block report will include the RBW replica, but will be
// queued on the StandbyNode.
cluster.TriggerBlockReports();
}
finally
{
IOUtils.CloseStream(@out);
}
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
BlockManagerTestUtil.UpdateState(nn2.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, TestFilePath);
}
public virtual void TestBlocksScheduledCounter()
{
cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
//open a file an write a few bytes:
FSDataOutputStream @out = fs.Create(new Path("/testBlockScheduledCounter"));
for (int i = 0; i < 1024; i++)
{
@out.Write(i);
}
// flush to make sure a block is allocated.
@out.Hflush();
AList <DatanodeDescriptor> dnList = new AList <DatanodeDescriptor>();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
dm.FetchDatanodes(dnList, dnList, false);
DatanodeDescriptor dn = dnList[0];
NUnit.Framework.Assert.AreEqual(1, dn.GetBlocksScheduled());
// close the file and the counter should go to zero.
@out.Close();
NUnit.Framework.Assert.AreEqual(0, dn.GetBlocksScheduled());
}
public virtual void TestAppendLessThanChecksumChunk()
{
byte[] buf = new byte[1024];
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes
(1).Build();
cluster.WaitActive();
try
{
using (DistributedFileSystem fs = cluster.GetFileSystem())
{
int len1 = 200;
int len2 = 300;
Path p = new Path("/foo");
FSDataOutputStream @out = fs.Create(p);
@out.Write(buf, 0, len1);
@out.Close();
@out = fs.Append(p);
@out.Write(buf, 0, len2);
// flush but leave open
@out.Hflush();
// read data to verify the replica's content and checksum are correct
FSDataInputStream @in = fs.Open(p);
int length = @in.Read(0, buf, 0, len1 + len2);
NUnit.Framework.Assert.IsTrue(length > 0);
@in.Close();
@out.Close();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestAppend()
{
int maxOldFileLen = 2 * BlockSize + 1;
int maxFlushedBytes = BlockSize;
byte[] contents = AppendTestUtil.InitBuffer(maxOldFileLen + 2 * maxFlushedBytes);
for (int oldFileLen = 0; oldFileLen <= maxOldFileLen; oldFileLen++)
{
for (int flushedBytes1 = 0; flushedBytes1 <= maxFlushedBytes; flushedBytes1++)
{
for (int flushedBytes2 = 0; flushedBytes2 <= maxFlushedBytes; flushedBytes2++)
{
int fileLen = oldFileLen + flushedBytes1 + flushedBytes2;
// create the initial file of oldFileLen
Path p = new Path("foo" + oldFileLen + "_" + flushedBytes1 + "_" + flushedBytes2);
Log.Info("Creating file " + p);
FSDataOutputStream @out = fs.Create(p, false, conf.GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), Replication, BlockSize);
@out.Write(contents, 0, oldFileLen);
@out.Close();
// append flushedBytes bytes to the file
@out = fs.Append(p);
@out.Write(contents, oldFileLen, flushedBytes1);
@out.Hflush();
// write another flushedBytes2 bytes to the file
@out.Write(contents, oldFileLen + flushedBytes1, flushedBytes2);
@out.Close();
// validate the file content
AppendTestUtil.CheckFullFile(fs, p, fileLen, contents, p.ToString());
fs.Delete(p, false);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestRbwReplicas(MiniDFSCluster cluster, bool isCorrupt)
{
FSDataOutputStream @out = null;
FileSystem fs = cluster.GetFileSystem();
Path src = new Path("/test.txt");
try
{
int fileLen = 515;
// create some rbw replicas on disk
byte[] writeBuf = new byte[fileLen];
new Random().NextBytes(writeBuf);
@out = fs.Create(src);
@out.Write(writeBuf);
@out.Hflush();
DataNode dn = cluster.GetDataNodes()[0];
foreach (FsVolumeSpi v in Dataset(dn).GetVolumes())
{
FsVolumeImpl volume = (FsVolumeImpl)v;
FilePath currentDir = volume.GetCurrentDir().GetParentFile().GetParentFile();
FilePath rbwDir = new FilePath(currentDir, "rbw");
foreach (FilePath file in rbwDir.ListFiles())
{
if (isCorrupt && Block.IsBlockFilename(file))
{
new RandomAccessFile(file, "rw").SetLength(fileLen - 1);
}
}
}
// corrupt
cluster.RestartDataNodes();
cluster.WaitActive();
dn = cluster.GetDataNodes()[0];
// check volumeMap: one rwr replica
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ReplicaMap replicas = Dataset(dn).volumeMap;
NUnit.Framework.Assert.AreEqual(1, replicas.Size(bpid));
ReplicaInfo replica = replicas.Replicas(bpid).GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rwr, replica.GetState
());
if (isCorrupt)
{
NUnit.Framework.Assert.AreEqual((fileLen - 1) / 512 * 512, replica.GetNumBytes());
}
else
{
NUnit.Framework.Assert.AreEqual(fileLen, replica.GetNumBytes());
}
Dataset(dn).Invalidate(bpid, new Block[] { replica });
}
finally
{
IOUtils.CloseStream(@out);
if (fs.Exists(src))
{
fs.Delete(src, false);
}
fs.Close();
}
}
public override void Run()
{
FSDataOutputStream @out = null;
int i = 0;
try
{
@out = fs.Create(filepath);
for (; running; i++)
{
System.Console.Out.WriteLine(GetName() + " writes " + i);
@out.Write(i);
@out.Hflush();
Sleep(100);
}
}
catch (Exception e)
{
System.Console.Out.WriteLine(GetName() + " dies: e=" + e);
}
finally
{
System.Console.Out.WriteLine(GetName() + ": i=" + i);
IOUtils.CloseStream(@out);
}
}
public virtual void TestHSyncBlockBoundary()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSyncBlockBoundary/foo");
int len = 1 << 16;
byte[] fileContents = AppendTestUtil.InitBuffer(len);
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null);
// fill exactly one block (tests the SYNC_BLOCK case) and flush
@out.Write(fileContents, 0, len);
@out.Hflush();
// the full block should have caused a sync
CheckSyncMetric(cluster, 1);
@out.Hsync();
// first on block again
CheckSyncMetric(cluster, 1);
// write one more byte and sync again
@out.Write(1);
@out.Hsync();
CheckSyncMetric(cluster, 2);
@out.Close();
CheckSyncMetric(cluster, 3);
cluster.Shutdown();
}
public virtual void TestHSyncWithReplication()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSyncWithReplication/foo");
int len = 1 << 16;
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)3, len, null);
@out.Write(1);
@out.Hflush();
CheckSyncMetric(cluster, 0, 0);
CheckSyncMetric(cluster, 1, 0);
CheckSyncMetric(cluster, 2, 0);
@out.Hsync();
CheckSyncMetric(cluster, 0, 1);
CheckSyncMetric(cluster, 1, 1);
CheckSyncMetric(cluster, 2, 1);
@out.Hsync();
CheckSyncMetric(cluster, 0, 2);
CheckSyncMetric(cluster, 1, 2);
CheckSyncMetric(cluster, 2, 2);
cluster.Shutdown();
}
public virtual void TestPipelineHeartbeat()
{
int DatanodeNum = 2;
int fileLen = 6;
Configuration conf = new HdfsConfiguration();
int timeout = 2000;
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, timeout);
Path p = new Path("/pipelineHeartbeat/foo");
System.Console.Out.WriteLine("p=" + p);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeNum
).Build();
try
{
DistributedFileSystem fs = cluster.GetFileSystem();
byte[] fileContents = AppendTestUtil.InitBuffer(fileLen);
// create a new file.
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, p, DatanodeNum);
stm.Write(fileContents, 0, 1);
Sharpen.Thread.Sleep(timeout);
stm.Hflush();
System.Console.Out.WriteLine("Wrote 1 byte and hflush " + p);
// write another byte
Sharpen.Thread.Sleep(timeout);
stm.Write(fileContents, 1, 1);
stm.Hflush();
stm.Write(fileContents, 2, 1);
Sharpen.Thread.Sleep(timeout);
stm.Hflush();
stm.Write(fileContents, 3, 1);
Sharpen.Thread.Sleep(timeout);
stm.Write(fileContents, 4, 1);
stm.Hflush();
stm.Write(fileContents, 5, 1);
Sharpen.Thread.Sleep(timeout);
stm.Close();
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, p, fileLen, fileContents, "Failed to slowly write to a file"
);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Append to a partial CRC chunk and the first write does not fill up the
/// partial CRC trunk
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void TestAppendToPartialChunk(bool appendToNewBlock)
{
Path p = new Path("/partialChunk/foo" + (appendToNewBlock ? "0" : "1"));
int fileLen = 513;
System.Console.Out.WriteLine("p=" + p);
byte[] fileContents = AppendTestUtil.InitBuffer(fileLen);
// create a new file.
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, p, 1);
// create 1 byte file
stm.Write(fileContents, 0, 1);
stm.Close();
System.Console.Out.WriteLine("Wrote 1 byte and closed the file " + p);
// append to file
stm = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null) : fs.Append(p);
// Append to a partial CRC trunk
stm.Write(fileContents, 1, 1);
stm.Hflush();
// The partial CRC trunk is not full yet and close the file
stm.Close();
System.Console.Out.WriteLine("Append 1 byte and closed the file " + p);
// write the remainder of the file
stm = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null) : fs.Append(p);
// ensure getPos is set to reflect existing size of the file
NUnit.Framework.Assert.AreEqual(2, stm.GetPos());
// append to a partial CRC trunk
stm.Write(fileContents, 2, 1);
// The partial chunk is not full yet, force to send a packet to DN
stm.Hflush();
System.Console.Out.WriteLine("Append and flush 1 byte");
// The partial chunk is not full yet, force to send another packet to DN
stm.Write(fileContents, 3, 2);
stm.Hflush();
System.Console.Out.WriteLine("Append and flush 2 byte");
// fill up the partial chunk and close the file
stm.Write(fileContents, 5, fileLen - 5);
stm.Close();
System.Console.Out.WriteLine("Flush 508 byte and closed the file " + p);
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, p, fileLen, fileContents, "Failed to append to a partial chunk"
);
}
/// <exception cref="System.Exception"/>
private void DoTestWriteOverFailoverWithDnFail(TestPipelinesFailover.TestScenario
scenario)
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(5).Build();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
Sharpen.Thread.Sleep(500);
Log.Info("Starting with NN 0 active");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
stm = fs.Create(TestPath);
// write a block and a half
AppendTestUtil.Write(stm, 0, BlockAndAHalf);
// Make sure all the blocks are written before failover
stm.Hflush();
Log.Info("Failing over to NN 1");
scenario.Run(cluster);
NUnit.Framework.Assert.IsTrue(fs.Exists(TestPath));
cluster.StopDataNode(0);
// write another block and a half
AppendTestUtil.Write(stm, BlockAndAHalf, BlockAndAHalf);
stm.Hflush();
Log.Info("Failing back to NN 0");
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
cluster.StopDataNode(1);
AppendTestUtil.Write(stm, BlockAndAHalf * 2, BlockAndAHalf);
stm.Hflush();
stm.Close();
stm = null;
AppendTestUtil.Check(fs, TestPath, BlockAndAHalf * 3);
}
finally
{
IOUtils.CloseStream(stm);
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
internal static FSDataOutputStream CreateAndHflush(FileSystem fs, Path file, byte
[] data, int length)
{
FSDataOutputStream @out = fs.Create(file, false, 4096, (short)3, 1024);
@out.Write(data, 0, length);
@out.Hflush();
return(@out);
}
public virtual void TestWrite()
{
MiniDFSCluster cluster = null;
int numDataNodes = 2;
Configuration conf = GetConf(numDataNodes);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
NameNode nn = cluster.GetNameNode();
BlockManager bm = nn.GetNamesystem().GetBlockManager();
BlockTokenSecretManager sm = bm.GetBlockTokenSecretManager();
// set a short token lifetime (1 second)
SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L);
Path fileToWrite = new Path(FileToWrite);
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream stm = WriteFile(fs, fileToWrite, (short)numDataNodes, BlockSize
);
// write a partial block
int mid = rawData.Length - 1;
stm.Write(rawData, 0, mid);
stm.Hflush();
/*
* wait till token used in stm expires
*/
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = DFSTestUtil.
GetBlockToken(stm);
while (!SecurityTestUtil.IsBlockTokenExpired(token))
{
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
}
// remove a datanode to force re-establishing pipeline
cluster.StopDataNode(0);
// write the rest of the file
stm.Write(rawData, mid, rawData.Length - mid);
stm.Close();
// check if write is successful
FSDataInputStream in4 = fs.Open(fileToWrite);
NUnit.Framework.Assert.IsTrue(CheckFile1(in4));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
private void DoWriteOverFailoverTest(TestPipelinesFailover.TestScenario scenario,
TestPipelinesFailover.MethodToTestIdempotence methodToTest)
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
// Don't check replication periodically.
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(3).Build();
try
{
int sizeWritten = 0;
cluster.WaitActive();
cluster.TransitionToActive(0);
Sharpen.Thread.Sleep(500);
Log.Info("Starting with NN 0 active");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
stm = fs.Create(TestPath);
// write a block and a half
AppendTestUtil.Write(stm, 0, BlockAndAHalf);
sizeWritten += BlockAndAHalf;
// Make sure all of the blocks are written out before failover.
stm.Hflush();
Log.Info("Failing over to NN 1");
scenario.Run(cluster);
// NOTE: explicitly do *not* make any further metadata calls
// to the NN here. The next IPC call should be to allocate the next
// block. Any other call would notice the failover and not test
// idempotence of the operation (HDFS-3031)
FSNamesystem ns1 = cluster.GetNameNode(1).GetNamesystem();
BlockManagerTestUtil.UpdateState(ns1.GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, ns1.GetPendingReplicationBlocks());
NUnit.Framework.Assert.AreEqual(0, ns1.GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, ns1.GetMissingBlocksCount());
// If we're testing allocateBlock()'s idempotence, write another
// block and a half, so we have to allocate a new block.
// Otherise, don't write anything, so our next RPC will be
// completeFile() if we're testing idempotence of that operation.
if (methodToTest == TestPipelinesFailover.MethodToTestIdempotence.AllocateBlock)
{
// write another block and a half
AppendTestUtil.Write(stm, sizeWritten, BlockAndAHalf);
sizeWritten += BlockAndAHalf;
}
stm.Close();
stm = null;
AppendTestUtil.Check(fs, TestPath, sizeWritten);
}
finally
{
IOUtils.CloseStream(stm);
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void DoMultithreadedWrites(Configuration conf, Path p, int numThreads
, int bufferSize, int numWrites, int replication)
{
InitBuffer(bufferSize);
// create a new file.
FileSystem fs = p.GetFileSystem(conf);
FSDataOutputStream stm = CreateFile(fs, p, replication);
System.Console.Out.WriteLine("Created file simpleFlush.dat");
// There have been a couple issues with flushing empty buffers, so do
// some empty flushes first.
stm.Hflush();
stm.Hflush();
stm.Write(1);
stm.Hflush();
stm.Hflush();
CountDownLatch countdown = new CountDownLatch(1);
AList <Sharpen.Thread> threads = new AList <Sharpen.Thread>();
AtomicReference <Exception> thrown = new AtomicReference <Exception>();
for (int i = 0; i < numThreads; i++)
{
Sharpen.Thread t = new TestMultiThreadedHflush.WriterThread(this, stm, thrown, countdown
, numWrites);
threads.AddItem(t);
t.Start();
}
// Start all the threads at the same time for maximum raciness!
countdown.CountDown();
foreach (Sharpen.Thread t_1 in threads)
{
t_1.Join();
}
if (thrown.Get() != null)
{
throw new RuntimeException("Deferred", thrown.Get());
}
stm.Close();
System.Console.Out.WriteLine("Closed file.");
}
// return the initial state of the configuration
/// <summary>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// </summary>
/// <remarks>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// In this case, the block report becomes delayed until after the
/// block is marked completed on the NN, and hence it reports an RBW
/// replica for a COMPLETE block. Such a report should not be marked
/// corrupt.
/// This is a regression test for HDFS-2791.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestOneReplicaRbwReportArrivesAfterBlockCompleted()
{
CountDownLatch brFinished = new CountDownLatch(1);
GenericTestUtils.DelayAnswer delayer = new _DelayAnswer_579(brFinished, Log);
// inform the test that our block report went through.
string MethodName = GenericTestUtils.GetMethodName();
Path filePath = new Path("/" + MethodName + ".dat");
// Start a second DN for this test -- we're checking
// what happens when one of the DNs is slowed for some reason.
ReplFactor = 2;
StartDNandWait(null, false);
NameNode nn = cluster.GetNameNode();
FSDataOutputStream @out = fs.Create(filePath, ReplFactor);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Set up a spy so that we can delay the block report coming
// from this node.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
nn);
Org.Mockito.Mockito.DoAnswer(delayer).When(spy).BlockReport(Org.Mockito.Mockito.AnyObject
<DatanodeRegistration>(), Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyObject
<StorageBlockReport[]>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
// Force a block report to be generated. The block report will have
// an RBW replica in it. Wait for the RPC to be sent, but block
// it before it gets to the NN.
dn.ScheduleAllBlockReport(0);
delayer.WaitForCall();
}
finally
{
IOUtils.CloseStream(@out);
}
// Now that the stream is closed, the NN will have the block in COMPLETE
// state.
delayer.Proceed();
brFinished.Await();
// Verify that no replicas are marked corrupt, and that the
// file is still readable.
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, filePath);
// Ensure that the file is readable even from the DN that we futzed with.
cluster.StopDataNode(1);
DFSTestUtil.ReadFile(fs, filePath);
}
public virtual void TestHardLeaseRecovery()
{
//create a file
string filestr = "/hardLeaseRecovery";
AppendTestUtil.Log.Info("filestr=" + filestr);
Path filepath = new Path(filestr);
FSDataOutputStream stm = dfs.Create(filepath, true, BufSize, ReplicationNum, BlockSize
);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
// write bytes into the file.
int size = AppendTestUtil.NextInt(FileSize);
AppendTestUtil.Log.Info("size=" + size);
stm.Write(buffer, 0, size);
// hflush file
AppendTestUtil.Log.Info("hflush");
stm.Hflush();
// kill the lease renewal thread
AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
dfs.dfs.GetLeaseRenewer().InterruptAndJoin();
// set the hard limit to be 1 second
cluster.SetLeasePeriod(LongLeasePeriod, ShortLeasePeriod);
// wait for lease recovery to complete
LocatedBlocks locatedBlocks;
do
{
Sharpen.Thread.Sleep(ShortLeasePeriod);
locatedBlocks = dfs.dfs.GetLocatedBlocks(filestr, 0L, size);
}while (locatedBlocks.IsUnderConstruction());
NUnit.Framework.Assert.AreEqual(size, locatedBlocks.GetFileLength());
// make sure that the writer thread gets killed
try
{
stm.Write('b');
stm.Close();
NUnit.Framework.Assert.Fail("Writer thread should have been killed");
}
catch (IOException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
// verify data
AppendTestUtil.Log.Info("File size is good. Now validating sizes from datanodes..."
);
AppendTestUtil.CheckFullFile(dfs, filepath, size, buffer, filestr);
}
public virtual void TestComplexFlush()
{
Configuration conf = new HdfsConfiguration();
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// create a new file.
Path file1 = new Path("/complexFlush.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file complexFlush.dat");
int start = 0;
for (start = 0; (start + 29) < AppendTestUtil.FileSize;)
{
stm.Write(fileContents, start, 29);
stm.Hflush();
start += 29;
}
stm.Write(fileContents, start, AppendTestUtil.FileSize - start);
// need to make sure we completely write out all full blocks before
// the checkFile() call (see FSOutputSummer#flush)
stm.Flush();
// verify that full blocks are sane
CheckFile(fs, file1, 1);
stm.Close();
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <summary>TC11: Racing rename</summary>
/// <exception cref="System.Exception"/>
private void TestTC11(bool appendToNewBlock)
{
Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append half block of data.
FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.
Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
int len2 = (int)BlockSize / 2;
AppendTestUtil.Write(out_1, len1, len2);
out_1.Hflush();
//c. Rename file to file.new.
Path pnew = new Path(p + ".new");
NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew));
//d. Close file handle that was opened in (b).
out_1.Close();
//check block sizes
long len = fs.GetFileStatus(pnew).GetLen();
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString
(), 0L, len);
int numblock = locatedblocks.LocatedBlockCount();
for (int i = 0; i < numblock; i++)
{
LocatedBlock lb = locatedblocks.Get(i);
ExtendedBlock blk = lb.GetBlock();
long size = lb.GetBlockSize();
if (i < numblock - 1)
{
NUnit.Framework.Assert.AreEqual(BlockSize, size);
}
foreach (DatanodeInfo datanodeinfo in lb.GetLocations())
{
DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort());
Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId
(), blk.GetBlockId());
NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes());
}
}
}
/// <summary>Test NN crash and client crash/stuck immediately after block allocation</summary>
/// <exception cref="System.Exception"/>
public virtual void TestOpenFileWhenNNAndClientCrashAfterAddBlock()
{
cluster.GetConfiguration(0).Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey,
"1.0f");
string testData = "testData";
// to make sure we write the full block before creating dummy block at NN.
cluster.GetConfiguration(0).SetInt("io.bytes.per.checksum", testData.Length);
cluster.RestartNameNode(0);
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
cluster.TransitionToStandby(1);
DistributedFileSystem dfs = cluster.GetFileSystem(0);
string pathString = "/tmp1.txt";
Path filePath = new Path(pathString);
FSDataOutputStream create = dfs.Create(filePath, FsPermission.GetDefault(), true,
1024, (short)3, testData.Length, null);
create.Write(Sharpen.Runtime.GetBytesForString(testData));
create.Hflush();
long fileId = ((DFSOutputStream)create.GetWrappedStream()).GetFileId();
FileStatus fileStatus = dfs.GetFileStatus(filePath);
DFSClient client = DFSClientAdapter.GetClient(dfs);
// add one dummy block at NN, but not write to DataNode
ExtendedBlock previousBlock = DFSClientAdapter.GetPreviousBlock(client, fileId);
DFSClientAdapter.GetNamenode(client).AddBlock(pathString, client.GetClientName(),
new ExtendedBlock(previousBlock), new DatanodeInfo[0], DFSClientAdapter.GetFileId
((DFSOutputStream)create.GetWrappedStream()), null);
cluster.RestartNameNode(0, true);
cluster.RestartDataNode(0);
cluster.TransitionToActive(0);
// let the block reports be processed.
Sharpen.Thread.Sleep(2000);
FSDataInputStream @is = dfs.Open(filePath);
@is.Close();
dfs.RecoverLease(filePath);
// initiate recovery
NUnit.Framework.Assert.IsTrue("Recovery also should be success", dfs.RecoverLease
(filePath));
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestLeaseRecoveryAndAppend()
{
Configuration conf = new Configuration();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Path file = new Path("/testLeaseRecovery");
DistributedFileSystem dfs = cluster.GetFileSystem();
// create a file with 0 bytes
FSDataOutputStream @out = dfs.Create(file);
@out.Hflush();
@out.Hsync();
// abort the original stream
((DFSOutputStream)@out.GetWrappedStream()).Abort();
DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster
.GetConfiguration(0));
// Append to a file , whose lease is held by another client should fail
try
{
newdfs.Append(file);
NUnit.Framework.Assert.Fail("Append to a file(lease is held by another client) should fail"
);
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("file lease is currently owned")
);
}
// Lease recovery on first try should be successful
bool recoverLease = newdfs.RecoverLease(file);
NUnit.Framework.Assert.IsTrue(recoverLease);
FSDataOutputStream append = newdfs.Append(file);
append.Write(Sharpen.Runtime.GetBytesForString("test"));
append.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
cluster = null;
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestTimeoutMetric()
{
Configuration conf = new HdfsConfiguration();
Path path = new Path("/test");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
IList <FSDataOutputStream> streams = Lists.NewArrayList();
try
{
FSDataOutputStream @out = cluster.GetFileSystem().Create(path, (short)2);
DataNodeFaultInjector injector = Org.Mockito.Mockito.Mock <DataNodeFaultInjector>(
);
Org.Mockito.Mockito.DoThrow(new IOException("mock IOException")).When(injector).WriteBlockAfterFlush
();
DataNodeFaultInjector.instance = injector;
streams.AddItem(@out);
@out.WriteBytes("old gs data\n");
@out.Hflush();
/* Test the metric. */
MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(cluster.GetDataNodes()
[0].GetMetrics().Name());
MetricsAsserts.AssertCounter("DatanodeNetworkErrors", 1L, dnMetrics);
/* Test JMX datanode network counts. */
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=DataNode,name=DataNodeInfo"
);
object dnc = mbs.GetAttribute(mxbeanName, "DatanodeNetworkCounts");
string allDnc = dnc.ToString();
NUnit.Framework.Assert.IsTrue("expected to see loopback address", allDnc.IndexOf(
"127.0.0.1") >= 0);
NUnit.Framework.Assert.IsTrue("expected to see networkErrors", allDnc.IndexOf("networkErrors"
) >= 0);
}
finally
{
IOUtils.Cleanup(Log, Sharpen.Collections.ToArray(streams, new IDisposable[0]));
if (cluster != null)
{
cluster.Shutdown();
}
DataNodeFaultInjector.instance = new DataNodeFaultInjector();
}
}
/// <summary>Tests lease recovery if a client crashes.</summary>
/// <remarks>
/// Tests lease recovery if a client crashes. This approximates the
/// use case of HBase WALs being recovered after a NN failover.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestLeaseRecoveryAfterFailover()
{
Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
Sharpen.Thread.Sleep(500);
Log.Info("Starting with NN 0 active");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
stm = fs.Create(TestPath);
// write a block and a half
AppendTestUtil.Write(stm, 0, BlockAndAHalf);
stm.Hflush();
Log.Info("Failing over to NN 1");
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.IsTrue(fs.Exists(TestPath));
FileSystem fsOtherUser = CreateFsAsOtherUser(cluster, conf);
LoopRecoverLease(fsOtherUser, TestPath);
AppendTestUtil.Check(fs, TestPath, BlockAndAHalf);
// Fail back to ensure that the block locations weren't lost on the
// original node.
cluster.TransitionToStandby(1);
cluster.TransitionToActive(0);
AppendTestUtil.Check(fs, TestPath, BlockAndAHalf);
}
finally
{
IOUtils.CloseStream(stm);
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private Path CreateFile(string filestr, int size, bool triggerLeaseRenewerInterrupt
)
{
AppendTestUtil.Log.Info("filestr=" + filestr);
Path filepath = new Path(filestr);
FSDataOutputStream stm = dfs.Create(filepath, true, BufSize, ReplicationNum, BlockSize
);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
AppendTestUtil.Log.Info("size=" + size);
stm.Write(buffer, 0, size);
// hflush file
AppendTestUtil.Log.Info("hflush");
stm.Hflush();
if (triggerLeaseRenewerInterrupt)
{
AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
dfs.dfs.GetLeaseRenewer().InterruptAndJoin();
}
return(filepath);
}