public virtual void TestHedgedReadLoopTooManyTimes()
{
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int hedgedReadTimeoutMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis
);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
int sleepMs = 100;
Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When
(injector).FetchFromDatanodeException();
Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay
();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
FSDataOutputStream output = null;
DFSInputStream input = null;
string filename = "/hedgedReadMaxOut.dat";
try
{
Path file = new Path(filename);
output = fileSys.Create(file, (short)2);
byte[] data = new byte[64 * 1024];
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Close();
byte[] buffer = new byte[64 * 1024];
input = dfsClient.Open(filename);
input.Read(0, buffer, 0, 1024);
input.Close();
NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting());
}
catch (BlockMissingException)
{
NUnit.Framework.Assert.IsTrue(false);
}
finally
{
Org.Mockito.Mockito.Reset(injector);
IOUtils.Cleanup(null, input);
IOUtils.Cleanup(null, output);
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void TestAppendLessThanChecksumChunk()
{
byte[] buf = new byte[1024];
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes
(1).Build();
cluster.WaitActive();
try
{
using (DistributedFileSystem fs = cluster.GetFileSystem())
{
int len1 = 200;
int len2 = 300;
Path p = new Path("/foo");
FSDataOutputStream @out = fs.Create(p);
@out.Write(buf, 0, len1);
@out.Close();
@out = fs.Append(p);
@out.Write(buf, 0, len2);
// flush but leave open
@out.Hflush();
// read data to verify the replica's content and checksum are correct
FSDataInputStream @in = fs.Open(p);
int length = @in.Read(0, buf, 0, len1 + len2);
NUnit.Framework.Assert.IsTrue(length > 0);
@in.Close();
@out.Close();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestAppend()
{
int maxOldFileLen = 2 * BlockSize + 1;
int maxFlushedBytes = BlockSize;
byte[] contents = AppendTestUtil.InitBuffer(maxOldFileLen + 2 * maxFlushedBytes);
for (int oldFileLen = 0; oldFileLen <= maxOldFileLen; oldFileLen++)
{
for (int flushedBytes1 = 0; flushedBytes1 <= maxFlushedBytes; flushedBytes1++)
{
for (int flushedBytes2 = 0; flushedBytes2 <= maxFlushedBytes; flushedBytes2++)
{
int fileLen = oldFileLen + flushedBytes1 + flushedBytes2;
// create the initial file of oldFileLen
Path p = new Path("foo" + oldFileLen + "_" + flushedBytes1 + "_" + flushedBytes2);
Log.Info("Creating file " + p);
FSDataOutputStream @out = fs.Create(p, false, conf.GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), Replication, BlockSize);
@out.Write(contents, 0, oldFileLen);
@out.Close();
// append flushedBytes bytes to the file
@out = fs.Append(p);
@out.Write(contents, oldFileLen, flushedBytes1);
@out.Hflush();
// write another flushedBytes2 bytes to the file
@out.Write(contents, oldFileLen + flushedBytes1, flushedBytes2);
@out.Close();
// validate the file content
AppendTestUtil.CheckFullFile(fs, p, fileLen, contents, p.ToString());
fs.Delete(p, false);
}
}
}
}
public virtual void TestHSyncBlockBoundary()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSyncBlockBoundary/foo");
int len = 1 << 16;
byte[] fileContents = AppendTestUtil.InitBuffer(len);
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null);
// fill exactly one block (tests the SYNC_BLOCK case) and flush
@out.Write(fileContents, 0, len);
@out.Hflush();
// the full block should have caused a sync
CheckSyncMetric(cluster, 1);
@out.Hsync();
// first on block again
CheckSyncMetric(cluster, 1);
// write one more byte and sync again
@out.Write(1);
@out.Hsync();
CheckSyncMetric(cluster, 2);
@out.Close();
CheckSyncMetric(cluster, 3);
cluster.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
public override void Write(Text key, Text value)
{
lock (this)
{
@out.Write(key.GetBytes(), 0, key.GetLength());
@out.Write(value.GetBytes(), 0, value.GetLength());
}
}
//
// writes to file
//
/// <exception cref="System.IO.IOException"/>
private static void WriteFile(FSDataOutputStream stm, long seed)
{
byte[] buffer = AppendTestUtil.RandomBytes(seed, fileSize);
int mid = fileSize / 2;
stm.Write(buffer, 0, mid);
stm.Write(buffer, mid, fileSize - mid);
}
/// <summary>
/// Test the case that remove a data volume on a particular DataNode when the
/// volume is actively being written.
/// </summary>
/// <param name="dataNodeIdx">the index of the DataNode to remove a volume.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.BrokenBarrierException"/>
private void TestRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
{
// Starts DFS cluster with 3 DataNodes to form a pipeline.
StartDFSCluster(1, 3);
short Replication = 3;
DataNode dn = cluster.GetDataNodes()[dataNodeIdx];
FileSystem fs = cluster.GetFileSystem();
Path testFile = new Path("/test");
long lastTimeDiskErrorCheck = dn.GetLastDiskErrorCheck();
FSDataOutputStream @out = fs.Create(testFile, Replication);
Random rb = new Random(0);
byte[] writeBuf = new byte[BlockSize / 2];
// half of the block.
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
// Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
// BlockReceiver releases volume reference before finalizeBlock(), the blocks
// on the volume will be removed, and finalizeBlock() throws IOE.
FsDatasetSpi <FsVolumeSpi> data = dn.data;
dn.data = Org.Mockito.Mockito.Spy(data);
Org.Mockito.Mockito.DoAnswer(new _Answer_599(data)).When(dn.data).FinalizeBlock(Matchers.Any
<ExtendedBlock>());
// Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
// the block is not removed, since the volume reference should not
// be released at this point.
CyclicBarrier barrier = new CyclicBarrier(2);
IList <string> oldDirs = GetDataDirs(dn);
string newDirs = oldDirs[1];
// Remove the first volume.
IList <Exception> exceptions = new AList <Exception>();
Sharpen.Thread reconfigThread = new _Thread_616(barrier, dn, newDirs, exceptions);
reconfigThread.Start();
barrier.Await();
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
@out.Close();
reconfigThread.Join();
// Verify the file has sufficient replications.
DFSTestUtil.WaitReplication(fs, testFile, Replication);
// Read the content back
byte[] content = DFSTestUtil.ReadFileBuffer(fs, testFile);
NUnit.Framework.Assert.AreEqual(BlockSize, content.Length);
// If an IOException thrown from BlockReceiver#run, it triggers
// DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
// to see whether there is IOException in BlockReceiver#run().
NUnit.Framework.Assert.AreEqual(lastTimeDiskErrorCheck, dn.GetLastDiskErrorCheck(
));
if (!exceptions.IsEmpty())
{
throw new IOException(exceptions[0].InnerException);
}
}
public virtual void TestWrite()
{
MiniDFSCluster cluster = null;
int numDataNodes = 2;
Configuration conf = GetConf(numDataNodes);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
NameNode nn = cluster.GetNameNode();
BlockManager bm = nn.GetNamesystem().GetBlockManager();
BlockTokenSecretManager sm = bm.GetBlockTokenSecretManager();
// set a short token lifetime (1 second)
SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L);
Path fileToWrite = new Path(FileToWrite);
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream stm = WriteFile(fs, fileToWrite, (short)numDataNodes, BlockSize
);
// write a partial block
int mid = rawData.Length - 1;
stm.Write(rawData, 0, mid);
stm.Hflush();
/*
* wait till token used in stm expires
*/
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = DFSTestUtil.
GetBlockToken(stm);
while (!SecurityTestUtil.IsBlockTokenExpired(token))
{
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
}
// remove a datanode to force re-establishing pipeline
cluster.StopDataNode(0);
// write the rest of the file
stm.Write(rawData, mid, rawData.Length - mid);
stm.Close();
// check if write is successful
FSDataInputStream in4 = fs.Open(fileToWrite);
NUnit.Framework.Assert.IsTrue(CheckFile1(in4));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void HSyncEndBlock_00()
{
int preferredBlockSize = 1024;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem fileSystem = cluster.GetFileSystem();
FSDataOutputStream stm = null;
try
{
Path path = new Path("/" + fName);
stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize);
System.Console.Out.WriteLine("Created file " + path.ToString());
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(0L, currentFileLength);
LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count);
// write a block and call hsync(end_block) at the block boundary
stm.Write(new byte[preferredBlockSize]);
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count);
// call hsync then call hsync(end_block) immediately
stm.Write(new byte[preferredBlockSize / 2]);
stm.Hsync();
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength
);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count);
stm.Write(new byte[preferredBlockSize / 4]);
stm.Hsync();
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize
/ 4, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count);
}
finally
{
IOUtils.Cleanup(null, stm, fileSystem);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestPacketTransmissionDelay()
{
// Make the first datanode to not relay heartbeat packet.
DataNodeFaultInjector dnFaultInjector = new _DataNodeFaultInjector_171();
DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.Get();
DataNodeFaultInjector.Set(dnFaultInjector);
// Setting the timeout to be 3 seconds. Normally heartbeat packet
// would be sent every 1.5 seconds if there is no data traffic.
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientSocketTimeoutKey, "3000");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("noheartbeat.dat"), (short)2);
@out.Write(unchecked ((int)(0x31)));
@out.Hflush();
DFSOutputStream dfsOut = (DFSOutputStream)@out.GetWrappedStream();
// original pipeline
DatanodeInfo[] orgNodes = dfsOut.GetPipeline();
// Cause the second datanode to timeout on reading packet
Sharpen.Thread.Sleep(3500);
@out.Write(unchecked ((int)(0x32)));
@out.Hflush();
// new pipeline
DatanodeInfo[] newNodes = dfsOut.GetPipeline();
@out.Close();
bool contains = false;
for (int i = 0; i < newNodes.Length; i++)
{
if (orgNodes[0].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
throw new IOException("The first datanode should have been replaced.");
}
if (orgNodes[1].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
contains = true;
}
}
NUnit.Framework.Assert.IsTrue(contains);
}
finally
{
DataNodeFaultInjector.Set(oldDnInjector);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestHSyncOperation(bool testWithAppend)
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSync/foo");
int len = 1 << 16;
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null);
if (testWithAppend)
{
// re-open the file with append call
@out.Close();
@out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.SyncBlock), 4096, null
);
}
@out.Hflush();
// hflush does not sync
CheckSyncMetric(cluster, 0);
@out.Hsync();
// hsync on empty file does nothing
CheckSyncMetric(cluster, 0);
@out.Write(1);
CheckSyncMetric(cluster, 0);
@out.Hsync();
CheckSyncMetric(cluster, 1);
// avoiding repeated hsyncs is a potential future optimization
@out.Hsync();
CheckSyncMetric(cluster, 2);
@out.Hflush();
// hflush still does not sync
CheckSyncMetric(cluster, 2);
@out.Close();
// close is sync'ing
CheckSyncMetric(cluster, 3);
// same with a file created with out SYNC_BLOCK
@out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite), 4096, (short)1, len, null);
@out.Hsync();
CheckSyncMetric(cluster, 3);
@out.Write(1);
CheckSyncMetric(cluster, 3);
@out.Hsync();
CheckSyncMetric(cluster, 4);
// repeated hsyncs
@out.Hsync();
CheckSyncMetric(cluster, 5);
@out.Close();
// close does not sync (not opened with SYNC_BLOCK)
CheckSyncMetric(cluster, 5);
cluster.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void CreateFileAndTestSpaceReservation(string fileNamePrefix, int fileBlockSize
)
{
// Enough for 1 block + meta files + some delta.
long configuredCapacity = fileBlockSize * 2 - 1;
StartCluster(BlockSize, 1, configuredCapacity);
FSDataOutputStream @out = null;
Path path = new Path("/" + fileNamePrefix + ".dat");
try
{
@out = fs.Create(path, false, 4096, (short)1, fileBlockSize);
byte[] buffer = new byte[rand.Next(fileBlockSize / 4)];
@out.Write(buffer);
@out.Hsync();
int bytesWritten = buffer.Length;
// Check that space was reserved for a full block minus the bytesWritten.
Assert.AssertThat(singletonVolume.GetReservedForRbw(), IS.Is((long)fileBlockSize
- bytesWritten));
@out.Close();
@out = null;
// Check that the reserved space has been released since we closed the
// file.
Assert.AssertThat(singletonVolume.GetReservedForRbw(), IS.Is(0L));
// Reopen the file for appends and write 1 more byte.
@out = fs.Append(path);
@out.Write(buffer);
@out.Hsync();
bytesWritten += buffer.Length;
// Check that space was again reserved for a full block minus the
// bytesWritten so far.
Assert.AssertThat(singletonVolume.GetReservedForRbw(), IS.Is((long)fileBlockSize
- bytesWritten));
// Write once again and again verify the available space. This ensures
// that the reserved space is progressively adjusted to account for bytes
// written to disk.
@out.Write(buffer);
@out.Hsync();
bytesWritten += buffer.Length;
Assert.AssertThat(singletonVolume.GetReservedForRbw(), IS.Is((long)fileBlockSize
- bytesWritten));
}
finally
{
if (@out != null)
{
@out.Close();
}
}
}
public virtual void TestSimpleFlush()
{
Configuration conf = new HdfsConfiguration();
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// create a new file.
Path file1 = new Path("/simpleFlush.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file simpleFlush.dat");
// write to file
int mid = AppendTestUtil.FileSize / 2;
stm.Write(fileContents, 0, mid);
stm.Hflush();
System.Console.Out.WriteLine("Wrote and Flushed first part of file.");
// write the remainder of the file
stm.Write(fileContents, mid, AppendTestUtil.FileSize - mid);
System.Console.Out.WriteLine("Written second part of file");
stm.Hflush();
stm.Hflush();
System.Console.Out.WriteLine("Wrote and Flushed second part of file.");
// verify that full blocks are sane
CheckFile(fs, file1, 1);
stm.Close();
System.Console.Out.WriteLine("Closed file.");
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestComplexFlush()
{
Configuration conf = new HdfsConfiguration();
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// create a new file.
Path file1 = new Path("/complexFlush.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file complexFlush.dat");
int start = 0;
for (start = 0; (start + 29) < AppendTestUtil.FileSize;)
{
stm.Write(fileContents, start, 29);
stm.Hflush();
start += 29;
}
stm.Write(fileContents, start, AppendTestUtil.FileSize - start);
// need to make sure we completely write out all full blocks before
// the checkFile() call (see FSOutputSummer#flush)
stm.Flush();
// verify that full blocks are sane
CheckFile(fs, file1, 1);
stm.Close();
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
}
catch (IOException e)
{
System.Console.Out.WriteLine("Exception :" + e);
throw;
}
catch (Exception e)
{
System.Console.Out.WriteLine("Throwable :" + e);
Sharpen.Runtime.PrintStackTrace(e);
throw new IOException("Throwable : " + e);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestHardLeaseRecovery()
{
//create a file
string filestr = "/hardLeaseRecovery";
AppendTestUtil.Log.Info("filestr=" + filestr);
Path filepath = new Path(filestr);
FSDataOutputStream stm = dfs.Create(filepath, true, BufSize, ReplicationNum, BlockSize
);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
// write bytes into the file.
int size = AppendTestUtil.NextInt(FileSize);
AppendTestUtil.Log.Info("size=" + size);
stm.Write(buffer, 0, size);
// hflush file
AppendTestUtil.Log.Info("hflush");
stm.Hflush();
// kill the lease renewal thread
AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
dfs.dfs.GetLeaseRenewer().InterruptAndJoin();
// set the hard limit to be 1 second
cluster.SetLeasePeriod(LongLeasePeriod, ShortLeasePeriod);
// wait for lease recovery to complete
LocatedBlocks locatedBlocks;
do
{
Sharpen.Thread.Sleep(ShortLeasePeriod);
locatedBlocks = dfs.dfs.GetLocatedBlocks(filestr, 0L, size);
}while (locatedBlocks.IsUnderConstruction());
NUnit.Framework.Assert.AreEqual(size, locatedBlocks.GetFileLength());
// make sure that the writer thread gets killed
try
{
stm.Write('b');
stm.Close();
NUnit.Framework.Assert.Fail("Writer thread should have been killed");
}
catch (IOException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
// verify data
AppendTestUtil.Log.Info("File size is good. Now validating sizes from datanodes..."
);
AppendTestUtil.CheckFullFile(dfs, filepath, size, buffer, filestr);
}
/* create a file, write data chunk by chunk */
/// <exception cref="System.Exception"/>
private void WriteFile2(Path name)
{
FSDataOutputStream stm = fileSys.Create(name, true, fileSys.GetConf().GetInt(CommonConfigurationKeysPublic
.IoFileBufferSizeKey, 4096), NumOfDatanodes, BlockSize);
int i = 0;
for (; i < FileSize - BytesPerChecksum; i += BytesPerChecksum)
{
stm.Write(expected, i, BytesPerChecksum);
}
stm.Write(expected, i, FileSize - 3 * BytesPerChecksum);
stm.Close();
CheckFile(name);
CleanupFile(name);
}
/// <exception cref="System.IO.IOException"/>
private Path InitFiles(FileSystem fs, int numFiles, int numBytes)
{
Path dir = new Path(Runtime.GetProperty("test.build.data", ".") + "/mapred");
Path multiFileDir = new Path(dir, "test.multifile");
fs.Delete(multiFileDir, true);
fs.Mkdirs(multiFileDir);
Log.Info("Creating " + numFiles + " file(s) in " + multiFileDir);
for (int i = 0; i < numFiles; i++)
{
Path path = new Path(multiFileDir, "file_" + i);
FSDataOutputStream @out = fs.Create(path);
if (numBytes == -1)
{
numBytes = rand.Next(MaxBytes);
}
for (int j = 0; j < numBytes; j++)
{
@out.Write(rand.Next());
}
@out.Close();
if (Log.IsDebugEnabled())
{
Log.Debug("Created file " + path + " with length " + numBytes);
}
lengths[path.GetName()] = System.Convert.ToInt64(numBytes);
}
FileInputFormat.SetInputPaths(job, multiFileDir);
return(multiFileDir);
}
/// <summary>
/// Test fsimage loading when 1) there is an empty file loaded from fsimage,
/// and 2) there is later an append operation to be applied from edit log.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestLoadImageWithEmptyFile()
{
// create an empty file
Path file = new Path(dir, "file");
FSDataOutputStream @out = hdfs.Create(file);
@out.Close();
// save namespace
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// append to the empty file
@out = hdfs.Append(file);
@out.Write(1);
@out.Close();
// restart cluster
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
FileStatus status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(1, status.GetLen());
}
public virtual void TestBlockTokenInLastLocatedBlock()
{
Configuration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 512);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
try
{
FileSystem fs = cluster.GetFileSystem();
string fileName = "/testBlockTokenInLastLocatedBlock";
Path filePath = new Path(fileName);
FSDataOutputStream @out = fs.Create(filePath, (short)1);
@out.Write(new byte[1000]);
// ensure that the first block is written out (see FSOutputSummer#flush)
@out.Flush();
LocatedBlocks locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName
, 0, 1000);
while (locatedBlocks.GetLastLocatedBlock() == null)
{
Sharpen.Thread.Sleep(100);
locatedBlocks = cluster.GetNameNodeRpc().GetBlockLocations(fileName, 0, 1000);
}
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = locatedBlocks
.GetLastLocatedBlock().GetBlockToken();
NUnit.Framework.Assert.AreEqual(BlockTokenIdentifier.KindName, token.GetKind());
@out.Close();
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
internal static void CreateHdfsFile(FileSystem fs, Path p, long length, bool dropBehind
)
{
FSDataOutputStream fos = null;
try
{
// create file with replication factor of 1
fos = fs.Create(p, (short)1);
if (dropBehind != null)
{
fos.SetDropBehind(dropBehind);
}
byte[] buf = new byte[8196];
while (length > 0)
{
int amt = (length > buf.Length) ? buf.Length : (int)length;
fos.Write(buf, 0, amt);
length -= amt;
}
}
catch (IOException e)
{
Log.Error("ioexception", e);
}
finally
{
if (fos != null)
{
fos.Close();
}
}
}
public virtual void TestBlocksScheduledCounter()
{
cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
//open a file an write a few bytes:
FSDataOutputStream @out = fs.Create(new Path("/testBlockScheduledCounter"));
for (int i = 0; i < 1024; i++)
{
@out.Write(i);
}
// flush to make sure a block is allocated.
@out.Hflush();
AList <DatanodeDescriptor> dnList = new AList <DatanodeDescriptor>();
DatanodeManager dm = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager
();
dm.FetchDatanodes(dnList, dnList, false);
DatanodeDescriptor dn = dnList[0];
NUnit.Framework.Assert.AreEqual(1, dn.GetBlocksScheduled());
// close the file and the counter should go to zero.
@out.Close();
NUnit.Framework.Assert.AreEqual(0, dn.GetBlocksScheduled());
}
/*
* In order to make this write atomic as a part of write we will first write
* data to .tmp file and then rename it. Here we are assuming that rename is
* atomic for underlying file system.
*/
/// <exception cref="System.Exception"/>
protected internal virtual void WriteFile(Path outputPath, byte[] data, bool makeUnradableByAdmin
)
{
Path tempPath = new Path(outputPath.GetParent(), outputPath.GetName() + ".tmp");
FSDataOutputStream fsOut = null;
// This file will be overwritten when app/attempt finishes for saving the
// final status.
try
{
fsOut = fs.Create(tempPath, true);
if (makeUnradableByAdmin)
{
SetUnreadableBySuperuserXattrib(tempPath);
}
fsOut.Write(data);
fsOut.Close();
fsOut = null;
fs.Rename(tempPath, outputPath);
}
finally
{
IOUtils.Cleanup(Log, fsOut);
}
}
public virtual void TestForEmptyFile()
{
Configuration conf = new Configuration();
FileSystem fileSys = FileSystem.Get(conf);
Path file = new Path("test" + "/file");
FSDataOutputStream @out = fileSys.Create(file, true, conf.GetInt("io.file.buffer.size"
, 4096), (short)1, (long)1024);
@out.Write(new byte[0]);
@out.Close();
// split it using a File input format
TestMRCJCFileInputFormat.DummyInputFormat inFormat = new TestMRCJCFileInputFormat.DummyInputFormat
(this);
Job job = Job.GetInstance(conf);
FileInputFormat.SetInputPaths(job, "test");
IList <InputSplit> splits = inFormat.GetSplits(job);
NUnit.Framework.Assert.AreEqual(1, splits.Count);
FileSplit fileSplit = (FileSplit)splits[0];
NUnit.Framework.Assert.AreEqual(0, fileSplit.GetLocations().Length);
NUnit.Framework.Assert.AreEqual(file.GetName(), fileSplit.GetPath().GetName());
NUnit.Framework.Assert.AreEqual(0, fileSplit.GetStart());
NUnit.Framework.Assert.AreEqual(0, fileSplit.GetLength());
fileSys.Delete(file.GetParent(), true);
}
/// <exception cref="System.IO.IOException"/>
protected internal void MakeTestFile(Path path, long length, bool isLazyPersist)
{
EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create);
if (isLazyPersist)
{
createFlags.AddItem(CreateFlag.LazyPersist);
}
FSDataOutputStream fos = null;
try
{
fos = fs.Create(path, FsPermission.GetFileDefault(), createFlags, BufferLength, ReplFactor
, BlockSize, null);
// Allocate a block.
byte[] buffer = new byte[BufferLength];
for (int bytesWritten = 0; bytesWritten < length;)
{
fos.Write(buffer, 0, buffer.Length);
bytesWritten += buffer.Length;
}
if (length > 0)
{
fos.Hsync();
}
}
finally
{
IOUtils.CloseQuietly(fos);
}
}
/// <summary>Create a file with a length of <code>fileSize</code>.</summary>
/// <remarks>
/// Create a file with a length of <code>fileSize</code>.
/// The file is filled with 'a'.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private void GenFile(Path file, long fileSize)
{
long startTime = Time.Now();
FSDataOutputStream @out = null;
try
{
@out = Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.fc.Create(file, EnumSet.Of
(CreateFlag.Create, CreateFlag.Overwrite), Options.CreateOpts.CreateParent(), Options.CreateOpts
.BufferSize(4096), Options.CreateOpts.RepFac((short)3));
this.executionTime[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.Create] += (Time
.Now() - startTime);
Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.numOfOps[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator
.Create]++;
long i = fileSize;
while (i > 0)
{
long s = Math.Min(fileSize, this._enclosing.WriteContents.Length);
@out.Write(this._enclosing.WriteContents, 0, (int)s);
i -= s;
}
startTime = Time.Now();
this.executionTime[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.WriteClose] +=
(Time.Now() - startTime);
Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.numOfOps[Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator
.WriteClose]++;
}
finally
{
IOUtils.Cleanup(Org.Apache.Hadoop.FS.LoadGenerator.LoadGenerator.Log, @out);
}
}
//Do Nothing
/// <exception cref="System.Exception"/>
public virtual void TestWebHdfsDoAs()
{
WebHdfsTestUtil.Log.Info("START: testWebHdfsDoAs()");
WebHdfsTestUtil.Log.Info("ugi.getShortUserName()=" + ugi.GetShortUserName());
WebHdfsFileSystem webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, config, WebHdfsFileSystem
.Scheme);
Path root = new Path("/");
cluster.GetFileSystem().SetPermission(root, new FsPermission((short)0x1ff));
Whitebox.SetInternalState(webhdfs, "ugi", proxyUgi);
{
Path responsePath = webhdfs.GetHomeDirectory();
WebHdfsTestUtil.Log.Info("responsePath=" + responsePath);
NUnit.Framework.Assert.AreEqual(webhdfs.GetUri() + "/user/" + ProxyUser, responsePath
.ToString());
}
Path f = new Path("/testWebHdfsDoAs/a.txt");
{
FSDataOutputStream @out = webhdfs.Create(f);
@out.Write(Sharpen.Runtime.GetBytesForString("Hello, webhdfs user!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
{
FSDataOutputStream @out = webhdfs.Append(f);
@out.Write(Sharpen.Runtime.GetBytesForString("\nHello again!"));
@out.Close();
FileStatus status = webhdfs.GetFileStatus(f);
WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner());
WebHdfsTestUtil.Log.Info("status.getLen() =" + status.GetLen());
NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner());
}
}
public virtual void TestFailureOpenRandomFile()
{
if (skip)
{
return;
}
CloseOutput();
// create an random file
path = new Path(fs.GetWorkingDirectory(), outputFile);
@out = fs.Create(path);
Random rand = new Random();
byte[] buf = new byte[K];
// fill with > 1MB data
for (int nx = 0; nx < K + 2; nx++)
{
rand.NextBytes(buf);
@out.Write(buf);
}
@out.Close();
try
{
new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen(), conf);
NUnit.Framework.Assert.Fail("Error on handling random files.");
}
catch (IOException)
{
}
}
public virtual void TestDeletionOfLaterBlocksWithZeroSizeFirstBlock()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
byte[] testData = Sharpen.Runtime.GetBytesForString("foo bar baz");
// Create a zero-length file.
DFSTestUtil.CreateFile(hdfs, bar, 0, Replication, 0L);
NUnit.Framework.Assert.AreEqual(0, fsdir.GetINode4Write(bar.ToString()).AsFile().
GetBlocks().Length);
// Create a snapshot that includes that file.
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s0");
// Extend that file.
FSDataOutputStream @out = hdfs.Append(bar);
@out.Write(testData);
@out.Close();
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(testData.Length, blks[0].GetNumBytes());
// Delete the file.
hdfs.Delete(bar, true);
// Now make sure that the NN can still save an fsimage successfully.
cluster.GetNameNode().GetRpcServer().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter
, false);
cluster.GetNameNode().GetRpcServer().SaveNamespace();
}
/// <exception cref="System.IO.IOException"/>
private void WriteFile(Path file, byte[] data)
{
int WriteBufferSize = 4096;
FSDataOutputStream @out = fs.Create(file, FilePermissions, true, WriteBufferSize,
fs.GetDefaultReplication(file), fs.GetDefaultBlockSize(file), null);
try
{
try
{
@out.Write(data);
@out.Close();
@out = null;
}
finally
{
IOUtils.Cleanup(Log, @out);
}
}
catch (IOException e)
{
fs.Delete(file, false);
throw;
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFavoredNodesEndToEndForAppend()
{
// create 10 files with random preferred nodes
for (int i = 0; i < NumFiles; i++)
{
Random rand = new Random(Runtime.CurrentTimeMillis() + i);
// pass a new created rand so as to get a uniform distribution each time
// without too much collisions (look at the do-while loop in getDatanodes)
IPEndPoint[] datanode = GetDatanodes(rand);
Path p = new Path("/filename" + i);
// create and close the file.
dfs.Create(p, FsPermission.GetDefault(), true, 4096, (short)3, 4096L, null, null)
.Close();
// re-open for append
FSDataOutputStream @out = dfs.Append(p, EnumSet.Of(CreateFlag.Append), 4096, null
, datanode);
@out.Write(SomeBytes);
@out.Close();
BlockLocation[] locations = GetBlockLocations(p);
// verify the files got created in the right nodes
foreach (BlockLocation loc in locations)
{
string[] hosts = loc.GetNames();
string[] hosts1 = GetStringForInetSocketAddrs(datanode);
NUnit.Framework.Assert.IsTrue(CompareNodes(hosts, hosts1));
}
}
}