/// <exception cref="System.Exception"/>
public virtual void TestFilesExceedMaxLockedMemory()
{
Log.Info("beginning testFilesExceedMaxLockedMemory");
// Create some test files that will exceed total cache capacity
int numFiles = 5;
long fileSize = CacheCapacity / (numFiles - 1);
Path[] testFiles = new Path[numFiles];
HdfsBlockLocation[][] fileLocs = new HdfsBlockLocation[numFiles][];
long[] fileSizes = new long[numFiles];
for (int i = 0; i < numFiles; i++)
{
testFiles[i] = new Path("/testFilesExceedMaxLockedMemory-" + i);
DFSTestUtil.CreateFile(fs, testFiles[i], fileSize, (short)1, unchecked ((long)(0xDFAl
)));
fileLocs[i] = (HdfsBlockLocation[])fs.GetFileBlockLocations(testFiles[i], 0, fileSize
);
// Get the file size (sum of blocks)
long[] sizes = GetBlockSizes(fileLocs[i]);
for (int j = 0; j < sizes.Length; j++)
{
fileSizes[i] += sizes[j];
}
}
// Cache the first n-1 files
long total = 0;
DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd);
for (int i_1 = 0; i_1 < numFiles - 1; i_1++)
{
SetHeartbeatResponse(CacheBlocks(fileLocs[i_1]));
total = DFSTestUtil.VerifyExpectedCacheUsage(rounder.Round(total + fileSizes[i_1]
), 4 * (i_1 + 1), fsd);
}
// nth file should hit a capacity exception
LogVerificationAppender appender = new LogVerificationAppender();
Logger logger = Logger.GetRootLogger();
logger.AddAppender(appender);
SetHeartbeatResponse(CacheBlocks(fileLocs[numFiles - 1]));
GenericTestUtils.WaitFor(new _Supplier_351(appender), 500, 30000);
// Also check the metrics for the failure
NUnit.Framework.Assert.IsTrue("Expected more than 0 failed cache attempts", fsd.GetNumBlocksFailedToCache
() > 0);
// Uncache the n-1 files
int curCachedBlocks = 16;
for (int i_2 = 0; i_2 < numFiles - 1; i_2++)
{
SetHeartbeatResponse(UncacheBlocks(fileLocs[i_2]));
long uncachedBytes = rounder.Round(fileSizes[i_2]);
total -= uncachedBytes;
curCachedBlocks -= uncachedBytes / BlockSize;
DFSTestUtil.VerifyExpectedCacheUsage(total, curCachedBlocks, fsd);
}
Log.Info("finishing testFilesExceedMaxLockedMemory");
}
/// <exception cref="System.Exception"/>
private void TestImageChecksum(bool compress)
{
MiniDFSCluster cluster = null;
if (compress)
{
config.SetBoolean(DFSConfigKeys.DfsImageCompressionCodecKey, true);
}
try
{
Log.Info("\n===========================================\n" + "Starting empty cluster"
);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(0).Format(true).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
fs.Mkdirs(new Path("/test"));
Log.Info("Shutting down cluster #1");
cluster.Shutdown();
cluster = null;
// Corrupt the md5 files in all the namedirs
CorruptFSImageMD5(true);
// Attach our own log appender so we can verify output
LogVerificationAppender appender = new LogVerificationAppender();
Logger logger = Logger.GetRootLogger();
logger.AddAppender(appender);
// Try to start a new cluster
Log.Info("\n===========================================\n" + "Starting same cluster after simulated crash"
);
try
{
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(0).Format(false).Build(
);
NUnit.Framework.Assert.Fail("Should not have successfully started with corrupt image"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Failed to load an FSImage file!", ioe);
int md5failures = appender.CountExceptionsWithMessage(" is corrupt with MD5 checksum of "
);
// Two namedirs, so should have seen two failures
NUnit.Framework.Assert.AreEqual(2, md5failures);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public _Supplier_351(LogVerificationAppender appender)
{
this.appender = appender;
}
/// <exception cref="System.Exception"/>
public virtual void TestXattrConfiguration()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try
{
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrSizeKey, -1);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(true).Build();
NUnit.Framework.Assert.Fail("Expected exception with negative xattr size");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Cannot set a negative value for the maximum size of an xattr"
, e);
}
finally
{
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrSizeKey, DFSConfigKeys.DfsNamenodeMaxXattrSizeDefault
);
if (cluster != null)
{
cluster.Shutdown();
}
}
try
{
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrsPerInodeKey, -1);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(true).Build();
NUnit.Framework.Assert.Fail("Expected exception with negative # xattrs per inode"
);
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Cannot set a negative limit on the number of xattrs per inode"
, e);
}
finally
{
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrsPerInodeKey, DFSConfigKeys.DfsNamenodeMaxXattrsPerInodeDefault
);
if (cluster != null)
{
cluster.Shutdown();
}
}
try
{
// Set up a logger to check log message
LogVerificationAppender appender = new LogVerificationAppender();
Logger logger = Logger.GetRootLogger();
logger.AddAppender(appender);
int count = appender.CountLinesWithMessage("Maximum size of an xattr: 0 (unlimited)"
);
NUnit.Framework.Assert.AreEqual("Expected no messages about unlimited xattr size"
, 0, count);
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrSizeKey, 0);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(true).Build();
count = appender.CountLinesWithMessage("Maximum size of an xattr: 0 (unlimited)");
// happens twice because we format then run
NUnit.Framework.Assert.AreEqual("Expected unlimited xattr size", 2, count);
}
finally
{
conf.SetInt(DFSConfigKeys.DfsNamenodeMaxXattrSizeKey, DFSConfigKeys.DfsNamenodeMaxXattrSizeDefault
);
if (cluster != null)
{
cluster.Shutdown();
}
}
}