public virtual void TestDiskSpaceUtilizationLimit()
{
string dirA = new FilePath(testDir, "dirA").GetPath();
string[] dirs = new string[] { dirA };
DirectoryCollection dc = new DirectoryCollection(dirs, 0.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFullDirs().Count);
dc = new DirectoryCollection(dirs, 100.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
dc = new DirectoryCollection(dirs, testDir.GetTotalSpace() / (1024 * 1024));
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFullDirs().Count);
dc = new DirectoryCollection(dirs, 100.0F, 0);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
}
public virtual void TestFailedDisksBecomingGoodAgain()
{
string dirA = new FilePath(testDir, "dirA").GetPath();
string[] dirs = new string[] { dirA };
DirectoryCollection dc = new DirectoryCollection(dirs, 0.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFullDirs().Count);
dc.SetDiskUtilizationPercentageCutoff(100.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
string dirB = new FilePath(testDir, "dirB").GetPath();
Path pathB = new Path(dirB);
FsPermission permDirB = new FsPermission((short)0x100);
localFs.Mkdir(pathB, null, true);
localFs.SetPermission(pathB, permDirB);
string[] dirs2 = new string[] { dirB };
dc = new DirectoryCollection(dirs2, 100.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
permDirB = new FsPermission((short)0x1c0);
localFs.SetPermission(pathB, permDirB);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
}
public virtual void TestConcurrentAccess()
{
// Initialize DirectoryCollection with a file instead of a directory
string[] dirs = new string[] { testFile.GetPath() };
DirectoryCollection dc = new DirectoryCollection(dirs, conf.GetFloat(YarnConfiguration
.NmMaxPerDiskUtilizationPercentage, YarnConfiguration.DefaultNmMaxPerDiskUtilizationPercentage
));
// Create an iterator before checkDirs is called to reliable test case
IList <string> list = dc.GetGoodDirs();
ListIterator <string> li = list.ListIterator();
// DiskErrorException will invalidate iterator of non-concurrent
// collections. ConcurrentModificationException will be thrown upon next
// use of the iterator.
NUnit.Framework.Assert.IsTrue("checkDirs did not remove test file from directory list"
, dc.CheckDirs());
// Verify no ConcurrentModification is thrown
li.Next();
}
private void CheckDirs()
{
bool disksStatusChange = false;
ICollection <string> failedLocalDirsPreCheck = new HashSet <string>(localDirs.GetFailedDirs
());
ICollection <string> failedLogDirsPreCheck = new HashSet <string>(logDirs.GetFailedDirs
());
if (localDirs.CheckDirs())
{
disksStatusChange = true;
}
if (logDirs.CheckDirs())
{
disksStatusChange = true;
}
ICollection <string> failedLocalDirsPostCheck = new HashSet <string>(localDirs.GetFailedDirs
());
ICollection <string> failedLogDirsPostCheck = new HashSet <string>(logDirs.GetFailedDirs
());
bool disksFailed = false;
bool disksTurnedGood = false;
disksFailed = DisksTurnedBad(failedLocalDirsPreCheck, failedLocalDirsPostCheck);
disksTurnedGood = DisksTurnedGood(failedLocalDirsPreCheck, failedLocalDirsPostCheck
);
// skip check if we have new failed or good local dirs since we're going to
// log anyway
if (!disksFailed)
{
disksFailed = DisksTurnedBad(failedLogDirsPreCheck, failedLogDirsPostCheck);
}
if (!disksTurnedGood)
{
disksTurnedGood = DisksTurnedGood(failedLogDirsPreCheck, failedLogDirsPostCheck);
}
LogDiskStatus(disksFailed, disksTurnedGood);
if (disksStatusChange)
{
UpdateDirsAfterTest();
}
lastDisksCheckTime = Runtime.CurrentTimeMillis();
}
