public virtual void TestConcurrentAccess()
{
// Initialize DirectoryCollection with a file instead of a directory
string[] dirs = new string[] { testFile.GetPath() };
DirectoryCollection dc = new DirectoryCollection(dirs, conf.GetFloat(YarnConfiguration
.NmMaxPerDiskUtilizationPercentage, YarnConfiguration.DefaultNmMaxPerDiskUtilizationPercentage
));
// Create an iterator before checkDirs is called to reliable test case
IList <string> list = dc.GetGoodDirs();
ListIterator <string> li = list.ListIterator();
// DiskErrorException will invalidate iterator of non-concurrent
// collections. ConcurrentModificationException will be thrown upon next
// use of the iterator.
NUnit.Framework.Assert.IsTrue("checkDirs did not remove test file from directory list"
, dc.CheckDirs());
// Verify no ConcurrentModification is thrown
li.Next();
}
public virtual void TestFailedDisksBecomingGoodAgain()
{
string dirA = new FilePath(testDir, "dirA").GetPath();
string[] dirs = new string[] { dirA };
DirectoryCollection dc = new DirectoryCollection(dirs, 0.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFullDirs().Count);
dc.SetDiskUtilizationPercentageCutoff(100.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
string dirB = new FilePath(testDir, "dirB").GetPath();
Path pathB = new Path(dirB);
FsPermission permDirB = new FsPermission((short)0x100);
localFs.Mkdir(pathB, null, true);
localFs.SetPermission(pathB, permDirB);
string[] dirs2 = new string[] { dirB };
dc = new DirectoryCollection(dirs2, 100.0F);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(0, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
permDirB = new FsPermission((short)0x1c0);
localFs.SetPermission(pathB, permDirB);
dc.CheckDirs();
NUnit.Framework.Assert.AreEqual(1, dc.GetGoodDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFailedDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dc.GetFullDirs().Count);
}
/// <summary>
/// Function to get the log dirs which should be considered when cleaning up
/// resources.
/// </summary>
/// <remarks>
/// Function to get the log dirs which should be considered when cleaning up
/// resources. Contains the good log dirs and the log dirs that have reached
/// the disk space limit
/// </remarks>
/// <returns>the log dirs which should be considered for cleaning up</returns>
public virtual IList <string> GetLogDirsForCleanup()
{
return(DirectoryCollection.Concat(logDirs.GetGoodDirs(), logDirs.GetFullDirs()));
}
/// <summary>
/// Check the health of current set of local directories(good and failed),
/// updating the list of valid directories if necessary.
/// </summary>
/// <returns>
/// <em>true</em> if there is a new disk-failure identified in this
/// checking or a failed directory passes the disk check <em>false</em>
/// otherwise.
/// </returns>
internal virtual bool CheckDirs()
{
lock (this)
{
bool setChanged = false;
ICollection <string> preCheckGoodDirs = new HashSet <string>(localDirs);
ICollection <string> preCheckFullDirs = new HashSet <string>(fullDirs);
ICollection <string> preCheckOtherErrorDirs = new HashSet <string>(errorDirs);
IList <string> failedDirs = DirectoryCollection.Concat(errorDirs, fullDirs);
IList <string> allLocalDirs = DirectoryCollection.Concat(localDirs, failedDirs);
IDictionary <string, DirectoryCollection.DiskErrorInformation> dirsFailedCheck = TestDirs
(allLocalDirs);
localDirs.Clear();
errorDirs.Clear();
fullDirs.Clear();
foreach (KeyValuePair <string, DirectoryCollection.DiskErrorInformation> entry in
dirsFailedCheck)
{
string dir = entry.Key;
DirectoryCollection.DiskErrorInformation errorInformation = entry.Value;
switch (entry.Value.cause)
{
case DirectoryCollection.DiskErrorCause.DiskFull:
{
fullDirs.AddItem(entry.Key);
break;
}
case DirectoryCollection.DiskErrorCause.Other:
{
errorDirs.AddItem(entry.Key);
break;
}
}
if (preCheckGoodDirs.Contains(dir))
{
Log.Warn("Directory " + dir + " error, " + errorInformation.message + ", removing from list of valid directories"
);
setChanged = true;
numFailures++;
}
}
foreach (string dir_1 in allLocalDirs)
{
if (!dirsFailedCheck.Contains(dir_1))
{
localDirs.AddItem(dir_1);
if (preCheckFullDirs.Contains(dir_1) || preCheckOtherErrorDirs.Contains(dir_1))
{
setChanged = true;
Log.Info("Directory " + dir_1 + " passed disk check, adding to list of valid directories."
);
}
}
}
ICollection <string> postCheckFullDirs = new HashSet <string>(fullDirs);
ICollection <string> postCheckOtherDirs = new HashSet <string>(errorDirs);
foreach (string dir_2 in preCheckFullDirs)
{
if (postCheckOtherDirs.Contains(dir_2))
{
Log.Warn("Directory " + dir_2 + " error " + dirsFailedCheck[dir_2].message);
}
}
foreach (string dir_3 in preCheckOtherErrorDirs)
{
if (postCheckFullDirs.Contains(dir_3))
{
Log.Warn("Directory " + dir_3 + " error " + dirsFailedCheck[dir_3].message);
}
}
return(setChanged);
}
}
/// <summary>
/// Function to get the local dirs which should be considered for reading
/// existing files on disk.
/// </summary>
/// <remarks>
/// Function to get the local dirs which should be considered for reading
/// existing files on disk. Contains the good local dirs and the local dirs
/// that have reached the disk space limit
/// </remarks>
/// <returns>the local dirs which should be considered for reading</returns>
public virtual IList <string> GetLocalDirsForRead()
{
return(DirectoryCollection.Concat(localDirs.GetGoodDirs(), localDirs.GetFullDirs(
)));
}
