/// <summary>Create the ShortCircuitShm.</summary>
/// <param name="shmId">The ID to use.</param>
/// <param name="stream">
/// The stream that we're going to use to create this
/// shared memory segment.
/// Although this is a FileInputStream, we are going to
/// assume that the underlying file descriptor is writable
/// as well as readable. It would be more appropriate to use
/// a RandomAccessFile here, but that class does not have
/// any public accessor which returns a FileDescriptor,
/// unlike FileInputStream.
/// </param>
/// <exception cref="System.IO.IOException"/>
public ShortCircuitShm(ShortCircuitShm.ShmId shmId, FileInputStream stream)
{
if (!NativeIO.IsAvailable())
{
throw new NotSupportedException("NativeIO is not available.");
}
if (Shell.Windows)
{
throw new NotSupportedException("DfsClientShm is not yet implemented for Windows."
);
}
if (@unsafe == null)
{
throw new NotSupportedException("can't use DfsClientShm because we failed to " +
"load misc.Unsafe.");
}
this.shmId = shmId;
this.mmappedLength = GetUsableLength(stream);
this.baseAddress = NativeIO.POSIX.Mmap(stream.GetFD(), NativeIO.POSIX.MmapProtRead
| NativeIO.POSIX.MmapProtWrite, true, mmappedLength);
this.slots = new ShortCircuitShm.Slot[mmappedLength / BytesPerSlot];
this.allocatedSlots = new BitSet(slots.Length);
if (Log.IsTraceEnabled())
{
Log.Trace("creating " + this.GetType().Name + "(shmId=" + shmId + ", mmappedLength="
+ mmappedLength + ", baseAddress=" + string.Format("%x", baseAddress) + ", slots.length="
+ slots.Length + ")");
}
}
/// <summary>Return the singleton instance for the current process.</summary>
public static Org.Apache.Hadoop.IO.ReadaheadPool GetInstance()
{
lock (typeof(Org.Apache.Hadoop.IO.ReadaheadPool))
{
if (instance == null && NativeIO.IsAvailable())
{
instance = new Org.Apache.Hadoop.IO.ReadaheadPool();
}
return(instance);
}
}
/// <summary>Use the command chmod to set permission.</summary>
/// <exception cref="System.IO.IOException"/>
public override void SetPermission(Path p, FsPermission permission)
{
if (NativeIO.IsAvailable())
{
NativeIO.POSIX.Chmod(PathToFile(p).GetCanonicalPath(), permission.ToShort());
}
else
{
string perm = string.Format("%04o", permission.ToShort());
Shell.ExecCommand(Shell.GetSetPermissionCommand(perm, false, FileUtil.MakeShellPath
(PathToFile(p), true)));
}
}
public static HdfsConfiguration InitZeroCopyTest()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
Assume.AssumeTrue(SystemUtils.IsOsUnix);
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetInt(DFSConfigKeys.DfsClientMmapCacheSize, 3);
conf.SetLong(DFSConfigKeys.DfsClientMmapCacheTimeoutMs, 100);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestRequestMmapAccess._PORT.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, true);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetLong(DFSConfigKeys.DfsCachereportIntervalMsecKey, 1000);
conf.SetLong(DFSConfigKeys.DfsNamenodePathBasedCacheRefreshIntervalMs, 1000);
return(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestMemlockLimit()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
long memlockLimit = NativeIO.POSIX.GetCacheManipulator().GetMemlockLimit();
// Can't increase the memlock limit past the maximum.
Assume.AssumeTrue(memlockLimit != long.MaxValue);
FilePath dataDir = new FilePath(BaseDir, "data").GetCanonicalFile();
Configuration conf = cluster.GetConfiguration(0);
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, MakeURI("file", null, Util.FileAsURI
(dataDir).GetPath()));
long prevLimit = conf.GetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, DFSConfigKeys
.DfsDatanodeMaxLockedMemoryDefault);
DataNode dn = null;
try
{
// Try starting the DN with limit configured to the ulimit
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, memlockLimit);
dn = DataNode.CreateDataNode(new string[] { }, conf);
dn.Shutdown();
dn = null;
// Try starting the DN with a limit > ulimit
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, memlockLimit + 1);
try
{
dn = DataNode.CreateDataNode(new string[] { }, conf);
}
catch (RuntimeException e)
{
GenericTestUtils.AssertExceptionContains("more than the datanode's available RLIMIT_MEMLOCK"
, e);
}
}
finally
{
if (dn != null)
{
dn.Shutdown();
}
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, prevLimit);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestReadIncorrectlyRestrictedWithSecurity()
{
// this will only run if libs are available
Assume.AssumeTrue(NativeIO.IsAvailable());
System.Console.Out.WriteLine("Running test with native libs...");
string invalidUser = "******";
// We need to make sure that forceSecure.. call works only if
// the file belongs to expectedOwner.
// InputStream
try
{
SecureIOUtils.ForceSecureOpenForRead(testFilePathIs, invalidUser, realGroup).Close
();
NUnit.Framework.Assert.Fail("Didn't throw expection for wrong user ownership!");
}
catch (IOException)
{
}
// expected
// FSDataInputStream
try
{
SecureIOUtils.ForceSecureOpenFSDataInputStream(testFilePathFadis, invalidUser, realGroup
).Close();
NUnit.Framework.Assert.Fail("Didn't throw expection for wrong user ownership!");
}
catch (IOException)
{
}
// expected
// RandomAccessFile
try
{
SecureIOUtils.ForceSecureOpenForRandomRead(testFilePathRaf, "r", invalidUser, realGroup
).Close();
NUnit.Framework.Assert.Fail("Didn't throw expection for wrong user ownership!");
}
catch (IOException)
{
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual bool MkOneDirWithMode(Path p, FilePath p2f, FsPermission
permission)
{
if (permission == null)
{
return(p2f.Mkdir());
}
else
{
if (Shell.Windows && NativeIO.IsAvailable())
{
try
{
NativeIO.Windows.CreateDirectoryWithMode(p2f, permission.ToShort());
return(true);
}
catch (IOException e)
{
if (Log.IsDebugEnabled())
{
Log.Debug(string.Format("NativeIO.createDirectoryWithMode error, path = %s, mode = %o"
, p2f, permission.ToShort()), e);
}
return(false);
}
}
else
{
bool b = p2f.Mkdir();
if (b)
{
SetPermission(p, permission);
}
return(b);
}
}
}
static SecureIOUtils()
{
bool shouldBeSecure = UserGroupInformation.IsSecurityEnabled();
bool canBeSecure = NativeIO.IsAvailable();
if (!canBeSecure && shouldBeSecure)
{
throw new RuntimeException("Secure IO is not possible without native code extensions."
);
}
// Pre-cache an instance of the raw FileSystem since we sometimes
// do secure IO in a shutdown hook, where this call could fail.
try
{
rawFilesystem = FileSystem.GetLocal(new Configuration()).GetRaw();
}
catch (IOException)
{
throw new RuntimeException("Couldn't obtain an instance of RawLocalFileSystem.");
}
// SecureIO just skips security checks in the case that security is
// disabled
skipSecurity = !canBeSecure;
}
/// <exception cref="System.IO.IOException"/>
private LocalFSFileOutputStream(RawLocalFileSystem _enclosing, Path f, bool append
, FsPermission permission)
{
this._enclosing = _enclosing;
FilePath file = this._enclosing.PathToFile(f);
if (permission == null)
{
this.fos = new FileOutputStream(file, append);
}
else
{
if (Shell.Windows && NativeIO.IsAvailable())
{
this.fos = NativeIO.Windows.CreateFileOutputStreamWithMode(file, append, permission
.ToShort());
}
else
{
this.fos = new FileOutputStream(file, append);
bool success = false;
try
{
this._enclosing.SetPermission(f, permission);
success = true;
}
finally
{
if (!success)
{
IOUtils.Cleanup(FileSystem.Log, this.fos);
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogsFileAccess()
{
// This test will run only if NativeIO is enabled as SecureIOUtils
// require it to be enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
FilePath workDir = new FilePath(testWorkDir, "testContainerLogsFileAccess1");
Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles");
string data = "Log File content for container : ";
// Creating files for container1. Log aggregator will try to read log files
// with illegal user.
ApplicationId applicationId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 1);
ContainerId testContainerId1 = ContainerId.NewContainerId(applicationAttemptId, 1
);
Path appDir = new Path(srcFileRoot, testContainerId1.GetApplicationAttemptId().GetApplicationId
().ToString());
Path srcFilePath1 = new Path(appDir, testContainerId1.ToString());
string stdout = "stdout";
string stderr = "stderr";
WriteSrcFile(srcFilePath1, stdout, data + testContainerId1.ToString() + stdout);
WriteSrcFile(srcFilePath1, stderr, data + testContainerId1.ToString() + stderr);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf,
remoteAppLogFile, ugi);
AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId1
);
string randomUser = "******";
AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue
(Collections.SingletonList(srcFileRoot.ToString()), testContainerId1, randomUser
));
// It is trying simulate a situation where first log file is owned by
// different user (probably symlink) and second one by the user itself.
// The first file should not be aggregated. Because this log file has the invalid
// user name.
Org.Mockito.Mockito.When(logValue.GetUser()).ThenReturn(randomUser).ThenReturn(ugi
.GetShortUserName());
logWriter.Append(logKey, logValue);
logWriter.Close();
BufferedReader @in = new BufferedReader(new FileReader(new FilePath(remoteAppLogFile
.ToUri().GetRawPath())));
string line;
StringBuilder sb = new StringBuilder(string.Empty);
while ((line = @in.ReadLine()) != null)
{
Log.Info(line);
sb.Append(line);
}
line = sb.ToString();
string expectedOwner = ugi.GetShortUserName();
if (Path.Windows)
{
string adminsGroupString = "Administrators";
if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString))
{
expectedOwner = adminsGroupString;
}
}
// This file: stderr should not be aggregated.
// And we will not aggregate the log message.
string stdoutFile1 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stderr }));
// The file: stdout is expected to be aggregated.
string stdoutFile2 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string
[] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId
().GetApplicationId().ToString(), testContainerId1.ToString(), stdout }));
string message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '"
+ ugi.GetShortUserName() + "'";
NUnit.Framework.Assert.IsFalse(line.Contains(message2));
NUnit.Framework.Assert.IsFalse(line.Contains(data + testContainerId1.ToString() +
stderr));
NUnit.Framework.Assert.IsTrue(line.Contains(data + testContainerId1.ToString() +
stdout));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogPageAccess()
{
// SecureIOUtils require Native IO to be enabled. This test will run
// only if it is enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
string user = "******" + Runtime.CurrentTimeMillis();
FilePath absLogDir = null;
FilePath appDir = null;
FilePath containerDir = null;
FilePath syslog = null;
try
{
// target log directory
absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
.GetAbsoluteFile();
absLogDir.Mkdir();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
// Making sure that application returns a random user. This is required
// for SecureIOUtils' file owner check.
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
// Testing secure read access for log files
// Creating application and container directory and syslog file.
appDir = new FilePath(absLogDir, appId.ToString());
appDir.Mkdir();
containerDir = new FilePath(appDir, container1.ToString());
containerDir.Mkdir();
syslog = new FilePath(containerDir, "syslog");
syslog.CreateNewFile();
BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
);
@out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
@out.Close();
Context context = Org.Mockito.Mockito.Mock <Context>();
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
appMap[appId] = app;
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
context.GetContainers()[container1] = container;
ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
(context);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = container1.ToString();
@params[YarnWebParams.ContainerLogType] = "syslog";
Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
ContainerLogsPage), cLogsBlock, @params, (Module[])null);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
+ user + "' doesn't own requested log file : syslog");
}
finally
{
if (syslog != null)
{
syslog.Delete();
}
if (containerDir != null)
{
containerDir.Delete();
}
if (appDir != null)
{
appDir.Delete();
}
if (absLogDir != null)
{
absLogDir.Delete();
}
}
}