/// <summary>Executes the filesystem operation.</summary>
/// <param name="fs">filesystem instance to use.</param>
/// <returns>The inputstream of the file.</returns>
/// <exception cref="System.IO.IOException">thrown if an IO error occured.</exception>
public virtual InputStream Execute(FileSystem fs)
{
int bufferSize = HttpFSServerWebApp.Get().GetConfig().GetInt("httpfs.buffer.size"
, 4096);
return(fs.Open(path, bufferSize));
}
/// <summary>Create an HttpFS Server to talk to the MiniDFSCluster we created.</summary>
/// <exception cref="System.Exception"/>
private void CreateHttpFSServer()
{
FilePath homeDir = TestDirHelper.GetTestDir();
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
TextWriter w = new FileWriter(secretFile);
w.Write("secret");
w.Close();
// HDFS configuration
FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
);
if (!hadoopConfDir.Mkdirs())
{
throw new IOException();
}
string fsDefaultName = nnConf.Get(CommonConfigurationKeysPublic.FsDefaultNameKey);
Configuration conf = new Configuration(false);
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
// Explicitly turn off XAttr support
conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, false);
FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.WriteXml(os);
os.Close();
// HTTPFS configuration
conf = new Configuration(false);
conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString());
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
, HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
, HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
());
FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
);
os = new FileOutputStream(httpfsSite);
conf.WriteXml(os);
os.Close();
ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();
Uri url = cl.GetResource("webapp");
if (url == null)
{
throw new IOException();
}
WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");
Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
server.AddHandler(context);
server.Start();
}
/// <summary>
/// Executes a
/// <see cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccess.FileSystemExecutor{T}"/
/// >
/// using a filesystem for the effective
/// user.
/// </summary>
/// <param name="ugi">user making the request.</param>
/// <param name="executor">FileSystemExecutor to execute.</param>
/// <returns>FileSystemExecutor response</returns>
/// <exception cref="System.IO.IOException">thrown if an IO error occurrs.</exception>
/// <exception cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccessException">
/// thrown if a FileSystemAccess releated error occurred. Thrown
/// exceptions are handled by
/// <see cref="HttpFSExceptionProvider"/>
/// .
/// </exception>
private T FsExecute <T>(UserGroupInformation ugi, FileSystemAccess.FileSystemExecutor
<T> executor)
{
FileSystemAccess fsAccess = HttpFSServerWebApp.Get().Get <FileSystemAccess>();
Configuration conf = HttpFSServerWebApp.Get().Get <FileSystemAccess>().GetFileSystemConfiguration
();
return(fsAccess.Execute(ugi.GetShortUserName(), conf, executor));
}
public virtual void Server()
{
string dir = TestDirHelper.GetTestDir().GetAbsolutePath();
Configuration httpfsConf = new Configuration(false);
HttpFSServerWebApp server = new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf
);
server.Init();
server.Destroy();
}
/// <summary>Returns a filesystem instance.</summary>
/// <remarks>
/// Returns a filesystem instance. The fileystem instance is wired for release at the completion of
/// the current Servlet request via the
/// <see cref="Org.Apache.Hadoop.Lib.Servlet.FileSystemReleaseFilter"/>
/// .
/// <p>
/// If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
/// <code>AccessControlException</code> will be thrown.
/// </remarks>
/// <param name="ugi">principal for whom the filesystem instance is.</param>
/// <returns>a filesystem for the specified user or do-as user.</returns>
/// <exception cref="System.IO.IOException">
/// thrown if an IO error occurred. Thrown exceptions are
/// handled by
/// <see cref="HttpFSExceptionProvider"/>
/// .
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccessException">
/// thrown if a FileSystemAccess releated error occurred. Thrown
/// exceptions are handled by
/// <see cref="HttpFSExceptionProvider"/>
/// .
/// </exception>
private FileSystem CreateFileSystem(UserGroupInformation ugi)
{
string hadoopUser = ugi.GetShortUserName();
FileSystemAccess fsAccess = HttpFSServerWebApp.Get().Get <FileSystemAccess>();
Configuration conf = HttpFSServerWebApp.Get().Get <FileSystemAccess>().GetFileSystemConfiguration
();
FileSystem fs = fsAccess.CreateFileSystem(hadoopUser, conf);
FileSystemReleaseFilter.SetFileSystem(fs);
return(fs);
}
protected override Configuration GetProxyuserConfiguration(FilterConfig filterConfig
)
{
IDictionary <string, string> proxyuserConf = HttpFSServerWebApp.Get().GetConfig().
GetValByRegex("httpfs\\.proxyuser\\.");
Configuration conf = new Configuration(false);
foreach (KeyValuePair <string, string> entry in proxyuserConf)
{
conf.Set(Sharpen.Runtime.Substring(entry.Key, "httpfs.".Length), entry.Value);
}
return(conf);
}
/// <exception cref="System.Exception"/>
private void CreateHttpFSServer()
{
FilePath homeDir = TestDirHelper.GetTestDir();
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
TextWriter w = new FileWriter(secretFile);
w.Write("secret");
w.Close();
//HDFS configuration
FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
);
hadoopConfDir.Mkdirs();
string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
.FsDefaultNameKey);
Configuration conf = new Configuration(false);
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.WriteXml(os);
os.Close();
conf = new Configuration(false);
conf.Set("httpfs.proxyuser.client.hosts", "*");
conf.Set("httpfs.proxyuser.client.groups", "*");
conf.Set("httpfs.authentication.type", "kerberos");
conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
());
FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
);
os = new FileOutputStream(httpfsSite);
conf.WriteXml(os);
os.Close();
ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();
Uri url = cl.GetResource("webapp");
WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");
Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
server.AddHandler(context);
server.Start();
HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
}
/// <summary>Returns the hadoop-auth configuration from HttpFSServer's configuration.
/// </summary>
/// <remarks>
/// Returns the hadoop-auth configuration from HttpFSServer's configuration.
/// <p>
/// It returns all HttpFSServer's configuration properties prefixed with
/// <code>httpfs.authentication</code>. The <code>httpfs.authentication</code>
/// prefix is removed from the returned property names.
/// </remarks>
/// <param name="configPrefix">parameter not used.</param>
/// <param name="filterConfig">parameter not used.</param>
/// <returns>hadoop-auth configuration read from HttpFSServer's configuration.</returns>
/// <exception cref="Javax.Servlet.ServletException"/>
protected override Properties GetConfiguration(string configPrefix, FilterConfig
filterConfig)
{
Properties props = new Properties();
Configuration conf = HttpFSServerWebApp.Get().GetConfig();
props.SetProperty(AuthenticationFilter.CookiePath, "/");
foreach (KeyValuePair <string, string> entry in conf)
{
string name = entry.Key;
if (name.StartsWith(ConfPrefix))
{
string value = conf.Get(name);
name = Sharpen.Runtime.Substring(name, ConfPrefix.Length);
props.SetProperty(name, value);
}
}
string signatureSecretFile = props.GetProperty(SignatureSecretFile, null);
if (signatureSecretFile == null)
{
throw new RuntimeException("Undefined property: " + SignatureSecretFile);
}
try
{
StringBuilder secret = new StringBuilder();
StreamReader reader = new InputStreamReader(new FileInputStream(signatureSecretFile
), Charsets.Utf8);
int c = reader.Read();
while (c > -1)
{
secret.Append((char)c);
c = reader.Read();
}
reader.Close();
props.SetProperty(AuthenticationFilter.SignatureSecret, secret.ToString());
}
catch (IOException)
{
throw new RuntimeException("Could not read HttpFS signature secret file: " + signatureSecretFile
);
}
return(props);
}
/// <summary>
/// Returns the
/// <see cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccess"/>
/// service to return the FileSystemAccess filesystem
/// instance to.
/// </summary>
/// <returns>the FileSystemAccess service.</returns>
protected internal override FileSystemAccess GetFileSystemAccess()
{
return(HttpFSServerWebApp.Get().Get <FileSystemAccess>());
}
/// <exception cref="System.Exception"/>
private void CreateHttpFSServer(bool addDelegationTokenAuthHandler)
{
FilePath homeDir = TestDirHelper.GetTestDir();
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
TextWriter w = new FileWriter(secretFile);
w.Write("secret");
w.Close();
//HDFS configuration
FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
);
hadoopConfDir.Mkdirs();
string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
.FsDefaultNameKey);
Configuration conf = new Configuration(false);
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, true);
FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.WriteXml(os);
os.Close();
//HTTPFS configuration
conf = new Configuration(false);
if (addDelegationTokenAuthHandler)
{
conf.Set("httpfs.authentication.type", typeof(HttpFSKerberosAuthenticationHandlerForTesting
).FullName);
}
conf.Set("httpfs.services.ext", typeof(TestHttpFSServer.MockGroups).FullName);
conf.Set("httpfs.admin.group", HadoopUsersConfTestHelper.GetHadoopUserGroups(HadoopUsersConfTestHelper
.GetHadoopUsers()[0])[0]);
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
, HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
, HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
());
conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString());
FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
);
os = new FileOutputStream(httpfsSite);
conf.WriteXml(os);
os.Close();
ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();
Uri url = cl.GetResource("webapp");
WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");
Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
server.AddHandler(context);
server.Start();
if (addDelegationTokenAuthHandler)
{
HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
}
}
public virtual Response Get(string path, HttpFSParametersProvider.OperationParam
op, Parameters @params, HttpServletRequest request)
{
UserGroupInformation user = HttpUserGroupInformation.Get();
Response response;
path = MakeAbsolute(path);
MDC.Put(HttpFSFileSystem.OpParam, op.Value().ToString());
MDC.Put("hostname", request.GetRemoteAddr());
switch (op.Value())
{
case HttpFSFileSystem.Operation.Open:
{
//Invoking the command directly using an unmanaged FileSystem that is
// released by the FileSystemReleaseFilter
FSOperations.FSOpen command = new FSOperations.FSOpen(path);
FileSystem fs = CreateFileSystem(user);
InputStream @is = command.Execute(fs);
long offset = @params.Get <HttpFSParametersProvider.OffsetParam>(HttpFSParametersProvider.OffsetParam
.Name);
long len = @params.Get <HttpFSParametersProvider.LenParam>(HttpFSParametersProvider.LenParam
.Name);
AuditLog.Info("[{}] offset [{}] len [{}]", new object[] { path, offset, len });
InputStreamEntity entity = new InputStreamEntity(@is, offset, len);
response = Response.Ok(entity).Type(MediaType.ApplicationOctetStream).Build();
break;
}
case HttpFSFileSystem.Operation.Getfilestatus:
{
FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path);
IDictionary json = FsExecute(user, command);
AuditLog.Info("[{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Liststatus:
{
string filter = @params.Get <HttpFSParametersProvider.FilterParam>(HttpFSParametersProvider.FilterParam
.Name);
FSOperations.FSListStatus command = new FSOperations.FSListStatus(path, filter);
IDictionary json = FsExecute(user, command);
AuditLog.Info("[{}] filter [{}]", path, (filter != null) ? filter : "-");
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Gethomedirectory:
{
EnforceRootPath(op.Value(), path);
FSOperations.FSHomeDir command = new FSOperations.FSHomeDir();
JSONObject json = FsExecute(user, command);
AuditLog.Info(string.Empty);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Instrumentation:
{
EnforceRootPath(op.Value(), path);
Groups groups = HttpFSServerWebApp.Get().Get <Groups>();
IList <string> userGroups = groups.GetGroups(user.GetShortUserName());
if (!userGroups.Contains(HttpFSServerWebApp.Get().GetAdminGroup()))
{
throw new AccessControlException("User not in HttpFSServer admin group");
}
Instrumentation instrumentation = HttpFSServerWebApp.Get().Get <Instrumentation>();
IDictionary snapshot = instrumentation.GetSnapshot();
response = Response.Ok(snapshot).Build();
break;
}
case HttpFSFileSystem.Operation.Getcontentsummary:
{
FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path);
IDictionary json = FsExecute(user, command);
AuditLog.Info("[{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Getfilechecksum:
{
FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path);
IDictionary json = FsExecute(user, command);
AuditLog.Info("[{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Getfileblocklocations:
{
response = Response.Status(Response.Status.BadRequest).Build();
break;
}
case HttpFSFileSystem.Operation.Getaclstatus:
{
FSOperations.FSAclStatus command = new FSOperations.FSAclStatus(path);
IDictionary json = FsExecute(user, command);
AuditLog.Info("ACL status for [{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Getxattrs:
{
IList <string> xattrNames = @params.GetValues <HttpFSParametersProvider.XAttrNameParam
>(HttpFSParametersProvider.XAttrNameParam.Name);
XAttrCodec encoding = @params.Get <HttpFSParametersProvider.XAttrEncodingParam>(HttpFSParametersProvider.XAttrEncodingParam
.Name);
FSOperations.FSGetXAttrs command = new FSOperations.FSGetXAttrs(path, xattrNames,
encoding);
IDictionary json = FsExecute(user, command);
AuditLog.Info("XAttrs for [{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
case HttpFSFileSystem.Operation.Listxattrs:
{
FSOperations.FSListXAttrs command = new FSOperations.FSListXAttrs(path);
IDictionary json = FsExecute(user, command);
AuditLog.Info("XAttr names for [{}]", path);
response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
break;
}
default:
{
throw new IOException(MessageFormat.Format("Invalid HTTP GET operation [{0}]", op
.Value()));
}
}
return(response);
}