public virtual void TestGetUri()
{
NUnit.Framework.Assert.AreEqual(NameNode.GetUri(new IPEndPoint("foo", 555)), URI.
Create("hdfs://foo:555"));
NUnit.Framework.Assert.AreEqual(NameNode.GetUri(new IPEndPoint("foo", NameNode.DefaultPort
)), URI.Create("hdfs://foo"));
}
/// <exception cref="System.Exception"/>
public virtual void TestSetupDistributedCacheConflictsFiles()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem
));
URI mockUri = URI.Create("mockfs://mock/");
FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem
();
URI file = new URI("mockfs://mock/tmp/something.zip#something");
Path filePath = new Path(file);
URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
Path file2Path = new Path(file2);
Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath);
Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path);
DistributedCache.AddCacheFile(file, conf);
DistributedCache.AddCacheFile(file2, conf);
conf.Set(MRJobConfig.CacheFileTimestamps, "10,11");
conf.Set(MRJobConfig.CacheFilesSizes, "10,11");
conf.Set(MRJobConfig.CacheFileVisibilities, "true,true");
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.AreEqual(1, localResources.Count);
LocalResource lr = localResources["something"];
//First one wins
NUnit.Framework.Assert.IsNotNull(lr);
NUnit.Framework.Assert.AreEqual(10l, lr.GetSize());
NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp());
NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType());
}
public virtual void TestDeleteOnExitPathHandling()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestChRootedFileSystem.MockFileSystem), typeof(
FileSystem));
URI chrootUri = URI.Create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem)chrootFs.GetRawFileSystem()).GetRawFileSystem
();
// ensure delete propagates the correct path
Path chrootPath = new Path("/c");
Path rawPath = new Path("/a/b/c");
chrootFs.Delete(chrootPath, false);
Org.Mockito.Mockito.Verify(mockFs).Delete(Eq(rawPath), Eq(false));
Org.Mockito.Mockito.Reset(mockFs);
// fake that the path exists for deleteOnExit
FileStatus stat = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(mockFs.GetFileStatus(Eq(rawPath))).ThenReturn(stat);
// ensure deleteOnExit propagates the correct path
chrootFs.DeleteOnExit(chrootPath);
chrootFs.Close();
Org.Mockito.Mockito.Verify(mockFs).Delete(Eq(rawPath), Eq(true));
}
public virtual void TestAclMethodsPathTranslation()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestChRootedFileSystem.MockFileSystem), typeof(
FileSystem));
URI chrootUri = URI.Create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem)chrootFs.GetRawFileSystem()).GetRawFileSystem
();
Path chrootPath = new Path("/c");
Path rawPath = new Path("/a/b/c");
IList <AclEntry> entries = Collections.EmptyList();
chrootFs.ModifyAclEntries(chrootPath, entries);
Org.Mockito.Mockito.Verify(mockFs).ModifyAclEntries(rawPath, entries);
chrootFs.RemoveAclEntries(chrootPath, entries);
Org.Mockito.Mockito.Verify(mockFs).RemoveAclEntries(rawPath, entries);
chrootFs.RemoveDefaultAcl(chrootPath);
Org.Mockito.Mockito.Verify(mockFs).RemoveDefaultAcl(rawPath);
chrootFs.RemoveAcl(chrootPath);
Org.Mockito.Mockito.Verify(mockFs).RemoveAcl(rawPath);
chrootFs.SetAcl(chrootPath, entries);
Org.Mockito.Mockito.Verify(mockFs).SetAcl(rawPath, entries);
chrootFs.GetAclStatus(chrootPath);
Org.Mockito.Mockito.Verify(mockFs).GetAclStatus(rawPath);
}
/// <summary>Resolves a URI reference against a base URI.</summary>
/// <remarks>
/// Resolves a URI reference against a base URI. Work-around for bugs in
/// java.net.URI (e.g. <http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4708535>)
/// </remarks>
/// <param name="baseURI">the base URI</param>
/// <param name="reference">the URI reference</param>
/// <returns>the resulting URI</returns>
public static URI Resolve(URI baseURI, URI reference)
{
Args.NotNull(baseURI, "Base URI");
Args.NotNull(reference, "Reference URI");
URI @ref = reference;
string s = @ref.ToString();
if (s.StartsWith("?"))
{
return(ResolveReferenceStartingWithQueryString(baseURI, @ref));
}
bool emptyReference = s.Length == 0;
if (emptyReference)
{
@ref = URI.Create("#");
}
URI resolved = baseURI.Resolve(@ref);
if (emptyReference)
{
string resolvedString = resolved.ToString();
resolved = URI.Create(Sharpen.Runtime.Substring(resolvedString, 0, resolvedString
.IndexOf('#')));
}
return(NormalizeSyntax(resolved));
}
/// <exception cref="System.IO.IOException"/>
public static void TrashNonDefaultFS(Configuration conf)
{
conf.SetLong(FsTrashIntervalKey, 10);
{
// 10 minute
// attempt non-default FileSystem trash
FileSystem lfs = FileSystem.GetLocal(conf);
Path p = TestDir;
Path f = new Path(p, "foo/bar");
if (lfs.Exists(p))
{
lfs.Delete(p, true);
}
try
{
FileSystemTestHelper.WriteFile(lfs, f, 10);
FileSystem.CloseAll();
FileSystem localFs = FileSystem.Get(URI.Create("file:///"), conf);
Trash lTrash = new Trash(localFs, conf);
lTrash.MoveToTrash(f.GetParent());
CheckTrash(localFs, lTrash.GetCurrentTrashDir(), f);
}
finally
{
if (lfs.Exists(p))
{
lfs.Delete(p, true);
}
}
}
}
public virtual void TestGetFilterLocalFsSetsConfs()
{
FilterFileSystem flfs = (FilterFileSystem)FileSystem.Get(URI.Create("flfs:/"), conf
);
CheckFsConf(flfs, conf, 3);
}
private FileSystem CreateMockFs(bool useConf)
{
FileSystem mockFs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(mockFs.GetUri()).ThenReturn(URI.Create("mock:/"));
Org.Mockito.Mockito.When(mockFs.GetConf()).ThenReturn(useConf ? conf : null);
return(mockFs);
}
// returns default file system
internal static FileSystem DefaultFileSystem()
{
// load default provider
FileSystemProvider provider = AccessController.doPrivileged(new PrivilegedActionAnonymousInnerClassHelper());
// return file system
return(provider.GetFileSystem(URI.Create("file:///")));
}
public virtual void TestCanonicalUriWithPath()
{
URI uri;
uri = NetUtils.GetCanonicalUri(URI.Create("path"), 2);
Assert.Equal("path", uri.ToString());
uri = NetUtils.GetCanonicalUri(URI.Create("/path"), 2);
Assert.Equal("/path", uri.ToString());
}
/// <summary>Resolves a reference starting with a query string.</summary>
/// <remarks>Resolves a reference starting with a query string.</remarks>
/// <param name="baseURI">the base URI</param>
/// <param name="reference">the URI reference starting with a query string</param>
/// <returns>the resulting URI</returns>
private static URI ResolveReferenceStartingWithQueryString(URI baseURI, URI reference
)
{
string baseUri = baseURI.ToString();
baseUri = baseUri.IndexOf('?') > -1 ? Sharpen.Runtime.Substring(baseUri, 0, baseUri
.IndexOf('?')) : baseUri;
return(URI.Create(baseUri + reference.ToString()));
}
public virtual void TestWebHdfsCustomDefaultPorts()
{
URI uri = URI.Create("webhdfs://localhost");
WebHdfsFileSystem fs = (WebHdfsFileSystem)FileSystem.Get(uri, conf);
NUnit.Framework.Assert.AreEqual(123, fs.GetDefaultPort());
NUnit.Framework.Assert.AreEqual(uri, fs.GetUri());
NUnit.Framework.Assert.AreEqual("127.0.0.1:123", fs.GetCanonicalServiceName());
}
public virtual void TestSwebHdfsCustomUriPortWithCustomDefaultPorts()
{
URI uri = URI.Create("swebhdfs://localhost:789");
SWebHdfsFileSystem fs = (SWebHdfsFileSystem)FileSystem.Get(uri, conf);
NUnit.Framework.Assert.AreEqual(456, fs.GetDefaultPort());
NUnit.Framework.Assert.AreEqual(uri, fs.GetUri());
NUnit.Framework.Assert.AreEqual("127.0.0.1:789", fs.GetCanonicalServiceName());
}
/// <exception cref="System.IO.IOException"/>
private WebHdfsFileSystem SpyWebhdfsInSecureSetup()
{
WebHdfsFileSystem fsOrig = new WebHdfsFileSystem();
fsOrig.Initialize(URI.Create("webhdfs://127.0.0.1:0"), conf);
WebHdfsFileSystem fs = Org.Mockito.Mockito.Spy(fsOrig);
return(fs);
}
public virtual void TestURIEmptyPath()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestChRootedFileSystem.MockFileSystem), typeof(
FileSystem));
URI chrootUri = URI.Create("mockfs://foo");
new ChRootedFileSystem(chrootUri, conf);
}
public virtual void TestInitFilterLocalFsSetsEmbedConf()
{
FilterFileSystem flfs = new TestFilterFileSystem.FilterLocalFileSystem();
Assert.Equal(typeof(LocalFileSystem), flfs.GetRawFileSystem().
GetType());
CheckFsConf(flfs, null, 3);
flfs.Initialize(URI.Create("flfs:/"), conf);
CheckFsConf(flfs, conf, 3);
}
/// <summary>Return the URI that locates the HTTP server.</summary>
internal virtual URI GetServerURI()
{
// getHttpClientScheme() only returns https for HTTPS_ONLY policy. This
// matches the behavior that the first connector is a HTTPS connector only
// for HTTPS_ONLY policy.
IPEndPoint addr = httpServer.GetConnectorAddress(0);
return(URI.Create(DFSUtil.GetHttpClientScheme(conf) + "://" + NetUtils.GetHostPortString
(addr)));
}
/// <summary>
/// 从一个文件路径创建一个uri对象。
/// 需要注意的是，在高版本Android上，由于系统限制直接在Uri暴露文件的绝对路径，因此返回的Uri会是诸如content://...的形式。
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
public Uri getUriForFile(string path)
{
if (path.StartsWith("file://"))
{
path = path.Substring(7);
}
var file = new Java.IO.File(URI.Create(path));
return(Uri.FromFile(file));
}
public virtual void TestHsftpCustomUriPortWithDefaultPorts()
{
Configuration conf = new Configuration();
URI uri = URI.Create("hsftp://localhost:123");
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.Get(uri, conf);
NUnit.Framework.Assert.AreEqual(DFSConfigKeys.DfsNamenodeHttpsPortDefault, fs.GetDefaultPort
());
NUnit.Framework.Assert.AreEqual(uri, fs.GetUri());
NUnit.Framework.Assert.AreEqual("127.0.0.1:123", fs.GetCanonicalServiceName());
}
public virtual void TestHftpCustomUriPortWithCustomDefaultPorts()
{
Configuration conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHttpPortKey, 123);
URI uri = URI.Create("hftp://localhost:789");
HftpFileSystem fs = (HftpFileSystem)FileSystem.Get(uri, conf);
NUnit.Framework.Assert.AreEqual(123, fs.GetDefaultPort());
NUnit.Framework.Assert.AreEqual(uri, fs.GetUri());
NUnit.Framework.Assert.AreEqual("127.0.0.1:789", fs.GetCanonicalServiceName());
}
public virtual void TestBuildDTServiceName()
{
SecurityUtil.SetTokenServiceUseIp(true);
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildDTServiceName(
URI.Create("test://LocalHost"), 123));
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildDTServiceName(
URI.Create("test://LocalHost:123"), 456));
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildDTServiceName(
URI.Create("test://127.0.0.1"), 123));
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildDTServiceName(
URI.Create("test://127.0.0.1:123"), 456));
}
// create a fs from the authority, then check its uri against the given uri
// and the canonical. then try to fetch paths using the canonical
/// <exception cref="System.Exception"/>
internal virtual FileSystem GetVerifiedFS(string authority, string canonical, Configuration
conf)
{
URI uri = URI.Create(authority);
URI canonicalUri = URI.Create(canonical);
FileSystem fs = new TestFileSystemCanonicalization.DummyFileSystem(uri, conf);
Assert.Equal(uri, fs.GetUri());
Assert.Equal(canonicalUri, fs.GetCanonicalUri());
VerifyCheckPath(fs, "/file", true);
return(fs);
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI name, Configuration conf)
{
base.Initialize(name, conf);
SetConf(conf);
this.uri = URI.Create(name.GetScheme() + "://" + name.GetAuthority());
tokenAspect = new TokenAspect <TestTokenAspect.DummyFs>(this, SecurityUtil.BuildTokenService
(uri), TokenKind);
if (emulateSecurityEnabled || UserGroupInformation.IsSecurityEnabled())
{
tokenAspect.InitDelegationToken(ugi);
}
}
public virtual void TestInitFilterFsSetsEmbedConf()
{
LocalFileSystem lfs = new LocalFileSystem();
CheckFsConf(lfs, null, 2);
FilterFileSystem ffs = new FilterFileSystem(lfs);
Assert.Equal(lfs, ffs.GetRawFileSystem());
CheckFsConf(ffs, null, 3);
ffs.Initialize(URI.Create("filter:/"), conf);
CheckFsConf(ffs, conf, 3);
}
public virtual void TestIsInSafemode()
{
// Check for the standby nn without client failover.
NameNode nn2 = cluster.GetNameNode(1);
NUnit.Framework.Assert.IsTrue("nn2 should be in standby state", nn2.IsStandbyState
());
IPEndPoint nameNodeAddress = nn2.GetNameNodeAddress();
Configuration conf = new Configuration();
DistributedFileSystem dfs = new DistributedFileSystem();
try
{
dfs.Initialize(URI.Create("hdfs://" + nameNodeAddress.GetHostName() + ":" + nameNodeAddress
.Port), conf);
dfs.IsInSafeMode();
NUnit.Framework.Assert.Fail("StandBy should throw exception for isInSafeMode");
}
catch (IOException e)
{
if (e is RemoteException)
{
IOException sbExcpetion = ((RemoteException)e).UnwrapRemoteException();
NUnit.Framework.Assert.IsTrue("StandBy nn should not support isInSafeMode", sbExcpetion
is StandbyException);
}
else
{
throw;
}
}
finally
{
if (null != dfs)
{
dfs.Close();
}
}
// Check with Client FailOver
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.GetNameNodeRpc(1).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
false);
DistributedFileSystem dfsWithFailOver = (DistributedFileSystem)fs;
NUnit.Framework.Assert.IsTrue("ANN should be in SafeMode", dfsWithFailOver.IsInSafeMode
());
cluster.GetNameNodeRpc(1).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
NUnit.Framework.Assert.IsFalse("ANN should be out of SafeMode", dfsWithFailOver.IsInSafeMode
());
}
public virtual void TestCanonicalUriWithNoAuthority()
{
URI uri;
uri = NetUtils.GetCanonicalUri(URI.Create("scheme:/"), 2);
Assert.Equal("scheme:/", uri.ToString());
uri = NetUtils.GetCanonicalUri(URI.Create("scheme:/path"), 2);
Assert.Equal("scheme:/path", uri.ToString());
uri = NetUtils.GetCanonicalUri(URI.Create("scheme:///"), 2);
Assert.Equal("scheme:///", uri.ToString());
uri = NetUtils.GetCanonicalUri(URI.Create("scheme:///path"), 2);
Assert.Equal("scheme:///path", uri.ToString());
}
public static void SetUp()
{
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsWebhdfsAuthenticationFilterKey, typeof(TestWebHdfsWithAuthenticationFilter.CustomizedFilter
).FullName);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "localhost:0");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
IPEndPoint addr = cluster.GetNameNode().GetHttpAddress();
fs = FileSystem.Get(URI.Create("webhdfs://" + NetUtils.GetHostPortString(addr)),
conf);
cluster.WaitActive();
}
private void ConstructHttpServerURI(QJournalProtocolProtos.GetJournalStateResponseProto
ret)
{
if (ret.HasFromURL())
{
URI uri = URI.Create(ret.GetFromURL());
httpServerURL = GetHttpServerURI(uri.GetScheme(), uri.GetPort());
}
else
{
httpServerURL = GetHttpServerURI("http", ret.GetHttpPort());
}
}
/// <summary>Parse the file system URI out of the provided token.</summary>
public static URI GetServiceUriFromToken <_T0>(string scheme, Org.Apache.Hadoop.Security.Token.Token
<_T0> token)
where _T0 : TokenIdentifier
{
string tokStr = token.GetService().ToString();
string prefix = BuildTokenServicePrefixForLogicalUri(scheme);
if (tokStr.StartsWith(prefix))
{
tokStr = tokStr.ReplaceFirst(prefix, string.Empty);
}
return(URI.Create(scheme + "://" + tokStr));
}
public virtual void TestTimeout()
{
Configuration conf = new Configuration();
URI uri = URI.Create("hftp://localhost");
HftpFileSystem fs = (HftpFileSystem)FileSystem.Get(uri, conf);
URLConnection conn = fs.connectionFactory.OpenConnection(new Uri("http://localhost"
));
NUnit.Framework.Assert.AreEqual(URLConnectionFactory.DefaultSocketTimeout, conn.GetConnectTimeout
());
NUnit.Framework.Assert.AreEqual(URLConnectionFactory.DefaultSocketTimeout, conn.GetReadTimeout
());
}