/// <summary>
/// Get default file system URI for the cluster (used to ensure consistency
/// of history done/staging locations) over different context
/// </summary>
/// <returns>Default file context</returns>
private static FileContext GetDefaultFileContext()
{
// If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore
// ignore it. This prevents defaulting history paths to file system specified
// by core-default.xml which would not make sense in any case. For a test
// case to exploit this functionality it should create core-site.xml
FileContext fc = null;
Configuration defaultConf = new Configuration();
string[] sources;
sources = defaultConf.GetPropertySources(CommonConfigurationKeysPublic.FsDefaultNameKey
);
if (sources != null && (!Arrays.AsList(sources).Contains("core-default.xml") || sources
.Length > 1))
{
try
{
fc = FileContext.GetFileContext(defaultConf);
Log.Info("Default file system [" + fc.GetDefaultFileSystem().GetUri() + "]");
}
catch (UnsupportedFileSystemException e)
{
Log.Error("Unable to create default file context [" + defaultConf.Get(CommonConfigurationKeysPublic
.FsDefaultNameKey) + "]", e);
}
}
else
{
Log.Info("Default file system is set solely " + "by core-default.xml therefore - ignoring"
);
}
return(fc);
}
public virtual void SetUp()
{
InitializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fcTarget.Mkdir(new Path(targetTestRoot, "user"), FileContext.DefaultPerm, true);
fcTarget.Mkdir(new Path(targetTestRoot, "data"), FileContext.DefaultPerm, true);
fcTarget.Mkdir(new Path(targetTestRoot, "dir2"), FileContext.DefaultPerm, true);
fcTarget.Mkdir(new Path(targetTestRoot, "dir3"), FileContext.DefaultPerm, true);
FileContextTestHelper.CreateFile(fcTarget, new Path(targetTestRoot, "aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = new Configuration();
ConfigUtil.AddLink(conf, "/targetRoot", targetTestRoot.ToUri());
ConfigUtil.AddLink(conf, "/user", new Path(targetTestRoot, "user").ToUri());
ConfigUtil.AddLink(conf, "/user2", new Path(targetTestRoot, "user").ToUri());
ConfigUtil.AddLink(conf, "/data", new Path(targetTestRoot, "data").ToUri());
ConfigUtil.AddLink(conf, "/internalDir/linkToDir2", new Path(targetTestRoot, "dir2"
).ToUri());
ConfigUtil.AddLink(conf, "/internalDir/internalDir2/linkToDir3", new Path(targetTestRoot
, "dir3").ToUri());
ConfigUtil.AddLink(conf, "/danglingLink", new Path(targetTestRoot, "missingTarget"
).ToUri());
ConfigUtil.AddLink(conf, "/linkToAFile", new Path(targetTestRoot, "aFile").ToUri(
));
fcView = FileContext.GetFileContext(FsConstants.ViewfsUri, conf);
}
public virtual void TestFcResolveAfs()
{
Configuration conf = new Configuration();
FileContext fcLocal = FileContext.GetLocalFSFileContext();
FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
string localTestRoot = helper.GetAbsoluteTestRootDir(fcLocal);
Path alphaLocalPath = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString()
, new FilePath(localTestRoot, "alpha").GetAbsolutePath());
DFSTestUtil.CreateFile(FileSystem.GetLocal(conf), alphaLocalPath, 16, (short)1, 2
);
Path linkTarget = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString(), localTestRoot
);
Path hdfsLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString(), "/tmp/link"
);
fcHdfs.CreateSymlink(linkTarget, hdfsLink, true);
Path alphaHdfsPathViaLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString
() + "/tmp/link/alpha");
ICollection <AbstractFileSystem> afsList = fcHdfs.ResolveAbstractFileSystems(alphaHdfsPathViaLink
);
NUnit.Framework.Assert.AreEqual(2, afsList.Count);
foreach (AbstractFileSystem afs in afsList)
{
if ((!afs.Equals(fcHdfs.GetDefaultFileSystem())) && (!afs.Equals(fcLocal.GetDefaultFileSystem
())))
{
NUnit.Framework.Assert.Fail("Failed to resolve AFS correctly");
}
}
}
/*
* return the ViewFS File context to be used for tests
*/
/// <exception cref="System.Exception"/>
public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper)
{
FileContext fsTarget = FileContext.GetLocalFSFileContext();
Path targetOfTests = helper.GetTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.Delete(targetOfTests, true);
fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true);
Configuration conf = new Configuration();
// Set up viewfs link for test dir as described above
string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath();
LinkUpFirstComponents(conf, testDir, fsTarget, "test dir");
// Set up viewfs link for home dir as described above
SetUpHomeDir(conf, fsTarget);
// the test path may be relative to working dir - we need to make that work:
// Set up viewfs link for wd as described above
string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath();
LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf);
fc.SetWorkingDirectory(new Path(wdDir));
// in case testdir relative to wd.
Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory());
//System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
//System.out.println("TargetOfTests = "+ targetOfTests.toUri());
return(fc);
}
public virtual void TestURIEmptyPath()
{
Configuration conf = new Configuration();
ConfigUtil.AddLink(conf, "/user", new URI("file://foo"));
FileContext.GetFileContext(FsConstants.ViewfsUri, conf);
}
public virtual void TestLinkTargetNonSymlink()
{
FileContext fc = null;
Path notSymlink = new Path("/notasymlink");
try
{
fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create));
DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration
(0));
try
{
client.GetLinkTarget(notSymlink.ToString());
NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("is not a symbolic link", e);
}
}
finally
{
if (fc != null)
{
fc.Delete(notSymlink, false);
}
}
}
/// <exception cref="Org.Apache.Hadoop.FS.UnsupportedFileSystemException"/>
protected internal static void PrintResults(TextWriter @out)
{
@out.WriteLine("Result of running LoadGenerator against fileSystem: " + FileContext
.GetFileContext().GetDefaultFileSystem().GetUri());
if (numOfOps[Open] != 0)
{
@out.WriteLine("Average open execution time: " + (double)executionTime[Open] / numOfOps
[Open] + "ms");
}
if (numOfOps[List] != 0)
{
@out.WriteLine("Average list execution time: " + (double)executionTime[List] / numOfOps
[List] + "ms");
}
if (numOfOps[Delete] != 0)
{
@out.WriteLine("Average deletion execution time: " + (double)executionTime[Delete
] / numOfOps[Delete] + "ms");
@out.WriteLine("Average create execution time: " + (double)executionTime[Create]
/ numOfOps[Create] + "ms");
@out.WriteLine("Average write_close execution time: " + (double)executionTime[WriteClose
] / numOfOps[WriteClose] + "ms");
}
if (totalTime != 0)
{
@out.WriteLine("Average operations per second: " + (double)totalOps / totalTime +
"ops/s");
}
@out.WriteLine();
}
// nothing to do or verify here
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint
nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler
)
{
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
CreateUserLocalDirs(localDirs, user);
CreateUserCacheDirs(localDirs, user);
CreateAppDirs(localDirs, user, appId);
CreateAppLogDirs(appId, logDirs, user);
// randomly choose the local directory
Path appStorageDir = GetWorkingDir(localDirs, user, appId);
string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
CopyFile(nmPrivateContainerTokensPath, tokenDst, user);
Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
FileContext localizerFc = FileContext.GetFileContext(lfs.GetDefaultFileSystem(),
GetConf());
localizerFc.SetUMask(lfs.GetUMask());
localizerFc.SetWorkingDirectory(appStorageDir);
Log.Info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.GetWorkingDirectory
());
ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId
, GetPaths(localDirs), RecordFactoryProvider.GetRecordFactory(GetConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.RunLocalization(nmAddr);
}
/// <exception cref="System.IO.IOException"/>
public object Run()
{
FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf);
myfc.Delete(TestPermissionSymlinks.link, false);
return(null);
}
/// <exception cref="System.IO.IOException"/>
public object Run()
{
FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf);
myfc.Open(TestPermissionSymlinks.link).Read();
return(null);
}
public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical
)
: base(typeof(Org.Apache.Hadoop.Mapred.LocalContainerLauncher).FullName)
{
this.context = context;
this.umbilical = umbilical;
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
// (TODO/FIXME: pointless to use RPC to talk to self; should create
// LocalTaskAttemptListener or similar: implement umbilical protocol
// but skip RPC stuff)
try
{
curFC = FileContext.GetFileContext(curDir.ToURI());
}
catch (UnsupportedFileSystemException)
{
Log.Error("Local filesystem " + curDir.ToURI().ToString() + " is unsupported?? (should never happen)"
);
}
// Save list of files/dirs that are supposed to be present so can delete
// any extras created by one task before starting subsequent task. Note
// that there's no protection against deleted or renamed localization;
// users who do that get what they deserve (and will have to disable
// uberization in order to run correctly).
FilePath[] curLocalFiles = curDir.ListFiles();
localizedFiles = new HashSet <FilePath>(curLocalFiles.Length);
for (int j = 0; j < curLocalFiles.Length; ++j)
{
localizedFiles.AddItem(curLocalFiles[j]);
}
}
/// <exception cref="System.Exception"/>
public FSDataOutputStream Run()
{
this._enclosing.fc = FileContext.GetFileContext(conf);
this._enclosing.fc.SetUMask(AggregatedLogFormat.AppLogFileUmask);
return(this._enclosing.fc.Create(remoteAppLogFile, EnumSet.Of(CreateFlag.Create,
CreateFlag.Overwrite), new Options.CreateOpts[] { }));
}
public virtual void TestFailedDirLogDeletion()
{
FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 7);
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i = 0; i < localLogDirs.Length; i++)
{
localLogDirPaths.AddItem(localLogDirs[i].GetAbsolutePath());
}
string localLogDirsString = StringUtils.Join(localLogDirPaths, ",");
conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString);
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false);
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l);
LocalDirsHandlerService mockDirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher,
mockDelService, mockDirsHandler, new NMNullStateStoreService());
NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler);
AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext
().GetDefaultFileSystem());
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA
<Configuration>());
logHandler.Init(conf);
logHandler.Start();
RunMockedFailedDirs(logHandler, appId, user, mockDelService, mockDirsHandler, conf
, spylfs, lfs, localLogDirs);
logHandler.Close();
}
public static void ClusterSetupAtBeginning()
{
cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology
(2)).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf);
fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf);
}
public static void ClusterSetupAtBegining()
{
SupportsBlocks = true;
Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
}
// Equivalence of @Before for cluster mode testing.
/// <exception cref="System.IO.IOException"/>
private void InitClusterModeTest()
{
Log = LogFactory.GetLog(typeof(TestWriteRead));
Log.Info("initClusterModeTest");
conf = new Configuration();
mfc = FileContext.GetFileContext();
mfs = FileSystem.Get(conf);
}
/// <exception cref="System.IO.IOException"/>
public Configuration LoadConfFile()
{
FileContext fc = FileContext.GetFileContext(configFile.ToUri(), conf);
Configuration jobConf = new Configuration(false);
jobConf.AddResource(fc.Open(configFile), configFile.ToString());
return(jobConf);
}
/// <exception cref="System.IO.IOException"/>
public object Run()
{
FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf);
Path newlink = new Path(TestPermissionSymlinks.targetParent, "newlink");
myfc.Rename(TestPermissionSymlinks.link, newlink, Options.Rename.None);
return(null);
}
public virtual void TestHarFsWithoutAuthority()
{
URI uri = harFileSystem.GetUri();
NUnit.Framework.Assert.IsNull("har uri authority not null: " + uri, uri.GetAuthority
());
FileContext.GetFileContext(uri, conf);
}
public static void ClusterSetupAtBeginning()
{
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology
(2)).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf);
fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf);
}
/// <exception cref="System.IO.IOException"/>
public LogReader(Configuration conf, Path remoteAppLogFile)
{
FileContext fileContext = FileContext.GetFileContext(conf);
this.fsDataIStream = fileContext.Open(remoteAppLogFile);
reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile
).GetLen(), conf);
this.scanner = reader.CreateScanner();
}
public virtual int DumpAContainersLogs(string appId, string containerId, string nodeId
, string jobOwner)
{
Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir
, YarnConfiguration.DefaultNmRemoteAppLogDir));
string suffix = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf());
Path remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, ConverterUtils
.ToApplicationId(appId), jobOwner, suffix);
RemoteIterator <FileStatus> nodeFiles;
try
{
Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir
);
nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus
(remoteAppLogDir);
}
catch (FileNotFoundException)
{
LogDirNotExist(remoteAppLogDir.ToString());
return(-1);
}
bool foundContainerLogs = false;
while (nodeFiles.HasNext())
{
FileStatus thisNodeFile = nodeFiles.Next();
string fileName = thisNodeFile.GetPath().GetName();
if (fileName.Contains(LogAggregationUtils.GetNodeString(nodeId)) && !fileName.EndsWith
(LogAggregationUtils.TmpFileSuffix))
{
AggregatedLogFormat.LogReader reader = null;
try
{
reader = new AggregatedLogFormat.LogReader(GetConf(), thisNodeFile.GetPath());
if (DumpAContainerLogs(containerId, reader, System.Console.Out, thisNodeFile.GetModificationTime
()) > -1)
{
foundContainerLogs = true;
}
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
}
if (!foundContainerLogs)
{
ContainerLogNotFound(containerId);
return(-1);
}
return(0);
}
public override void SetUp()
{
Configuration S3Conf = new Configuration();
Configuration localConf = new Configuration();
S3Conf.Set(CommonConfigurationKeysPublic.FsDefaultNameDefault, S3Conf.Get("test.fs.s3.name"
));
fc1 = FileContext.GetFileContext(S3Conf);
fc2 = FileContext.GetFileContext(localConf);
}
public static void ClusterSetupAtBegining()
{
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
fc = FileContext.GetFileContext(cluster.GetURI(0), conf);
defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation
.GetCurrentUser().GetShortUserName()));
fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true);
}
/// <exception cref="System.IO.IOException"/>
private void RestartCluster(bool format)
{
if (cluster != null)
{
cluster.Shutdown();
cluster = null;
}
cluster = new MiniDFSCluster.Builder(Conf).Format(format).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
}
/// <exception cref="System.IO.IOException"/>
public object Run()
{
FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf);
Path newlink = new Path(TestPermissionSymlinks.linkParent, "newlink");
myfc.Rename(TestPermissionSymlinks.link, newlink, Options.Rename.None);
Path linkTarget = myfc.GetLinkTarget(newlink);
NUnit.Framework.Assert.AreEqual("Expected link's target to match target!", TestPermissionSymlinks
.target, linkTarget);
return(null);
}
public static void ClusterSetupAtBegining()
{
SupportsBlocks = true;
Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
Path defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation
.GetCurrentUser().GetShortUserName()));
fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true);
}
public override void SetUp()
{
// create the test root on local_fs
fcTarget = FileContext.GetLocalFSFileContext();
base.SetUp();
// this sets up conf (and fcView which we replace)
// Now create a viewfs using a mount table called "default"
// hence viewfs://default/
schemeWithAuthority = new URI(FsConstants.ViewfsScheme, "default", "/", null, null
);
fcView = FileContext.GetFileContext(schemeWithAuthority, conf);
}
public static void TestSetUp()
{
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsListLimit, 2);
cluster = new MiniDFSCluster.Builder(conf).Build();
fs = cluster.GetFileSystem();
fc = FileContext.GetFileContext(cluster.GetURI(0), conf);
hftpfs = cluster.GetHftpFileSystem(0);
dfsClient = new DFSClient(NameNode.GetAddress(conf), conf);
file1 = new Path("filestatus.dat");
WriteFile(fs, file1, 1, fileSize, blockSize);
}
public virtual void TestFcDelegationToken()
{
FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
AbstractFileSystem afs = fcHdfs.GetDefaultFileSystem();
IList <Org.Apache.Hadoop.Security.Token.Token <object> > tokenList = afs.GetDelegationTokens
(UserGroupInformation.GetCurrentUser().GetUserName());
((Org.Apache.Hadoop.FS.Hdfs)afs).RenewDelegationToken((Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>)tokenList[0]);
((Org.Apache.Hadoop.FS.Hdfs)afs).CancelDelegationToken((Org.Apache.Hadoop.Security.Token.Token
<AbstractDelegationTokenIdentifier>)tokenList[0]);
}
