/// <summary> /// Get default file system URI for the cluster (used to ensure consistency /// of history done/staging locations) over different context /// </summary> /// <returns>Default file context</returns> private static FileContext GetDefaultFileContext() { // If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore // ignore it. This prevents defaulting history paths to file system specified // by core-default.xml which would not make sense in any case. For a test // case to exploit this functionality it should create core-site.xml FileContext fc = null; Configuration defaultConf = new Configuration(); string[] sources; sources = defaultConf.GetPropertySources(CommonConfigurationKeysPublic.FsDefaultNameKey ); if (sources != null && (!Arrays.AsList(sources).Contains("core-default.xml") || sources .Length > 1)) { try { fc = FileContext.GetFileContext(defaultConf); Log.Info("Default file system [" + fc.GetDefaultFileSystem().GetUri() + "]"); } catch (UnsupportedFileSystemException e) { Log.Error("Unable to create default file context [" + defaultConf.Get(CommonConfigurationKeysPublic .FsDefaultNameKey) + "]", e); } } else { Log.Info("Default file system is set solely " + "by core-default.xml therefore - ignoring" ); } return(fc); }
public virtual void SetUp() { InitializeTargetTestRoot(); // Make user and data dirs - we creates links to them in the mount table fcTarget.Mkdir(new Path(targetTestRoot, "user"), FileContext.DefaultPerm, true); fcTarget.Mkdir(new Path(targetTestRoot, "data"), FileContext.DefaultPerm, true); fcTarget.Mkdir(new Path(targetTestRoot, "dir2"), FileContext.DefaultPerm, true); fcTarget.Mkdir(new Path(targetTestRoot, "dir3"), FileContext.DefaultPerm, true); FileContextTestHelper.CreateFile(fcTarget, new Path(targetTestRoot, "aFile")); // Now we use the mount fs to set links to user and dir // in the test root // Set up the defaultMT in the config with our mount point links conf = new Configuration(); ConfigUtil.AddLink(conf, "/targetRoot", targetTestRoot.ToUri()); ConfigUtil.AddLink(conf, "/user", new Path(targetTestRoot, "user").ToUri()); ConfigUtil.AddLink(conf, "/user2", new Path(targetTestRoot, "user").ToUri()); ConfigUtil.AddLink(conf, "/data", new Path(targetTestRoot, "data").ToUri()); ConfigUtil.AddLink(conf, "/internalDir/linkToDir2", new Path(targetTestRoot, "dir2" ).ToUri()); ConfigUtil.AddLink(conf, "/internalDir/internalDir2/linkToDir3", new Path(targetTestRoot , "dir3").ToUri()); ConfigUtil.AddLink(conf, "/danglingLink", new Path(targetTestRoot, "missingTarget" ).ToUri()); ConfigUtil.AddLink(conf, "/linkToAFile", new Path(targetTestRoot, "aFile").ToUri( )); fcView = FileContext.GetFileContext(FsConstants.ViewfsUri, conf); }
public virtual void TestFcResolveAfs() { Configuration conf = new Configuration(); FileContext fcLocal = FileContext.GetLocalFSFileContext(); FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri()); string localTestRoot = helper.GetAbsoluteTestRootDir(fcLocal); Path alphaLocalPath = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString() , new FilePath(localTestRoot, "alpha").GetAbsolutePath()); DFSTestUtil.CreateFile(FileSystem.GetLocal(conf), alphaLocalPath, 16, (short)1, 2 ); Path linkTarget = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString(), localTestRoot ); Path hdfsLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString(), "/tmp/link" ); fcHdfs.CreateSymlink(linkTarget, hdfsLink, true); Path alphaHdfsPathViaLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString () + "/tmp/link/alpha"); ICollection <AbstractFileSystem> afsList = fcHdfs.ResolveAbstractFileSystems(alphaHdfsPathViaLink ); NUnit.Framework.Assert.AreEqual(2, afsList.Count); foreach (AbstractFileSystem afs in afsList) { if ((!afs.Equals(fcHdfs.GetDefaultFileSystem())) && (!afs.Equals(fcLocal.GetDefaultFileSystem ()))) { NUnit.Framework.Assert.Fail("Failed to resolve AFS correctly"); } } }
/* * return the ViewFS File context to be used for tests */ /// <exception cref="System.Exception"/> public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper) { FileContext fsTarget = FileContext.GetLocalFSFileContext(); Path targetOfTests = helper.GetTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.Delete(targetOfTests, true); fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true); Configuration conf = new Configuration(); // Set up viewfs link for test dir as described above string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath(); LinkUpFirstComponents(conf, testDir, fsTarget, "test dir"); // Set up viewfs link for home dir as described above SetUpHomeDir(conf, fsTarget); // the test path may be relative to working dir - we need to make that work: // Set up viewfs link for wd as described above string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath(); LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf); fc.SetWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory()); //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test")); //System.out.println("TargetOfTests = "+ targetOfTests.toUri()); return(fc); }
public virtual void TestURIEmptyPath() { Configuration conf = new Configuration(); ConfigUtil.AddLink(conf, "/user", new URI("file://foo")); FileContext.GetFileContext(FsConstants.ViewfsUri, conf); }
public virtual void TestLinkTargetNonSymlink() { FileContext fc = null; Path notSymlink = new Path("/notasymlink"); try { fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri()); fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create)); DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration (0)); try { client.GetLinkTarget(notSymlink.ToString()); NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink"); } catch (IOException e) { GenericTestUtils.AssertExceptionContains("is not a symbolic link", e); } } finally { if (fc != null) { fc.Delete(notSymlink, false); } } }
/// <exception cref="Org.Apache.Hadoop.FS.UnsupportedFileSystemException"/> protected internal static void PrintResults(TextWriter @out) { @out.WriteLine("Result of running LoadGenerator against fileSystem: " + FileContext .GetFileContext().GetDefaultFileSystem().GetUri()); if (numOfOps[Open] != 0) { @out.WriteLine("Average open execution time: " + (double)executionTime[Open] / numOfOps [Open] + "ms"); } if (numOfOps[List] != 0) { @out.WriteLine("Average list execution time: " + (double)executionTime[List] / numOfOps [List] + "ms"); } if (numOfOps[Delete] != 0) { @out.WriteLine("Average deletion execution time: " + (double)executionTime[Delete ] / numOfOps[Delete] + "ms"); @out.WriteLine("Average create execution time: " + (double)executionTime[Create] / numOfOps[Create] + "ms"); @out.WriteLine("Average write_close execution time: " + (double)executionTime[WriteClose ] / numOfOps[WriteClose] + "ms"); } if (totalTime != 0) { @out.WriteLine("Average operations per second: " + (double)totalOps / totalTime + "ops/s"); } @out.WriteLine(); }
// nothing to do or verify here /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler ) { IList <string> localDirs = dirsHandler.GetLocalDirs(); IList <string> logDirs = dirsHandler.GetLogDirs(); CreateUserLocalDirs(localDirs, user); CreateUserCacheDirs(localDirs, user); CreateAppDirs(localDirs, user, appId); CreateAppLogDirs(appId, logDirs, user); // randomly choose the local directory Path appStorageDir = GetWorkingDir(localDirs, user, appId); string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId); Path tokenDst = new Path(appStorageDir, tokenFn); CopyFile(nmPrivateContainerTokensPath, tokenDst, user); Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); FileContext localizerFc = FileContext.GetFileContext(lfs.GetDefaultFileSystem(), GetConf()); localizerFc.SetUMask(lfs.GetUMask()); localizerFc.SetWorkingDirectory(appStorageDir); Log.Info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.GetWorkingDirectory ()); ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId , GetPaths(localDirs), RecordFactoryProvider.GetRecordFactory(GetConf())); // TODO: DO it over RPC for maintaining similarity? localizer.RunLocalization(nmAddr); }
/// <exception cref="System.IO.IOException"/> public object Run() { FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf); myfc.Delete(TestPermissionSymlinks.link, false); return(null); }
/// <exception cref="System.IO.IOException"/> public object Run() { FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf); myfc.Open(TestPermissionSymlinks.link).Read(); return(null); }
public LocalContainerLauncher(AppContext context, TaskUmbilicalProtocol umbilical ) : base(typeof(Org.Apache.Hadoop.Mapred.LocalContainerLauncher).FullName) { this.context = context; this.umbilical = umbilical; // umbilical: MRAppMaster creates (taskAttemptListener), passes to us // (TODO/FIXME: pointless to use RPC to talk to self; should create // LocalTaskAttemptListener or similar: implement umbilical protocol // but skip RPC stuff) try { curFC = FileContext.GetFileContext(curDir.ToURI()); } catch (UnsupportedFileSystemException) { Log.Error("Local filesystem " + curDir.ToURI().ToString() + " is unsupported?? (should never happen)" ); } // Save list of files/dirs that are supposed to be present so can delete // any extras created by one task before starting subsequent task. Note // that there's no protection against deleted or renamed localization; // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). FilePath[] curLocalFiles = curDir.ListFiles(); localizedFiles = new HashSet <FilePath>(curLocalFiles.Length); for (int j = 0; j < curLocalFiles.Length; ++j) { localizedFiles.AddItem(curLocalFiles[j]); } }
/// <exception cref="System.Exception"/> public FSDataOutputStream Run() { this._enclosing.fc = FileContext.GetFileContext(conf); this._enclosing.fc.SetUMask(AggregatedLogFormat.AppLogFileUmask); return(this._enclosing.fc.Create(remoteAppLogFile, EnumSet.Of(CreateFlag.Create, CreateFlag.Overwrite), new Options.CreateOpts[] { })); }
public virtual void TestFailedDirLogDeletion() { FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 7); IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length); for (int i = 0; i < localLogDirs.Length; i++) { localLogDirPaths.AddItem(localLogDirs[i].GetAbsolutePath()); } string localLogDirsString = StringUtils.Join(localLogDirPaths, ","); conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString); conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l); LocalDirsHandlerService mockDirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, mockDirsHandler, new NMNullStateStoreService()); NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler); AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext ().GetDefaultFileSystem()); FileContext lfs = FileContext.GetFileContext(spylfs, conf); Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA <Configuration>()); logHandler.Init(conf); logHandler.Start(); RunMockedFailedDirs(logHandler, appId, user, mockDelService, mockDirsHandler, conf , spylfs, lfs, localLogDirs); logHandler.Close(); }
public static void ClusterSetupAtBeginning() { cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology (2)).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf); fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf); }
public static void ClusterSetupAtBegining() { SupportsBlocks = true; Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); }
// Equivalence of @Before for cluster mode testing. /// <exception cref="System.IO.IOException"/> private void InitClusterModeTest() { Log = LogFactory.GetLog(typeof(TestWriteRead)); Log.Info("initClusterModeTest"); conf = new Configuration(); mfc = FileContext.GetFileContext(); mfs = FileSystem.Get(conf); }
/// <exception cref="System.IO.IOException"/> public Configuration LoadConfFile() { FileContext fc = FileContext.GetFileContext(configFile.ToUri(), conf); Configuration jobConf = new Configuration(false); jobConf.AddResource(fc.Open(configFile), configFile.ToString()); return(jobConf); }
/// <exception cref="System.IO.IOException"/> public object Run() { FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf); Path newlink = new Path(TestPermissionSymlinks.targetParent, "newlink"); myfc.Rename(TestPermissionSymlinks.link, newlink, Options.Rename.None); return(null); }
public virtual void TestHarFsWithoutAuthority() { URI uri = harFileSystem.GetUri(); NUnit.Framework.Assert.IsNull("har uri authority not null: " + uri, uri.GetAuthority ()); FileContext.GetFileContext(uri, conf); }
public static void ClusterSetupAtBeginning() { clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology (2)).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf); fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf); }
/// <exception cref="System.IO.IOException"/> public LogReader(Configuration conf, Path remoteAppLogFile) { FileContext fileContext = FileContext.GetFileContext(conf); this.fsDataIStream = fileContext.Open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile ).GetLen(), conf); this.scanner = reader.CreateScanner(); }
public virtual int DumpAContainersLogs(string appId, string containerId, string nodeId , string jobOwner) { Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir , YarnConfiguration.DefaultNmRemoteAppLogDir)); string suffix = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf()); Path remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, ConverterUtils .ToApplicationId(appId), jobOwner, suffix); RemoteIterator <FileStatus> nodeFiles; try { Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir ); nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus (remoteAppLogDir); } catch (FileNotFoundException) { LogDirNotExist(remoteAppLogDir.ToString()); return(-1); } bool foundContainerLogs = false; while (nodeFiles.HasNext()) { FileStatus thisNodeFile = nodeFiles.Next(); string fileName = thisNodeFile.GetPath().GetName(); if (fileName.Contains(LogAggregationUtils.GetNodeString(nodeId)) && !fileName.EndsWith (LogAggregationUtils.TmpFileSuffix)) { AggregatedLogFormat.LogReader reader = null; try { reader = new AggregatedLogFormat.LogReader(GetConf(), thisNodeFile.GetPath()); if (DumpAContainerLogs(containerId, reader, System.Console.Out, thisNodeFile.GetModificationTime ()) > -1) { foundContainerLogs = true; } } finally { if (reader != null) { reader.Close(); } } } } if (!foundContainerLogs) { ContainerLogNotFound(containerId); return(-1); } return(0); }
public override void SetUp() { Configuration S3Conf = new Configuration(); Configuration localConf = new Configuration(); S3Conf.Set(CommonConfigurationKeysPublic.FsDefaultNameDefault, S3Conf.Get("test.fs.s3.name" )); fc1 = FileContext.GetFileContext(S3Conf); fc2 = FileContext.GetFileContext(localConf); }
public static void ClusterSetupAtBegining() { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); fc = FileContext.GetFileContext(cluster.GetURI(0), conf); defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation .GetCurrentUser().GetShortUserName())); fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true); }
/// <exception cref="System.IO.IOException"/> private void RestartCluster(bool format) { if (cluster != null) { cluster.Shutdown(); cluster = null; } cluster = new MiniDFSCluster.Builder(Conf).Format(format).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); }
/// <exception cref="System.IO.IOException"/> public object Run() { FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf); Path newlink = new Path(TestPermissionSymlinks.linkParent, "newlink"); myfc.Rename(TestPermissionSymlinks.link, newlink, Options.Rename.None); Path linkTarget = myfc.GetLinkTarget(newlink); NUnit.Framework.Assert.AreEqual("Expected link's target to match target!", TestPermissionSymlinks .target, linkTarget); return(null); }
public static void ClusterSetupAtBegining() { SupportsBlocks = true; Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build(); cluster.WaitClusterUp(); fc = FileContext.GetFileContext(cluster.GetURI(0), Conf); Path defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation .GetCurrentUser().GetShortUserName())); fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true); }
public override void SetUp() { // create the test root on local_fs fcTarget = FileContext.GetLocalFSFileContext(); base.SetUp(); // this sets up conf (and fcView which we replace) // Now create a viewfs using a mount table called "default" // hence viewfs://default/ schemeWithAuthority = new URI(FsConstants.ViewfsScheme, "default", "/", null, null ); fcView = FileContext.GetFileContext(schemeWithAuthority, conf); }
public static void TestSetUp() { conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsListLimit, 2); cluster = new MiniDFSCluster.Builder(conf).Build(); fs = cluster.GetFileSystem(); fc = FileContext.GetFileContext(cluster.GetURI(0), conf); hftpfs = cluster.GetHftpFileSystem(0); dfsClient = new DFSClient(NameNode.GetAddress(conf), conf); file1 = new Path("filestatus.dat"); WriteFile(fs, file1, 1, fileSize, blockSize); }
public virtual void TestFcDelegationToken() { FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri()); AbstractFileSystem afs = fcHdfs.GetDefaultFileSystem(); IList <Org.Apache.Hadoop.Security.Token.Token <object> > tokenList = afs.GetDelegationTokens (UserGroupInformation.GetCurrentUser().GetUserName()); ((Org.Apache.Hadoop.FS.Hdfs)afs).RenewDelegationToken((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)tokenList[0]); ((Org.Apache.Hadoop.FS.Hdfs)afs).CancelDelegationToken((Org.Apache.Hadoop.Security.Token.Token <AbstractDelegationTokenIdentifier>)tokenList[0]); }