public virtual void TestGetFullDirs()
{
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext localFs = FileContext.GetLocalFSFileContext(conf);
string localDir1 = new FilePath(testDir, "localDir1").GetPath();
string localDir2 = new FilePath(testDir, "localDir2").GetPath();
string logDir1 = new FilePath(testDir, "logDir1").GetPath();
string logDir2 = new FilePath(testDir, "logDir2").GetPath();
Path localDir1Path = new Path(localDir1);
Path logDir1Path = new Path(logDir1);
FsPermission dirPermissions = new FsPermission((short)0x108);
localFs.Mkdir(localDir1Path, dirPermissions, true);
localFs.Mkdir(logDir1Path, dirPermissions, true);
conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2);
conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2);
conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
dirSvc.Init(conf);
NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLocalDirs().Count);
NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLogDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLocalDirs().Count);
NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLogDirs().Count);
FileUtils.DeleteDirectory(new FilePath(localDir1));
FileUtils.DeleteDirectory(new FilePath(localDir2));
FileUtils.DeleteDirectory(new FilePath(logDir1));
FileUtils.DeleteDirectory(new FilePath(logDir1));
dirSvc.Close();
}
/// <exception cref="System.IO.IOException"/>
internal virtual byte[] CreateTmpFile(Path dst, Random r, int len)
{
// use unmodified local context
FileContext lfs = FileContext.GetLocalFSFileContext();
dst = lfs.MakeQualified(dst);
lfs.Mkdir(dst.GetParent(), null, true);
byte[] bytes = new byte[len];
FSDataOutputStream @out = null;
try
{
@out = lfs.Create(dst, EnumSet.Of(CreateFlag.Create, CreateFlag.Overwrite));
r.NextBytes(bytes);
@out.Write(bytes);
}
finally
{
if (@out != null)
{
@out.Close();
}
}
return(bytes);
}
public virtual void TestFcResolveAfs()
{
Configuration conf = new Configuration();
FileContext fcLocal = FileContext.GetLocalFSFileContext();
FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
string localTestRoot = helper.GetAbsoluteTestRootDir(fcLocal);
Path alphaLocalPath = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString()
, new FilePath(localTestRoot, "alpha").GetAbsolutePath());
DFSTestUtil.CreateFile(FileSystem.GetLocal(conf), alphaLocalPath, 16, (short)1, 2
);
Path linkTarget = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString(), localTestRoot
);
Path hdfsLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString(), "/tmp/link"
);
fcHdfs.CreateSymlink(linkTarget, hdfsLink, true);
Path alphaHdfsPathViaLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString
() + "/tmp/link/alpha");
ICollection <AbstractFileSystem> afsList = fcHdfs.ResolveAbstractFileSystems(alphaHdfsPathViaLink
);
NUnit.Framework.Assert.AreEqual(2, afsList.Count);
foreach (AbstractFileSystem afs in afsList)
{
if ((!afs.Equals(fcHdfs.GetDefaultFileSystem())) && (!afs.Equals(fcLocal.GetDefaultFileSystem
())))
{
NUnit.Framework.Assert.Fail("Failed to resolve AFS correctly");
}
}
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps
, int reduces, bool autoComplete, string testName, bool cleanOnStart, int startCount
, Clock clock, bool unregistered, string assignedQueue)
: base(appAttemptId, amContainerId, NmHost, NmPort, NmHttpPort, clock, Runtime.CurrentTimeMillis
())
{
this.testWorkDir = new FilePath("target", testName);
testAbsPath = new Path(testWorkDir.GetAbsolutePath());
Log.Info("PathUsed: " + testAbsPath);
if (cleanOnStart)
{
testAbsPath = new Path(testWorkDir.GetAbsolutePath());
try
{
FileContext.GetLocalFSFileContext().Delete(testAbsPath, true);
}
catch (Exception e)
{
Log.Warn("COULD NOT CLEANUP: " + testAbsPath, e);
throw new YarnRuntimeException("could not cleanup test dir", e);
}
}
this.maps = maps;
this.reduces = reduces;
this.autoComplete = autoComplete;
// If safeToReportTerminationToUser is set to true, we can verify whether
// the job can reaches the final state when MRAppMaster shuts down.
this.successfullyUnregistered.Set(unregistered);
this.assignedQueue = assignedQueue;
}
public virtual void TestContainerLocalizerClosesFilesystems()
{
// mocked generics
// verify filesystems are closed when localizer doesn't fail
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
// verify filesystems are closed when localizer fails
localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer
).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService
>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
}
/// <summary>delete the test directory in the target local fs</summary>
/// <exception cref="System.Exception"/>
public static void TearDownForViewFsLocalFs(FileContextTestHelper helper)
{
FileContext fclocal = FileContext.GetLocalFSFileContext();
Path targetOfTests = helper.GetTestRootPath(fclocal);
fclocal.Delete(targetOfTests, true);
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
/*
* return the ViewFS File context to be used for tests
*/
/// <exception cref="System.Exception"/>
public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper)
{
FileContext fsTarget = FileContext.GetLocalFSFileContext();
Path targetOfTests = helper.GetTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.Delete(targetOfTests, true);
fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true);
Configuration conf = new Configuration();
// Set up viewfs link for test dir as described above
string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath();
LinkUpFirstComponents(conf, testDir, fsTarget, "test dir");
// Set up viewfs link for home dir as described above
SetUpHomeDir(conf, fsTarget);
// the test path may be relative to working dir - we need to make that work:
// Set up viewfs link for wd as described above
string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath();
LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf);
fc.SetWorkingDirectory(new Path(wdDir));
// in case testdir relative to wd.
Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory());
//System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
//System.out.println("TargetOfTests = "+ targetOfTests.toUri());
return(fc);
}
public virtual void TestRenameMapOutputForReduce()
{
JobConf conf = new JobConf();
MROutputFiles mrOutputFiles = new MROutputFiles();
mrOutputFiles.SetConf(conf);
// make sure both dirs are distinct
//
conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);
conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);
Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
GetParent());
// make both dirs part of LOCAL_DIR
conf.SetStrings(MRConfig.LocalDir, localDirs);
FileContext lfc = FileContext.GetLocalFSFileContext(conf);
lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0);
LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
}
public virtual void TestFailedDirLogDeletion()
{
FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 7);
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i = 0; i < localLogDirs.Length; i++)
{
localLogDirPaths.AddItem(localLogDirs[i].GetAbsolutePath());
}
string localLogDirsString = StringUtils.Join(localLogDirPaths, ",");
conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString);
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false);
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l);
LocalDirsHandlerService mockDirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
>();
NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher,
mockDelService, mockDirsHandler, new NMNullStateStoreService());
NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler);
AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext
().GetDefaultFileSystem());
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA
<Configuration>());
logHandler.Init(conf);
logHandler.Start();
RunMockedFailedDirs(logHandler, appId, user, mockDelService, mockDirsHandler, conf
, spylfs, lfs, localLogDirs);
logHandler.Close();
}
/// <summary>
/// Tests that an edits log created using CreateEditsLog is valid and can be
/// loaded successfully by a namenode.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCanLoadCreatedEditsLog()
{
// Format namenode.
HdfsConfiguration conf = new HdfsConfiguration();
FilePath nameDir = new FilePath(HdfsDir, "name");
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Util.FileAsURI(nameDir).ToString());
DFSTestUtil.FormatNameNode(conf);
// Call CreateEditsLog and move the resulting edits to the name dir.
CreateEditsLog.Main(new string[] { "-f", "1000", "0", "1", "-d", TestDir.GetAbsolutePath
() });
Path editsWildcard = new Path(TestDir.GetAbsolutePath(), "*");
FileContext localFc = FileContext.GetLocalFSFileContext();
foreach (FileStatus edits in localFc.Util().GlobStatus(editsWildcard))
{
Path src = edits.GetPath();
Path dst = new Path(new FilePath(nameDir, "current").GetAbsolutePath(), src.GetName
());
localFc.Rename(src, dst);
}
// Start a namenode to try to load the edits.
cluster = new MiniDFSCluster.Builder(conf).Format(false).ManageNameDfsDirs(false)
.WaitSafeMode(false).Build();
cluster.WaitClusterUp();
}
public virtual void SetupForTests()
{
conf = new Configuration();
localFs = FileContext.GetLocalFSFileContext(conf);
testDir.Mkdirs();
testFile.CreateNewFile();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
lock (this)
{
if (this._enclosing.resourceManagers[this.index] != null)
{
this.WaitForAppMastersToFinish(5000);
this._enclosing.resourceManagers[this.index].Stop();
}
if (Shell.Windows)
{
// On Windows, clean up the short temporary symlink that was created to
// work around path length limitation.
string testWorkDirPath = this._enclosing.testWorkDir.GetAbsolutePath();
try
{
FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDirPath), true);
}
catch (IOException)
{
MiniYARNCluster.Log.Warn("could not cleanup symlink: " + this._enclosing.testWorkDir
.GetAbsolutePath());
}
}
base.ServiceStop();
}
}
public virtual void TearDown()
{
if (testRootDir.Exists())
{
FileContext.GetLocalFSFileContext().Delete(new Path(testRootDir.GetAbsolutePath()
), true);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.ExecutionException"/>
public virtual void TestDownloadPublicWithStatCache()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
// if test directory doesn't have ancestor permission, skip this test
FileSystem f = basedir.GetFileSystem(conf);
Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
int size = 512;
ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger
>();
CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader
(conf);
LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build
(new _CacheLoader_328(counts, loader));
// increment the count
// use the default loader
// test FSDownload.isPublic() concurrently
int fileCount = 3;
IList <Callable <bool> > tasks = new AList <Callable <bool> >();
for (int i = 0; i < fileCount; i++)
{
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
Path path = new Path(basedir, "test-file-" + i);
CreateFile(files, path, size, rand);
FileSystem fs = path.GetFileSystem(conf);
FileStatus sStat = fs.GetFileStatus(path);
tasks.AddItem(new _Callable_358(fs, path, sStat, statCache));
}
ExecutorService exec = Executors.NewFixedThreadPool(fileCount);
try
{
IList <Future <bool> > futures = exec.InvokeAll(tasks);
// files should be public
foreach (Future <bool> future in futures)
{
NUnit.Framework.Assert.IsTrue(future.Get());
}
// for each path exactly one file status call should be made
foreach (AtomicInteger count in counts.Values)
{
NUnit.Framework.Assert.AreSame(count.Get(), 1);
}
}
finally
{
exec.Shutdown();
}
}
public virtual void Setup()
{
localFS = FileContext.GetLocalFSFileContext();
tmpDir.Mkdirs();
logsDir.Mkdirs();
remoteLogsDir.Mkdirs();
nmLocalDir.Mkdirs();
syncBarrier = new CyclicBarrier(2);
}
public virtual void Setup()
{
localFS = FileContext.GetLocalFSFileContext();
tmpDir.Mkdirs();
logsDir.Mkdirs();
remoteLogsDir.Mkdirs();
nmLocalDir.Mkdirs();
// Construct the Container-id
cId = CreateContainerId();
}
public static void Setup()
{
//Do not error out if metrics are inited multiple times
DefaultMetricsSystem.SetMiniClusterMode(true);
FilePath dir = new FilePath(stagingDir);
stagingDir = dir.GetAbsolutePath();
localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(testDir.GetAbsolutePath()), true);
testDir.Mkdir();
}
public DefaultContainerExecutor()
{
try
{
this.lfs = FileContext.GetLocalFSFileContext();
}
catch (UnsupportedFileSystemException e)
{
throw new RuntimeException(e);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDownloadBadPublic()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = 512;
LocalResourceVisibility vis = LocalResourceVisibility.Public;
Path path = new Path(basedir, "test-file");
LocalResource rsrc = CreateFile(files, path, size, rand, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone());
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
p.Value.Get();
NUnit.Framework.Assert.Fail("We localized a file that is not public.");
}
}
catch (ExecutionException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is IOException);
}
}
internal virtual FileContext GetLocalFileContext(Configuration conf)
{
try
{
return(FileContext.GetLocalFSFileContext(conf));
}
catch (IOException)
{
throw new YarnRuntimeException("Failed to access local fs");
}
}
internal static FileContext GetLfs()
{
try
{
return(FileContext.GetLocalFSFileContext());
}
catch (UnsupportedFileSystemException e)
{
throw new RuntimeException(e);
}
}
public override void SetUp()
{
// create the test root on local_fs
fcTarget = FileContext.GetLocalFSFileContext();
base.SetUp();
// this sets up conf (and fcView which we replace)
// Now create a viewfs using a mount table called "default"
// hence viewfs://default/
schemeWithAuthority = new URI(FsConstants.ViewfsScheme, "default", "/", null, null
);
fcView = FileContext.GetFileContext(schemeWithAuthority, conf);
}
public virtual void Setup()
{
fsPath = new FilePath("target", GetType().Name + "-tmpDir").GetAbsoluteFile();
fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true);
conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceRecoveryEnabled, true);
conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(LeveldbTimelineStateStore
), typeof(TimelineStateStore));
conf.Set(YarnConfiguration.TimelineServiceLeveldbStateStorePath, fsPath.GetAbsolutePath
());
}
public static void DeleteTmpFiles()
{
FileContext lfs = FileContext.GetLocalFSFileContext();
try
{
lfs.Delete(BaseTmpPath, true);
}
catch (FileNotFoundException)
{
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
try
{
//TODO Is this required?
FileContext.GetLocalFSFileContext(conf);
}
catch (UnsupportedFileSystemException e)
{
throw new YarnRuntimeException("Failed to start ContainersLauncher", e);
}
base.ServiceInit(conf);
}
public virtual void TestLocalizerTokenIsGettingRemoved()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(spylfs, Org.Mockito.Mockito.Times(1)).Delete(tokenPath
, false);
}
public DockerContainerExecutor()
{
// This validates that the image is a proper docker image and would not crash docker.
try
{
this.lfs = FileContext.GetLocalFSFileContext();
this.dockerImagePattern = Sharpen.Pattern.Compile(DockerImagePattern);
}
catch (UnsupportedFileSystemException e)
{
throw new RuntimeException(e);
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void SetupInternal(int numNodeManager)
{
Log.Info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
conf.Set("yarn.log.dir", "target");
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
if (yarnCluster == null)
{
yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
, 1, 1);
yarnCluster.Init(conf);
yarnCluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
WaitForNMsToRegister();
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
);
if (url == null)
{
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
);
}
Configuration yarnClusterConfig = yarnCluster.GetConfig();
yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
).GetParent());
//write the document to a buffer (not directly to the file, as that
//can cause the file being written to get read -which will then fail.
ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
yarnClusterConfig.WriteXml(bytesOut);
bytesOut.Close();
//write the bytes to the file in the classpath
OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
os.Write(bytesOut.ToByteArray());
os.Close();
}
FileContext fsContext = FileContext.GetLocalFSFileContext();
fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
)), true);
try
{
Sharpen.Thread.Sleep(2000);
}
catch (Exception e)
{
Log.Info("setup thread sleep interrupted. message=" + e.Message);
}
}
public virtual void SetUp()
{
// create the test root on local_fs
fcTarget = FileContext.GetLocalFSFileContext();
chrootedTo = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget);
// In case previous test was killed before cleanup
fcTarget.Delete(chrootedTo, true);
fcTarget.Mkdir(chrootedTo, FileContext.DefaultPerm, true);
Configuration conf = new Configuration();
// ChRoot to the root of the testDirectory
fc = FileContext.GetFileContext(new ChRootedFs(fcTarget.GetDefaultFileSystem(), chrootedTo
), conf);
}
