public virtual void TestContainerLocalizerClosesFilesystems()
{
// mocked generics
// verify filesystems are closed when localizer doesn't fail
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
// verify filesystems are closed when localizer fails
localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer
).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService
>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
}
public virtual void TestLocalizerTokenIsGettingRemoved()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(spylfs, Org.Mockito.Mockito.Times(1)).Delete(tokenPath
, false);
}
/// <exception cref="System.Exception"/>
private ContainerLocalizer SetupContainerLocalizerForTest()
{
// mocked generics
// don't actually create dirs
Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA
<FsPermission>(), Matchers.AnyBoolean());
Configuration conf = new Configuration();
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
localDirs = new AList <Path>();
for (int i = 0; i < 4; ++i)
{
localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty)));
}
RecordFactory mockRF = GetMockLocalizerRecordFactory();
ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId
, localDirs, mockRF);
ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc);
// return credential stream instead of opening local file
random = new Random();
long seed = random.NextLong();
System.Console.Out.WriteLine("SEED: " + seed);
random.SetSeed(seed);
DataInputBuffer appTokens = CreateFakeCredentials(random, 10);
tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt
, containerId)));
Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens
))).When(spylfs).Open(tokenPath);
nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>();
Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr);
Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt());
// return result instantly for deterministic test
ExecutorService syncExec = Org.Mockito.Mockito.Mock <ExecutorService>();
CompletionService <Path> cs = Org.Mockito.Mockito.Mock <CompletionService>();
Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279
());
Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool();
Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec
);
return(localizer);
}
public _PrivilegedAction_151(ContainerLocalizer _enclosing, IPEndPoint nmAddr)
{
this._enclosing = _enclosing;
this.nmAddr = nmAddr;
}
public virtual void TestContainerLocalizerMain()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
// verify created cache
IList <Path> privCacheList = new AList <Path>();
IList <Path> appCacheList = new AList <Path>();
foreach (Path p in localDirs)
{
Path @base = new Path(new Path(p, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
privCacheList.AddItem(privcache);
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
appCacheList.AddItem(appcache);
}
// mock heartbeat responses from NM
ResourceLocalizationSpec rsrcA = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcB = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcC = GetMockRsrc(random, LocalResourceVisibility.Application
, appCacheList[0]);
ResourceLocalizationSpec rsrcD = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
Org.Mockito.Mockito.When(nmProxy.Heartbeat(Matchers.IsA <LocalizerStatus>())).ThenReturn
(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.SingletonList
(rsrcA))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections
.SingletonList(rsrcB))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Live, Sharpen.Collections.SingletonList(rsrcC))).ThenReturn(new MockLocalizerHeartbeatResponse
(LocalizerAction.Live, Sharpen.Collections.SingletonList(rsrcD))).ThenReturn(new
MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.EmptyList
<ResourceLocalizationSpec>())).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Die, null));
LocalResource tRsrcA = rsrcA.GetResource();
LocalResource tRsrcB = rsrcB.GetResource();
LocalResource tRsrcC = rsrcC.GetResource();
LocalResource tRsrcD = rsrcD.GetResource();
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcA.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcA), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcB.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcB), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcC.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcC), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcD.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcD), Matchers.IsA <UserGroupInformation>());
// run localization
NUnit.Framework.Assert.AreEqual(0, localizer.RunLocalization(nmAddr));
foreach (Path p_1 in localDirs)
{
Path @base = new Path(new Path(p_1, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
// $x/usercache/$user/filecache
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(privcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
// $x/usercache/$user/appcache/$appId/filecache
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(appcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
}
// verify tokens read at expected location
Org.Mockito.Mockito.Verify(spylfs).Open(tokenPath);
// verify downloaded resources reported to NM
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcA.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcB.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcC.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcD.GetResource())));
// verify all HB use localizerID provided
Org.Mockito.Mockito.Verify(nmProxy, Org.Mockito.Mockito.Never()).Heartbeat(Matchers.ArgThat
(new _ArgumentMatcher_193()));
}
