public virtual void TestContainerLocalizerClosesFilesystems() { // mocked generics // verify filesystems are closed when localizer doesn't fail FileContext fs = FileContext.GetLocalFSFileContext(); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); ContainerLocalizer localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol >(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>()); Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems (Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation >()); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); // verify filesystems are closed when localizer fails localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer ).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService >(), Matchers.Any <UserGroupInformation>()); Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems (Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation >()); }
public virtual void TestLocalizerTokenIsGettingRemoved() { FileContext fs = FileContext.GetLocalFSFileContext(); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); ContainerLocalizer localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol >(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(spylfs, Org.Mockito.Mockito.Times(1)).Delete(tokenPath , false); }
/// <exception cref="System.Exception"/> private ContainerLocalizer SetupContainerLocalizerForTest() { // mocked generics // don't actually create dirs Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA <FsPermission>(), Matchers.AnyBoolean()); Configuration conf = new Configuration(); FileContext lfs = FileContext.GetFileContext(spylfs, conf); localDirs = new AList <Path>(); for (int i = 0; i < 4; ++i) { localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty))); } RecordFactory mockRF = GetMockLocalizerRecordFactory(); ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId , localDirs, mockRF); ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc); // return credential stream instead of opening local file random = new Random(); long seed = random.NextLong(); System.Console.Out.WriteLine("SEED: " + seed); random.SetSeed(seed); DataInputBuffer appTokens = CreateFakeCredentials(random, 10); tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt , containerId))); Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens ))).When(spylfs).Open(tokenPath); nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>(); Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr); Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt()); // return result instantly for deterministic test ExecutorService syncExec = Org.Mockito.Mockito.Mock <ExecutorService>(); CompletionService <Path> cs = Org.Mockito.Mockito.Mock <CompletionService>(); Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279 ()); Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool(); Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec ); return(localizer); }
public _PrivilegedAction_151(ContainerLocalizer _enclosing, IPEndPoint nmAddr) { this._enclosing = _enclosing; this.nmAddr = nmAddr; }
public virtual void TestContainerLocalizerMain() { FileContext fs = FileContext.GetLocalFSFileContext(); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); ContainerLocalizer localizer = SetupContainerLocalizerForTest(); // verify created cache IList <Path> privCacheList = new AList <Path>(); IList <Path> appCacheList = new AList <Path>(); foreach (Path p in localDirs) { Path @base = new Path(new Path(p, ContainerLocalizer.Usercache), appUser); Path privcache = new Path(@base, ContainerLocalizer.Filecache); privCacheList.AddItem(privcache); Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId)); Path appcache = new Path(appDir, ContainerLocalizer.Filecache); appCacheList.AddItem(appcache); } // mock heartbeat responses from NM ResourceLocalizationSpec rsrcA = GetMockRsrc(random, LocalResourceVisibility.Private , privCacheList[0]); ResourceLocalizationSpec rsrcB = GetMockRsrc(random, LocalResourceVisibility.Private , privCacheList[0]); ResourceLocalizationSpec rsrcC = GetMockRsrc(random, LocalResourceVisibility.Application , appCacheList[0]); ResourceLocalizationSpec rsrcD = GetMockRsrc(random, LocalResourceVisibility.Private , privCacheList[0]); Org.Mockito.Mockito.When(nmProxy.Heartbeat(Matchers.IsA <LocalizerStatus>())).ThenReturn (new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.SingletonList (rsrcA))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections .SingletonList(rsrcB))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction .Live, Sharpen.Collections.SingletonList(rsrcC))).ThenReturn(new MockLocalizerHeartbeatResponse (LocalizerAction.Live, Sharpen.Collections.SingletonList(rsrcD))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.EmptyList <ResourceLocalizationSpec>())).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction .Die, null)); LocalResource tRsrcA = rsrcA.GetResource(); LocalResource tRsrcB = rsrcB.GetResource(); LocalResource tRsrcC = rsrcC.GetResource(); LocalResource tRsrcD = rsrcD.GetResource(); Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcA.GetResource ().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>() , Matchers.Eq(tRsrcA), Matchers.IsA <UserGroupInformation>()); Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcB.GetResource ().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>() , Matchers.Eq(tRsrcB), Matchers.IsA <UserGroupInformation>()); Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcC.GetResource ().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>() , Matchers.Eq(tRsrcC), Matchers.IsA <UserGroupInformation>()); Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcD.GetResource ().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>() , Matchers.Eq(tRsrcD), Matchers.IsA <UserGroupInformation>()); // run localization NUnit.Framework.Assert.AreEqual(0, localizer.RunLocalization(nmAddr)); foreach (Path p_1 in localDirs) { Path @base = new Path(new Path(p_1, ContainerLocalizer.Usercache), appUser); Path privcache = new Path(@base, ContainerLocalizer.Filecache); // $x/usercache/$user/filecache Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(privcache), Matchers.Eq(CacheDirPerm ), Matchers.Eq(false)); Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId)); // $x/usercache/$user/appcache/$appId/filecache Path appcache = new Path(appDir, ContainerLocalizer.Filecache); Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(appcache), Matchers.Eq(CacheDirPerm ), Matchers.Eq(false)); } // verify tokens read at expected location Org.Mockito.Mockito.Verify(spylfs).Open(tokenPath); // verify downloaded resources reported to NM Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches (rsrcA.GetResource()))); Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches (rsrcB.GetResource()))); Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches (rsrcC.GetResource()))); Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches (rsrcD.GetResource()))); // verify all HB use localizerID provided Org.Mockito.Mockito.Verify(nmProxy, Org.Mockito.Mockito.Never()).Heartbeat(Matchers.ArgThat (new _ArgumentMatcher_193())); }