/// <exception cref="System.IO.IOException"/>
protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService
<Path> cs, UserGroupInformation ugi)
{
while (true)
{
try
{
LocalizerStatus status = CreateStatus();
LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status);
switch (response.GetLocalizerAction())
{
case LocalizerAction.Live:
{
IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs();
foreach (ResourceLocalizationSpec newRsrc in newRsrcs)
{
if (!pendingResources.Contains(newRsrc.GetResource()))
{
pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory
().GetFile()), newRsrc.GetResource(), ugi));
}
}
break;
}
case LocalizerAction.Die:
{
// killall running localizations
foreach (Future <Path> pending in pendingResources.Values)
{
pending.Cancel(true);
}
status = CreateStatus();
// ignore response
try
{
nodemanager.Heartbeat(status);
}
catch (YarnException)
{
}
return;
}
}
cs.Poll(1000, TimeUnit.Milliseconds);
}
catch (Exception)
{
return;
}
catch (YarnException)
{
// TODO cleanup
return;
}
}
}
Heartbeat(RpcController controller, YarnServerNodemanagerServiceProtos.LocalizerStatusProto
proto)
{
LocalizerStatusPBImpl request = new LocalizerStatusPBImpl(proto);
try
{
LocalizerHeartbeatResponse response = real.Heartbeat(request);
return(((LocalizerHeartbeatResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
public virtual void TestLocalizerRPC()
{
IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040);
TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService
(locAddr);
try
{
server.Start();
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol
), locAddr, conf);
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId("localizer0");
LocalizerHeartbeatResponse response = client.Heartbeat(status);
NUnit.Framework.Assert.AreEqual(DieHBResponse(), response);
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.IsTrue(true);
}
public virtual void TestContainerLocalizerMain()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
// verify created cache
IList <Path> privCacheList = new AList <Path>();
IList <Path> appCacheList = new AList <Path>();
foreach (Path p in localDirs)
{
Path @base = new Path(new Path(p, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
privCacheList.AddItem(privcache);
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
appCacheList.AddItem(appcache);
}
// mock heartbeat responses from NM
ResourceLocalizationSpec rsrcA = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcB = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcC = GetMockRsrc(random, LocalResourceVisibility.Application
, appCacheList[0]);
ResourceLocalizationSpec rsrcD = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
Org.Mockito.Mockito.When(nmProxy.Heartbeat(Matchers.IsA <LocalizerStatus>())).ThenReturn
(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.SingletonList
(rsrcA))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections
.SingletonList(rsrcB))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Live, Sharpen.Collections.SingletonList(rsrcC))).ThenReturn(new MockLocalizerHeartbeatResponse
(LocalizerAction.Live, Sharpen.Collections.SingletonList(rsrcD))).ThenReturn(new
MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.EmptyList
<ResourceLocalizationSpec>())).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Die, null));
LocalResource tRsrcA = rsrcA.GetResource();
LocalResource tRsrcB = rsrcB.GetResource();
LocalResource tRsrcC = rsrcC.GetResource();
LocalResource tRsrcD = rsrcD.GetResource();
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcA.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcA), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcB.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcB), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcC.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcC), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcD.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcD), Matchers.IsA <UserGroupInformation>());
// run localization
NUnit.Framework.Assert.AreEqual(0, localizer.RunLocalization(nmAddr));
foreach (Path p_1 in localDirs)
{
Path @base = new Path(new Path(p_1, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
// $x/usercache/$user/filecache
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(privcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
// $x/usercache/$user/appcache/$appId/filecache
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(appcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
}
// verify tokens read at expected location
Org.Mockito.Mockito.Verify(spylfs).Open(tokenPath);
// verify downloaded resources reported to NM
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcA.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcB.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcC.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcD.GetResource())));
// verify all HB use localizerID provided
Org.Mockito.Mockito.Verify(nmProxy, Org.Mockito.Mockito.Never()).Heartbeat(Matchers.ArgThat
(new _ArgumentMatcher_193()));
}
