/// <exception cref="System.IO.IOException"/>
protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService
<Path> cs, UserGroupInformation ugi)
{
while (true)
{
try
{
LocalizerStatus status = CreateStatus();
LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status);
switch (response.GetLocalizerAction())
{
case LocalizerAction.Live:
{
IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs();
foreach (ResourceLocalizationSpec newRsrc in newRsrcs)
{
if (!pendingResources.Contains(newRsrc.GetResource()))
{
pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory
().GetFile()), newRsrc.GetResource(), ugi));
}
}
break;
}
case LocalizerAction.Die:
{
// killall running localizations
foreach (Future <Path> pending in pendingResources.Values)
{
pending.Cancel(true);
}
status = CreateStatus();
// ignore response
try
{
nodemanager.Heartbeat(status);
}
catch (YarnException)
{
}
return;
}
}
cs.Poll(1000, TimeUnit.Milliseconds);
}
catch (Exception)
{
return;
}
catch (YarnException)
{
// TODO cleanup
return;
}
}
}
/// <exception cref="System.Exception"/>
private ContainerLocalizer SetupContainerLocalizerForTest()
{
// mocked generics
// don't actually create dirs
Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA
<FsPermission>(), Matchers.AnyBoolean());
Configuration conf = new Configuration();
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
localDirs = new AList <Path>();
for (int i = 0; i < 4; ++i)
{
localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty)));
}
RecordFactory mockRF = GetMockLocalizerRecordFactory();
ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId
, localDirs, mockRF);
ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc);
// return credential stream instead of opening local file
random = new Random();
long seed = random.NextLong();
System.Console.Out.WriteLine("SEED: " + seed);
random.SetSeed(seed);
DataInputBuffer appTokens = CreateFakeCredentials(random, 10);
tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt
, containerId)));
Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens
))).When(spylfs).Open(tokenPath);
nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>();
Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr);
Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt());
// return result instantly for deterministic test
ExecutorService syncExec = Org.Mockito.Mockito.Mock <ExecutorService>();
CompletionService <Path> cs = Org.Mockito.Mockito.Mock <CompletionService>();
Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279
());
Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool();
Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec
);
return(localizer);
}
private void TestPbClientFactory()
{
IPEndPoint addr = new IPEndPoint(0);
System.Console.Error.WriteLine(addr.GetHostName() + addr.Port);
Configuration conf = new Configuration();
LocalizationProtocol instance = new TestRPCFactories.LocalizationProtocolTestImpl
(this);
Org.Apache.Hadoop.Ipc.Server server = null;
try
{
server = RpcServerFactoryPBImpl.Get().GetServer(typeof(LocalizationProtocol), instance
, addr, conf, null, 1);
server.Start();
System.Console.Error.WriteLine(server.GetListenerAddress());
System.Console.Error.WriteLine(NetUtils.GetConnectAddress(server));
try
{
LocalizationProtocol client = (LocalizationProtocol)RpcClientFactoryPBImpl.Get().
GetClient(typeof(LocalizationProtocol), 1, NetUtils.GetConnectAddress(server), conf
);
NUnit.Framework.Assert.IsNotNull(client);
}
catch (YarnRuntimeException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Failed to create client");
}
}
catch (YarnRuntimeException e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Failed to create server");
}
finally
{
server.Stop();
}
}
public virtual void TestLocalizerRPC()
{
IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040);
TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService
(locAddr);
try
{
server.Start();
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol
), locAddr, conf);
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId("localizer0");
LocalizerHeartbeatResponse response = client.Heartbeat(status);
NUnit.Framework.Assert.AreEqual(DieHBResponse(), response);
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.IsTrue(true);
}
public LocalizationProtocolPBServiceImpl(LocalizationProtocol impl)
{
this.real = impl;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual int RunLocalization(IPEndPoint nmAddr)
{
// load credentials
InitDirs(conf, user, appId, lfs, localDirs);
Credentials creds = new Credentials();
DataInputStream credFile = null;
try
{
// assume credentials in cwd
// TODO: Fix
Path tokenPath = new Path(string.Format(TokenFileNameFmt, localizerId));
credFile = lfs.Open(tokenPath);
creds.ReadTokenStorageStream(credFile);
// Explicitly deleting token file.
lfs.Delete(tokenPath, false);
}
finally
{
if (credFile != null)
{
credFile.Close();
}
}
// create localizer context
UserGroupInformation remoteUser = UserGroupInformation.CreateRemoteUser(user);
remoteUser.AddToken(creds.GetToken(LocalizerTokenIdentifier.Kind));
LocalizationProtocol nodeManager = remoteUser.DoAs(new _PrivilegedAction_151(this
, nmAddr));
// create user context
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in creds.GetAllTokens
())
{
ugi.AddToken(token);
}
ExecutorService exec = null;
try
{
exec = CreateDownloadThreadPool();
CompletionService <Path> ecs = CreateCompletionService(exec);
LocalizeFiles(nodeManager, ecs, ugi);
return(0);
}
catch (Exception e)
{
// Print traces to stdout so that they can be logged by the NM address
// space.
Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
return(-1);
}
finally
{
try
{
if (exec != null)
{
exec.ShutdownNow();
}
LocalDirAllocator.RemoveContext(appCacheDirContextName);
}
finally
{
CloseFileSystems(ugi);
}
}
}
public ContainerLocalizationImpl(Dispatcher dispatcher, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, LocalizationProtocol localization)
{
}