/// <exception cref="System.IO.IOException"/>
protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService
<Path> cs, UserGroupInformation ugi)
{
while (true)
{
try
{
LocalizerStatus status = CreateStatus();
LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status);
switch (response.GetLocalizerAction())
{
case LocalizerAction.Live:
{
IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs();
foreach (ResourceLocalizationSpec newRsrc in newRsrcs)
{
if (!pendingResources.Contains(newRsrc.GetResource()))
{
pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory
().GetFile()), newRsrc.GetResource(), ugi));
}
}
break;
}
case LocalizerAction.Die:
{
// killall running localizations
foreach (Future <Path> pending in pendingResources.Values)
{
pending.Cancel(true);
}
status = CreateStatus();
// ignore response
try
{
nodemanager.Heartbeat(status);
}
catch (YarnException)
{
}
return;
}
}
cs.Poll(1000, TimeUnit.Milliseconds);
}
catch (Exception)
{
return;
}
catch (YarnException)
{
// TODO cleanup
return;
}
}
}
/// <exception cref="System.Exception"/>
private ContainerLocalizer SetupContainerLocalizerForTest()
{
// mocked generics
// don't actually create dirs
Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA
<FsPermission>(), Matchers.AnyBoolean());
Configuration conf = new Configuration();
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
localDirs = new AList <Path>();
for (int i = 0; i < 4; ++i)
{
localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty)));
}
RecordFactory mockRF = GetMockLocalizerRecordFactory();
ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId
, localDirs, mockRF);
ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc);
// return credential stream instead of opening local file
random = new Random();
long seed = random.NextLong();
System.Console.Out.WriteLine("SEED: " + seed);
random.SetSeed(seed);
DataInputBuffer appTokens = CreateFakeCredentials(random, 10);
tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt
, containerId)));
Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens
))).When(spylfs).Open(tokenPath);
nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>();
Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr);
Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt());
// return result instantly for deterministic test
ExecutorService syncExec = Org.Mockito.Mockito.Mock <ExecutorService>();
CompletionService <Path> cs = Org.Mockito.Mockito.Mock <CompletionService>();
Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279
());
Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool();
Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec
);
return(localizer);
}
