/// <exception cref="System.Exception"/>
public virtual void TestLocalizerStatusSerDe()
{
LocalizerStatus rsrcS = CreateLocalizerStatus();
NUnit.Framework.Assert.IsTrue(rsrcS is LocalizerStatusPBImpl);
LocalizerStatusPBImpl rsrcPb = (LocalizerStatusPBImpl)rsrcS;
DataOutputBuffer @out = new DataOutputBuffer();
rsrcPb.GetProto().WriteDelimitedTo(@out);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), 0, @out.GetLength());
YarnServerNodemanagerServiceProtos.LocalizerStatusProto rsrcPbD = YarnServerNodemanagerServiceProtos.LocalizerStatusProto
.ParseDelimitedFrom(@in);
NUnit.Framework.Assert.IsNotNull(rsrcPbD);
LocalizerStatus rsrcD = new LocalizerStatusPBImpl(rsrcPbD);
NUnit.Framework.Assert.AreEqual(rsrcS, rsrcD);
NUnit.Framework.Assert.AreEqual("localizer0", rsrcS.GetLocalizerId());
NUnit.Framework.Assert.AreEqual("localizer0", rsrcD.GetLocalizerId());
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcS.GetResourceStatus
(0));
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcD.GetResourceStatus
(0));
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService
<Path> cs, UserGroupInformation ugi)
{
while (true)
{
try
{
LocalizerStatus status = CreateStatus();
LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status);
switch (response.GetLocalizerAction())
{
case LocalizerAction.Live:
{
IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs();
foreach (ResourceLocalizationSpec newRsrc in newRsrcs)
{
if (!pendingResources.Contains(newRsrc.GetResource()))
{
pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory
().GetFile()), newRsrc.GetResource(), ugi));
}
}
break;
}
case LocalizerAction.Die:
{
// killall running localizations
foreach (Future <Path> pending in pendingResources.Values)
{
pending.Cancel(true);
}
status = CreateStatus();
// ignore response
try
{
nodemanager.Heartbeat(status);
}
catch (YarnException)
{
}
return;
}
}
cs.Poll(1000, TimeUnit.Milliseconds);
}
catch (Exception)
{
return;
}
catch (YarnException)
{
// TODO cleanup
return;
}
}
}
internal static LocalizerStatus CreateLocalizerStatus()
{
LocalizerStatus ret = recordFactory.NewRecordInstance <LocalizerStatus>();
NUnit.Framework.Assert.IsTrue(ret is LocalizerStatusPBImpl);
ret.SetLocalizerId("localizer0");
ret.AddResourceStatus(CreateLocalResourceStatus());
return(ret);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status)
{
YarnServerNodemanagerServiceProtos.LocalizerStatusProto statusProto = ((LocalizerStatusPBImpl
)status).GetProto();
try
{
return(new LocalizerHeartbeatResponsePBImpl(proxy.Heartbeat(null, statusProto)));
}
catch (ServiceException e)
{
RPCUtil.UnwrapAndThrowException(e);
return(null);
}
}
/// <summary>Create the payload for the HeartBeat.</summary>
/// <remarks>
/// Create the payload for the HeartBeat. Mainly the list of
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalResourceStatus
/// "/>
/// es
/// </remarks>
/// <returns>
/// a
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalizerStatus
/// "/>
/// that can be sent via heartbeat.
/// </returns>
/// <exception cref="System.Exception"/>
private LocalizerStatus CreateStatus()
{
IList <LocalResourceStatus> currentResources = new AList <LocalResourceStatus>();
// TODO: Synchronization??
for (IEnumerator <LocalResource> i = pendingResources.Keys.GetEnumerator(); i.HasNext
();)
{
LocalResource rsrc = i.Next();
LocalResourceStatus stat = recordFactory.NewRecordInstance <LocalResourceStatus>();
stat.SetResource(rsrc);
Future <Path> fPath = pendingResources[rsrc];
if (fPath.IsDone())
{
try
{
Path localPath = fPath.Get();
stat.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(localPath));
stat.SetLocalSize(FileUtil.GetDU(new FilePath(localPath.GetParent().ToUri())));
stat.SetStatus(ResourceStatusType.FetchSuccess);
}
catch (ExecutionException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e.InnerException));
}
catch (CancellationException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e));
}
// TODO shouldn't remove until ACK
i.Remove();
}
else
{
stat.SetStatus(ResourceStatusType.FetchPending);
}
currentResources.AddItem(stat);
}
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId(localizerId);
status.AddAllResources(currentResources);
return(status);
}
public virtual void TestLocalizerRPC()
{
IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040);
TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService
(locAddr);
try
{
server.Start();
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol
), locAddr, conf);
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId("localizer0");
LocalizerHeartbeatResponse response = client.Heartbeat(status);
NUnit.Framework.Assert.AreEqual(DieHBResponse(), response);
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.IsTrue(true);
}
public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status)
{
return(DieHBResponse());
}
public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status)
{
return(null);
}
