/// <exception cref="System.Exception"/> public virtual void TestLocalizerStatusSerDe() { LocalizerStatus rsrcS = CreateLocalizerStatus(); NUnit.Framework.Assert.IsTrue(rsrcS is LocalizerStatusPBImpl); LocalizerStatusPBImpl rsrcPb = (LocalizerStatusPBImpl)rsrcS; DataOutputBuffer @out = new DataOutputBuffer(); rsrcPb.GetProto().WriteDelimitedTo(@out); DataInputBuffer @in = new DataInputBuffer(); @in.Reset(@out.GetData(), 0, @out.GetLength()); YarnServerNodemanagerServiceProtos.LocalizerStatusProto rsrcPbD = YarnServerNodemanagerServiceProtos.LocalizerStatusProto .ParseDelimitedFrom(@in); NUnit.Framework.Assert.IsNotNull(rsrcPbD); LocalizerStatus rsrcD = new LocalizerStatusPBImpl(rsrcPbD); NUnit.Framework.Assert.AreEqual(rsrcS, rsrcD); NUnit.Framework.Assert.AreEqual("localizer0", rsrcS.GetLocalizerId()); NUnit.Framework.Assert.AreEqual("localizer0", rsrcD.GetLocalizerId()); NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcS.GetResourceStatus (0)); NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcD.GetResourceStatus (0)); }
/// <exception cref="System.IO.IOException"/> protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService <Path> cs, UserGroupInformation ugi) { while (true) { try { LocalizerStatus status = CreateStatus(); LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status); switch (response.GetLocalizerAction()) { case LocalizerAction.Live: { IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs(); foreach (ResourceLocalizationSpec newRsrc in newRsrcs) { if (!pendingResources.Contains(newRsrc.GetResource())) { pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory ().GetFile()), newRsrc.GetResource(), ugi)); } } break; } case LocalizerAction.Die: { // killall running localizations foreach (Future <Path> pending in pendingResources.Values) { pending.Cancel(true); } status = CreateStatus(); // ignore response try { nodemanager.Heartbeat(status); } catch (YarnException) { } return; } } cs.Poll(1000, TimeUnit.Milliseconds); } catch (Exception) { return; } catch (YarnException) { // TODO cleanup return; } } }
internal static LocalizerStatus CreateLocalizerStatus() { LocalizerStatus ret = recordFactory.NewRecordInstance <LocalizerStatus>(); NUnit.Framework.Assert.IsTrue(ret is LocalizerStatusPBImpl); ret.SetLocalizerId("localizer0"); ret.AddResourceStatus(CreateLocalResourceStatus()); return(ret); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status) { YarnServerNodemanagerServiceProtos.LocalizerStatusProto statusProto = ((LocalizerStatusPBImpl )status).GetProto(); try { return(new LocalizerHeartbeatResponsePBImpl(proxy.Heartbeat(null, statusProto))); } catch (ServiceException e) { RPCUtil.UnwrapAndThrowException(e); return(null); } }
/// <summary>Create the payload for the HeartBeat.</summary> /// <remarks> /// Create the payload for the HeartBeat. Mainly the list of /// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalResourceStatus /// "/> /// es /// </remarks> /// <returns> /// a /// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalizerStatus /// "/> /// that can be sent via heartbeat. /// </returns> /// <exception cref="System.Exception"/> private LocalizerStatus CreateStatus() { IList <LocalResourceStatus> currentResources = new AList <LocalResourceStatus>(); // TODO: Synchronization?? for (IEnumerator <LocalResource> i = pendingResources.Keys.GetEnumerator(); i.HasNext ();) { LocalResource rsrc = i.Next(); LocalResourceStatus stat = recordFactory.NewRecordInstance <LocalResourceStatus>(); stat.SetResource(rsrc); Future <Path> fPath = pendingResources[rsrc]; if (fPath.IsDone()) { try { Path localPath = fPath.Get(); stat.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(localPath)); stat.SetLocalSize(FileUtil.GetDU(new FilePath(localPath.GetParent().ToUri()))); stat.SetStatus(ResourceStatusType.FetchSuccess); } catch (ExecutionException e) { stat.SetStatus(ResourceStatusType.FetchFailure); stat.SetException(SerializedException.NewInstance(e.InnerException)); } catch (CancellationException e) { stat.SetStatus(ResourceStatusType.FetchFailure); stat.SetException(SerializedException.NewInstance(e)); } // TODO shouldn't remove until ACK i.Remove(); } else { stat.SetStatus(ResourceStatusType.FetchPending); } currentResources.AddItem(stat); } LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>(); status.SetLocalizerId(localizerId); status.AddAllResources(currentResources); return(status); }
public virtual void TestLocalizerRPC() { IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040); TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService (locAddr); try { server.Start(); Configuration conf = new Configuration(); YarnRPC rpc = YarnRPC.Create(conf); LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol ), locAddr, conf); LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>(); status.SetLocalizerId("localizer0"); LocalizerHeartbeatResponse response = client.Heartbeat(status); NUnit.Framework.Assert.AreEqual(DieHBResponse(), response); } finally { server.Stop(); } NUnit.Framework.Assert.IsTrue(true); }
public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status) { return(DieHBResponse()); }
public virtual LocalizerHeartbeatResponse Heartbeat(LocalizerStatus status) { return(null); }