/// <summary>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it.
/// </summary>
/// <remarks>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it. This method is first called by messageReceived()
/// maxSessionOpenFiles times and then on the completion of every
/// sendMapOutput operation. This limits the number of open files on a node,
/// which can get really large(exhausting file descriptors on the NM) if all
/// sendMapOutputs are called in one go, as was done previous to this change.
/// </remarks>
/// <param name="reduceContext">used to call sendMapOutput with correct params.</param>
/// <returns>the ChannelFuture of the sendMapOutput, can be null.</returns>
/// <exception cref="System.Exception"/>
public virtual ChannelFuture SendMap(ShuffleHandler.ReduceContext reduceContext)
{
ChannelFuture nextMap = null;
if (reduceContext.GetMapsToSend().Get() < reduceContext.GetMapIds().Count)
{
int nextIndex = reduceContext.GetMapsToSend().GetAndIncrement();
string mapId = reduceContext.GetMapIds()[nextIndex];
try
{
ShuffleHandler.Shuffle.MapOutputInfo info = reduceContext.GetInfoMap()[mapId];
if (info == null)
{
info = this.GetMapOutputInfo(reduceContext.GetOutputBasePathStr() + mapId, mapId,
reduceContext.GetReduceId(), reduceContext.GetUser());
}
nextMap = this.SendMapOutput(reduceContext.GetCtx(), reduceContext.GetCtx().GetChannel
(), reduceContext.GetUser(), mapId, reduceContext.GetReduceId(), info);
if (null == nextMap)
{
this.SendError(reduceContext.GetCtx(), HttpResponseStatus.NotFound);
return null;
}
nextMap.AddListener(new ShuffleHandler.ReduceMapFileCount(this, reduceContext));
}
catch (IOException e)
{
ShuffleHandler.Log.Error("Shuffle error :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(reduceContext.GetCtx(), errorMessage, HttpResponseStatus.InternalServerError
);
return null;
}
}
return nextMap;
}
protected override void ChannelRead0(ChannelHandlerContext ctx, HttpRequest req)
{
uri = req.GetUri();
IO.Netty.Channel.Channel client = ctx.Channel();
IO.Netty.Bootstrap.Bootstrap proxiedServer = new IO.Netty.Bootstrap.Bootstrap().Group
(client.EventLoop()).Channel(typeof(NioSocketChannel)).Handler(new _ChannelInitializer_106
(this, client));
ChannelFuture f = proxiedServer.Connect(host);
proxiedChannel = f.Channel();
f.AddListener(new _ChannelFutureListener_115(this, ctx, req, client));
}