/// <summary>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it.
/// </summary>
/// <remarks>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it. This method is first called by messageReceived()
/// maxSessionOpenFiles times and then on the completion of every
/// sendMapOutput operation. This limits the number of open files on a node,
/// which can get really large(exhausting file descriptors on the NM) if all
/// sendMapOutputs are called in one go, as was done previous to this change.
/// </remarks>
/// <param name="reduceContext">used to call sendMapOutput with correct params.</param>
/// <returns>the ChannelFuture of the sendMapOutput, can be null.</returns>
/// <exception cref="System.Exception"/>
public virtual ChannelFuture SendMap(ShuffleHandler.ReduceContext reduceContext)
{
ChannelFuture nextMap = null;
if (reduceContext.GetMapsToSend().Get() < reduceContext.GetMapIds().Count)
{
int nextIndex = reduceContext.GetMapsToSend().GetAndIncrement();
string mapId = reduceContext.GetMapIds()[nextIndex];
try
{
ShuffleHandler.Shuffle.MapOutputInfo info = reduceContext.GetInfoMap()[mapId];
if (info == null)
{
info = this.GetMapOutputInfo(reduceContext.GetOutputBasePathStr() + mapId, mapId,
reduceContext.GetReduceId(), reduceContext.GetUser());
}
nextMap = this.SendMapOutput(reduceContext.GetCtx(), reduceContext.GetCtx().GetChannel
(), reduceContext.GetUser(), mapId, reduceContext.GetReduceId(), info);
if (null == nextMap)
{
this.SendError(reduceContext.GetCtx(), HttpResponseStatus.NotFound);
return null;
}
nextMap.AddListener(new ShuffleHandler.ReduceMapFileCount(this, reduceContext));
}
catch (IOException e)
{
ShuffleHandler.Log.Error("Shuffle error :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(reduceContext.GetCtx(), errorMessage, HttpResponseStatus.InternalServerError
);
return null;
}
}
return nextMap;
}
/// <exception cref="System.Exception"/>
public override void MessageReceived(ChannelHandlerContext ctx, MessageEvent evt)
{
HttpRequest request = (HttpRequest)evt.GetMessage();
if (request.GetMethod() != HttpMethod.Get)
{
this.SendError(ctx, HttpResponseStatus.MethodNotAllowed);
return;
}
// Check whether the shuffle version is compatible
if (!ShuffleHeader.DefaultHttpHeaderName.Equals(request.GetHeader(ShuffleHeader.HttpHeaderName
)) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(request.GetHeader(ShuffleHeader
.HttpHeaderVersion)))
{
this.SendError(ctx, "Incompatible shuffle request version", HttpResponseStatus.BadRequest
);
}
IDictionary<string, IList<string>> q = new QueryStringDecoder(request.GetUri()).GetParameters
();
IList<string> keepAliveList = q["keepAlive"];
bool keepAliveParam = false;
if (keepAliveList != null && keepAliveList.Count == 1)
{
keepAliveParam = Sharpen.Extensions.ValueOf(keepAliveList[0]);
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("KeepAliveParam : " + keepAliveList + " : " + keepAliveParam
);
}
}
IList<string> mapIds = this.SplitMaps(q["map"]);
IList<string> reduceQ = q["reduce"];
IList<string> jobQ = q["job"];
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("RECV: " + request.GetUri() + "\n mapId: " + mapIds + "\n reduceId: "
+ reduceQ + "\n jobId: " + jobQ + "\n keepAlive: " + keepAliveParam);
}
if (mapIds == null || reduceQ == null || jobQ == null)
{
this.SendError(ctx, "Required param job, map and reduce", HttpResponseStatus.BadRequest
);
return;
}
if (reduceQ.Count != 1 || jobQ.Count != 1)
{
this.SendError(ctx, "Too many job/reduce parameters", HttpResponseStatus.BadRequest
);
return;
}
int reduceId;
string jobId;
try
{
reduceId = System.Convert.ToInt32(reduceQ[0]);
jobId = jobQ[0];
}
catch (FormatException)
{
this.SendError(ctx, "Bad reduce parameter", HttpResponseStatus.BadRequest);
return;
}
catch (ArgumentException)
{
this.SendError(ctx, "Bad job parameter", HttpResponseStatus.BadRequest);
return;
}
string reqUri = request.GetUri();
if (null == reqUri)
{
// TODO? add upstream?
this.SendError(ctx, HttpResponseStatus.Forbidden);
return;
}
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
try
{
this.VerifyRequest(jobId, ctx, request, response, new Uri("http", string.Empty, this
.port, reqUri));
}
catch (IOException e)
{
ShuffleHandler.Log.Warn("Shuffle failure ", e);
this.SendError(ctx, e.Message, HttpResponseStatus.Unauthorized);
return;
}
IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap = new
Dictionary<string, ShuffleHandler.Shuffle.MapOutputInfo>();
Org.Jboss.Netty.Channel.Channel ch = evt.GetChannel();
string user = this._enclosing.userRsrc[jobId];
// $x/$user/appcache/$appId/output/$mapId
// TODO: Once Shuffle is out of NM, this can use MR APIs to convert
// between App and Job
string outputBasePathStr = this.GetBaseLocation(jobId, user);
try
{
this.PopulateHeaders(mapIds, outputBasePathStr, user, reduceId, request, response
, keepAliveParam, mapOutputInfoMap);
}
catch (IOException e)
{
ch.Write(response);
ShuffleHandler.Log.Error("Shuffle error in populating headers :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(ctx, errorMessage, HttpResponseStatus.InternalServerError);
return;
}
ch.Write(response);
//Initialize one ReduceContext object per messageReceived call
ShuffleHandler.ReduceContext reduceContext = new ShuffleHandler.ReduceContext(mapIds
, reduceId, ctx, user, mapOutputInfoMap, outputBasePathStr);
for (int i = 0; i < Math.Min(this._enclosing.maxSessionOpenFiles, mapIds.Count);
i++)
{
ChannelFuture nextMap = this.SendMap(reduceContext);
if (nextMap == null)
{
return;
}
}
}
public ReduceMapFileCount(ShuffleHandler _enclosing, ShuffleHandler.ReduceContext
rc)
{
this._enclosing = _enclosing;
this.reduceContext = rc;
}