/// <exception cref="System.IO.IOException"/>
protected internal virtual void PopulateHeaders(IList<string> mapIds, string outputBaseStr
, string user, int reduce, HttpRequest request, HttpResponse response, bool keepAliveParam
, IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap)
{
long contentLength = 0;
foreach (string mapId in mapIds)
{
string @base = outputBaseStr + mapId;
ShuffleHandler.Shuffle.MapOutputInfo outputInfo = this.GetMapOutputInfo(@base, mapId
, reduce, user);
if (mapOutputInfoMap.Count < this._enclosing.mapOutputMetaInfoCacheSize)
{
mapOutputInfoMap[mapId] = outputInfo;
}
// Index file
Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index",
this.conf);
IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName
, user);
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
contentLength += info.partLength;
contentLength += dob.GetLength();
}
// Now set the response headers.
this.SetResponseHeaders(response, keepAliveParam, contentLength);
}
/// <summary>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it.
/// </summary>
/// <remarks>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it. This method is first called by messageReceived()
/// maxSessionOpenFiles times and then on the completion of every
/// sendMapOutput operation. This limits the number of open files on a node,
/// which can get really large(exhausting file descriptors on the NM) if all
/// sendMapOutputs are called in one go, as was done previous to this change.
/// </remarks>
/// <param name="reduceContext">used to call sendMapOutput with correct params.</param>
/// <returns>the ChannelFuture of the sendMapOutput, can be null.</returns>
/// <exception cref="System.Exception"/>
public virtual ChannelFuture SendMap(ShuffleHandler.ReduceContext reduceContext)
{
ChannelFuture nextMap = null;
if (reduceContext.GetMapsToSend().Get() < reduceContext.GetMapIds().Count)
{
int nextIndex = reduceContext.GetMapsToSend().GetAndIncrement();
string mapId = reduceContext.GetMapIds()[nextIndex];
try
{
ShuffleHandler.Shuffle.MapOutputInfo info = reduceContext.GetInfoMap()[mapId];
if (info == null)
{
info = this.GetMapOutputInfo(reduceContext.GetOutputBasePathStr() + mapId, mapId,
reduceContext.GetReduceId(), reduceContext.GetUser());
}
nextMap = this.SendMapOutput(reduceContext.GetCtx(), reduceContext.GetCtx().GetChannel
(), reduceContext.GetUser(), mapId, reduceContext.GetReduceId(), info);
if (null == nextMap)
{
this.SendError(reduceContext.GetCtx(), HttpResponseStatus.NotFound);
return null;
}
nextMap.AddListener(new ShuffleHandler.ReduceMapFileCount(this, reduceContext));
}
catch (IOException e)
{
ShuffleHandler.Log.Error("Shuffle error :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(reduceContext.GetCtx(), errorMessage, HttpResponseStatus.InternalServerError
);
return null;
}
}
return nextMap;
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx,
Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo
mapOutputInfo)
{
IndexRecord info = mapOutputInfo.indexRecord;
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength()));
FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString());
RandomAccessFile spill;
try
{
spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null);
}
catch (FileNotFoundException)
{
ShuffleHandler.Log.Info(spillfile + " not found");
return null;
}
ChannelFuture writeFuture;
if (ch.GetPipeline().Get<SslHandler>() == null)
{
FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info
.partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this
._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize
, this._enclosing.shuffleTransferToAllowed);
writeFuture = ch.Write(partition);
writeFuture.AddListener(new _ChannelFutureListener_1135(partition));
}
else
{
// TODO error handling; distinguish IO/connection failures,
// attribute to appropriate spill output
// HTTPS cannot be done with zero copy.
FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info
.partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this
._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath
());
writeFuture = ch.Write(chunk);
}
this._enclosing.metrics.shuffleConnections.Incr();
this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength);
// optimistic
return writeFuture;
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual ShuffleHandler.Shuffle.MapOutputInfo GetMapOutputInfo(
string @base, string mapId, int reduce, string user)
{
// Index file
Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index",
this.conf);
IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName
, user);
Path mapOutputFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out", this
.conf);
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug(@base + " : " + mapOutputFileName + " : " + indexFileName
);
}
ShuffleHandler.Shuffle.MapOutputInfo outputInfo = new ShuffleHandler.Shuffle.MapOutputInfo
(this, mapOutputFileName, info);
return outputInfo;
}
