/// <exception cref="System.IO.IOException"/> protected internal virtual void PopulateHeaders(IList<string> mapIds, string outputBaseStr , string user, int reduce, HttpRequest request, HttpResponse response, bool keepAliveParam , IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap) { long contentLength = 0; foreach (string mapId in mapIds) { string @base = outputBaseStr + mapId; ShuffleHandler.Shuffle.MapOutputInfo outputInfo = this.GetMapOutputInfo(@base, mapId , reduce, user); if (mapOutputInfoMap.Count < this._enclosing.mapOutputMetaInfoCacheSize) { mapOutputInfoMap[mapId] = outputInfo; } // Index file Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index", this.conf); IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName , user); ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce); DataOutputBuffer dob = new DataOutputBuffer(); header.Write(dob); contentLength += info.partLength; contentLength += dob.GetLength(); } // Now set the response headers. this.SetResponseHeaders(response, keepAliveParam, contentLength); }
/// <summary> /// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend /// and increments it. /// </summary> /// <remarks> /// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend /// and increments it. This method is first called by messageReceived() /// maxSessionOpenFiles times and then on the completion of every /// sendMapOutput operation. This limits the number of open files on a node, /// which can get really large(exhausting file descriptors on the NM) if all /// sendMapOutputs are called in one go, as was done previous to this change. /// </remarks> /// <param name="reduceContext">used to call sendMapOutput with correct params.</param> /// <returns>the ChannelFuture of the sendMapOutput, can be null.</returns> /// <exception cref="System.Exception"/> public virtual ChannelFuture SendMap(ShuffleHandler.ReduceContext reduceContext) { ChannelFuture nextMap = null; if (reduceContext.GetMapsToSend().Get() < reduceContext.GetMapIds().Count) { int nextIndex = reduceContext.GetMapsToSend().GetAndIncrement(); string mapId = reduceContext.GetMapIds()[nextIndex]; try { ShuffleHandler.Shuffle.MapOutputInfo info = reduceContext.GetInfoMap()[mapId]; if (info == null) { info = this.GetMapOutputInfo(reduceContext.GetOutputBasePathStr() + mapId, mapId, reduceContext.GetReduceId(), reduceContext.GetUser()); } nextMap = this.SendMapOutput(reduceContext.GetCtx(), reduceContext.GetCtx().GetChannel (), reduceContext.GetUser(), mapId, reduceContext.GetReduceId(), info); if (null == nextMap) { this.SendError(reduceContext.GetCtx(), HttpResponseStatus.NotFound); return null; } nextMap.AddListener(new ShuffleHandler.ReduceMapFileCount(this, reduceContext)); } catch (IOException e) { ShuffleHandler.Log.Error("Shuffle error :", e); string errorMessage = this.GetErrorMessage(e); this.SendError(reduceContext.GetCtx(), errorMessage, HttpResponseStatus.InternalServerError ); return null; } } return nextMap; }
/// <exception cref="System.IO.IOException"/> protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx, Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo mapOutputInfo) { IndexRecord info = mapOutputInfo.indexRecord; ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce); DataOutputBuffer dob = new DataOutputBuffer(); header.Write(dob); ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength())); FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString()); RandomAccessFile spill; try { spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null); } catch (FileNotFoundException) { ShuffleHandler.Log.Info(spillfile + " not found"); return null; } ChannelFuture writeFuture; if (ch.GetPipeline().Get<SslHandler>() == null) { FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info .partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this ._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize , this._enclosing.shuffleTransferToAllowed); writeFuture = ch.Write(partition); writeFuture.AddListener(new _ChannelFutureListener_1135(partition)); } else { // TODO error handling; distinguish IO/connection failures, // attribute to appropriate spill output // HTTPS cannot be done with zero copy. FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info .partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this ._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath ()); writeFuture = ch.Write(chunk); } this._enclosing.metrics.shuffleConnections.Incr(); this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength); // optimistic return writeFuture; }
/// <exception cref="System.IO.IOException"/> protected internal virtual ShuffleHandler.Shuffle.MapOutputInfo GetMapOutputInfo( string @base, string mapId, int reduce, string user) { // Index file Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index", this.conf); IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName , user); Path mapOutputFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out", this .conf); if (ShuffleHandler.Log.IsDebugEnabled()) { ShuffleHandler.Log.Debug(@base + " : " + mapOutputFileName + " : " + indexFileName ); } ShuffleHandler.Shuffle.MapOutputInfo outputInfo = new ShuffleHandler.Shuffle.MapOutputInfo (this, mapOutputFileName, info); return outputInfo; }