/// <exception cref="System.IO.IOException"/> protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx, Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo mapOutputInfo) { IndexRecord info = mapOutputInfo.indexRecord; ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce); DataOutputBuffer dob = new DataOutputBuffer(); header.Write(dob); ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength())); FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString()); RandomAccessFile spill; try { spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null); } catch (FileNotFoundException) { ShuffleHandler.Log.Info(spillfile + " not found"); return null; } ChannelFuture writeFuture; if (ch.GetPipeline().Get<SslHandler>() == null) { FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info .partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this ._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize , this._enclosing.shuffleTransferToAllowed); writeFuture = ch.Write(partition); writeFuture.AddListener(new _ChannelFutureListener_1135(partition)); } else { // TODO error handling; distinguish IO/connection failures, // attribute to appropriate spill output // HTTPS cannot be done with zero copy. FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info .partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this ._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath ()); writeFuture = ch.Write(chunk); } this._enclosing.metrics.shuffleConnections.Incr(); this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength); // optimistic return writeFuture; }
/// <exception cref="System.IO.IOException"/> public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token, string serviceUrl) { NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token); HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus .Ok); channel.Write(response).AddListener(ChannelFutureListener.Close); }
/// <exception cref="System.IO.IOException"/> public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token, string serviceUrl) { NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token); byte[] bytes = Sharpen.Runtime.GetBytesForString(TestDelegationTokenRemoteFetcher .ExpDate); ChannelBuffer cbuffer = ChannelBuffers.Buffer(bytes.Length); cbuffer.WriteBytes(bytes); HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus .Ok); response.SetHeader(HttpHeaders.Names.ContentLength, bytes.Length.ToString()); response.SetContent(cbuffer); channel.Write(response).AddListener(ChannelFutureListener.Close); }
/// <exception cref="System.IO.IOException"/> public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token, string serviceUrl) { NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token); Credentials creds = new Credentials(); creds.AddToken(new Text(serviceUrl), token); DataOutputBuffer @out = new DataOutputBuffer(); creds.Write(@out); int fileLength = @out.GetData().Length; ChannelBuffer cbuffer = ChannelBuffers.Buffer(fileLength); cbuffer.WriteBytes(@out.GetData()); HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus .Ok); response.SetHeader(HttpHeaders.Names.ContentLength, fileLength.ToString()); response.SetContent(cbuffer); channel.Write(response).AddListener(ChannelFutureListener.Close); }
/// <exception cref="System.Exception"/> public override void MessageReceived(ChannelHandlerContext ctx, MessageEvent evt) { HttpRequest request = (HttpRequest)evt.GetMessage(); if (request.GetMethod() != HttpMethod.Get) { this.SendError(ctx, HttpResponseStatus.MethodNotAllowed); return; } // Check whether the shuffle version is compatible if (!ShuffleHeader.DefaultHttpHeaderName.Equals(request.GetHeader(ShuffleHeader.HttpHeaderName )) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(request.GetHeader(ShuffleHeader .HttpHeaderVersion))) { this.SendError(ctx, "Incompatible shuffle request version", HttpResponseStatus.BadRequest ); } IDictionary<string, IList<string>> q = new QueryStringDecoder(request.GetUri()).GetParameters (); IList<string> keepAliveList = q["keepAlive"]; bool keepAliveParam = false; if (keepAliveList != null && keepAliveList.Count == 1) { keepAliveParam = Sharpen.Extensions.ValueOf(keepAliveList[0]); if (ShuffleHandler.Log.IsDebugEnabled()) { ShuffleHandler.Log.Debug("KeepAliveParam : " + keepAliveList + " : " + keepAliveParam ); } } IList<string> mapIds = this.SplitMaps(q["map"]); IList<string> reduceQ = q["reduce"]; IList<string> jobQ = q["job"]; if (ShuffleHandler.Log.IsDebugEnabled()) { ShuffleHandler.Log.Debug("RECV: " + request.GetUri() + "\n mapId: " + mapIds + "\n reduceId: " + reduceQ + "\n jobId: " + jobQ + "\n keepAlive: " + keepAliveParam); } if (mapIds == null || reduceQ == null || jobQ == null) { this.SendError(ctx, "Required param job, map and reduce", HttpResponseStatus.BadRequest ); return; } if (reduceQ.Count != 1 || jobQ.Count != 1) { this.SendError(ctx, "Too many job/reduce parameters", HttpResponseStatus.BadRequest ); return; } int reduceId; string jobId; try { reduceId = System.Convert.ToInt32(reduceQ[0]); jobId = jobQ[0]; } catch (FormatException) { this.SendError(ctx, "Bad reduce parameter", HttpResponseStatus.BadRequest); return; } catch (ArgumentException) { this.SendError(ctx, "Bad job parameter", HttpResponseStatus.BadRequest); return; } string reqUri = request.GetUri(); if (null == reqUri) { // TODO? add upstream? this.SendError(ctx, HttpResponseStatus.Forbidden); return; } HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus .Ok); try { this.VerifyRequest(jobId, ctx, request, response, new Uri("http", string.Empty, this .port, reqUri)); } catch (IOException e) { ShuffleHandler.Log.Warn("Shuffle failure ", e); this.SendError(ctx, e.Message, HttpResponseStatus.Unauthorized); return; } IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap = new Dictionary<string, ShuffleHandler.Shuffle.MapOutputInfo>(); Org.Jboss.Netty.Channel.Channel ch = evt.GetChannel(); string user = this._enclosing.userRsrc[jobId]; // $x/$user/appcache/$appId/output/$mapId // TODO: Once Shuffle is out of NM, this can use MR APIs to convert // between App and Job string outputBasePathStr = this.GetBaseLocation(jobId, user); try { this.PopulateHeaders(mapIds, outputBasePathStr, user, reduceId, request, response , keepAliveParam, mapOutputInfoMap); } catch (IOException e) { ch.Write(response); ShuffleHandler.Log.Error("Shuffle error in populating headers :", e); string errorMessage = this.GetErrorMessage(e); this.SendError(ctx, errorMessage, HttpResponseStatus.InternalServerError); return; } ch.Write(response); //Initialize one ReduceContext object per messageReceived call ShuffleHandler.ReduceContext reduceContext = new ShuffleHandler.ReduceContext(mapIds , reduceId, ctx, user, mapOutputInfoMap, outputBasePathStr); for (int i = 0; i < Math.Min(this._enclosing.maxSessionOpenFiles, mapIds.Count); i++) { ChannelFuture nextMap = this.SendMap(reduceContext); if (nextMap == null) { return; } } }