/// <exception cref="System.IO.IOException"/>
protected internal virtual ChannelFuture SendMapOutput(ChannelHandlerContext ctx,
Org.Jboss.Netty.Channel.Channel ch, string user, string mapId, int reduce, ShuffleHandler.Shuffle.MapOutputInfo
mapOutputInfo)
{
IndexRecord info = mapOutputInfo.indexRecord;
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
ch.Write(ChannelBuffers.WrappedBuffer(dob.GetData(), 0, dob.GetLength()));
FilePath spillfile = new FilePath(mapOutputInfo.mapOutputFileName.ToString());
RandomAccessFile spill;
try
{
spill = SecureIOUtils.OpenForRandomRead(spillfile, "r", user, null);
}
catch (FileNotFoundException)
{
ShuffleHandler.Log.Info(spillfile + " not found");
return null;
}
ChannelFuture writeFuture;
if (ch.GetPipeline().Get<SslHandler>() == null)
{
FadvisedFileRegion partition = new FadvisedFileRegion(spill, info.startOffset, info
.partLength, this._enclosing.manageOsCache, this._enclosing.readaheadLength, this
._enclosing.readaheadPool, spillfile.GetAbsolutePath(), this._enclosing.shuffleBufferSize
, this._enclosing.shuffleTransferToAllowed);
writeFuture = ch.Write(partition);
writeFuture.AddListener(new _ChannelFutureListener_1135(partition));
}
else
{
// TODO error handling; distinguish IO/connection failures,
// attribute to appropriate spill output
// HTTPS cannot be done with zero copy.
FadvisedChunkedFile chunk = new FadvisedChunkedFile(spill, info.startOffset, info
.partLength, this._enclosing.sslFileBufferSize, this._enclosing.manageOsCache, this
._enclosing.readaheadLength, this._enclosing.readaheadPool, spillfile.GetAbsolutePath
());
writeFuture = ch.Write(chunk);
}
this._enclosing.metrics.shuffleConnections.Incr();
this._enclosing.metrics.shuffleOutputBytes.Incr(info.partLength);
// optimistic
return writeFuture;
}
/// <exception cref="System.IO.IOException"/>
public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier> token, string serviceUrl)
{
NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token);
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
channel.Write(response).AddListener(ChannelFutureListener.Close);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier> token, string serviceUrl)
{
NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token);
byte[] bytes = Sharpen.Runtime.GetBytesForString(TestDelegationTokenRemoteFetcher
.ExpDate);
ChannelBuffer cbuffer = ChannelBuffers.Buffer(bytes.Length);
cbuffer.WriteBytes(bytes);
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
response.SetHeader(HttpHeaders.Names.ContentLength, bytes.Length.ToString());
response.SetContent(cbuffer);
channel.Write(response).AddListener(ChannelFutureListener.Close);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Handle(Org.Jboss.Netty.Channel.Channel channel, Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier> token, string serviceUrl)
{
NUnit.Framework.Assert.AreEqual(this._enclosing.testToken, token);
Credentials creds = new Credentials();
creds.AddToken(new Text(serviceUrl), token);
DataOutputBuffer @out = new DataOutputBuffer();
creds.Write(@out);
int fileLength = @out.GetData().Length;
ChannelBuffer cbuffer = ChannelBuffers.Buffer(fileLength);
cbuffer.WriteBytes(@out.GetData());
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
response.SetHeader(HttpHeaders.Names.ContentLength, fileLength.ToString());
response.SetContent(cbuffer);
channel.Write(response).AddListener(ChannelFutureListener.Close);
}
/// <exception cref="System.Exception"/>
public override void MessageReceived(ChannelHandlerContext ctx, MessageEvent evt)
{
HttpRequest request = (HttpRequest)evt.GetMessage();
if (request.GetMethod() != HttpMethod.Get)
{
this.SendError(ctx, HttpResponseStatus.MethodNotAllowed);
return;
}
// Check whether the shuffle version is compatible
if (!ShuffleHeader.DefaultHttpHeaderName.Equals(request.GetHeader(ShuffleHeader.HttpHeaderName
)) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(request.GetHeader(ShuffleHeader
.HttpHeaderVersion)))
{
this.SendError(ctx, "Incompatible shuffle request version", HttpResponseStatus.BadRequest
);
}
IDictionary<string, IList<string>> q = new QueryStringDecoder(request.GetUri()).GetParameters
();
IList<string> keepAliveList = q["keepAlive"];
bool keepAliveParam = false;
if (keepAliveList != null && keepAliveList.Count == 1)
{
keepAliveParam = Sharpen.Extensions.ValueOf(keepAliveList[0]);
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("KeepAliveParam : " + keepAliveList + " : " + keepAliveParam
);
}
}
IList<string> mapIds = this.SplitMaps(q["map"]);
IList<string> reduceQ = q["reduce"];
IList<string> jobQ = q["job"];
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("RECV: " + request.GetUri() + "\n mapId: " + mapIds + "\n reduceId: "
+ reduceQ + "\n jobId: " + jobQ + "\n keepAlive: " + keepAliveParam);
}
if (mapIds == null || reduceQ == null || jobQ == null)
{
this.SendError(ctx, "Required param job, map and reduce", HttpResponseStatus.BadRequest
);
return;
}
if (reduceQ.Count != 1 || jobQ.Count != 1)
{
this.SendError(ctx, "Too many job/reduce parameters", HttpResponseStatus.BadRequest
);
return;
}
int reduceId;
string jobId;
try
{
reduceId = System.Convert.ToInt32(reduceQ[0]);
jobId = jobQ[0];
}
catch (FormatException)
{
this.SendError(ctx, "Bad reduce parameter", HttpResponseStatus.BadRequest);
return;
}
catch (ArgumentException)
{
this.SendError(ctx, "Bad job parameter", HttpResponseStatus.BadRequest);
return;
}
string reqUri = request.GetUri();
if (null == reqUri)
{
// TODO? add upstream?
this.SendError(ctx, HttpResponseStatus.Forbidden);
return;
}
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
try
{
this.VerifyRequest(jobId, ctx, request, response, new Uri("http", string.Empty, this
.port, reqUri));
}
catch (IOException e)
{
ShuffleHandler.Log.Warn("Shuffle failure ", e);
this.SendError(ctx, e.Message, HttpResponseStatus.Unauthorized);
return;
}
IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap = new
Dictionary<string, ShuffleHandler.Shuffle.MapOutputInfo>();
Org.Jboss.Netty.Channel.Channel ch = evt.GetChannel();
string user = this._enclosing.userRsrc[jobId];
// $x/$user/appcache/$appId/output/$mapId
// TODO: Once Shuffle is out of NM, this can use MR APIs to convert
// between App and Job
string outputBasePathStr = this.GetBaseLocation(jobId, user);
try
{
this.PopulateHeaders(mapIds, outputBasePathStr, user, reduceId, request, response
, keepAliveParam, mapOutputInfoMap);
}
catch (IOException e)
{
ch.Write(response);
ShuffleHandler.Log.Error("Shuffle error in populating headers :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(ctx, errorMessage, HttpResponseStatus.InternalServerError);
return;
}
ch.Write(response);
//Initialize one ReduceContext object per messageReceived call
ShuffleHandler.ReduceContext reduceContext = new ShuffleHandler.ReduceContext(mapIds
, reduceId, ctx, user, mapOutputInfoMap, outputBasePathStr);
for (int i = 0; i < Math.Min(this._enclosing.maxSessionOpenFiles, mapIds.Count);
i++)
{
ChannelFuture nextMap = this.SendMap(reduceContext);
if (nextMap == null)
{
return;
}
}
}
