//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public void connect() throws Exception
public override void Connect()
{
ChannelFuture channelFuture = Bootstrap.connect(DestinationConflict.socketAddress());
NettyChannel = channelFuture.sync().channel();
NettyChannel.closeFuture().addListener((ChannelFutureListener)future => Handler.onClose());
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldLogOnlyTheFirstCaughtException() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldLogOnlyTheFirstCaughtException()
{
AssertableLogProvider logProvider = new AssertableLogProvider();
BoltConnection connection = mock(typeof(BoltConnection));
HouseKeeper houseKeeper = new HouseKeeper(connection, logProvider.GetLog(typeof(HouseKeeper)));
Bootstrap bootstrap = NewBootstrap(houseKeeper);
Exception error1 = new Exception("error #1");
Exception error2 = new Exception("error #2");
Exception error3 = new Exception("error #3");
try
{
using (ServerSocket serverSocket = new ServerSocket(0))
{
ChannelFuture future = bootstrap.connect("localhost", serverSocket.LocalPort).sync();
Channel channel = future.channel();
// fire multiple errors
channel.pipeline().fireExceptionCaught(error1);
channel.pipeline().fireExceptionCaught(error2);
channel.pipeline().fireExceptionCaught(error3);
// await for the channel to be closed by the HouseKeeper
channel.closeFuture().sync();
}
}
finally
{
// make sure event loop group is always terminated
bootstrap.config().group().shutdownGracefully().sync();
}
logProvider.AssertExactly(inLog(typeof(HouseKeeper)).error(startsWith("Fatal error occurred when handling a client connection"), equalTo(error1)));
}
/// <summary>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it.
/// </summary>
/// <remarks>
/// Calls sendMapOutput for the mapId pointed by ReduceContext.mapsToSend
/// and increments it. This method is first called by messageReceived()
/// maxSessionOpenFiles times and then on the completion of every
/// sendMapOutput operation. This limits the number of open files on a node,
/// which can get really large(exhausting file descriptors on the NM) if all
/// sendMapOutputs are called in one go, as was done previous to this change.
/// </remarks>
/// <param name="reduceContext">used to call sendMapOutput with correct params.</param>
/// <returns>the ChannelFuture of the sendMapOutput, can be null.</returns>
/// <exception cref="System.Exception"/>
public virtual ChannelFuture SendMap(ShuffleHandler.ReduceContext reduceContext)
{
ChannelFuture nextMap = null;
if (reduceContext.GetMapsToSend().Get() < reduceContext.GetMapIds().Count)
{
int nextIndex = reduceContext.GetMapsToSend().GetAndIncrement();
string mapId = reduceContext.GetMapIds()[nextIndex];
try
{
ShuffleHandler.Shuffle.MapOutputInfo info = reduceContext.GetInfoMap()[mapId];
if (info == null)
{
info = this.GetMapOutputInfo(reduceContext.GetOutputBasePathStr() + mapId, mapId,
reduceContext.GetReduceId(), reduceContext.GetUser());
}
nextMap = this.SendMapOutput(reduceContext.GetCtx(), reduceContext.GetCtx().GetChannel
(), reduceContext.GetUser(), mapId, reduceContext.GetReduceId(), info);
if (null == nextMap)
{
this.SendError(reduceContext.GetCtx(), HttpResponseStatus.NotFound);
return null;
}
nextMap.AddListener(new ShuffleHandler.ReduceMapFileCount(this, reduceContext));
}
catch (IOException e)
{
ShuffleHandler.Log.Error("Shuffle error :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(reduceContext.GetCtx(), errorMessage, HttpResponseStatus.InternalServerError
);
return null;
}
}
return nextMap;
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldNotLogExceptionsWhenEvenLoopIsShuttingDown() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldNotLogExceptionsWhenEvenLoopIsShuttingDown()
{
AssertableLogProvider logProvider = new AssertableLogProvider();
BoltConnection connection = mock(typeof(BoltConnection));
HouseKeeper houseKeeper = new HouseKeeper(connection, logProvider.GetLog(typeof(HouseKeeper)));
Bootstrap bootstrap = NewBootstrap(houseKeeper);
try
{
using (ServerSocket serverSocket = new ServerSocket(0))
{
ChannelFuture future = bootstrap.connect("localhost", serverSocket.LocalPort).sync();
Channel channel = future.channel();
// write some messages without flushing
for (int i = 0; i < 100; i++)
{
// use void promise which should redirect all write errors back to the pipeline and the HouseKeeper
channel.write(writeUtf8(channel.alloc(), "Hello"), channel.voidPromise());
}
// stop the even loop to make all pending writes fail
bootstrap.config().group().shutdownGracefully();
// await for the channel to be closed by the HouseKeeper
channel.closeFuture().sync();
}
}
finally
{
// make sure event loop group is always terminated
bootstrap.config().group().shutdownGracefully().sync();
}
logProvider.AssertNoLoggingOccurred();
}
public override void OperationComplete(ChannelFuture future)
{
if (future.IsSuccess())
{
partition.TransferSuccessful();
}
partition.ReleaseExternalResources();
}
public virtual void Connect(int port)
{
ChannelFuture channelFuture = _bootstrap.connect("localhost", port).awaitUninterruptibly();
_channel = channelFuture.channel();
if (!channelFuture.Success)
{
throw new Exception("Failed to connect", channelFuture.cause());
}
}
public void OperationComplete(ChannelFuture future)
{
if (future.IsSuccess())
{
ctx.Channel().Read();
}
else
{
SimpleHttpProxyHandler.Log.Debug("Proxy failed. Cause: ", future.Cause());
future.Channel().Close();
}
}
// pattern to identify errors related to the client closing the socket early
// idea borrowed from Netty SslHandler
//seconds
// 0 implies no limit
// 0 implies Netty default of 2 * number of available processors
/* the maximum number of files a single GET request can
open simultaneously during shuffle
*/
/// <exception cref="System.Exception"/>
public override void OperationComplete(ChannelFuture future)
{
if (future.IsSuccess())
{
shuffleOutputsOK.Incr();
}
else
{
shuffleOutputsFailed.Incr();
}
shuffleConnections.Decr();
}
protected override void ChannelRead0(ChannelHandlerContext ctx, HttpRequest req)
{
uri = req.GetUri();
IO.Netty.Channel.Channel client = ctx.Channel();
IO.Netty.Bootstrap.Bootstrap proxiedServer = new IO.Netty.Bootstrap.Bootstrap().Group
(client.EventLoop()).Channel(typeof(NioSocketChannel)).Handler(new _ChannelInitializer_106
(this, client));
ChannelFuture f = proxiedServer.Connect(host);
proxiedChannel = f.Channel();
f.AddListener(new _ChannelFutureListener_115(this, ctx, req, client));
}
private void WriteCurrentChunk()
{
if (!_channel.Open || !_channel.Connected || !_channel.Bound)
{
throw new ComException("Channel has been closed, so no need to try to write to it anymore. Client closed it?");
}
WaitForClientToCatchUpOnReadingChunks();
ChannelFuture future = _channel.write(_buffer);
future.addListener(NewChannelFutureListener(_buffer));
_writeAheadCounter.incrementAndGet();
}
public override void OperationComplete(ChannelFuture future)
{
if (!future.Done)
{
throw new ComException("This should not be possible because we waited for the future to be done");
}
if (!future.Success || future.Cancelled)
{
future.Channel.close();
}
_writeAheadCounter.decrementAndGet();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public void start() throws Throwable
public override void Start()
{
bool useEpoll = _useEpoll && Epoll.Available;
ServerConfigurationProvider configurationProvider = useEpoll ? EpollConfigurationProvider.INSTANCE : NioConfigurationProvider.INSTANCE;
_bossGroup = configurationProvider.CreateEventLoopGroup(1, _tf);
// These threads handle live channels. Each thread has a set of channels it is responsible for, and it will
// continuously run a #select() loop to react to new events on these channels.
_selectorGroup = configurationProvider.CreateEventLoopGroup(_numSelectorThreads, _tf);
// Bootstrap the various ports and protocols we want to handle
foreach (KeyValuePair <BoltConnector, ProtocolInitializer> bootstrapEntry in _bootstrappersMap.SetOfKeyValuePairs())
{
try
{
ProtocolInitializer protocolInitializer = bootstrapEntry.Value;
BoltConnector boltConnector = bootstrapEntry.Key;
ServerBootstrap serverBootstrap = CreateServerBootstrap(configurationProvider, protocolInitializer);
ChannelFuture channelFuture = serverBootstrap.bind(protocolInitializer.Address().socketAddress()).sync();
InetSocketAddress localAddress = ( InetSocketAddress )channelFuture.channel().localAddress();
_connectionRegister.register(boltConnector.Key(), localAddress);
string host = protocolInitializer.Address().Hostname;
int port = localAddress.Port;
if (host.Contains(":"))
{
// IPv6
_log.info("Bolt enabled on [%s]:%s.", host, port);
}
else
{
// IPv4
_log.info("Bolt enabled on %s:%s.", host, port);
}
}
catch (Exception e)
{
// We catch throwable here because netty uses clever tricks to have method signatures that look like they do not
// throw checked exceptions, but they actually do. The compiler won't let us catch them explicitly because in theory
// they shouldn't be possible, so we have to catch Throwable and do our own checks to grab them
throw new PortBindException(bootstrapEntry.Value.address(), e);
}
}
}
/// <exception cref="System.Exception"/>
public override void OperationComplete(ChannelFuture future)
{
if (!future.IsSuccess())
{
future.GetChannel().Close();
return;
}
int waitCount = this.reduceContext.GetMapsToWait().DecrementAndGet();
if (waitCount == 0)
{
this._enclosing.metrics.OperationComplete(future);
future.GetChannel().Close();
}
else
{
this._enclosing.pipelineFact.GetSHUFFLE().SendMap(this.reduceContext);
}
}
public virtual void Start()
{
if (httpServer != null)
{
ChannelFuture f = httpServer.Bind(DataNode.GetInfoAddr(conf));
f.SyncUninterruptibly();
httpAddress = (IPEndPoint)f.Channel().LocalAddress();
Log.Info("Listening HTTP traffic on " + httpAddress);
}
if (httpsServer != null)
{
IPEndPoint secInfoSocAddr = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys
.DfsDatanodeHttpsAddressKey, DFSConfigKeys.DfsDatanodeHttpsAddressDefault));
ChannelFuture f = httpsServer.Bind(secInfoSocAddr);
f.SyncUninterruptibly();
httpsAddress = (IPEndPoint)f.Channel().LocalAddress();
Log.Info("Listening HTTPS traffic on " + httpsAddress);
}
}
protected internal override ChannelContext create()
{
_outerInstance.msgLog.info(threadInfo() + "Trying to open a new channel from " + _outerInstance.origin + " to " + _outerInstance.destination);
// We must specify the origin address in case the server has multiple IPs per interface
ChannelFuture channelFuture = _outerInstance.bootstrap.connect(_outerInstance.destination, _outerInstance.origin);
channelFuture.awaitUninterruptibly(5, TimeUnit.SECONDS);
if (channelFuture.Success)
{
_outerInstance.msgLog.info(threadInfo() + "Opened a new channel from " + channelFuture.Channel.LocalAddress + " to " + channelFuture.Channel.RemoteAddress);
return(new ChannelContext(channelFuture.Channel, ChannelBuffers.dynamicBuffer(), ByteBuffer.allocate(1024 * 1024)));
}
Exception cause = channelFuture.Cause;
string msg = _outerInstance.GetType().Name + " could not connect from " + _outerInstance.origin + " to " + _outerInstance.destination;
_outerInstance.msgLog.debug(msg);
throw outerInstance.traceComException(new ComException(msg, cause), "Client.start");
}
public virtual void Run()
{
// Configure the client.
ChannelFactory factory = new NioClientSocketChannelFactory(Executors.NewCachedThreadPool
(), Executors.NewCachedThreadPool(), 1, 1);
ClientBootstrap bootstrap = new ClientBootstrap(factory);
// Set up the pipeline factory.
bootstrap.SetPipelineFactory(SetPipelineFactory());
bootstrap.SetOption("tcpNoDelay", true);
bootstrap.SetOption("keepAlive", true);
// Start the connection attempt.
ChannelFuture future = bootstrap.Connect(new IPEndPoint(host, port));
if (oneShot)
{
// Wait until the connection is closed or the connection attempt fails.
future.GetChannel().GetCloseFuture().AwaitUninterruptibly();
// Shut down thread pools to exit.
bootstrap.ReleaseExternalResources();
}
}
/// <exception cref="System.Exception"/>
public void OperationComplete(ChannelFuture future)
{
if (future.IsSuccess())
{
ctx.Channel().Pipeline().Remove <HttpResponseEncoder>();
HttpRequest newReq = new DefaultFullHttpRequest(HttpVersion.Http11, req.GetMethod
(), req.GetUri());
newReq.Headers().Add(req.Headers());
newReq.Headers().Set(HttpHeaders.Names.Connection, HttpHeaders.Values.Close);
future.Channel().WriteAndFlush(newReq);
}
else
{
DefaultHttpResponse resp = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.InternalServerError);
resp.Headers().Set(HttpHeaders.Names.Connection, HttpHeaders.Values.Close);
SimpleHttpProxyHandler.Log.Info("Proxy " + this._enclosing.uri + " failed. Cause: "
, future.Cause());
ctx.WriteAndFlush(resp).AddListener(ChannelFutureListener.Close);
client.Close();
}
}
private Channel OpenChannel(URI clusterUri)
{
SocketAddress destination = new InetSocketAddress(clusterUri.Host, clusterUri.Port == -1 ? _config.defaultPort() : clusterUri.Port);
// We must specify the origin address in case the server has multiple IPs per interface
SocketAddress origin = new InetSocketAddress(_me.Host, 0);
_msgLog.info("Attempting to connect from " + origin + " to " + destination);
ChannelFuture channelFuture = _clientBootstrap.connect(destination, origin);
channelFuture.awaitUninterruptibly(5, TimeUnit.SECONDS);
if (channelFuture.Success)
{
Channel channel = channelFuture.Channel;
_msgLog.info("Connected from " + channel.LocalAddress + " to " + channel.RemoteAddress);
return(channel);
}
Exception cause = channelFuture.Cause;
_msgLog.info("Failed to connect to " + destination + " due to: " + cause);
throw new ChannelOpenFailedException(cause);
}
/// <exception cref="System.Exception"/>
public override void MessageReceived(ChannelHandlerContext ctx, MessageEvent evt)
{
HttpRequest request = (HttpRequest)evt.GetMessage();
if (request.GetMethod() != HttpMethod.Get)
{
this.SendError(ctx, HttpResponseStatus.MethodNotAllowed);
return;
}
// Check whether the shuffle version is compatible
if (!ShuffleHeader.DefaultHttpHeaderName.Equals(request.GetHeader(ShuffleHeader.HttpHeaderName
)) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(request.GetHeader(ShuffleHeader
.HttpHeaderVersion)))
{
this.SendError(ctx, "Incompatible shuffle request version", HttpResponseStatus.BadRequest
);
}
IDictionary<string, IList<string>> q = new QueryStringDecoder(request.GetUri()).GetParameters
();
IList<string> keepAliveList = q["keepAlive"];
bool keepAliveParam = false;
if (keepAliveList != null && keepAliveList.Count == 1)
{
keepAliveParam = Sharpen.Extensions.ValueOf(keepAliveList[0]);
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("KeepAliveParam : " + keepAliveList + " : " + keepAliveParam
);
}
}
IList<string> mapIds = this.SplitMaps(q["map"]);
IList<string> reduceQ = q["reduce"];
IList<string> jobQ = q["job"];
if (ShuffleHandler.Log.IsDebugEnabled())
{
ShuffleHandler.Log.Debug("RECV: " + request.GetUri() + "\n mapId: " + mapIds + "\n reduceId: "
+ reduceQ + "\n jobId: " + jobQ + "\n keepAlive: " + keepAliveParam);
}
if (mapIds == null || reduceQ == null || jobQ == null)
{
this.SendError(ctx, "Required param job, map and reduce", HttpResponseStatus.BadRequest
);
return;
}
if (reduceQ.Count != 1 || jobQ.Count != 1)
{
this.SendError(ctx, "Too many job/reduce parameters", HttpResponseStatus.BadRequest
);
return;
}
int reduceId;
string jobId;
try
{
reduceId = System.Convert.ToInt32(reduceQ[0]);
jobId = jobQ[0];
}
catch (FormatException)
{
this.SendError(ctx, "Bad reduce parameter", HttpResponseStatus.BadRequest);
return;
}
catch (ArgumentException)
{
this.SendError(ctx, "Bad job parameter", HttpResponseStatus.BadRequest);
return;
}
string reqUri = request.GetUri();
if (null == reqUri)
{
// TODO? add upstream?
this.SendError(ctx, HttpResponseStatus.Forbidden);
return;
}
HttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus
.Ok);
try
{
this.VerifyRequest(jobId, ctx, request, response, new Uri("http", string.Empty, this
.port, reqUri));
}
catch (IOException e)
{
ShuffleHandler.Log.Warn("Shuffle failure ", e);
this.SendError(ctx, e.Message, HttpResponseStatus.Unauthorized);
return;
}
IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap = new
Dictionary<string, ShuffleHandler.Shuffle.MapOutputInfo>();
Org.Jboss.Netty.Channel.Channel ch = evt.GetChannel();
string user = this._enclosing.userRsrc[jobId];
// $x/$user/appcache/$appId/output/$mapId
// TODO: Once Shuffle is out of NM, this can use MR APIs to convert
// between App and Job
string outputBasePathStr = this.GetBaseLocation(jobId, user);
try
{
this.PopulateHeaders(mapIds, outputBasePathStr, user, reduceId, request, response
, keepAliveParam, mapOutputInfoMap);
}
catch (IOException e)
{
ch.Write(response);
ShuffleHandler.Log.Error("Shuffle error in populating headers :", e);
string errorMessage = this.GetErrorMessage(e);
this.SendError(ctx, errorMessage, HttpResponseStatus.InternalServerError);
return;
}
ch.Write(response);
//Initialize one ReduceContext object per messageReceived call
ShuffleHandler.ReduceContext reduceContext = new ShuffleHandler.ReduceContext(mapIds
, reduceId, ctx, user, mapOutputInfoMap, outputBasePathStr);
for (int i = 0; i < Math.Min(this._enclosing.maxSessionOpenFiles, mapIds.Count);
i++)
{
ChannelFuture nextMap = this.SendMap(reduceContext);
if (nextMap == null)
{
return;
}
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @SuppressWarnings("SameParameterValue") void connect(int port)
internal virtual void Connect(int port)
{
ChannelFuture channelFuture = Bootstrap.connect("localhost", port).syncUninterruptibly();
Channel = channelFuture.channel();
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected:
//ORIGINAL LINE: private synchronized void send(final org.neo4j.cluster.com.message.Message message)
private void Send(Message message)
{
lock (this)
{
_monitor.queuedMessage(message);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.net.URI to = java.net.URI.create(message.getHeader(org.neo4j.cluster.com.message.Message.HEADER_TO));
URI to = URI.create(message.getHeader(Message.HEADER_TO));
ExecutorService senderExecutor = _senderExecutors.computeIfAbsent(to, t => Executors.newSingleThreadExecutor(new NamedThreadFactory("Cluster Sender " + t.toASCIIString(), _monitor)));
senderExecutor.submit(() =>
{
Channel channel = GetChannel(to);
try
{
if (channel == null)
{
channel = OpenChannel(to);
OpenedChannel(to, channel);
// Instance could be connected to, remove any marker of it being failed
_failedInstances.remove(to);
}
}
catch (Exception e)
{
// Only print out failure message on first fail
if (!_failedInstances.Contains(to))
{
_msgLog.warn(e.Message);
_failedInstances.Add(to);
}
return;
}
try
{
// Set HEADER_FROM header
message.setHeader(Message.HEADER_FROM, _me.toASCIIString());
_msgLog.debug("Sending to " + to + ": " + message);
ChannelFuture future = channel.write(message);
future.addListener(future1 =>
{
_monitor.sentMessage(message);
if (!future1.Success)
{
_msgLog.debug("Unable to write " + message + " to " + future1.Channel, future1.Cause);
ClosedChannel(future1.Channel);
// Try again
Send(message);
}
});
}
catch (Exception e)
{
if (Exceptions.contains(e, typeof(ClosedChannelException)))
{
_msgLog.warn("Could not send message, because the connection has been closed.");
}
else
{
_msgLog.warn("Could not send message", e);
}
channel.close();
}
});
}
}
