public async ValueTask DisposeAsync()
{
if (!_disposed)
{
using (_logger.BeginScope(_connectionId))
{
_logger.LogDebug("Disposing resources");
await DisconnectAsync();
await _stream.DisposeAsync();
_socket.Dispose();
}
}
}
/// <summary>
/// Asynchronously ends the server connection
/// </summary>
public async Task EndAsync()
{
Status = StatusFlag.NotActive;
try {
await netSWriter.DisposeAsync();
netSReader.Dispose();
await netStream.DisposeAsync();
tcpClient.Dispose();
tcpListener.Stop();
} catch (Exception e) {
throw new _Exception.Session.EndAsync("Failed to end the session!", e);
}
}
/// <summary>
/// Write supplied <paramref name="connectionNumber"/> to supplied <paramref name="socket"/>.
/// </summary>
/// <param name="socket"><see cref="Socket"/> to write <paramref name="connectionNumber"/> to.</param>
/// <param name="connectionNumber"><see cref="long"/> to write to <paramref name="socket"/>.</param>
/// <returns>Task handle.</returns>
private static async Task WriteConnectionNumber(Socket socket, long connectionNumber)
{
// Console.WriteLine($"Writing connection number {connectionNumber} to socket...");
NetworkStream stream = new NetworkStream(socket, false);
byte[] bytes = BitConverter.GetBytes(connectionNumber);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(bytes);
}
await stream.WriteAsync(bytes.AsMemory(0, 8));
await stream.DisposeAsync();
// Console.WriteLine($"Connection number {connectionNumber} written to socket.");
}
public override async ValueTask DisposeAsync()
{
await _networkStream.DisposeAsync().ConfigureAwait(false);
await base.DisposeAsync().ConfigureAwait(false);
}
public async Task StartAsync(Tye.Hosting.Model.Application application)
{
_host = new HostBuilder()
.ConfigureServer(server =>
{
server.UseSockets(sockets =>
{
foreach (var service in application.Services.Values)
{
if (service.Description.RunInfo == null)
{
// We eventually want to proxy everything, this is temporary
continue;
}
static int GetNextPort()
{
// Let the OS assign the next available port. Unless we cycle through all ports
// on a test run, the OS will always increment the port number when making these calls.
// This prevents races in parallel test runs where a test is already bound to
// a given port, and a new test is able to bind to the same port due to port
// reuse being enabled by default by the OS.
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
return(((IPEndPoint)socket.LocalEndPoint).Port);
}
foreach (var binding in service.Description.Bindings)
{
if (binding.Port == null && !binding.AutoAssignPort)
{
continue;
}
if (binding.Port == null)
{
binding.Port = GetNextPort();
}
if (binding.Protocol == "http" || (binding.Protocol == null && service.ServiceType == Model.ServiceType.Project))
{
binding.ContainerPort = 80;
}
else if (binding.Protocol == "https")
{
binding.ContainerPort = 443;
}
if (service.Description.Replicas == 1)
{
// No need to proxy
service.PortMap[binding.Port.Value] = new List <int> {
binding.Port.Value
};
continue;
}
var ports = new List <int>();
for (var i = 0; i < service.Description.Replicas; i++)
{
// Reserve a port for each replica
var port = GetNextPort();
ports.Add(port);
}
_logger.LogInformation(
"Mapping external port {ExternalPort} to internal port(s) {InternalPorts} for {ServiceName} binding {BindingName}",
binding.Port,
string.Join(", ", ports.Select(p => p.ToString())),
service.Description.Name,
binding.Name ?? binding.Protocol);
service.PortMap[binding.Port.Value] = ports;
sockets.Listen(IPAddress.Loopback, binding.Port.Value, o =>
{
long count = 0;
// o.UseConnectionLogging("Tye.Proxy");
o.Run(async connection =>
{
var notificationFeature = connection.Features.Get <IConnectionLifetimeNotificationFeature>();
var next = (int)(Interlocked.Increment(ref count) % ports.Count);
NetworkStream?targetStream = null;
try
{
var target = new Socket(SocketType.Stream, ProtocolType.Tcp)
{
NoDelay = true
};
var port = ports[next];
_logger.LogDebug("Attempting to connect to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
await target.ConnectAsync(IPAddress.Loopback, port);
_logger.LogDebug("Successfully connected to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
targetStream = new NetworkStream(target, ownsSocket: true);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Proxy error for service {ServiceName}", service.Description.Name);
if (targetStream is object)
{
await targetStream.DisposeAsync();
}
connection.Abort();
return;
}
try
{
_logger.LogDebug("Proxying traffic to {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]);
// external -> internal
var reading = Task.Run(() => connection.Transport.Input.CopyToAsync(targetStream, notificationFeature.ConnectionClosedRequested));
// internal -> external
var writing = Task.Run(() => targetStream.CopyToAsync(connection.Transport.Output, notificationFeature.ConnectionClosedRequested));
await Task.WhenAll(reading, writing);
}
catch (ConnectionResetException)
{
// Connection was reset
}
catch (IOException)
{
// Reset can also appear as an IOException with an inner SocketException
}
catch (OperationCanceledException ex)
{
if (!notificationFeature.ConnectionClosedRequested.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
}
catch (Exception ex)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
finally
{
await targetStream.DisposeAsync();
}
// This needs to reconnect to the target port(s) until its bound
// it has to stop if the service is no longer running
});
});
}
}
public static async Task RunAsync(Application application, string[] args)
{
var options = new JsonSerializerOptions()
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true,
};
options.Converters.Add(ReplicaStatus.JsonConverter);
using var host = Host.CreateDefaultBuilder(args)
.UseSerilog((context, configuration) =>
{
// Logging for this application
configuration
.MinimumLevel.Verbose()
.Filter.ByExcluding(Matching.FromSource("Microsoft"))
.Enrich
.FromLogContext()
.WriteTo
.Console();
})
.ConfigureWebHostDefaults(web =>
{
web.ConfigureServices(services =>
{
services.AddRazorPages();
services.AddServerSideBlazor();
services.AddOptions <StaticFileOptions>()
.PostConfigure(o =>
{
// Make sure we don't remove the other file providers (blazor needs this)
o.FileProvider = new CompositeFileProvider(o.FileProvider, new ManifestEmbeddedFileProvider(typeof(MicronetesHost).Assembly, "wwwroot"));
});
services.AddSingleton(application);
});
web.ConfigureKestrel(options =>
{
var logger = options.ApplicationServices.GetRequiredService <ILogger <MicronetesHost> >();
var config = options.ApplicationServices.GetRequiredService <IConfiguration>();
if (config["port"] != null && int.TryParse(config["port"], out int cpPort))
{
// Use the specified port
options.Listen(IPAddress.Loopback, cpPort);
}
else
{
// This is lame but it allows running multiple versions of this
// we should also allow ports to be specified as input
options.Listen(IPAddress.Loopback, 0);
}
foreach (var service in application.Services.Values)
{
if (service.Description.External)
{
// We eventually want to proxy everything, this is temporary
continue;
}
static int GetNextPort()
{
// Let the OS assign the next available port. Unless we cycle through all ports
// on a test run, the OS will always increment the port number when making these calls.
// This prevents races in parallel test runs where a test is already bound to
// a given port, and a new test is able to bind to the same port due to port
// reuse being enabled by default by the OS.
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
return(((IPEndPoint)socket.LocalEndPoint).Port);
}
foreach (var binding in service.Description.Bindings)
{
if (binding.Port == null)
{
continue;
}
if (service.Description.Replicas == 1)
{
// No need to proxy
service.PortMap[binding.Port.Value] = new List <int> {
binding.Port.Value
};
continue;
}
var ports = new List <int>();
for (int i = 0; i < service.Description.Replicas; i++)
{
// Reserve a port for each replica
var port = GetNextPort();
ports.Add(port);
}
logger.LogInformation("Mapping external port {ExternalPort} to internal port(s) {InternalPorts} for {ServiceName}", binding.Port, string.Join(", ", ports.Select(p => p.ToString())), service.Description.Name);
service.PortMap[binding.Port.Value] = ports;
options.Listen(IPAddress.Loopback, binding.Port.Value, o =>
{
long count = 0;
// o.UseConnectionLogging("Micronetes.Proxy");
o.Run(async connection =>
{
var notificationFeature = connection.Features.Get <IConnectionLifetimeNotificationFeature>();
var next = (int)(Interlocked.Increment(ref count) % ports.Count);
NetworkStream targetStream = null;
try
{
var target = new Socket(SocketType.Stream, ProtocolType.Tcp)
{
NoDelay = true
};
var port = ports[next];
logger.LogDebug("Attempting to connect to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
await target.ConnectAsync(IPAddress.Loopback, port);
logger.LogDebug("Successfully connected to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
targetStream = new NetworkStream(target, ownsSocket: true);
}
catch (Exception ex)
{
logger.LogDebug(ex, "Proxy error for service {ServiceName}", service.Description.Name);
await targetStream.DisposeAsync();
connection.Abort();
return;
}
try
{
logger.LogDebug("Proxying traffic to {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]);
// external -> internal
var reading = Task.Run(() => connection.Transport.Input.CopyToAsync(targetStream, notificationFeature.ConnectionClosedRequested));
// internal -> external
var writing = Task.Run(() => targetStream.CopyToAsync(connection.Transport.Output, notificationFeature.ConnectionClosedRequested));
await Task.WhenAll(reading, writing);
}
catch (ConnectionResetException)
{
// Connection was reset
}
catch (OperationCanceledException ex)
{
if (!notificationFeature.ConnectionClosedRequested.IsCancellationRequested)
{
logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
}
catch (Exception ex)
{
logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
finally
{
await targetStream.DisposeAsync();
}
// This needs to reconnect to the target port(s) until its bound
// it has to stop if the service is no longer running
});
});
}
}
public Task StartAsync(Application application)
{
_host = new HostBuilder()
.ConfigureServer(server =>
{
server.UseSockets(sockets =>
{
foreach (var service in application.Services.Values)
{
if (service.Description.RunInfo == null)
{
continue;
}
service.Items[typeof(Subscription)] = service.ReplicaEvents.Subscribe(OnReplicaEvent);
foreach (var binding in service.Description.Bindings)
{
if (binding.Port == null)
{
// There's no port so nothing to proxy
continue;
}
if (service.Description.Readiness == null && service.Description.Replicas == 1)
{
// No need to proxy for a single replica, we may want to do this later but right now we skip it
continue;
}
var ports = binding.ReplicaPorts;
// We need to bind to all interfaces on linux since the container -> host communication won't work
// if we use the IP address to reach out of the host. This works fine on osx and windows
// but doesn't work on linux.
var host = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? IPAddress.Any : IPAddress.Loopback;
sockets.Listen(host, binding.Port.Value, o =>
{
long count = 0;
// o.UseConnectionLogging("Tye.Proxy");
o.Run(async connection =>
{
var notificationFeature = connection.Features.Get <IConnectionLifetimeNotificationFeature>();
var next = (int)(Interlocked.Increment(ref count) % ports.Count);
if (!_cancellationsByReplicaPort.TryGetValue(ports[next], out var cts))
{
// replica in ready state <=> it's ports have cancellation tokens in the dictionary
// if replica is not in ready state, we don't forward traffic, but return instead
return;
}
using var _ = cts.Token.Register(() => notificationFeature.RequestClose());
NetworkStream?targetStream = null;
try
{
var target = new Socket(SocketType.Stream, ProtocolType.Tcp)
{
NoDelay = true
};
var port = ports[next];
_logger.LogDebug("Attempting to connect to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
await target.ConnectAsync(IPAddress.Loopback, port);
_logger.LogDebug("Successfully connected to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
targetStream = new NetworkStream(target, ownsSocket: true);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Proxy error for service {ServiceName}", service.Description.Name);
if (targetStream is object)
{
await targetStream.DisposeAsync();
}
connection.Abort();
return;
}
try
{
_logger.LogDebug("Proxying traffic to {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]);
// external -> internal
var reading = Task.Run(() => connection.Transport.Input.CopyToAsync(targetStream, notificationFeature.ConnectionClosedRequested));
// internal -> external
var writing = Task.Run(() => targetStream.CopyToAsync(connection.Transport.Output, notificationFeature.ConnectionClosedRequested));
await Task.WhenAll(reading, writing);
}
catch (ConnectionResetException)
{
// Connection was reset
}
catch (IOException)
{
// Reset can also appear as an IOException with an inner SocketException
}
catch (OperationCanceledException ex)
{
if (!notificationFeature.ConnectionClosedRequested.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
_logger.LogDebug("Existing proxy {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]);
}
catch (Exception ex)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
finally
{
await targetStream.DisposeAsync();
}
// This needs to reconnect to the target port(s) until its bound
// it has to stop if the service is no longer running
});
});
}
}
});
})
.Build();
return(_host.StartAsync());
}
public virtual async ValueTask DisposeAsync()
{
await NetworkStream.DisposeAsync().ConfigureAwait(false);
lockStream.Dispose();
}
public Task StartAsync(Application application)
{
_host = new HostBuilder()
.ConfigureServer(server =>
{
server.UseSockets(sockets =>
{
foreach (var service in application.Services.Values)
{
if (service.Description.RunInfo == null)
{
continue;
}
foreach (var binding in service.Description.Bindings)
{
if (binding.Port == null)
{
// There's no port so nothing to proxy
continue;
}
if (service.Description.Replicas == 1)
{
// No need to proxy for a single replica, we may want to do this later but right now we skip it
continue;
}
var ports = binding.ReplicaPorts;
sockets.Listen(IPAddress.Loopback, binding.Port.Value, o =>
{
long count = 0;
// o.UseConnectionLogging("Tye.Proxy");
o.Run(async connection =>
{
var notificationFeature = connection.Features.Get <IConnectionLifetimeNotificationFeature>();
var next = (int)(Interlocked.Increment(ref count) % ports.Count);
NetworkStream?targetStream = null;
try
{
var target = new Socket(SocketType.Stream, ProtocolType.Tcp)
{
NoDelay = true
};
var port = ports[next];
_logger.LogDebug("Attempting to connect to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
await target.ConnectAsync(IPAddress.Loopback, port);
_logger.LogDebug("Successfully connected to {ServiceName} listening on {ExternalPort}:{Port}", service.Description.Name, binding.Port, port);
targetStream = new NetworkStream(target, ownsSocket: true);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Proxy error for service {ServiceName}", service.Description.Name);
if (targetStream is object)
{
await targetStream.DisposeAsync();
}
connection.Abort();
return;
}
try
{
_logger.LogDebug("Proxying traffic to {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]);
// external -> internal
var reading = Task.Run(() => connection.Transport.Input.CopyToAsync(targetStream, notificationFeature.ConnectionClosedRequested));
// internal -> external
var writing = Task.Run(() => targetStream.CopyToAsync(connection.Transport.Output, notificationFeature.ConnectionClosedRequested));
await Task.WhenAll(reading, writing);
}
catch (ConnectionResetException)
{
// Connection was reset
}
catch (IOException)
{
// Reset can also appear as an IOException with an inner SocketException
}
catch (OperationCanceledException ex)
{
if (!notificationFeature.ConnectionClosedRequested.IsCancellationRequested)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
}
catch (Exception ex)
{
_logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name);
}
finally
{
await targetStream.DisposeAsync();
}
// This needs to reconnect to the target port(s) until its bound
// it has to stop if the service is no longer running
});
});
}
}
});
})
.Build();
return(_host.StartAsync());
}
