private async Task <VostokApplicationRunResult> RunInternalAsync()
{
if (settings.ConfigureThreadPool)
{
ThreadPoolUtility.Setup(settings.ThreadPoolTuningMultiplier);
}
var result = BuildEnvironment();
if (result != null)
{
return(result);
}
using (environment)
using (new ApplicationDisposable(settings.Application, environment, log))
{
result = WarmupEnvironment();
if (result != null)
{
return(result);
}
result = await InitializeApplicationAsync().ConfigureAwait(false);
if (result.State == VostokApplicationState.Initialized)
{
result = await RunApplicationAsync().ConfigureAwait(false);
}
return(result);
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
timer = new Timer(
x => Log.For("ThreadPool").Information(ThreadPoolUtility.GetThreadPoolState().ToString()), null,
period: TimeSpan.FromMinutes(1), dueTime: TimeSpan.Zero);
return(Task.CompletedTask);
}
public bool Start(HostControl hostControl)
{
try
{
log.Information("Configuring container");
ConfigureContainer();
log.Information("Starting service");
ProcessPriorityHelper.SetMemoryPriority(ProcessMemoryPriority.Normal, new SerilogLog(log));
ProcessPriorityHelper.SetProcessPriorityClass(ProcessPriorityClass.Normal, new SerilogLog(log));
if (!settings.TryGetInt("ThreadMultiplier", out var threadMultiplier))
{
threadMultiplier = 16;
log.Warning("`ThreadMultiplier` setting not found. Use multiplier {multiplier}", threadMultiplier);
}
ThreadPoolUtility.SetUp(new SerilogLog(log), threadMultiplier);
StartChildServices();
log.Information("Service is started");
Console.WriteLine("Service is started");
return(true);
}
catch (Exception e)
{
log.Error(e, "Unexpected error while starting service");
(log as IDisposable)?.Dispose();
throw;
}
}
public void ReportAndFixIfNeeded(ILog log)
{
var state = ThreadPoolUtility.GetPoolState();
if (state.UsedWorkerThreads < state.MinWorkerThreads &&
state.UsedIocpThreads < state.MinIocpThreads)
{
return;
}
var currentTimestamp = DateTime.UtcNow;
lock (syncObject)
{
if (currentTimestamp - lastReportTimestamp < minReportInterval)
{
return;
}
lastReportTimestamp = currentTimestamp;
}
log.Warn(
"Looks like you're kinda low on ThreadPool, buddy. " +
$"Workers: {state.UsedWorkerThreads}/{state.MinWorkerThreads}/{state.MaxWorkerThreads}, " +
$"IOCP: {state.UsedIocpThreads}/{state.MinIocpThreads}/{state.MaxIocpThreads} (busy/min/max).");
var currentMultiplier = Math.Min(state.MinWorkerThreads / Environment.ProcessorCount, state.MinIocpThreads / Environment.ProcessorCount);
if (currentMultiplier < 128)
{
log.Info("I will configure ThreadPool for you, buddy!");
ThreadPoolUtility.Setup(log);
}
}
protected TransportTestsBase()
{
ILog log = new ConsoleLog();
Transport = ClusterClientConfigurationHelper.CreateTransport(log);
ThreadPoolUtility.Setup(log);
}
private void ConfigureHostBeforeRun()
{
var cpuUnitsLimit = environment.ApplicationLimits.CpuUnits;
if (settings.ConfigureThreadPool && cpuUnitsLimit.HasValue)
{
ThreadPoolUtility.Setup(settings.ThreadPoolTuningMultiplier, cpuUnitsLimit.Value);
}
}
public SystemMetrics GetSystemMetrics()
{
var threadPoolState = ThreadPoolUtility.GetPoolState();
var sample = requestCounters.NextSample();
return(new SystemMetrics
{
UsedThreads = threadPoolState.UsedThreads,
UsedIocpThreads = threadPoolState.UsedIocpThreads,
WorkingRequests = sample.WorkingRequests,
RequestsPerSecond = sample.AverageRps
});
}
private Task <VostokMultiHostRunResult> StartInternalAsync()
{
// NOTE: Configure thread pool once there if necessary.
if (settings.ConfigureThreadPool)
{
ThreadPoolUtility.Setup(settings.ThreadPoolTuningMultiplier);
}
BuildCommonEnvironment()?.EnsureSuccess();
StartAddedApplications();
return(Task.FromResult(ReturnResult(VostokMultiHostState.Running)));
}
public static void ReportAndFixIfNeeded(ILog log)
{
var state = ThreadPoolUtility.GetPoolState();
if (state.UsedWorkerThreads < state.MinWorkerThreads &&
state.UsedIocpThreads < state.MinIocpThreads)
{
return;
}
var currentTimestamp = DateTime.UtcNow;
lock (syncObject)
{
if (currentTimestamp - lastReportTimestamp < MinReportInterval)
{
return;
}
lastReportTimestamp = currentTimestamp;
}
log = log.ForContext(typeof(ThreadPoolMonitor));
log.Warn(
"Looks like you're kinda low on ThreadPool, buddy. " +
"Workers: {UsedWorkerThreads}/{MinWorkerThreads}, " +
"IOCP: {UsedIocpThreads}/{MinIocpThreads} (busy/min).",
state.UsedWorkerThreads,
state.MinWorkerThreads,
state.UsedIocpThreads,
state.MinIocpThreads);
var currentMultiplier = Math.Min(
state.MinWorkerThreads / Environment.ProcessorCount,
state.MinIocpThreads / Environment.ProcessorCount);
if (currentMultiplier < TargetMultiplier)
{
log.Info("I will configure ThreadPool for you, buddy!");
ThreadPoolUtility.Setup(TargetMultiplier);
var newState = ThreadPoolUtility.GetPoolState();
log.Info("New min worker threads = {MinWorkerThreads}", newState.MinWorkerThreads);
log.Info("New min IOCP threads = {MinIocpThreads}", newState.MinIocpThreads);
}
}
private void CheckAndUpdate()
{
var newThreadPoolMultiplier = settingsProvider(configProvider).ThreadPoolMultiplier;
var newCpuUnits = applicationLimits.CpuUnits;
if (WereSettingsUpdated(newThreadPoolMultiplier, newCpuUnits))
{
ThreadPoolUtility.Setup(newThreadPoolMultiplier, newCpuUnits);
previousThreadPoolMultiplier = newThreadPoolMultiplier;
previousCpuUnits = newCpuUnits;
LogThreadPoolSettings(newThreadPoolMultiplier, newCpuUnits);
}
}
public Task <HealthCheckResult> CheckAsync(CancellationToken cancellationToken)
{
var state = ThreadPoolUtility.GetPoolState();
if (state.UsedWorkerThreads >= state.MinWorkerThreads)
{
return(Task.FromResult(HealthCheckResult.Degraded($"Worker threads in thread pool are exhausted: {state.UsedWorkerThreads}/{state.MinWorkerThreads} (used/min).")));
}
if (state.UsedIocpThreads >= state.MinIocpThreads)
{
return(Task.FromResult(HealthCheckResult.Degraded($"IOCP threads in thread pool are exhausted: {state.UsedIocpThreads}/{state.MinIocpThreads} (used/min).")));
}
return(Task.FromResult(HealthCheckResult.Healthy()));
}
/// <summary>
/// <para>Launches the provided <see cref="IVostokApplication"/>.</para>
/// <para>Performs following operations:</para>
/// <list type="bullet">
/// <item><description>Creates an instance of <see cref="IVostokHostingEnvironment"/> using <see cref="VostokHostSettings.EnvironmentSetup"/>.</description></item>
/// <item><description>Configures thread pool if <see cref="VostokHostSettings.ConfigureThreadPool"/> specified.</description></item>
/// <item><description>Configures static providers if <see cref="VostokHostSettings.ConfigureStaticProviders"/> specified.</description></item>
/// <item><description>Calls <see cref="IVostokApplication.InitializeAsync"/>.</description></item>
/// <item><description>Calls <see cref="IVostokApplication.RunAsync"/>.</description></item>
/// <item><description>Stops application if <see cref="ShutdownTokenSource"/> has been canceled.</description></item>
/// </list>
/// <para>May throw an exception if an error occurs during environment creation.</para>
/// <para>Does not rethrow exceptions from <see cref="IVostokApplication"/>, stores them in result's <see cref="VostokApplicationRunResult.Error"/> property.</para>
/// </summary>
public virtual async Task <VostokApplicationRunResult> RunAsync()
{
if (!launchedOnce.TrySetTrue())
{
throw new InvalidOperationException("Application can't be launched multiple times.");
}
if (settings.ConfigureThreadPool)
{
ThreadPoolUtility.Setup(settings.ThreadPoolTuningMultiplier);
}
var result = BuildEnvironment();
if (result != null)
{
return(result);
}
using (environment)
using (settings.Application as IDisposable)
{
result = WarmupEnvironment();
if (result != null)
{
return(result);
}
result = await InitializeApplicationAsync().ConfigureAwait(false);
if (result.State == VostokApplicationState.Initialized)
{
result = await RunApplicationAsync().ConfigureAwait(false);
}
return(result);
}
}
private void SetUpServices(IAppBuilder appBuilder, Container container)
{
var log = container.GetInstance <ILog>();
var serviceSettings = container.GetInstance <IServiceSettings>();
ProcessPriorityHelper.SetMemoryPriority(ProcessMemoryPriority.Normal, log);
ProcessPriorityHelper.SetProcessPriorityClass(ProcessPriorityClass.Normal, log);
ThreadPoolUtility.SetUp(log, serviceSettings.GetThreadMultiplier());
var serviceBeacon = new ServiceBeacon(log, serviceSettings.GetPort());
serviceBeacon.Start();
var metricsWorker = container.GetInstance <MetricsWorker>();
metricsWorker.Start();
appBuilder.OnDisposing(
() =>
{
serviceBeacon.Stop();
metricsWorker.Stop();
});
}
private void LogThreadPoolSettings()
{
var state = ThreadPoolUtility.GetPoolState();
log.Info("Thread pool configuration: {MinWorkerThreads} min workers, {MinIOCPThreads} min IOCP.", state.MinWorkerThreads, state.MinIocpThreads);
}
public async Task RunAsync(
string workerName,
int processCount,
int maxWorkerPerProcess, // TODO: remove it
int executePerWorker, // TODO: remove it
ConsoleAppFramework.ConsoleAppContext consoleAppContext) // TODO:remove this context.
{
var cancellationToken = consoleAppContext.CancellationToken;
var args = consoleAppContext.Arguments;
ThreadPoolUtility.SetMinThread(maxWorkerPerProcess);
// validate worker is exists.
if (!workers.TryGetWorker(workerName, out var workerInfo))
{
throw new InvalidOperationException($"Worker:{workerName} does not found in assembly.");
}
var failSignal = new TaskFailSignal();
using (masterHost = StartMasterHost(args, cancellationToken))
await using (options.ScalingProvider)
{
var workerConnection = masterHost.Services.GetRequiredService <WorkerConnectionGroupContext>();
workerConnection.Initialize(processCount, options.WorkerDisconnectedBehaviour == WorkerDisconnectedBehaviour.Stop);
logger.LogInformation("Starting worker nodes.");
await options.ScalingProvider.StartWorkerAsync(options, processCount, provider, failSignal, cancellationToken).WithTimeoutAndCancellationAndTaskSignal(options.Timeout, cancellationToken, failSignal.Task);
// wait worker is connected
await workerConnection.WaitAllConnectedWithTimeoutAsync(options.Timeout, cancellationToken, failSignal.Task);
var broadcaster = workerConnection.Broadcaster;
Master?master = default;
if (workerInfo.MasterType != null)
{
master = provider.GetRequiredService(workerInfo.MasterType) as Master;
}
// MasterSetup
if (master != null)
{
logger.LogTrace("Invoke MasterSetup");
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
linkedToken.CancelAfter(options.Timeout);
await master.SetupAsync(linkedToken.Token);
}
}
await CreateCoWorkerAndExecuteAsync(broadcaster, workerConnection, workerName, cancellationToken, failSignal);
// Worker Teardown
logger.LogTrace("Send SetTeardownup command to workers and wait complete message.");
broadcaster.Teardown();
await workerConnection.OnTeardown.WaitWithTimeoutAsync(options.Timeout, cancellationToken, failSignal.Task);
options.OnExecuteResult?.Invoke(workerConnection.ExecuteResult.ToArray(), options, new ExecuteScenario(workerName, processCount, maxWorkerPerProcess, executePerWorker));
// Worker Shutdown
broadcaster.Shutdown();
// MasterTeardown
if (master != null)
{
logger.LogTrace("Invoke MasterTeardown");
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
linkedToken.CancelAfter(options.Timeout);
await master.TeardownAsync(linkedToken.Token);
}
}
await Task.Delay(TimeSpan.FromSeconds(1)); // wait Shutdown complete?
}
logger.LogInformation("Master shutdown.");
}
public void OneTimeSetUp()
{
ThreadPoolUtility.Setup();
LogProvider.Configure(new SynchronousConsoleLog());
}
public void OneTimeSetUp()
{
ThreadPoolUtility.Setup();
}
