static public IWorker CreateWorker(ClientJob clientJob)
{
IWorker worker = null;
switch (clientJob.Client)
{
case Worker.Wrk:
worker = new WrkWorker();
break;
case Worker.SignalR:
worker = new SignalRWorker();
break;
case Worker.Wait:
worker = new WaitWorker();
break;
case Worker.H2Load:
worker = new H2LoadWorker();
break;
case Worker.Bombardier:
worker = new BombardierWorker();
break;
}
return(worker);
}
public async Task StartJobAsync(ClientJob job)
{
_job = job;
byte[] requestBody = null;
if (job.ClientProperties.TryGetValue("RequestBody", out var requestBodyText))
{
requestBody = Convert.FromBase64String(requestBodyText);
_requestBodyLength = requestBody.Length;
// h2load takes a file as the request body
// write the body to a temporary file that is deleted in stop job
_requestBodyTempFile = Path.GetTempFileName();
await File.WriteAllBytesAsync(_requestBodyTempFile, requestBody);
}
InitializeJob();
await MeasureFirstRequestLatencyAsync(_job, requestBody);
_job.State = ClientState.Running;
_job.LastDriverCommunicationUtc = DateTime.UtcNow;
_process = StartProcess(_job, _requestBodyTempFile);
}
private static HttpRequestMessage CreateHttpMessage(ClientJob job, byte[] requestBody)
{
var requestMessage = new HttpRequestMessage(new HttpMethod(job.Method), job.ServerBenchmarkUri);
requestMessage.Version = new Version(2, 0);
var headers = job.Headers.ToList();
if (requestBody != null)
{
requestMessage.Content = new ByteArrayContent(requestBody);
foreach (var header in headers.Where(h => ContentHeaders.Contains(h.Key)))
{
requestMessage.Content.Headers.Add(header.Key, header.Value);
}
// Prune content headers. An error will be thrown if they are added to request headers.
headers.RemoveAll(h => ContentHeaders.Contains(h.Key));
}
foreach (var header in headers)
{
requestMessage.Headers.Add(header.Key, header.Value);
}
return(requestMessage);
}
private Task WriteJobsToSql(ServerJob serverJob, ClientJob clientJob, DateTime utcNow, string connectionString, string tableName, string path, string session, string description, string dimension, double value)
{
return(RetryOnExceptionAsync(5, () =>
WriteResultsToSql(
utcNow,
connectionString: connectionString,
tableName: tableName,
scenario: serverJob.Scenario,
session: session,
description: description,
aspnetCoreVersion: serverJob.AspNetCoreVersion,
runtimeVersion: serverJob.RuntimeVersion,
hardware: serverJob.Hardware.Value,
hardwareVersion: serverJob.HardwareVersion,
operatingSystem: serverJob.OperatingSystem.Value,
scheme: serverJob.Scheme,
source: serverJob.Source,
connectionFilter: serverJob.ConnectionFilter,
webHost: serverJob.WebHost,
kestrelThreadCount: serverJob.KestrelThreadCount,
clientThreads: clientJob.Threads,
connections: clientJob.Connections,
duration: clientJob.Duration,
pipelineDepth: clientJob.PipelineDepth,
path: path,
method: clientJob.Method,
headers: clientJob.Headers,
dimension: dimension,
value: value,
runtimeStore: serverJob.UseRuntimeStore)
, 5000));
}
public async Task StartJobAsync(ClientJob job)
{
_job = job;
Log($"Starting Job");
await InitializeJob();
// start connections
var tasks = new List <Task>(_clients.Count);
foreach (var client in _clients)
{
tasks.Add(client.ConnectAsync(new Uri(_job.ServerBenchmarkUri)));
}
await Task.WhenAll(tasks);
_job.State = ClientState.Running;
_job.LastDriverCommunicationUtc = DateTime.UtcNow;
_cancelationTokenSource = new CancellationTokenSource();
_cancelationTokenSource.CancelAfter(TimeSpan.FromSeconds(_job.Duration));
_workTimer.Restart();
try
{
switch (_scenario)
{
case "Basic":
await Basic(_cancelationTokenSource.Token);
break;
case "FormInput":
await FormInput(_cancelationTokenSource.Token);
break;
case "BackgroundUpdates":
await BackgroundUpdates(_cancelationTokenSource.Token);
break;
default:
throw new Exception($"Scenario '{_scenario}' is not a known scenario.");
}
}
catch (Exception ex)
{
var text = "Exception from test: " + ex;
Log(text);
_job.Error += Environment.NewLine + text;
}
_cancelationTokenSource.Token.WaitHandle.WaitOne();
await StopJobAsync();
}
static public IResultsSerializer CreateResultSerializer(ClientJob clientJob)
{
if (ResultSerializers.TryGetValue(clientJob.Client, out var serializerFactory))
{
return(serializerFactory());
}
return(null);
}
private static HttpRequestMessage CreateHttpMessage(ClientJob job)
{
var requestMessage = new HttpRequestMessage(new HttpMethod(job.Method), job.ServerBenchmarkUri);
foreach (var header in job.Headers)
{
requestMessage.Headers.Add(header.Key, header.Value);
}
return(requestMessage);
}
public IActionResult Create([FromBody] ClientJob job)
{
if (job == null || job.Id != 0 || job.State != ClientState.Waiting)
{
return(BadRequest());
}
job = _jobs.Add(job);
Response.Headers["Location"] = $"/jobs/{job.Id}";
return(new StatusCodeResult((int)HttpStatusCode.Accepted));
}
public JobInfoViewModel(ClientJob clientJob)
{
_clientJob = clientJob;
StartDate = DateTime.Now;
JobInput = clientJob.Job.GetInput();
_statusPropertyChangedProxy = new PropertyChangedProxy <ClientJob, string>(clientJob, m => m.HandlerName, newValue =>
{
OnPropertyChanged(() => HandlerName);
});
}
public async Task StartJobAsync(ClientJob job)
{
_job = job;
InitializeJob();
await MeasureFirstRequestLatencyAsync(_job);
_job.State = ClientState.Running;
_job.LastDriverCommunicationUtc = DateTime.UtcNow;
_process = StartProcess(_job);
}
static public BaseWorker CreateWorker(ClientJob clientJob)
{
BaseWorker worker = new BaseWorker(clientJob);
//switch (clientJob.Client)
//{
// //case Worker.SignalRCoreEcho:
// // worker = new SignalRCoreEchoWorker(clientJob);
// // break;
// default:
// worker = new BaseWorker(clientJob);
//}
return(worker);
}
public Task StartJobAsync(ClientJob job)
{
_job = job;
_job.State = ClientState.Running;
_job.LastDriverCommunicationUtc = DateTime.UtcNow;
_cts = new CancellationTokenSource();
_task = Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(_job.Duration));
_job.State = ClientState.Completed;
}, _cts.Token);
return(Task.CompletedTask);
}
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job)
{
// Ignoring startup latency when using h2c as HttpClient doesn't support
if (job.ServerBenchmarkUri.StartsWith("http:", StringComparison.OrdinalIgnoreCase))
{
Log("Ignoring first request latencies on h2c");
return;
}
if (job.SkipStartupLatencies)
{
return;
}
Log($"Measuring first request latency on {job.ServerBenchmarkUri}");
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var response = await _httpClient.SendAsync(CreateHttpMessage(job)))
{
var responseContent = await response.Content.ReadAsStringAsync();
job.LatencyFirstRequest = stopwatch.Elapsed;
}
Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms");
Log("Measuring subsequent requests latency");
for (var i = 0; i < 10; i++)
{
stopwatch.Restart();
using (var response = await _httpClient.SendAsync(CreateHttpMessage(job)))
{
var responseContent = await response.Content.ReadAsStringAsync();
// We keep the last measure to simulate a warmup phase.
job.LatencyNoLoad = stopwatch.Elapsed;
}
}
Log($"{job.LatencyNoLoad.TotalMilliseconds} ms");
}
public async Task WriteJobResultsToSqlAsync(
ServerJob serverJob,
ClientJob clientJob,
string sqlConnectionString,
string tableName,
string path,
string session,
string description,
Statistics statistics,
bool longRunning)
{
var utcNow = DateTime.UtcNow;
var scenario = serverJob.Scenario;
foreach (var result in CsvResults)
{
serverJob.Scenario = $"{scenario}.{result.Class}.{result.Method}{result.Params ?? ""}";
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "OperationsPerSecond",
value : result.OperationsPerSecond);
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Allocated (KB)",
value : result.Allocated);
}
serverJob.Scenario = scenario;
}
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job)
{
if (job.SkipStartupLatencies)
{
return;
}
Log($"Measuring first request latency on {job.ServerBenchmarkUri}");
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var message = CreateHttpMessage(job))
{
using (var response = await _httpClient.SendAsync(message))
{
var responseContent = await response.Content.ReadAsStringAsync();
job.LatencyFirstRequest = stopwatch.Elapsed;
}
}
Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms");
Log("Measuring subsequent requests latency");
for (var i = 0; i < 10; i++)
{
stopwatch.Restart();
using (var message = CreateHttpMessage(job))
{
using (var response = await _httpClient.SendAsync(message))
{
var responseContent = await response.Content.ReadAsStringAsync();
// We keep the last measure to simulate a warmup phase.
job.LatencyNoLoad = stopwatch.Elapsed;
}
}
}
Log($"{job.LatencyNoLoad.TotalMilliseconds} ms");
}
static public IWorker CreateWorker(ClientJob clientJob)
{
IWorker worker;
switch (clientJob.Client)
{
case Worker.Wrk:
worker = new WrkWorker();
break;
case Worker.Wrk2:
worker = new Wrk2Worker();
break;
case Worker.SignalR:
worker = new SignalRWorker();
break;
case Worker.Grpc:
worker = new GrpcWorker();
break;
case Worker.Wait:
worker = new WaitWorker();
break;
case Worker.H2Load:
worker = new H2LoadWorker();
break;
case Worker.Bombardier:
worker = new BombardierWorker();
break;
case Worker.BlazorIgnitor:
worker = new BlazorIgnitor();
break;
default:
throw new InvalidOperationException($"Unknown worker {clientJob.Client}.");
}
return(worker);
}
public IActionResult Create([FromBody] ClientJob job)
{
if (job == null)
{
Log("Invalid job");
return(BadRequest("Invalid job"));
}
if (job.Id != 0)
{
Log("Can't create an existing job");
return(BadRequest("Job.Id must not specified."));
}
if (job.State != ClientState.Initializing)
{
Log("Job should have ne in Initializing state");
return(BadRequest("Job should have ne in Initializing state"));
}
job = _jobs.Add(job);
return(AcceptedAtAction(nameof(GetById), new { id = job.Id }, job));
}
private static async Task ProcessJobs(CancellationToken cancellationToken)
{
IWorker worker = null;
ClientJob job = null;
var whenLastJobCompleted = DateTime.MinValue;
var waitForMoreJobs = false;
while (!cancellationToken.IsCancellationRequested)
{
var allJobs = _jobs.GetAll();
// Dequeue the first job. We will only pass jobs that have
// the same SpanId to the current worker.
job = allJobs.FirstOrDefault(newJob =>
{
// If the job is null then we don't have a span id to match against.
// Otherwise we want to pick jobs with the same span id.
return(job == null || string.Equals(newJob.SpanId, job.SpanId, StringComparison.OrdinalIgnoreCase));
});
if (job != null)
{
// A spanId means that a span is defined and we might run
// multiple jobs.
if (!string.IsNullOrEmpty(job.SpanId))
{
waitForMoreJobs = true;
}
if (job.State == ClientState.Waiting)
{
Log($"Starting '{job.Client}' worker");
Log($"Current Job SpanId '{job.SpanId}'");
job.State = ClientState.Starting;
try
{
if (worker == null)
{
worker = WorkerFactory.CreateWorker(job);
}
if (worker == null)
{
Log($"Error while creating the worker");
job.State = ClientState.Deleting;
whenLastJobCompleted = DateTime.UtcNow;
}
else
{
await worker.StartJobAsync(job);
}
}
catch (Exception e)
{
Log($"An unexpected error occurred while starting the job {job.Id}");
Log(e.ToString());
job.State = ClientState.Deleting;
}
}
else if (job.State == ClientState.Running || job.State == ClientState.Completed)
{
var now = DateTime.UtcNow;
// Clean the job in case the driver is not running
if (now - job.LastDriverCommunicationUtc > TimeSpan.FromSeconds(30))
{
Log($"Driver didn't communicate for {now - job.LastDriverCommunicationUtc}. Halting job.");
job.State = ClientState.Deleting;
}
}
else if (job.State == ClientState.Deleting)
{
Log($"Deleting job {worker?.JobLogText ?? "no worker found"}");
try
{
if (worker != null)
{
await worker.StopJobAsync();
// Reset the last job completed indicator.
whenLastJobCompleted = DateTime.UtcNow;
}
}
finally
{
_jobs.Remove(job.Id);
job = null;
}
}
}
await Task.Delay(100);
// job will be null if there aren't any more jobs with the same spanId.
if (job == null)
{
// Currently no jobs with the same span id exist so we check if we can
// clear out the worker to signal to the worker factory to create
// a new one.
if (worker != null)
{
var now = DateTime.UtcNow;
// Disposing the worker conditions
// 1. A span isn't defined so there won't be any more jobs for this worker
// 2. We check that whenLastJob completed is something other that it's default value
// and 10 seconds have passed since the last job was completed.
if (!waitForMoreJobs || (whenLastJobCompleted != DateTime.MinValue && now - whenLastJobCompleted > TimeSpan.FromSeconds(10)))
{
Log("Job queuing timeout reached. Disposing worker.");
waitForMoreJobs = false;
await worker.DisposeAsync();
worker = null;
}
}
}
}
}
public async Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning)
{
var utcNow = DateTime.UtcNow;
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "RequestsPerSecond", statistics.RequestsPerSecond);
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "CPU", statistics.Cpu);
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "WorkingSet (MB)", statistics.WorkingSet);
if (statistics.LatencyAverage != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency Average (ms)", statistics.LatencyAverage);
}
if (statistics.Latency50Percentile != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency50Percentile (ms)", statistics.Latency50Percentile);
}
if (statistics.Latency75Percentile != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency75Percentile (ms)", statistics.Latency75Percentile);
}
if (statistics.Latency90Percentile != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency90Percentile (ms)", statistics.Latency90Percentile);
}
if (statistics.Latency99Percentile != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency99Percentile (ms)", statistics.Latency99Percentile);
}
if (statistics.MaxLatency != -1)
{
await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "MaxLatency (ms)", statistics.MaxLatency);
}
if (statistics.Other.Any())
{
foreach (var counter in Program.Counters)
{
if (!statistics.Other.ContainsKey(counter.Name))
{
continue;
}
if (statistics.Other[counter.Name] != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : connectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : counter.DisplayName,
value : statistics.Other[counter.Name]);
}
}
}
}
private static Process StartProcess(ClientJob job)
{
var tcs = new TaskCompletionSource <bool>();
var command = $"wrk -c {job.Connections} -t {job.Threads} -d {job.Duration}";
if (job.Headers != null)
{
foreach (var header in job.Headers)
{
command += $" -H \"{header}\"";
}
}
if (job.PipelineDepth > 0)
{
command += $" -s scripts/pipeline.lua";
}
command += $" {job.ServerBenchmarkUri}";
if (job.PipelineDepth > 0)
{
command += $" -- {job.PipelineDepth}";
}
var process = new Process()
{
StartInfo =
{
FileName = "stdbuf",
Arguments = $"-oL {command}",
WorkingDirectory = Path.GetDirectoryName(typeof(Startup).GetTypeInfo().Assembly.Location),
RedirectStandardOutput = true,
RedirectStandardError = true
},
EnableRaisingEvents = true
};
process.OutputDataReceived += (_, e) =>
{
Log(e.Data);
job.Output += (e.Data + Environment.NewLine);
};
process.ErrorDataReceived += (_, e) =>
{
Log(e.Data);
job.Error += (e.Data + Environment.NewLine);
};
process.Exited += (_, __) =>
{
double rps = -1;
var match = Regex.Match(job.Output, @"Requests/sec:\s*([\d.]*)");
if (match.Success && match.Groups.Count == 2)
{
double.TryParse(match.Groups[1].Value, out rps);
}
job.RequestsPerSecond = rps;
job.State = ClientState.Completed;
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
return(process);
}
public async Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning)
{
var utcNow = DateTime.UtcNow;
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "RequestsPerSecond", statistics.RequestsPerSecond, retry);
});
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "CPU", statistics.Cpu, retry);
});
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "WorkingSet (MB)", statistics.WorkingSet, retry);
});
if (statistics.LatencyAverage != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency Average (ms)", statistics.LatencyAverage, retry);
});
}
if (statistics.Latency50Percentile != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency50Percentile (ms)", statistics.Latency50Percentile, retry);
});
}
if (statistics.Latency75Percentile != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency75Percentile (ms)", statistics.Latency75Percentile, retry);
});
}
if (statistics.Latency90Percentile != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency90Percentile (ms)", statistics.Latency90Percentile, retry);
});
}
if (statistics.Latency99Percentile != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency99Percentile (ms)", statistics.Latency99Percentile, retry);
});
}
if (statistics.MaxLatency != -1)
{
await RetryOnExceptionAsync(5, async (retry) =>
{
await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "MaxLatency (ms)", statistics.MaxLatency, retry);
});
}
}
private async Task WriteJobResultToSqlAsync(ServerJob serverJob, ClientJob clientJob, DateTime utcNow, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning, string dimension, double value, bool checkExisting)
{
if (checkExisting)
{
if (await CheckForRow(connectionString, tableName, utcNow, dimension, session) == true)
{
return;
}
}
string insertCmd =
@"
INSERT INTO [dbo].[" + tableName + @"]
([DateTime]
,[Session]
,[Description]
,[AspNetCoreVersion]
,[RuntimeVersion]
,[Scenario]
,[Hardware]
,[HardwareVersion]
,[OperatingSystem]
,[Framework]
,[RuntimeStore]
,[Scheme]
,[Sources]
,[WebHost]
,[Transport]
,[HubProtocol]
,[ClientProperties]
,[Connections]
,[Duration]
,[Path]
,[Headers]
,[Dimension]
,[Value])
VALUES
(@DateTime
,@Session
,@Description
,@AspNetCoreVersion
,@RuntimeVersion
,@Scenario
,@Hardware
,@HardwareVersion
,@OperatingSystem
,@Framework
,@RuntimeStore
,@Scheme
,@Sources
,@WebHost
,@Transport
,@HubProtocol
,@ClientProperties
,@Connections
,@Duration
,@Path
,@Headers
,@Dimension
,@Value)
";
using (var connection = new SqlConnection(connectionString))
{
await connection.OpenAsync();
var transaction = connection.BeginTransaction();
try
{
var command = new SqlCommand(insertCmd, connection, transaction);
var p = command.Parameters;
p.AddWithValue("@DateTime", utcNow);
p.AddWithValue("@Session", session);
p.AddWithValue("@Description", description);
p.AddWithValue("@AspNetCoreVersion", serverJob.AspNetCoreVersion);
p.AddWithValue("@RuntimeVersion", serverJob.RuntimeVersion);
p.AddWithValue("@Scenario", serverJob.Scenario.ToString());
p.AddWithValue("@Hardware", serverJob.Hardware.ToString());
p.AddWithValue("@HardwareVersion", serverJob.HardwareVersion);
p.AddWithValue("@OperatingSystem", serverJob.OperatingSystem.ToString());
p.AddWithValue("@Framework", "Core");
p.AddWithValue("@RuntimeStore", serverJob.UseRuntimeStore);
p.AddWithValue("@Scheme", serverJob.Scheme.ToString().ToLowerInvariant());
p.AddWithValue("@Sources", serverJob.Source != null ? ConvertToSqlString(serverJob.Source) : (object)DBNull.Value);
p.AddWithValue("@WebHost", serverJob.WebHost.ToString());
p.AddWithValue("@Transport", clientJob.ClientProperties["TransportType"]);
p.AddWithValue("@HubProtocol", clientJob.ClientProperties["HubProtocol"]);
p.AddWithValue("@ClientProperties", JsonConvert.SerializeObject(clientJob.ClientProperties));
p.AddWithValue("@Connections", clientJob.Connections);
p.AddWithValue("@Duration", clientJob.Duration);
p.AddWithValue("@Path", string.IsNullOrEmpty(path) ? (object)DBNull.Value : path);
p.AddWithValue("@Headers", clientJob.Headers.Any() ? JsonConvert.SerializeObject(clientJob.Headers) : (object)DBNull.Value);
p.AddWithValue("@Dimension", dimension);
p.AddWithValue("@Value", value);
await command.ExecuteNonQueryAsync();
transaction.Commit();
}
catch
{
transaction.Rollback();
throw;
}
finally
{
transaction.Dispose();
}
}
}
public BaseWorker(ClientJob job)
{
_pkg.Job = job;
}
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job)
{
if (job.SkipStartupLatencies)
{
return;
}
Log($"Measuring first request latency on {job.ServerBenchmarkUri}");
var stopwatch = new Stopwatch();
stopwatch.Start();
using (var message = CreateHttpMessage(job))
{
var cts = new CancellationTokenSource();
var token = cts.Token;
cts.CancelAfter(FirstRequestTimeout);
token.ThrowIfCancellationRequested();
try
{
using (var response = await _httpClient.SendAsync(message, token))
{
job.LatencyFirstRequest = stopwatch.Elapsed;
}
}
catch (OperationCanceledException)
{
Log("A timeout occurred while measuring the first request: " + FirstRequestTimeout.ToString());
}
finally
{
cts.Dispose();
}
}
Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms");
Log("Measuring subsequent requests latency");
for (var i = 0; i < 10; i++)
{
stopwatch.Restart();
using (var message = CreateHttpMessage(job))
{
var cts = new CancellationTokenSource();
var token = cts.Token;
cts.CancelAfter(LatencyTimeout);
token.ThrowIfCancellationRequested();
try
{
using (var response = await _httpClient.SendAsync(message))
{
// We keep the last measure to simulate a warmup phase.
job.LatencyNoLoad = stopwatch.Elapsed;
}
}
catch (OperationCanceledException)
{
Log("A timeout occurred while measuring the latency, skipping ...");
break;
}
finally
{
cts.Dispose();
}
}
}
Log($"{job.LatencyNoLoad.TotalMilliseconds} ms");
}
public async Task WriteJobResultsToSqlAsync(
ServerJob serverJob,
ClientJob clientJob,
string sqlConnectionString,
string tableName,
string path,
string session,
string description,
Statistics statistics,
bool longRunning)
{
var utcNow = DateTime.UtcNow;
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "RequestsPerSecond",
value : statistics.RequestsPerSecond);
if (statistics.StartupMain != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Startup Main (ms)",
value : statistics.StartupMain);
}
if (statistics.BuildTime != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Build Time (ms)",
value : statistics.BuildTime);
}
if (statistics.PublishedSize != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Published Size (KB)",
value : statistics.PublishedSize);
}
if (statistics.FirstRequest != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "First Request (ms)",
value : statistics.FirstRequest);
}
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "WorkingSet (MB)",
value : statistics.WorkingSet);
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "CPU",
value : statistics.Cpu);
if (statistics.Latency != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
session : session,
description : description,
path : serverJob.Path,
dimension : "Latency (ms)",
value : statistics.Latency);
}
if (statistics.LatencyAverage != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "LatencyAverage (ms)",
value : statistics.LatencyAverage);
}
if (statistics.Latency50Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency50Percentile (ms)",
value : statistics.Latency50Percentile);
}
if (statistics.Latency75Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency75Percentile (ms)",
value : statistics.Latency75Percentile);
}
if (statistics.Latency90Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency90Percentile (ms)",
value : statistics.Latency90Percentile);
}
if (statistics.Latency99Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency99Percentile (ms)",
value : statistics.Latency99Percentile);
}
if (statistics.MaxLatency != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "MaxLatency (ms)",
value : statistics.MaxLatency);
}
if (statistics.SocketErrors != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "SocketErrors",
value : statistics.SocketErrors);
}
if (statistics.BadResponses != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "BadResponses",
value : statistics.BadResponses);
}
if (statistics.TotalRequests != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "TotalRequests",
value : statistics.TotalRequests);
}
if (statistics.Duration != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Duration (ms)",
value : statistics.Duration);
}
if (statistics.Other.Any())
{
foreach (var counter in Program.Counters)
{
if (!statistics.Other.ContainsKey(counter.Name))
{
continue;
}
if (statistics.Other[counter.Name] != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : counter.DisplayName,
value : statistics.Other[counter.Name]);
}
}
}
}
private static async Task <ClientJob> RunClientJob(Scenario scenario, Uri clientUri, string serverBenchmarkUri)
{
var clientJob = new ClientJob(_clientJobs[scenario])
{
ServerBenchmarkUri = serverBenchmarkUri
};
Uri clientJobUri = null;
try
{
Log($"Starting scenario {scenario} on benchmark client...");
var clientJobsUri = new Uri(clientUri, "/jobs");
var clientContent = JsonConvert.SerializeObject(clientJob);
LogVerbose($"POST {clientJobsUri} {clientContent}...");
var response = await _httpClient.PostAsync(clientJobsUri, new StringContent(clientContent, Encoding.UTF8, "application/json"));
var responseContent = await response.Content.ReadAsStringAsync();
LogVerbose($"{(int)response.StatusCode} {response.StatusCode}");
response.EnsureSuccessStatusCode();
clientJobUri = new Uri(clientUri, response.Headers.Location);
while (true)
{
LogVerbose($"GET {clientJobUri}...");
response = await _httpClient.GetAsync(clientJobUri);
responseContent = await response.Content.ReadAsStringAsync();
LogVerbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}");
clientJob = JsonConvert.DeserializeObject <ClientJob>(responseContent);
if (clientJob.State == ClientState.Running)
{
break;
}
else
{
await Task.Delay(1000);
}
}
while (true)
{
LogVerbose($"GET {clientJobUri}...");
response = await _httpClient.GetAsync(clientJobUri);
responseContent = await response.Content.ReadAsStringAsync();
LogVerbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}");
clientJob = JsonConvert.DeserializeObject <ClientJob>(responseContent);
if (clientJob.State == ClientState.Completed)
{
Log($"Scenario {scenario} completed on benchmark client");
LogVerbose($"Output: {clientJob.Output}");
LogVerbose($"Error: {clientJob.Error}");
Log($"RPS: {clientJob.RequestsPerSecond}");
break;
}
else
{
await Task.Delay(1000);
}
}
}
finally
{
if (clientJobUri != null)
{
Log($"Stopping scenario {scenario} on benchmark client...");
LogVerbose($"DELETE {clientJobUri}...");
var response = _httpClient.DeleteAsync(clientJobUri).Result;
LogVerbose($"{(int)response.StatusCode} {response.StatusCode}");
response.EnsureSuccessStatusCode();
}
}
return(clientJob);
}
private static Process StartProcess(ClientJob job)
{
var customScripts = new List <string>();
// Copying custom scripts
foreach (var script in job.Attachments)
{
if (!Directory.Exists("scripts/custom"))
{
Directory.CreateDirectory("scripts/custom");
}
Log("Copying script: " + script.Filename);
var destination = Path.Combine(Path.GetDirectoryName(typeof(WrkWorker).GetTypeInfo().Assembly.Location), "scripts/custom/" + script.Filename);
Directory.CreateDirectory(Path.GetDirectoryName(destination));
if (File.Exists(destination))
{
File.Delete(destination);
}
File.Copy(script.TempFilename, destination, true);
File.Delete(script.TempFilename);
customScripts.Add("scripts/custom/" + script.Filename);
}
var command = "wrk";
if (job.Headers != null)
{
foreach (var header in job.Headers)
{
command += $" -H \"{header.Key}: {header.Value}\"";
}
}
command += $" --latency -d {job.Duration} -c {job.Connections} --timeout {job.Timeout} -t {job.Threads} {job.ServerBenchmarkUri}{job.Query}";
foreach (var customScript in customScripts)
{
command += $" -s {customScript}";
}
if (job.ClientProperties.TryGetValue("ScriptName", out var scriptName) && !string.IsNullOrEmpty(scriptName))
{
command += $" -s scripts/{scriptName}.lua --";
var pipeLineDepth = int.Parse(job.ClientProperties["PipelineDepth"]);
if (pipeLineDepth > 0)
{
command += $" {pipeLineDepth}";
}
if (job.Method != "GET")
{
command += $" {job.Method}";
}
}
Log(command);
var process = new Process()
{
StartInfo =
{
FileName = "stdbuf",
Arguments = $"-oL {command}",
WorkingDirectory = Path.GetDirectoryName(typeof(WrkWorker).GetTypeInfo().Assembly.Location),
RedirectStandardOutput = true,
RedirectStandardError = true
},
EnableRaisingEvents = true
};
process.OutputDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
job.Output += (e.Data + Environment.NewLine);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
job.Error += (e.Data + Environment.NewLine);
}
};
process.Exited += (_, __) =>
{
// Wait for all Output messages to be flushed and available in job.Output
Thread.Sleep(100);
var rpsMatch = Regex.Match(job.Output, @"Requests/sec:\s*([\d\.]*)");
if (rpsMatch.Success && rpsMatch.Groups.Count == 2)
{
job.RequestsPerSecond = double.Parse(rpsMatch.Groups[1].Value);
}
const string LatencyPattern = @"\s+{0}\s+([\d\.]+)(\w+)";
var latencyMatch = Regex.Match(job.Output, String.Format(LatencyPattern, "Latency"));
job.Latency.Average = ReadLatency(latencyMatch);
var p50Match = Regex.Match(job.Output, String.Format(LatencyPattern, "50%"));
job.Latency.Within50thPercentile = ReadLatency(p50Match);
var p75Match = Regex.Match(job.Output, String.Format(LatencyPattern, "75%"));
job.Latency.Within75thPercentile = ReadLatency(p75Match);
var p90Match = Regex.Match(job.Output, String.Format(LatencyPattern, "90%"));
job.Latency.Within90thPercentile = ReadLatency(p90Match);
var p99Match = Regex.Match(job.Output, String.Format(LatencyPattern, "99%"));
job.Latency.Within99thPercentile = ReadLatency(p99Match);
var p100Match = Regex.Match(job.Output, @"\s+Latency\s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
job.Latency.MaxLatency = ReadLatency(p100Match);
var socketErrorsMatch = Regex.Match(job.Output, @"Socket errors: connect ([\d\.]*), read ([\d\.]*), write ([\d\.]*), timeout ([\d\.]*)");
job.SocketErrors = CountSocketErrors(socketErrorsMatch);
var badResponsesMatch = Regex.Match(job.Output, @"Non-2xx or 3xx responses: ([\d\.]*)");
job.BadResponses = ReadBadReponses(badResponsesMatch);
var requestsCountMatch = Regex.Match(job.Output, @"([\d\.]*) requests in ([\d\.]*)(\w*)");
job.Requests = ReadRequests(requestsCountMatch);
job.ActualDuration = ReadDuration(requestsCountMatch);
job.State = ClientState.Completed;
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
return(process);
}
private static Process StartProcess(ClientJob job)
{
var command = $"h2load {job.ServerBenchmarkUri}{job.Query}";
if (job.Headers != null)
{
foreach (var header in job.Headers)
{
command += $" -H \"{header.Key}: {header.Value}\"";
}
}
command += $" -D {job.Duration} -c {job.Connections} -T {job.Timeout} -t {job.Threads}";
if (job.ClientProperties.TryGetValue("Streams", out var m))
{
command += $" -m {m}";
}
Log(command);
var process = new Process()
{
StartInfo =
{
FileName = "stdbuf",
Arguments = $"-oL {command}",
WorkingDirectory = Path.GetDirectoryName(typeof(H2LoadWorker).GetTypeInfo().Assembly.Location),
RedirectStandardOutput = true,
RedirectStandardError = true
},
EnableRaisingEvents = true
};
process.OutputDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
job.Output += (e.Data + Environment.NewLine);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
job.Error += (e.Data + Environment.NewLine);
}
};
process.Exited += (_, __) =>
{
// Wait for all Output messages to be flushed and available in job.Output
Thread.Sleep(100);
var rpsMatch = Regex.Match(job.Output, @"([\d\.]+) req/s");
if (rpsMatch.Success && rpsMatch.Groups.Count == 2)
{
job.RequestsPerSecond = double.Parse(rpsMatch.Groups[1].Value);
}
var latencyMatch = Regex.Match(job.Output, @"time to 1st byte: \s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
job.Latency.Average = ReadLatency(latencyMatch);
job.Latency.Within50thPercentile = -1;
job.Latency.Within75thPercentile = -1;
job.Latency.Within90thPercentile = -1;
job.Latency.Within99thPercentile = -1;
var p100Match = Regex.Match(job.Output, @"time to 1st byte: \s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
job.Latency.MaxLatency = ReadLatency(p100Match);
var socketErrorsMatch = Regex.Match(job.Output, @"([\d\.]+) failed, ([\d\.]+) errored, ([\d\.]+) timeout");
job.SocketErrors = CountSocketErrors(socketErrorsMatch);
var badResponsesMatch = Regex.Match(job.Output, @"status codes: ([\d\.]+) 2xx, ([\d\.]+) 3xx, ([\d\.]+) 4xx, ([\d\.]+) 5xx");
job.BadResponses = ReadBadReponses(badResponsesMatch);
var requestsCountMatch = Regex.Match(job.Output, @"requests: ([\d\.]+) total");
job.Requests = ReadRequests(requestsCountMatch);
var durationMatch = Regex.Match(job.Output, @"finished in ([\d\.]+)(\w+)");
job.ActualDuration = ReadDuration(durationMatch);
job.State = ClientState.Completed;
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
return(process);
}
public async Task WriteJobResultsToSqlAsync(
ServerJob serverJob,
ClientJob clientJob,
string sqlConnectionString,
string tableName,
string path,
string session,
string description,
Statistics statistics,
bool longRunning)
{
var utcNow = DateTime.UtcNow;
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "RequestsPerSecond",
value : statistics.RequestsPerSecond);
if (statistics.StartupMain != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Startup Main (ms)",
value : statistics.StartupMain);
}
if (statistics.FirstRequest != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "First Request (ms)",
value : statistics.FirstRequest);
}
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "WorkingSet (MB)",
value : statistics.WorkingSet);
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "CPU",
value : statistics.Cpu);
if (statistics.Latency != -1 && !longRunning)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
session : session,
description : description,
path : serverJob.Path,
dimension : "Latency (ms)",
value : statistics.Latency);
}
if (statistics.LatencyAverage != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "LatencyAverage (ms)",
value : statistics.LatencyAverage);
}
if (statistics.Latency50Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency50Percentile (ms)",
value : statistics.Latency50Percentile);
}
if (statistics.Latency75Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency75Percentile (ms)",
value : statistics.Latency75Percentile);
}
if (statistics.Latency90Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency90Percentile (ms)",
value : statistics.Latency90Percentile);
}
if (statistics.Latency99Percentile != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Latency99Percentile (ms)",
value : statistics.Latency99Percentile);
}
if (statistics.MaxLatency != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "MaxLatency (ms)",
value : statistics.MaxLatency);
}
if (statistics.SocketErrors != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "SocketErrors",
value : statistics.SocketErrors);
}
if (statistics.BadResponses != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "BadResponses",
value : statistics.BadResponses);
}
if (statistics.TotalRequests != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "TotalRequests",
value : statistics.TotalRequests);
}
if (statistics.Duration != -1)
{
await WriteJobsToSql(
serverJob : serverJob,
clientJob : clientJob,
utcNow : utcNow,
connectionString : sqlConnectionString,
tableName : tableName,
path : serverJob.Path,
session : session,
description : description,
dimension : "Duration (ms)",
value : statistics.Duration);
}
}
public Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning)
{
return(Task.CompletedTask);
}
