static public IWorker CreateWorker(ClientJob clientJob) { IWorker worker = null; switch (clientJob.Client) { case Worker.Wrk: worker = new WrkWorker(); break; case Worker.SignalR: worker = new SignalRWorker(); break; case Worker.Wait: worker = new WaitWorker(); break; case Worker.H2Load: worker = new H2LoadWorker(); break; case Worker.Bombardier: worker = new BombardierWorker(); break; } return(worker); }
public async Task StartJobAsync(ClientJob job) { _job = job; byte[] requestBody = null; if (job.ClientProperties.TryGetValue("RequestBody", out var requestBodyText)) { requestBody = Convert.FromBase64String(requestBodyText); _requestBodyLength = requestBody.Length; // h2load takes a file as the request body // write the body to a temporary file that is deleted in stop job _requestBodyTempFile = Path.GetTempFileName(); await File.WriteAllBytesAsync(_requestBodyTempFile, requestBody); } InitializeJob(); await MeasureFirstRequestLatencyAsync(_job, requestBody); _job.State = ClientState.Running; _job.LastDriverCommunicationUtc = DateTime.UtcNow; _process = StartProcess(_job, _requestBodyTempFile); }
private static HttpRequestMessage CreateHttpMessage(ClientJob job, byte[] requestBody) { var requestMessage = new HttpRequestMessage(new HttpMethod(job.Method), job.ServerBenchmarkUri); requestMessage.Version = new Version(2, 0); var headers = job.Headers.ToList(); if (requestBody != null) { requestMessage.Content = new ByteArrayContent(requestBody); foreach (var header in headers.Where(h => ContentHeaders.Contains(h.Key))) { requestMessage.Content.Headers.Add(header.Key, header.Value); } // Prune content headers. An error will be thrown if they are added to request headers. headers.RemoveAll(h => ContentHeaders.Contains(h.Key)); } foreach (var header in headers) { requestMessage.Headers.Add(header.Key, header.Value); } return(requestMessage); }
private Task WriteJobsToSql(ServerJob serverJob, ClientJob clientJob, DateTime utcNow, string connectionString, string tableName, string path, string session, string description, string dimension, double value) { return(RetryOnExceptionAsync(5, () => WriteResultsToSql( utcNow, connectionString: connectionString, tableName: tableName, scenario: serverJob.Scenario, session: session, description: description, aspnetCoreVersion: serverJob.AspNetCoreVersion, runtimeVersion: serverJob.RuntimeVersion, hardware: serverJob.Hardware.Value, hardwareVersion: serverJob.HardwareVersion, operatingSystem: serverJob.OperatingSystem.Value, scheme: serverJob.Scheme, source: serverJob.Source, connectionFilter: serverJob.ConnectionFilter, webHost: serverJob.WebHost, kestrelThreadCount: serverJob.KestrelThreadCount, clientThreads: clientJob.Threads, connections: clientJob.Connections, duration: clientJob.Duration, pipelineDepth: clientJob.PipelineDepth, path: path, method: clientJob.Method, headers: clientJob.Headers, dimension: dimension, value: value, runtimeStore: serverJob.UseRuntimeStore) , 5000)); }
public async Task StartJobAsync(ClientJob job) { _job = job; Log($"Starting Job"); await InitializeJob(); // start connections var tasks = new List <Task>(_clients.Count); foreach (var client in _clients) { tasks.Add(client.ConnectAsync(new Uri(_job.ServerBenchmarkUri))); } await Task.WhenAll(tasks); _job.State = ClientState.Running; _job.LastDriverCommunicationUtc = DateTime.UtcNow; _cancelationTokenSource = new CancellationTokenSource(); _cancelationTokenSource.CancelAfter(TimeSpan.FromSeconds(_job.Duration)); _workTimer.Restart(); try { switch (_scenario) { case "Basic": await Basic(_cancelationTokenSource.Token); break; case "FormInput": await FormInput(_cancelationTokenSource.Token); break; case "BackgroundUpdates": await BackgroundUpdates(_cancelationTokenSource.Token); break; default: throw new Exception($"Scenario '{_scenario}' is not a known scenario."); } } catch (Exception ex) { var text = "Exception from test: " + ex; Log(text); _job.Error += Environment.NewLine + text; } _cancelationTokenSource.Token.WaitHandle.WaitOne(); await StopJobAsync(); }
static public IResultsSerializer CreateResultSerializer(ClientJob clientJob) { if (ResultSerializers.TryGetValue(clientJob.Client, out var serializerFactory)) { return(serializerFactory()); } return(null); }
private static HttpRequestMessage CreateHttpMessage(ClientJob job) { var requestMessage = new HttpRequestMessage(new HttpMethod(job.Method), job.ServerBenchmarkUri); foreach (var header in job.Headers) { requestMessage.Headers.Add(header.Key, header.Value); } return(requestMessage); }
public IActionResult Create([FromBody] ClientJob job) { if (job == null || job.Id != 0 || job.State != ClientState.Waiting) { return(BadRequest()); } job = _jobs.Add(job); Response.Headers["Location"] = $"/jobs/{job.Id}"; return(new StatusCodeResult((int)HttpStatusCode.Accepted)); }
public JobInfoViewModel(ClientJob clientJob) { _clientJob = clientJob; StartDate = DateTime.Now; JobInput = clientJob.Job.GetInput(); _statusPropertyChangedProxy = new PropertyChangedProxy <ClientJob, string>(clientJob, m => m.HandlerName, newValue => { OnPropertyChanged(() => HandlerName); }); }
public async Task StartJobAsync(ClientJob job) { _job = job; InitializeJob(); await MeasureFirstRequestLatencyAsync(_job); _job.State = ClientState.Running; _job.LastDriverCommunicationUtc = DateTime.UtcNow; _process = StartProcess(_job); }
static public BaseWorker CreateWorker(ClientJob clientJob) { BaseWorker worker = new BaseWorker(clientJob); //switch (clientJob.Client) //{ // //case Worker.SignalRCoreEcho: // // worker = new SignalRCoreEchoWorker(clientJob); // // break; // default: // worker = new BaseWorker(clientJob); //} return(worker); }
public Task StartJobAsync(ClientJob job) { _job = job; _job.State = ClientState.Running; _job.LastDriverCommunicationUtc = DateTime.UtcNow; _cts = new CancellationTokenSource(); _task = Task.Run(async() => { await Task.Delay(TimeSpan.FromSeconds(_job.Duration)); _job.State = ClientState.Completed; }, _cts.Token); return(Task.CompletedTask); }
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job) { // Ignoring startup latency when using h2c as HttpClient doesn't support if (job.ServerBenchmarkUri.StartsWith("http:", StringComparison.OrdinalIgnoreCase)) { Log("Ignoring first request latencies on h2c"); return; } if (job.SkipStartupLatencies) { return; } Log($"Measuring first request latency on {job.ServerBenchmarkUri}"); var stopwatch = new Stopwatch(); stopwatch.Start(); using (var response = await _httpClient.SendAsync(CreateHttpMessage(job))) { var responseContent = await response.Content.ReadAsStringAsync(); job.LatencyFirstRequest = stopwatch.Elapsed; } Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms"); Log("Measuring subsequent requests latency"); for (var i = 0; i < 10; i++) { stopwatch.Restart(); using (var response = await _httpClient.SendAsync(CreateHttpMessage(job))) { var responseContent = await response.Content.ReadAsStringAsync(); // We keep the last measure to simulate a warmup phase. job.LatencyNoLoad = stopwatch.Elapsed; } } Log($"{job.LatencyNoLoad.TotalMilliseconds} ms"); }
public async Task WriteJobResultsToSqlAsync( ServerJob serverJob, ClientJob clientJob, string sqlConnectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { var utcNow = DateTime.UtcNow; var scenario = serverJob.Scenario; foreach (var result in CsvResults) { serverJob.Scenario = $"{scenario}.{result.Class}.{result.Method}{result.Params ?? ""}"; await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "OperationsPerSecond", value : result.OperationsPerSecond); await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Allocated (KB)", value : result.Allocated); } serverJob.Scenario = scenario; }
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job) { if (job.SkipStartupLatencies) { return; } Log($"Measuring first request latency on {job.ServerBenchmarkUri}"); var stopwatch = new Stopwatch(); stopwatch.Start(); using (var message = CreateHttpMessage(job)) { using (var response = await _httpClient.SendAsync(message)) { var responseContent = await response.Content.ReadAsStringAsync(); job.LatencyFirstRequest = stopwatch.Elapsed; } } Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms"); Log("Measuring subsequent requests latency"); for (var i = 0; i < 10; i++) { stopwatch.Restart(); using (var message = CreateHttpMessage(job)) { using (var response = await _httpClient.SendAsync(message)) { var responseContent = await response.Content.ReadAsStringAsync(); // We keep the last measure to simulate a warmup phase. job.LatencyNoLoad = stopwatch.Elapsed; } } } Log($"{job.LatencyNoLoad.TotalMilliseconds} ms"); }
static public IWorker CreateWorker(ClientJob clientJob) { IWorker worker; switch (clientJob.Client) { case Worker.Wrk: worker = new WrkWorker(); break; case Worker.Wrk2: worker = new Wrk2Worker(); break; case Worker.SignalR: worker = new SignalRWorker(); break; case Worker.Grpc: worker = new GrpcWorker(); break; case Worker.Wait: worker = new WaitWorker(); break; case Worker.H2Load: worker = new H2LoadWorker(); break; case Worker.Bombardier: worker = new BombardierWorker(); break; case Worker.BlazorIgnitor: worker = new BlazorIgnitor(); break; default: throw new InvalidOperationException($"Unknown worker {clientJob.Client}."); } return(worker); }
public IActionResult Create([FromBody] ClientJob job) { if (job == null) { Log("Invalid job"); return(BadRequest("Invalid job")); } if (job.Id != 0) { Log("Can't create an existing job"); return(BadRequest("Job.Id must not specified.")); } if (job.State != ClientState.Initializing) { Log("Job should have ne in Initializing state"); return(BadRequest("Job should have ne in Initializing state")); } job = _jobs.Add(job); return(AcceptedAtAction(nameof(GetById), new { id = job.Id }, job)); }
private static async Task ProcessJobs(CancellationToken cancellationToken) { IWorker worker = null; ClientJob job = null; var whenLastJobCompleted = DateTime.MinValue; var waitForMoreJobs = false; while (!cancellationToken.IsCancellationRequested) { var allJobs = _jobs.GetAll(); // Dequeue the first job. We will only pass jobs that have // the same SpanId to the current worker. job = allJobs.FirstOrDefault(newJob => { // If the job is null then we don't have a span id to match against. // Otherwise we want to pick jobs with the same span id. return(job == null || string.Equals(newJob.SpanId, job.SpanId, StringComparison.OrdinalIgnoreCase)); }); if (job != null) { // A spanId means that a span is defined and we might run // multiple jobs. if (!string.IsNullOrEmpty(job.SpanId)) { waitForMoreJobs = true; } if (job.State == ClientState.Waiting) { Log($"Starting '{job.Client}' worker"); Log($"Current Job SpanId '{job.SpanId}'"); job.State = ClientState.Starting; try { if (worker == null) { worker = WorkerFactory.CreateWorker(job); } if (worker == null) { Log($"Error while creating the worker"); job.State = ClientState.Deleting; whenLastJobCompleted = DateTime.UtcNow; } else { await worker.StartJobAsync(job); } } catch (Exception e) { Log($"An unexpected error occurred while starting the job {job.Id}"); Log(e.ToString()); job.State = ClientState.Deleting; } } else if (job.State == ClientState.Running || job.State == ClientState.Completed) { var now = DateTime.UtcNow; // Clean the job in case the driver is not running if (now - job.LastDriverCommunicationUtc > TimeSpan.FromSeconds(30)) { Log($"Driver didn't communicate for {now - job.LastDriverCommunicationUtc}. Halting job."); job.State = ClientState.Deleting; } } else if (job.State == ClientState.Deleting) { Log($"Deleting job {worker?.JobLogText ?? "no worker found"}"); try { if (worker != null) { await worker.StopJobAsync(); // Reset the last job completed indicator. whenLastJobCompleted = DateTime.UtcNow; } } finally { _jobs.Remove(job.Id); job = null; } } } await Task.Delay(100); // job will be null if there aren't any more jobs with the same spanId. if (job == null) { // Currently no jobs with the same span id exist so we check if we can // clear out the worker to signal to the worker factory to create // a new one. if (worker != null) { var now = DateTime.UtcNow; // Disposing the worker conditions // 1. A span isn't defined so there won't be any more jobs for this worker // 2. We check that whenLastJob completed is something other that it's default value // and 10 seconds have passed since the last job was completed. if (!waitForMoreJobs || (whenLastJobCompleted != DateTime.MinValue && now - whenLastJobCompleted > TimeSpan.FromSeconds(10))) { Log("Job queuing timeout reached. Disposing worker."); waitForMoreJobs = false; await worker.DisposeAsync(); worker = null; } } } } }
public async Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { var utcNow = DateTime.UtcNow; await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "RequestsPerSecond", statistics.RequestsPerSecond); await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "CPU", statistics.Cpu); await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "WorkingSet (MB)", statistics.WorkingSet); if (statistics.LatencyAverage != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency Average (ms)", statistics.LatencyAverage); } if (statistics.Latency50Percentile != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency50Percentile (ms)", statistics.Latency50Percentile); } if (statistics.Latency75Percentile != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency75Percentile (ms)", statistics.Latency75Percentile); } if (statistics.Latency90Percentile != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency90Percentile (ms)", statistics.Latency90Percentile); } if (statistics.Latency99Percentile != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "Latency99Percentile (ms)", statistics.Latency99Percentile); } if (statistics.MaxLatency != -1) { await WriteJobsToSql(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, "MaxLatency (ms)", statistics.MaxLatency); } if (statistics.Other.Any()) { foreach (var counter in Program.Counters) { if (!statistics.Other.ContainsKey(counter.Name)) { continue; } if (statistics.Other[counter.Name] != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : connectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : counter.DisplayName, value : statistics.Other[counter.Name]); } } } }
private static Process StartProcess(ClientJob job) { var tcs = new TaskCompletionSource <bool>(); var command = $"wrk -c {job.Connections} -t {job.Threads} -d {job.Duration}"; if (job.Headers != null) { foreach (var header in job.Headers) { command += $" -H \"{header}\""; } } if (job.PipelineDepth > 0) { command += $" -s scripts/pipeline.lua"; } command += $" {job.ServerBenchmarkUri}"; if (job.PipelineDepth > 0) { command += $" -- {job.PipelineDepth}"; } var process = new Process() { StartInfo = { FileName = "stdbuf", Arguments = $"-oL {command}", WorkingDirectory = Path.GetDirectoryName(typeof(Startup).GetTypeInfo().Assembly.Location), RedirectStandardOutput = true, RedirectStandardError = true }, EnableRaisingEvents = true }; process.OutputDataReceived += (_, e) => { Log(e.Data); job.Output += (e.Data + Environment.NewLine); }; process.ErrorDataReceived += (_, e) => { Log(e.Data); job.Error += (e.Data + Environment.NewLine); }; process.Exited += (_, __) => { double rps = -1; var match = Regex.Match(job.Output, @"Requests/sec:\s*([\d.]*)"); if (match.Success && match.Groups.Count == 2) { double.TryParse(match.Groups[1].Value, out rps); } job.RequestsPerSecond = rps; job.State = ClientState.Completed; }; process.Start(); process.BeginOutputReadLine(); process.BeginErrorReadLine(); return(process); }
public async Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { var utcNow = DateTime.UtcNow; await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "RequestsPerSecond", statistics.RequestsPerSecond, retry); }); await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "CPU", statistics.Cpu, retry); }); await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "WorkingSet (MB)", statistics.WorkingSet, retry); }); if (statistics.LatencyAverage != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency Average (ms)", statistics.LatencyAverage, retry); }); } if (statistics.Latency50Percentile != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency50Percentile (ms)", statistics.Latency50Percentile, retry); }); } if (statistics.Latency75Percentile != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency75Percentile (ms)", statistics.Latency75Percentile, retry); }); } if (statistics.Latency90Percentile != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency90Percentile (ms)", statistics.Latency90Percentile, retry); }); } if (statistics.Latency99Percentile != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "Latency99Percentile (ms)", statistics.Latency99Percentile, retry); }); } if (statistics.MaxLatency != -1) { await RetryOnExceptionAsync(5, async (retry) => { await WriteJobResultToSqlAsync(serverJob, clientJob, utcNow, connectionString, tableName, path, session, description, statistics, longRunning, "MaxLatency (ms)", statistics.MaxLatency, retry); }); } }
private async Task WriteJobResultToSqlAsync(ServerJob serverJob, ClientJob clientJob, DateTime utcNow, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning, string dimension, double value, bool checkExisting) { if (checkExisting) { if (await CheckForRow(connectionString, tableName, utcNow, dimension, session) == true) { return; } } string insertCmd = @" INSERT INTO [dbo].[" + tableName + @"] ([DateTime] ,[Session] ,[Description] ,[AspNetCoreVersion] ,[RuntimeVersion] ,[Scenario] ,[Hardware] ,[HardwareVersion] ,[OperatingSystem] ,[Framework] ,[RuntimeStore] ,[Scheme] ,[Sources] ,[WebHost] ,[Transport] ,[HubProtocol] ,[ClientProperties] ,[Connections] ,[Duration] ,[Path] ,[Headers] ,[Dimension] ,[Value]) VALUES (@DateTime ,@Session ,@Description ,@AspNetCoreVersion ,@RuntimeVersion ,@Scenario ,@Hardware ,@HardwareVersion ,@OperatingSystem ,@Framework ,@RuntimeStore ,@Scheme ,@Sources ,@WebHost ,@Transport ,@HubProtocol ,@ClientProperties ,@Connections ,@Duration ,@Path ,@Headers ,@Dimension ,@Value) "; using (var connection = new SqlConnection(connectionString)) { await connection.OpenAsync(); var transaction = connection.BeginTransaction(); try { var command = new SqlCommand(insertCmd, connection, transaction); var p = command.Parameters; p.AddWithValue("@DateTime", utcNow); p.AddWithValue("@Session", session); p.AddWithValue("@Description", description); p.AddWithValue("@AspNetCoreVersion", serverJob.AspNetCoreVersion); p.AddWithValue("@RuntimeVersion", serverJob.RuntimeVersion); p.AddWithValue("@Scenario", serverJob.Scenario.ToString()); p.AddWithValue("@Hardware", serverJob.Hardware.ToString()); p.AddWithValue("@HardwareVersion", serverJob.HardwareVersion); p.AddWithValue("@OperatingSystem", serverJob.OperatingSystem.ToString()); p.AddWithValue("@Framework", "Core"); p.AddWithValue("@RuntimeStore", serverJob.UseRuntimeStore); p.AddWithValue("@Scheme", serverJob.Scheme.ToString().ToLowerInvariant()); p.AddWithValue("@Sources", serverJob.Source != null ? ConvertToSqlString(serverJob.Source) : (object)DBNull.Value); p.AddWithValue("@WebHost", serverJob.WebHost.ToString()); p.AddWithValue("@Transport", clientJob.ClientProperties["TransportType"]); p.AddWithValue("@HubProtocol", clientJob.ClientProperties["HubProtocol"]); p.AddWithValue("@ClientProperties", JsonConvert.SerializeObject(clientJob.ClientProperties)); p.AddWithValue("@Connections", clientJob.Connections); p.AddWithValue("@Duration", clientJob.Duration); p.AddWithValue("@Path", string.IsNullOrEmpty(path) ? (object)DBNull.Value : path); p.AddWithValue("@Headers", clientJob.Headers.Any() ? JsonConvert.SerializeObject(clientJob.Headers) : (object)DBNull.Value); p.AddWithValue("@Dimension", dimension); p.AddWithValue("@Value", value); await command.ExecuteNonQueryAsync(); transaction.Commit(); } catch { transaction.Rollback(); throw; } finally { transaction.Dispose(); } } }
public BaseWorker(ClientJob job) { _pkg.Job = job; }
private static async Task MeasureFirstRequestLatencyAsync(ClientJob job) { if (job.SkipStartupLatencies) { return; } Log($"Measuring first request latency on {job.ServerBenchmarkUri}"); var stopwatch = new Stopwatch(); stopwatch.Start(); using (var message = CreateHttpMessage(job)) { var cts = new CancellationTokenSource(); var token = cts.Token; cts.CancelAfter(FirstRequestTimeout); token.ThrowIfCancellationRequested(); try { using (var response = await _httpClient.SendAsync(message, token)) { job.LatencyFirstRequest = stopwatch.Elapsed; } } catch (OperationCanceledException) { Log("A timeout occurred while measuring the first request: " + FirstRequestTimeout.ToString()); } finally { cts.Dispose(); } } Log($"{job.LatencyFirstRequest.TotalMilliseconds} ms"); Log("Measuring subsequent requests latency"); for (var i = 0; i < 10; i++) { stopwatch.Restart(); using (var message = CreateHttpMessage(job)) { var cts = new CancellationTokenSource(); var token = cts.Token; cts.CancelAfter(LatencyTimeout); token.ThrowIfCancellationRequested(); try { using (var response = await _httpClient.SendAsync(message)) { // We keep the last measure to simulate a warmup phase. job.LatencyNoLoad = stopwatch.Elapsed; } } catch (OperationCanceledException) { Log("A timeout occurred while measuring the latency, skipping ..."); break; } finally { cts.Dispose(); } } } Log($"{job.LatencyNoLoad.TotalMilliseconds} ms"); }
public async Task WriteJobResultsToSqlAsync( ServerJob serverJob, ClientJob clientJob, string sqlConnectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { var utcNow = DateTime.UtcNow; await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "RequestsPerSecond", value : statistics.RequestsPerSecond); if (statistics.StartupMain != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Startup Main (ms)", value : statistics.StartupMain); } if (statistics.BuildTime != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Build Time (ms)", value : statistics.BuildTime); } if (statistics.PublishedSize != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Published Size (KB)", value : statistics.PublishedSize); } if (statistics.FirstRequest != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "First Request (ms)", value : statistics.FirstRequest); } await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "WorkingSet (MB)", value : statistics.WorkingSet); await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "CPU", value : statistics.Cpu); if (statistics.Latency != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, session : session, description : description, path : serverJob.Path, dimension : "Latency (ms)", value : statistics.Latency); } if (statistics.LatencyAverage != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "LatencyAverage (ms)", value : statistics.LatencyAverage); } if (statistics.Latency50Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency50Percentile (ms)", value : statistics.Latency50Percentile); } if (statistics.Latency75Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency75Percentile (ms)", value : statistics.Latency75Percentile); } if (statistics.Latency90Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency90Percentile (ms)", value : statistics.Latency90Percentile); } if (statistics.Latency99Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency99Percentile (ms)", value : statistics.Latency99Percentile); } if (statistics.MaxLatency != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "MaxLatency (ms)", value : statistics.MaxLatency); } if (statistics.SocketErrors != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "SocketErrors", value : statistics.SocketErrors); } if (statistics.BadResponses != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "BadResponses", value : statistics.BadResponses); } if (statistics.TotalRequests != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "TotalRequests", value : statistics.TotalRequests); } if (statistics.Duration != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Duration (ms)", value : statistics.Duration); } if (statistics.Other.Any()) { foreach (var counter in Program.Counters) { if (!statistics.Other.ContainsKey(counter.Name)) { continue; } if (statistics.Other[counter.Name] != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : counter.DisplayName, value : statistics.Other[counter.Name]); } } } }
private static async Task <ClientJob> RunClientJob(Scenario scenario, Uri clientUri, string serverBenchmarkUri) { var clientJob = new ClientJob(_clientJobs[scenario]) { ServerBenchmarkUri = serverBenchmarkUri }; Uri clientJobUri = null; try { Log($"Starting scenario {scenario} on benchmark client..."); var clientJobsUri = new Uri(clientUri, "/jobs"); var clientContent = JsonConvert.SerializeObject(clientJob); LogVerbose($"POST {clientJobsUri} {clientContent}..."); var response = await _httpClient.PostAsync(clientJobsUri, new StringContent(clientContent, Encoding.UTF8, "application/json")); var responseContent = await response.Content.ReadAsStringAsync(); LogVerbose($"{(int)response.StatusCode} {response.StatusCode}"); response.EnsureSuccessStatusCode(); clientJobUri = new Uri(clientUri, response.Headers.Location); while (true) { LogVerbose($"GET {clientJobUri}..."); response = await _httpClient.GetAsync(clientJobUri); responseContent = await response.Content.ReadAsStringAsync(); LogVerbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}"); clientJob = JsonConvert.DeserializeObject <ClientJob>(responseContent); if (clientJob.State == ClientState.Running) { break; } else { await Task.Delay(1000); } } while (true) { LogVerbose($"GET {clientJobUri}..."); response = await _httpClient.GetAsync(clientJobUri); responseContent = await response.Content.ReadAsStringAsync(); LogVerbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}"); clientJob = JsonConvert.DeserializeObject <ClientJob>(responseContent); if (clientJob.State == ClientState.Completed) { Log($"Scenario {scenario} completed on benchmark client"); LogVerbose($"Output: {clientJob.Output}"); LogVerbose($"Error: {clientJob.Error}"); Log($"RPS: {clientJob.RequestsPerSecond}"); break; } else { await Task.Delay(1000); } } } finally { if (clientJobUri != null) { Log($"Stopping scenario {scenario} on benchmark client..."); LogVerbose($"DELETE {clientJobUri}..."); var response = _httpClient.DeleteAsync(clientJobUri).Result; LogVerbose($"{(int)response.StatusCode} {response.StatusCode}"); response.EnsureSuccessStatusCode(); } } return(clientJob); }
private static Process StartProcess(ClientJob job) { var customScripts = new List <string>(); // Copying custom scripts foreach (var script in job.Attachments) { if (!Directory.Exists("scripts/custom")) { Directory.CreateDirectory("scripts/custom"); } Log("Copying script: " + script.Filename); var destination = Path.Combine(Path.GetDirectoryName(typeof(WrkWorker).GetTypeInfo().Assembly.Location), "scripts/custom/" + script.Filename); Directory.CreateDirectory(Path.GetDirectoryName(destination)); if (File.Exists(destination)) { File.Delete(destination); } File.Copy(script.TempFilename, destination, true); File.Delete(script.TempFilename); customScripts.Add("scripts/custom/" + script.Filename); } var command = "wrk"; if (job.Headers != null) { foreach (var header in job.Headers) { command += $" -H \"{header.Key}: {header.Value}\""; } } command += $" --latency -d {job.Duration} -c {job.Connections} --timeout {job.Timeout} -t {job.Threads} {job.ServerBenchmarkUri}{job.Query}"; foreach (var customScript in customScripts) { command += $" -s {customScript}"; } if (job.ClientProperties.TryGetValue("ScriptName", out var scriptName) && !string.IsNullOrEmpty(scriptName)) { command += $" -s scripts/{scriptName}.lua --"; var pipeLineDepth = int.Parse(job.ClientProperties["PipelineDepth"]); if (pipeLineDepth > 0) { command += $" {pipeLineDepth}"; } if (job.Method != "GET") { command += $" {job.Method}"; } } Log(command); var process = new Process() { StartInfo = { FileName = "stdbuf", Arguments = $"-oL {command}", WorkingDirectory = Path.GetDirectoryName(typeof(WrkWorker).GetTypeInfo().Assembly.Location), RedirectStandardOutput = true, RedirectStandardError = true }, EnableRaisingEvents = true }; process.OutputDataReceived += (_, e) => { if (e.Data != null) { Log(e.Data); job.Output += (e.Data + Environment.NewLine); } }; process.ErrorDataReceived += (_, e) => { if (e.Data != null) { Log(e.Data); job.Error += (e.Data + Environment.NewLine); } }; process.Exited += (_, __) => { // Wait for all Output messages to be flushed and available in job.Output Thread.Sleep(100); var rpsMatch = Regex.Match(job.Output, @"Requests/sec:\s*([\d\.]*)"); if (rpsMatch.Success && rpsMatch.Groups.Count == 2) { job.RequestsPerSecond = double.Parse(rpsMatch.Groups[1].Value); } const string LatencyPattern = @"\s+{0}\s+([\d\.]+)(\w+)"; var latencyMatch = Regex.Match(job.Output, String.Format(LatencyPattern, "Latency")); job.Latency.Average = ReadLatency(latencyMatch); var p50Match = Regex.Match(job.Output, String.Format(LatencyPattern, "50%")); job.Latency.Within50thPercentile = ReadLatency(p50Match); var p75Match = Regex.Match(job.Output, String.Format(LatencyPattern, "75%")); job.Latency.Within75thPercentile = ReadLatency(p75Match); var p90Match = Regex.Match(job.Output, String.Format(LatencyPattern, "90%")); job.Latency.Within90thPercentile = ReadLatency(p90Match); var p99Match = Regex.Match(job.Output, String.Format(LatencyPattern, "99%")); job.Latency.Within99thPercentile = ReadLatency(p99Match); var p100Match = Regex.Match(job.Output, @"\s+Latency\s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)"); job.Latency.MaxLatency = ReadLatency(p100Match); var socketErrorsMatch = Regex.Match(job.Output, @"Socket errors: connect ([\d\.]*), read ([\d\.]*), write ([\d\.]*), timeout ([\d\.]*)"); job.SocketErrors = CountSocketErrors(socketErrorsMatch); var badResponsesMatch = Regex.Match(job.Output, @"Non-2xx or 3xx responses: ([\d\.]*)"); job.BadResponses = ReadBadReponses(badResponsesMatch); var requestsCountMatch = Regex.Match(job.Output, @"([\d\.]*) requests in ([\d\.]*)(\w*)"); job.Requests = ReadRequests(requestsCountMatch); job.ActualDuration = ReadDuration(requestsCountMatch); job.State = ClientState.Completed; }; process.Start(); process.BeginOutputReadLine(); process.BeginErrorReadLine(); return(process); }
private static Process StartProcess(ClientJob job) { var command = $"h2load {job.ServerBenchmarkUri}{job.Query}"; if (job.Headers != null) { foreach (var header in job.Headers) { command += $" -H \"{header.Key}: {header.Value}\""; } } command += $" -D {job.Duration} -c {job.Connections} -T {job.Timeout} -t {job.Threads}"; if (job.ClientProperties.TryGetValue("Streams", out var m)) { command += $" -m {m}"; } Log(command); var process = new Process() { StartInfo = { FileName = "stdbuf", Arguments = $"-oL {command}", WorkingDirectory = Path.GetDirectoryName(typeof(H2LoadWorker).GetTypeInfo().Assembly.Location), RedirectStandardOutput = true, RedirectStandardError = true }, EnableRaisingEvents = true }; process.OutputDataReceived += (_, e) => { if (e.Data != null) { Log(e.Data); job.Output += (e.Data + Environment.NewLine); } }; process.ErrorDataReceived += (_, e) => { if (e.Data != null) { Log(e.Data); job.Error += (e.Data + Environment.NewLine); } }; process.Exited += (_, __) => { // Wait for all Output messages to be flushed and available in job.Output Thread.Sleep(100); var rpsMatch = Regex.Match(job.Output, @"([\d\.]+) req/s"); if (rpsMatch.Success && rpsMatch.Groups.Count == 2) { job.RequestsPerSecond = double.Parse(rpsMatch.Groups[1].Value); } var latencyMatch = Regex.Match(job.Output, @"time to 1st byte: \s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)"); job.Latency.Average = ReadLatency(latencyMatch); job.Latency.Within50thPercentile = -1; job.Latency.Within75thPercentile = -1; job.Latency.Within90thPercentile = -1; job.Latency.Within99thPercentile = -1; var p100Match = Regex.Match(job.Output, @"time to 1st byte: \s+[\d\.]+\w+\s+([\d\.]+)(\w+)"); job.Latency.MaxLatency = ReadLatency(p100Match); var socketErrorsMatch = Regex.Match(job.Output, @"([\d\.]+) failed, ([\d\.]+) errored, ([\d\.]+) timeout"); job.SocketErrors = CountSocketErrors(socketErrorsMatch); var badResponsesMatch = Regex.Match(job.Output, @"status codes: ([\d\.]+) 2xx, ([\d\.]+) 3xx, ([\d\.]+) 4xx, ([\d\.]+) 5xx"); job.BadResponses = ReadBadReponses(badResponsesMatch); var requestsCountMatch = Regex.Match(job.Output, @"requests: ([\d\.]+) total"); job.Requests = ReadRequests(requestsCountMatch); var durationMatch = Regex.Match(job.Output, @"finished in ([\d\.]+)(\w+)"); job.ActualDuration = ReadDuration(durationMatch); job.State = ClientState.Completed; }; process.Start(); process.BeginOutputReadLine(); process.BeginErrorReadLine(); return(process); }
public async Task WriteJobResultsToSqlAsync( ServerJob serverJob, ClientJob clientJob, string sqlConnectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { var utcNow = DateTime.UtcNow; await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "RequestsPerSecond", value : statistics.RequestsPerSecond); if (statistics.StartupMain != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Startup Main (ms)", value : statistics.StartupMain); } if (statistics.FirstRequest != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "First Request (ms)", value : statistics.FirstRequest); } await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "WorkingSet (MB)", value : statistics.WorkingSet); await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "CPU", value : statistics.Cpu); if (statistics.Latency != -1 && !longRunning) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, session : session, description : description, path : serverJob.Path, dimension : "Latency (ms)", value : statistics.Latency); } if (statistics.LatencyAverage != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "LatencyAverage (ms)", value : statistics.LatencyAverage); } if (statistics.Latency50Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency50Percentile (ms)", value : statistics.Latency50Percentile); } if (statistics.Latency75Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency75Percentile (ms)", value : statistics.Latency75Percentile); } if (statistics.Latency90Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency90Percentile (ms)", value : statistics.Latency90Percentile); } if (statistics.Latency99Percentile != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Latency99Percentile (ms)", value : statistics.Latency99Percentile); } if (statistics.MaxLatency != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "MaxLatency (ms)", value : statistics.MaxLatency); } if (statistics.SocketErrors != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "SocketErrors", value : statistics.SocketErrors); } if (statistics.BadResponses != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "BadResponses", value : statistics.BadResponses); } if (statistics.TotalRequests != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "TotalRequests", value : statistics.TotalRequests); } if (statistics.Duration != -1) { await WriteJobsToSql( serverJob : serverJob, clientJob : clientJob, utcNow : utcNow, connectionString : sqlConnectionString, tableName : tableName, path : serverJob.Path, session : session, description : description, dimension : "Duration (ms)", value : statistics.Duration); } }
public Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) { return(Task.CompletedTask); }