static async Task Main(string[] args)
{
Console.WriteLine("Bombardier Client");
Console.WriteLine("args: " + String.Join(' ', args));
Console.Write("Measuring first request ... ");
await MeasureFirstRequest(args);
// Extracting parameters
var argsList = args.ToList();
TryGetArgumentValue("-w", argsList, out int warmup);
TryGetArgumentValue("-d", argsList, out int duration);
TryGetArgumentValue("-n", argsList, out int requests);
if (duration == 0 && requests == 0)
{
Console.WriteLine("Couldn't find valid -d and -n arguments (integers)");
return;
}
TryGetArgumentValue("-w", argsList, out warmup);
args = argsList.ToArray();
var bombardierUrl = _bombardierUrls[Environment.OSVersion.Platform];
var bombardierFileName = Path.GetFileName(bombardierUrl);
using (var downloadStream = await _httpClient.GetStreamAsync(bombardierUrl))
using (var fileStream = File.Create(bombardierFileName))
{
await downloadStream.CopyToAsync(fileStream);
if (Environment.OSVersion.Platform == PlatformID.Unix)
{
Process.Start("chmod", "+x " + bombardierFileName);
}
}
var baseArguments = String.Join(' ', args.ToArray()) + " --print r --format json";
var process = new Process()
{
StartInfo =
{
FileName = bombardierFileName,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (warmup > 0)
{
process.StartInfo.Arguments = $" -d {warmup}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
process.Start();
process.WaitForExit();
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
process.StartInfo.Arguments =
requests > 0
? $" -n {requests} {baseArguments}"
: $" -d {duration}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
var document = JObject.Parse(output);
BenchmarksEventSource.Log.Metadata("bombardier/requests", "max", "sum", "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/badresponses", "max", "sum", "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/latency/mean", "max", "sum", "Mean latency (us)", "Mean latency (us)", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/latency/max", "max", "sum", "Max latency (us)", "Max latency (us)", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/rps/mean", "max", "sum", "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/rps/max", "max", "sum", "Requests/sec (max)", "Max requests per second", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/raw", "all", "all", "Raw results", "Raw results", "json");
var total =
document["result"]["req1xx"].Value <int>()
+ document["result"]["req2xx"].Value <int>()
+ document["result"]["req3xx"].Value <int>()
+ document["result"]["req3xx"].Value <int>()
+ document["result"]["req4xx"].Value <int>()
+ document["result"]["req5xx"].Value <int>()
+ document["result"]["others"].Value <int>();
var success = document["result"]["req2xx"].Value <int>() + document["result"]["req3xx"].Value <int>();
BenchmarksEventSource.Measure("bombardier/requests", total);
BenchmarksEventSource.Measure("bombardier/badresponses", total - success);
BenchmarksEventSource.Measure("bombardier/latency/mean", document["result"]["latency"]["mean"].Value <double>());
BenchmarksEventSource.Measure("bombardier/latency/max", document["result"]["latency"]["max"].Value <double>());
BenchmarksEventSource.Measure("bombardier/rps/max", document["result"]["rps"]["max"].Value <double>());
BenchmarksEventSource.Measure("bombardier/rps/mean", document["result"]["rps"]["mean"].Value <double>());
BenchmarksEventSource.Measure("bombardier/raw", output);
}
static int RunCore(string fileName, string[] args, bool parseLatency)
{
// Extracting duration parameters
string warmup = "";
string duration = "";
var argsList = args.ToList();
var durationIndex = argsList.FindIndex(x => String.Equals(x, "-d", StringComparison.OrdinalIgnoreCase));
if (durationIndex >= 0)
{
duration = argsList[durationIndex + 1];
argsList.RemoveAt(durationIndex);
argsList.RemoveAt(durationIndex);
}
else
{
Console.WriteLine("Couldn't find -d argument");
return(-1);
}
var warmupIndex = argsList.FindIndex(x => String.Equals(x, "-w", StringComparison.OrdinalIgnoreCase));
if (warmupIndex >= 0)
{
warmup = argsList[warmupIndex + 1];
argsList.RemoveAt(warmupIndex);
argsList.RemoveAt(warmupIndex);
}
args = argsList.ToArray();
var baseArguments = String.Join(' ', args.ToArray());
var process = new Process()
{
StartInfo =
{
FileName = fileName,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (!String.IsNullOrEmpty(warmup) && warmup != "0s")
{
process.StartInfo.Arguments = $"-d {warmup} {baseArguments}";
Console.WriteLine("> wrk " + process.StartInfo.Arguments);
process.Start();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
process.StartInfo.Arguments = $"-d {duration} {baseArguments}";
Console.WriteLine("> wrk " + process.StartInfo.Arguments);
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
BenchmarksEventSource.Log.Metadata("wrk/rps/mean", "max", "sum", "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Log.Metadata("wrk/requests", "max", "sum", "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Log.Metadata("wrk/latency/mean", "max", "sum", "Mean latency (ms)", "Mean latency (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk/latency/max", "max", "sum", "Max latency (ms)", "Max latency (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk/errors/badresponses", "max", "sum", "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Log.Metadata("wrk/errors/socketerrors", "max", "sum", "Socket errors", "Socket errors", "n0");
const string LatencyPattern = @"\s+{0}\s+([\d\.]+)(\w+)";
var latencyMatch = Regex.Match(output, String.Format(LatencyPattern, "Latency"));
BenchmarksEventSource.Measure("wrk/latency/mean", ReadLatency(latencyMatch));
var rpsMatch = Regex.Match(output, @"Requests/sec:\s*([\d\.]*)");
if (rpsMatch.Success && rpsMatch.Groups.Count == 2)
{
BenchmarksEventSource.Measure("wrk/rps/mean", double.Parse(rpsMatch.Groups[1].Value));
}
// Max latency is 3rd number after "Latency "
var maxLatencyMatch = Regex.Match(output, @"\s+Latency\s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
BenchmarksEventSource.Measure("wrk/latency/max", ReadLatency(maxLatencyMatch));
var requestsCountMatch = Regex.Match(output, @"([\d\.]*) requests in ([\d\.]*)(\w*)");
BenchmarksEventSource.Measure("wrk/requests", ReadRequests(requestsCountMatch));
var badResponsesMatch = Regex.Match(output, @"Non-2xx or 3xx responses: ([\d\.]*)");
BenchmarksEventSource.Measure("wrk/errors/badresponses", ReadBadReponses(badResponsesMatch));
var socketErrorsMatch = Regex.Match(output, @"Socket errors: connect ([\d\.]*), read ([\d\.]*), write ([\d\.]*), timeout ([\d\.]*)");
BenchmarksEventSource.Measure("wrk/errors/socketerrors", CountSocketErrors(socketErrorsMatch));
if (parseLatency)
{
BenchmarksEventSource.Log.Metadata("wrk/latency/50", "max", "avg", "Latency 50th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk/latency/75", "max", "avg", "Latency 75th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk/latency/90", "max", "avg", "Latency 90th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk/latency/99", "max", "avg", "Latency 99th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Measure("wrk/latency/50", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "50%"))));
BenchmarksEventSource.Measure("wrk/latency/75", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "75%"))));
BenchmarksEventSource.Measure("wrk/latency/90", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "90%"))));
BenchmarksEventSource.Measure("wrk/latency/99", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "99%"))));
}
return(0);
}
static async Task <int> Main(string[] args)
{
Console.WriteLine("Bombardier Client");
Console.WriteLine("args: " + String.Join(' ', args));
Console.Write("Measuring first request ... ");
await MeasureFirstRequest(args);
// Extracting parameters
var argsList = args.ToList();
TryGetArgumentValue("-w", argsList, out int warmup);
TryGetArgumentValue("-d", argsList, out int duration);
TryGetArgumentValue("-n", argsList, out int requests);
TryGetArgumentValue("-o", argsList, out string outputFormat);
TryGetArgumentValue("-f", argsList, out string bodyFile);
if (duration == 0 && requests == 0)
{
Console.WriteLine("Couldn't find valid -d and -n arguments (integers)");
return(-1);
}
if (string.IsNullOrEmpty(outputFormat))
{
outputFormat = "json";
}
else if ((outputFormat != "json") && (outputFormat != "plain-text"))
{
Console.WriteLine("Value value for -o is json or plain-text");
return(-1);
}
args = argsList.ToArray();
string bombardierUrl = null;
string bombardierVersion = "1.2.5";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.5/bombardier-windows-amd64.exe";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
switch (RuntimeInformation.ProcessArchitecture)
{
case Architecture.Arm64: bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.5/bombardier-linux-arm64"; break;
case Architecture.Arm: bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.5/bombardier-linux-arm"; break;
default: bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.5/bombardier-linux-amd64"; break;
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.5/bombardier-darwin-amd64";
}
else
{
Console.WriteLine("Unsupported platform");
return(-1);
}
var bombardierFileName = Path.Combine(Path.GetTempPath(), ".crank", bombardierVersion, Path.GetFileName(bombardierUrl));
if (!File.Exists(bombardierFileName))
{
Directory.CreateDirectory(Path.GetDirectoryName(bombardierFileName));
Console.WriteLine($"Downloading bombardier from {bombardierUrl} to {bombardierFileName}");
using (var downloadStream = await _httpClient.GetStreamAsync(bombardierUrl))
using (var fileStream = File.Create(bombardierFileName))
{
await downloadStream.CopyToAsync(fileStream);
await fileStream.FlushAsync();
await downloadStream.FlushAsync();
}
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine($"Setting execute permission on executable {bombardierFileName}");
Process.Start("chmod", "+x " + bombardierFileName);
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine($"Allow running bombardier");
Process.Start("spctl", "--add " + bombardierFileName);
}
if (!String.IsNullOrEmpty(bodyFile))
{
if (bodyFile.StartsWith("http", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine($"Downloading body file {bodyFile}");
var tempFile = Path.GetTempFileName();
using (var downloadStream = await _httpClient.GetStreamAsync(bodyFile))
using (var fileStream = File.Create(tempFile))
{
await downloadStream.CopyToAsync(fileStream);
}
bodyFile = tempFile;
}
argsList.Add("-f");
argsList.Add(bodyFile);
}
args = argsList.Select(Quote).ToArray();
var baseArguments = String.Join(' ', args) + $" --print r --format {outputFormat}";
var process = new Process()
{
StartInfo =
{
FileName = bombardierFileName,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (warmup > 0)
{
process.StartInfo.Arguments = $" -d {warmup}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
await StartProcessWithRetriesAsync(process);
process.WaitForExit();
if (process.ExitCode != 0)
{
Console.WriteLine("Failed to run bombardier.");
return(process.ExitCode);
}
}
else
{
Console.WriteLine("Warmup skipped");
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
if (duration > 0)
{
process.StartInfo.Arguments =
requests > 0
? $" -n {requests} {baseArguments}"
: $" -d {duration}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
await StartProcessWithRetriesAsync(process);
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
if (outputFormat != "json")
{
return(0);
}
var document = JObject.Parse(output);
BenchmarksEventSource.Register("bombardier/requests;http/requests", Operations.Max, Operations.Sum, "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Register("bombardier/badresponses;http/requests/badresponses", Operations.Max, Operations.Sum, "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Register("bombardier/latency/50;http/latency/50", Operations.Max, Operations.Max, "Latency 50th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/75;http/latency/75", Operations.Max, Operations.Max, "Latency 75th (ms)", "Latency 75th (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/90;http/latency/90", Operations.Max, Operations.Max, "Latency 90th (ms)", "Latency 90th (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/95;http/latency/95", Operations.Max, Operations.Max, "Latency 95th (ms)", "Latency 95th (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/99;http/latency/99", Operations.Max, Operations.Max, "Latency 99th (ms)", "Latency 99th (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/mean;http/latency/mean", Operations.Max, Operations.Avg, "Mean latency (ms)", "Mean latency (ms)", "n2");
BenchmarksEventSource.Register("bombardier/latency/max;http/latency/max", Operations.Max, Operations.Max, "Max latency (ms)", "Max latency (ms)", "n2");
BenchmarksEventSource.Register("bombardier/rps/mean;http/rps/mean", Operations.Max, Operations.Sum, "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Register("bombardier/rps/max;http/rps/max", Operations.Max, Operations.Sum, "Requests/sec (max)", "Max requests per second", "n0");
BenchmarksEventSource.Register("bombardier/throughput;http/throughput", Operations.Max, Operations.Sum, "Read throughput (MB/s)", "Read throughput (MB/s)", "n2");
BenchmarksEventSource.Register("bombardier/raw", Operations.All, Operations.All, "Raw results", "Raw results", "json");
var total =
document["result"]["req1xx"].Value <long>()
+ document["result"]["req2xx"].Value <long>()
+ document["result"]["req3xx"].Value <long>()
+ document["result"]["req4xx"].Value <long>()
+ document["result"]["req5xx"].Value <long>()
+ document["result"]["others"].Value <long>();
var success = document["result"]["req2xx"].Value <long>() + document["result"]["req3xx"].Value <long>();
BenchmarksEventSource.Measure("bombardier/requests;http/requests", total);
BenchmarksEventSource.Measure("bombardier/badresponses;http/requests/badresponses", total - success);
BenchmarksEventSource.Measure("bombardier/latency/50;http/latency/50", document["result"]["latency"]["percentiles"]["50"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/75;http/latency/75", document["result"]["latency"]["percentiles"]["75"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/90;http/latency/90", document["result"]["latency"]["percentiles"]["90"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/95;http/latency/95", document["result"]["latency"]["percentiles"]["95"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/99;http/latency/99", document["result"]["latency"]["percentiles"]["99"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/mean;http/latency/mean", document["result"]["latency"]["mean"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/latency/max;http/latency/max", document["result"]["latency"]["max"].Value <double>().ToMilliseconds());
BenchmarksEventSource.Measure("bombardier/rps/max;http/rps/max", document["result"]["rps"]["max"].Value <double>());
BenchmarksEventSource.Measure("bombardier/rps/mean;http/rps/mean", document["result"]["rps"]["mean"].Value <double>());
BenchmarksEventSource.Measure("bombardier/raw", output);
var bytesPerSecond = document["result"]["bytesRead"].Value <long>() / document["result"]["timeTakenSeconds"].Value <double>();
// B/s to MB/s
BenchmarksEventSource.Measure("bombardier/throughput;http/throughput", bytesPerSecond / 1024 / 1024);
}
else
{
Console.WriteLine("Benchmark skipped");
}
// Clean temporary files
try
{
File.Delete(bodyFile);
}
catch
{
}
return(0);
}
static async Task Main(string[] args)
{
Console.WriteLine("WRK2 Client");
Console.WriteLine("args: " + String.Join(' ', args));
var wrk2Filename = await DownloadWrk2Async();
Console.Write("Measuring first request ... ");
await MeasureFirstRequest(args);
// Do we need to parse latency?
var parseLatency = args.Any(x => x == "--latency" || x == "-L");
// Extracting duration parameters
string warmup = "";
string duration = "";
var argsList = args.ToList();
var durationIndex = argsList.FindIndex(x => String.Equals(x, "-d", StringComparison.OrdinalIgnoreCase));
if (durationIndex >= 0)
{
duration = argsList[durationIndex + 1];
argsList.RemoveAt(durationIndex);
argsList.RemoveAt(durationIndex);
}
else
{
Console.WriteLine("Couldn't find -d argument");
return;
}
var warmupIndex = argsList.FindIndex(x => String.Equals(x, "-w", StringComparison.OrdinalIgnoreCase));
if (warmupIndex >= 0)
{
warmup = argsList[warmupIndex + 1];
argsList.RemoveAt(warmupIndex);
argsList.RemoveAt(warmupIndex);
}
args = argsList.ToArray();
var baseArguments = String.Join(' ', args.ToArray());
var process = new Process()
{
StartInfo =
{
FileName = wrk2Filename,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (!String.IsNullOrEmpty(warmup) && warmup != "0s")
{
process.StartInfo.Arguments = $" -d {warmup} {baseArguments}";
Console.WriteLine("> wrk2 " + process.StartInfo.Arguments);
process.Start();
process.WaitForExit();
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
process.StartInfo.Arguments = $" -d {duration} {baseArguments}";
Console.WriteLine("> wrk2 " + process.StartInfo.Arguments);
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
BenchmarksEventSource.Log.Metadata("wrk2/rps/mean", "max", "sum", "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Log.Metadata("wrk2/requests", "max", "sum", "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Log.Metadata("wrk2/latency/mean", "max", "sum", "Mean latency (ms)", "Mean latency (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/max", "max", "sum", "Max latency (ms)", "Max latency (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/errors/badresponses", "max", "sum", "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Log.Metadata("wrk2/errors/socketerrors", "max", "sum", "Socket errors", "Socket errors", "n0");
var rpsMatch = Regex.Match(output, @"Requests/sec:\s*([\d\.]*)");
if (rpsMatch.Success && rpsMatch.Groups.Count == 2)
{
BenchmarksEventSource.Measure("wrk2/rps/mean", double.Parse(rpsMatch.Groups[1].Value));
}
const string LatencyPattern = @"\s+{0}\s*([\d\.]+)([a-z]+)";
var avgLatencyMatch = Regex.Match(output, String.Format(LatencyPattern, "Latency"));
BenchmarksEventSource.Measure("wrk2/latency/mean", ReadLatency(avgLatencyMatch));
// Max latency is 3rd number after "Latency "
var maxLatencyMatch = Regex.Match(output, @"\s+Latency\s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
BenchmarksEventSource.Measure("wrk2/latency/max", ReadLatency(maxLatencyMatch));
var requestsCountMatch = Regex.Match(output, @"([\d\.]*) requests in ([\d\.]*)(\w*)");
BenchmarksEventSource.Measure("wrk2/requests", ReadRequests(requestsCountMatch));
var badResponsesMatch = Regex.Match(output, @"Non-2xx or 3xx responses: ([\d\.]*)");
BenchmarksEventSource.Measure("wrk2/errors/badresponses", ReadBadReponses(badResponsesMatch));
var socketErrorsMatch = Regex.Match(output, @"Socket errors: connect ([\d\.]*), read ([\d\.]*), write ([\d\.]*), timeout ([\d\.]*)");
BenchmarksEventSource.Measure("wrk2/errors/socketerrors", CountSocketErrors(socketErrorsMatch));
if (parseLatency)
{
BenchmarksEventSource.Log.Metadata("wrk2/latency/50", "max", "avg", "Latency 50th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/75", "max", "avg", "Latency 75th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/90", "max", "avg", "Latency 90th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/99", "max", "avg", "Latency 99th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/99.9", "max", "avg", "Latency 99.9th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/99.99", "max", "avg", "Latency 99.99th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/99.999", "max", "avg", "Latency 99.999th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/100", "max", "avg", "Latency 100th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Log.Metadata("wrk2/latency/distribution", "all", "all", "Latency distribution", "Latency distribution", "json");
BenchmarksEventSource.Measure("wrk2/latency/50", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "50\\.000%"))));
BenchmarksEventSource.Measure("wrk2/latency/75", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "75\\.000%"))));
BenchmarksEventSource.Measure("wrk2/latency/90", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "90\\.000%"))));
BenchmarksEventSource.Measure("wrk2/latency/99", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "99\\.000%"))));
BenchmarksEventSource.Measure("wrk2/latency/99.9", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "99\\.900%"))));
BenchmarksEventSource.Measure("wrk2/latency/99.99", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "99\\.990%"))));
BenchmarksEventSource.Measure("wrk2/latency/99.999", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "99\\.999%"))));
BenchmarksEventSource.Measure("wrk2/latency/100", ReadLatency(Regex.Match(output, String.Format(LatencyPattern, "100\\.000%"))));
using (var sr = new StringReader(output))
{
var line = "";
do
{
line = sr.ReadLine();
} while (line != null && !line.Contains("Detailed Percentile spectrum:"));
var doc = new JArray();
if (line != null)
{
sr.ReadLine();
sr.ReadLine();
line = sr.ReadLine();
while (line != null && !line.StartsWith("#"))
{
Console.WriteLine("Analyzing: " + line);
var values = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
doc.Add(
new JObject(
new JProperty("latency_us", decimal.Parse(values[0], CultureInfo.InvariantCulture)),
new JProperty("count", decimal.Parse(values[2], CultureInfo.InvariantCulture)),
new JProperty("percentile", decimal.Parse(values[1], CultureInfo.InvariantCulture))
));
line = sr.ReadLine();
}
}
BenchmarksEventSource.Measure("wrk2/latency/distribution", doc.ToString());
}
}
}
static async Task <int> Main(string[] args)
{
Console.WriteLine("Bombardier Client");
Console.WriteLine("args: " + String.Join(' ', args));
Console.Write("Measuring first request ... ");
await MeasureFirstRequest(args);
// Extracting parameters
var argsList = args.ToList();
TryGetArgumentValue("-w", argsList, out int warmup);
TryGetArgumentValue("-d", argsList, out int duration);
TryGetArgumentValue("-n", argsList, out int requests);
if (duration == 0 && requests == 0)
{
Console.WriteLine("Couldn't find valid -d and -n arguments (integers)");
return(-1);
}
TryGetArgumentValue("-w", argsList, out warmup);
args = argsList.ToArray();
string bombardierUrl = null;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.4/bombardier-windows-amd64.exe";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.4/bombardier-linux-amd64";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
bombardierUrl = "https://github.com/codesenberg/bombardier/releases/download/v1.2.4/bombardier-darwin-amd64";
}
else
{
Console.WriteLine("Unsupported platform");
return(-1);
}
var bombardierFileName = Path.GetFileName(bombardierUrl);
Console.WriteLine($"Downloading bombardier from {bombardierUrl} to {bombardierFileName}");
using (var downloadStream = await _httpClient.GetStreamAsync(bombardierUrl))
using (var fileStream = File.Create(bombardierFileName))
{
await downloadStream.CopyToAsync(fileStream);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine($"Setting execute permission on executable {bombardierFileName}");
Process.Start("chmod", "+x " + bombardierFileName);
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine($"Allow running bombardier");
Process.Start("spctl", "--add " + bombardierFileName);
}
}
var baseArguments = String.Join(' ', args.ToArray()) + " --print r --format json";
var process = new Process()
{
StartInfo =
{
FileName = bombardierFileName,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (warmup > 0)
{
process.StartInfo.Arguments = $" -d {warmup}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
process.Start();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
process.StartInfo.Arguments =
requests > 0
? $" -n {requests} {baseArguments}"
: $" -d {duration}s {baseArguments}";
Console.WriteLine("> bombardier " + process.StartInfo.Arguments);
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
var document = JObject.Parse(output);
BenchmarksEventSource.Log.Metadata("bombardier/requests", "max", "sum", "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/badresponses", "max", "sum", "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/latency/mean", "max", "sum", "Mean latency (us)", "Mean latency (us)", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/latency/max", "max", "sum", "Max latency (us)", "Max latency (us)", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/rps/mean", "max", "sum", "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/rps/max", "max", "sum", "Requests/sec (max)", "Max requests per second", "n0");
BenchmarksEventSource.Log.Metadata("bombardier/raw", "all", "all", "Raw results", "Raw results", "json");
var total =
document["result"]["req1xx"].Value <int>()
+ document["result"]["req2xx"].Value <int>()
+ document["result"]["req3xx"].Value <int>()
+ document["result"]["req3xx"].Value <int>()
+ document["result"]["req4xx"].Value <int>()
+ document["result"]["req5xx"].Value <int>()
+ document["result"]["others"].Value <int>();
var success = document["result"]["req2xx"].Value <int>() + document["result"]["req3xx"].Value <int>();
BenchmarksEventSource.Measure("bombardier/requests", total);
BenchmarksEventSource.Measure("bombardier/badresponses", total - success);
BenchmarksEventSource.Measure("bombardier/latency/mean", document["result"]["latency"]["mean"].Value <double>());
BenchmarksEventSource.Measure("bombardier/latency/max", document["result"]["latency"]["max"].Value <double>());
BenchmarksEventSource.Measure("bombardier/rps/max", document["result"]["rps"]["max"].Value <double>());
BenchmarksEventSource.Measure("bombardier/rps/mean", document["result"]["rps"]["mean"].Value <double>());
BenchmarksEventSource.Measure("bombardier/raw", output);
return(0);
}
static int RunCore(string fileName, string[] args, bool parseLatency)
{
// Extracting duration parameters
string warmup = "";
string duration = "";
var argsList = args.ToList();
var durationIndex = argsList.FindIndex(x => String.Equals(x, "-d", StringComparison.OrdinalIgnoreCase));
if (durationIndex >= 0)
{
duration = argsList[durationIndex + 1];
argsList.RemoveAt(durationIndex);
argsList.RemoveAt(durationIndex);
}
else
{
Console.WriteLine("Couldn't find -d argument");
return(-1);
}
var warmupIndex = argsList.FindIndex(x => String.Equals(x, "-w", StringComparison.OrdinalIgnoreCase));
if (warmupIndex >= 0)
{
warmup = argsList[warmupIndex + 1];
argsList.RemoveAt(warmupIndex);
argsList.RemoveAt(warmupIndex);
}
args = argsList.Select(Quote).ToArray();
var baseArguments = String.Join(' ', args);
var process = new Process()
{
StartInfo =
{
FileName = fileName,
RedirectStandardOutput = true,
UseShellExecute = false,
},
EnableRaisingEvents = true
};
var stringBuilder = new StringBuilder();
process.OutputDataReceived += (_, e) =>
{
if (e != null && e.Data != null)
{
Console.WriteLine(e.Data);
lock (stringBuilder)
{
stringBuilder.AppendLine(e.Data);
}
}
};
// Warmup
if (!String.IsNullOrEmpty(warmup) && warmup != "0s")
{
process.StartInfo.Arguments = $"-d {warmup} {baseArguments}";
Console.WriteLine("> wrk " + process.StartInfo.Arguments);
process.Start();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
}
else
{
Console.WriteLine("Warmup skipped");
}
lock (stringBuilder)
{
stringBuilder.Clear();
}
if (!String.IsNullOrEmpty(duration) && duration != "0s")
{
process.StartInfo.Arguments = $"-d {duration} {baseArguments}";
Console.WriteLine("> wrk " + process.StartInfo.Arguments);
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.WaitForExit();
if (process.ExitCode != 0)
{
return(process.ExitCode);
}
string output;
lock (stringBuilder)
{
output = stringBuilder.ToString();
}
BenchmarksEventSource.Register("wrk/rps/mean;http/rps/mean", Operations.Max, Operations.Sum, "Requests/sec", "Requests per second", "n0");
BenchmarksEventSource.Register("wrk/requests;http/requests", Operations.Max, Operations.Sum, "Requests", "Total number of requests", "n0");
BenchmarksEventSource.Register("wrk/latency/mean;http/latency/mean", Operations.Max, Operations.Avg, "Mean latency (ms)", "Mean latency (ms)", "n2");
BenchmarksEventSource.Register("wrk/latency/max;http/latency/max", Operations.Max, Operations.Max, "Max latency (ms)", "Max latency (ms)", "n2");
BenchmarksEventSource.Register("wrk/errors/badresponses;http/requests/badresponses", Operations.Max, Operations.Sum, "Bad responses", "Non-2xx or 3xx responses", "n0");
BenchmarksEventSource.Register("wrk/errors/socketerrors;http/requests/errors", Operations.Max, Operations.Sum, "Socket errors", "Socket errors", "n0");
BenchmarksEventSource.Register("wrk/throughput;http/throughput", Operations.Max, Operations.Sum, "Read throughput (MB/s)", "Read throughput (MB/s)", "n2");
/* Sample output
* > wrk -d 15s -c 1024 http://10.0.0.102:5000/plaintext --latency -t 32
* Running 15s test @ http://10.0.0.102:5000/plaintext
* 32 threads and 1024 connections
* Thread Stats Avg Stdev Max +/- Stdev
* Latency 0.93ms 1.55ms 63.85ms 97.35%
* Req/Sec 364.39k 34.02k 1.08M 89.87%
* Latency Distribution
* 50% 717.00us
* 75% 1.05ms
* 90% 1.47ms
* 99% 6.80ms
* 174818486 requests in 15.10s, 20.51GB read
* Requests/sec: 11577304.21
* Transfer/sec: 1.36GB
*/
const string LatencyPattern = @"\s+{0}\s+([\d\.]+)(\w+)";
var latencyMatch = Regex.Match(output, String.Format(LatencyPattern, "Latency"));
BenchmarksEventSource.Measure("wrk/latency/mean;http/latency/mean", ReadLatency(latencyMatch));
var rpsMatch = Regex.Match(output, @"Requests/sec:\s*([\d\.]*)");
if (rpsMatch.Success && rpsMatch.Groups.Count == 2)
{
BenchmarksEventSource.Measure("wrk/rps/mean;http/rps/mean", double.Parse(rpsMatch.Groups[1].Value));
}
var throughputMatch = Regex.Match(output, @"Transfer/sec:\s+([\d\.]+)(\w+)");
BenchmarksEventSource.Measure("wrk/throughput;http/throughput", ReadThroughput(throughputMatch));
// Max latency is 3rd number after "Latency "
var maxLatencyMatch = Regex.Match(output, @"\s+Latency\s+[\d\.]+\w+\s+[\d\.]+\w+\s+([\d\.]+)(\w+)");
BenchmarksEventSource.Measure("wrk/latency/max;http/latency/max", ReadLatency(maxLatencyMatch));
var requestsCountMatch = Regex.Match(output, @"([\d\.]*) requests in ([\d\.]*)(\w*)");
BenchmarksEventSource.Measure("wrk/requests;http/requests", ReadRequests(requestsCountMatch));
var badResponsesMatch = Regex.Match(output, @"Non-2xx or 3xx responses: ([\d\.]*)");
BenchmarksEventSource.Measure("wrk/errors/badresponses;http/requests/badresponses", ReadBadResponses(badResponsesMatch));
var socketErrorsMatch = Regex.Match(output, @"Socket errors: connect ([\d\.]*), read ([\d\.]*), write ([\d\.]*), timeout ([\d\.]*)");
BenchmarksEventSource.Measure("wrk/errors/socketerrors;http/requests/errors", CountSocketErrors(socketErrorsMatch));
if (parseLatency)
{
BenchmarksEventSource.Register("wrk/latency/50;http/latency/50", Operations.Max, Operations.Max, "Latency 50th (ms)", "Latency 50th (ms)", "n2");
BenchmarksEventSource.Register("wrk/latency/75;http/latency/75", Operations.Max, Operations.Max, "Latency 75th (ms)", "Latency 75th (ms)", "n2");
BenchmarksEventSource.Register("wrk/latency/90;http/latency/90", Operations.Max, Operations.Max, "Latency 90th (ms)", "Latency 90th (ms)", "n2");
BenchmarksEventSource.Register("wrk/latency/99;http/latency/99", Operations.Max, Operations.Max, "Latency 99th (ms)", "Latency 99th (ms)", "n2");
BenchmarksEventSource.Measure("wrk/latency/50;http/latency/50", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "50%"))));
BenchmarksEventSource.Measure("wrk/latency/75;http/latency/75", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "75%"))));
BenchmarksEventSource.Measure("wrk/latency/90;http/latency/90", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "90%"))));
BenchmarksEventSource.Measure("wrk/latency/99;http/latency/99", ReadLatency(Regex.Match(output, string.Format(LatencyPattern, "99%"))));
}
}
else
{
Console.WriteLine("Benchmark skipped");
}
return(0);
}
private static Process StartProcess()
{
var command = $"h2load {ServerUrl}";
foreach (var header in Headers)
{
command += $" -H \"{header.Key}: {header.Value}\"";
}
command += $" -c {Connections} -T {Timeout} -t {Threads} -m {Streams} --warm-up-time {Warmup}";
command += Requests > 0 ? $" -n {Requests}" : $" -D {Duration}";
switch (Protocol)
{
case "http": command += " --h1"; break;
case "https": command += " --h1"; break;
case "h2": break;
case "h2c": command += " --no-tls-proto=h2c"; break;
default: throw new InvalidOperationException("Unknown protocol: " + Protocol);
}
if (RequestBodyFile != null)
{
command += $" -d \"{RequestBodyFile}\"";
}
Log(command);
var process = new Process()
{
StartInfo =
{
FileName = "stdbuf",
Arguments = $"-oL {command}",
WorkingDirectory = Path.GetDirectoryName(typeof(Program).Assembly.Location),
RedirectStandardOutput = true,
RedirectStandardError = true
},
EnableRaisingEvents = true
};
process.OutputDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
Output += (e.Data + Environment.NewLine);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data != null)
{
Log(e.Data);
Error += (e.Data + Environment.NewLine);
}
};
process.Start();
BenchmarksEventSource.SetChildProcessId(process.Id);
process.BeginOutputReadLine();
process.BeginErrorReadLine();
return(process);
}