public void Sync_Continous_load() {
var statsCollector = new ClacksInstrumentation();
var dispatcher = new SyncCommandRepository();
dispatcher.AddCommand("BIN", request => Response.Create("OK").WithData(request.Data), DataExpectation.Auto);
var clientHandlerFactory = new FaultingSyncClientHandlerFactory(dispatcher);
using(new ClacksServer(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _port), statsCollector, clientHandlerFactory)) {
Console.WriteLine("created server");
var r = new Random();
var n = 50;
var startSignal = new ManualResetEvent(false);
var readySignals = new List<WaitHandle>();
var workers = new List<Task>();
var workerRequests = new int[n];
var pool = ConnectionPool.Create("127.0.0.1", _port);
var poolUseCount = 0;
for(var i = 0; i < n; i++) {
var ready = new ManualResetEvent(false);
readySignals.Add(ready);
var myPool = pool;
var workerId = i;
workers.Add(Task.Factory.StartNew(() => {
ready.Set();
startSignal.WaitOne();
var localUseCount = 0;
var maxPoolUse = r.Next(50, 150);
while(true) {
localUseCount++;
var globalUseCount = Interlocked.Increment(ref poolUseCount);
if(globalUseCount > r.Next(200, 300)) {
Console.WriteLine("{0}: creating new pool", workerId);
Interlocked.Exchange(ref poolUseCount, 0);
pool = ConnectionPool.Create("127.0.0.1", _port);
myPool = pool;
} else if(myPool != pool && localUseCount > maxPoolUse) {
Console.WriteLine("{0}: picking new pool", workerId);
myPool = pool;
localUseCount = 0;
}
using(var client = new ClacksClient(myPool)) {
for(var k = 0; k < 1000; k++) {
var payload = new StringBuilder();
payload.AppendFormat("id:{0},data:", workerId);
var size = r.Next(2, 100);
for(var l = 0; l < size; l++) {
payload.Append(Guid.NewGuid().ToString());
}
var data = payload.ToString();
var bytes = Encoding.ASCII.GetBytes(data);
Client.Response response;
try {
response = client.Exec(new Client.Request("BIN").WithData(bytes).ExpectData("OK"));
} catch(Exception) {
Console.WriteLine("request failed with data:\r\n{0}", payload);
throw;
}
Assert.AreEqual("OK", response.Status);
Assert.AreEqual(1, response.Arguments.Length);
var responseData = Encoding.ASCII.GetString(response.Data);
if(data != responseData) {
lock(workers) {
Console.WriteLine("bad data response for worker {0}", workerId);
Console.WriteLine("sent: {0}", data);
Console.WriteLine("received: {0}", responseData);
}
}
workerRequests[workerId]++;
Thread.Sleep(50);
}
}
}
}));
}
Console.WriteLine("waiting for workers to get ready");
WaitHandle.WaitAll(readySignals.ToArray());
startSignal.Set();
Console.WriteLine("starting work");
var t = Stopwatch.StartNew();
Task.Factory.StartNew(() => {
var lastRequests = new int[n];
Thread.Sleep(TimeSpan.FromMinutes(2));
while(true) {
var currentRequests = new int[n];
workerRequests.CopyTo(currentRequests, 0);
for(var i = 0; i < n; i++) {
if(currentRequests[i] == lastRequests[i]) {
Console.WriteLine("worker {0} has not made new requests: {1} == {2}", i, currentRequests[i], lastRequests[i]);
}
}
lastRequests = currentRequests;
Console.WriteLine("{0} Processed {1} requests with {2} faults via {3}/{4} total/active connections at {5,6:0} requests/second and {6:0.000}ms/request",
DateTime.Now,
statsCollector.Requests,
clientHandlerFactory.Faults,
statsCollector.Connected,
statsCollector.Connected - statsCollector.Disconnected,
statsCollector.Requests / t.Elapsed.TotalSeconds,
TimeSpan.FromTicks(statsCollector.RequestTicks).TotalMilliseconds / statsCollector.Requests
);
Thread.Sleep(TimeSpan.FromMinutes(1));
}
});
Task.WaitAny(workers.ToArray());
foreach(var worker in workers) {
if(worker.IsFaulted) {
foreach(var exception in worker.Exception.Flatten().InnerExceptions)
Console.WriteLine(exception);
}
}
t.Stop();
Console.WriteLine("{0} Processed {1} requests with {2} faults via {3}/{4} total/active connections at {5,6:0} requests/second and {6:0.000}ms/request",
DateTime.Now,
statsCollector.Requests,
clientHandlerFactory.Faults,
statsCollector.Connected,
statsCollector.Connected - statsCollector.Disconnected,
statsCollector.Requests / t.Elapsed.TotalSeconds,
TimeSpan.FromTicks(statsCollector.RequestTicks).TotalMilliseconds / statsCollector.Requests
);
}
}
private static Task RunServer(string host, int port, int faultInterval) {
return Task.Factory.StartNew(() => {
var t = Stopwatch.StartNew();
var instrumentation = new ClacksInstrumentation();
var dispatcher = new SyncCommandRepository();
dispatcher.AddCommand("BIN", request => Response.Create("OK").WithData(request.Data), DataExpectation.Auto);
var clientHandlerFactory = new FaultingSyncClientHandlerFactory(dispatcher, faultInterval);
using(new ClacksServer(new IPEndPoint(IPAddress.Parse(host), port), instrumentation, clientHandlerFactory)) {
Console.WriteLine("created server");
Thread.Sleep(TimeSpan.FromMinutes(1));
var rates = new Queue<double>();
var totalRequests = 0;
while(true) {
lock(instrumentation.Connections) {
var requestCount = instrumentation.Requests - totalRequests;
totalRequests += requestCount;
rates.Enqueue(requestCount / t.Elapsed.TotalSeconds);
if(rates.Count > 5) {
rates.Dequeue();
}
t.Restart();
Console.WriteLine("{0} Processed {1} requests with {2} faults via {3}/{4} total/active connections at {5,6:0} requests/second and {6:0.000}ms/request",
DateTime.Now,
instrumentation.Requests,
clientHandlerFactory.Faults,
instrumentation.Connected,
instrumentation.Connections.Count,
rates.Average(),
TimeSpan.FromTicks(instrumentation.RequestTicks).TotalMilliseconds / instrumentation.Requests
);
foreach(var statusGroup in from con in instrumentation.Connections
group con by con.Value.Status into grouped
select new { Status = grouped.Key, Connections = grouped }) {
Console.WriteLine(" {0} in state '{1}': {2}",
statusGroup.Connections.Count(),
statusGroup.Status,
string.Join(", ", from con in statusGroup.Connections orderby con.Value.Id select con.Value.Id)
);
}
}
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
});
}
public SyncFluentCommandRegistration(ServerBuilder serverBuilder, SyncCommandRepository repository, string command) {
_serverBuilder = serverBuilder;
_repository = repository;
_command = command;
}