public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
var maxCount = 10000;
if (args.Length > 0)
{
int.TryParse(args[0], out maxCount);
}
var sw = Stopwatch.StartNew();
var records = context.Client.EventStores.ReadAllEvents(EventStoreOffset.Zero, maxCount);
int msgCount = 0;
long dataSize = 0;
foreach (var record in records)
{
msgCount += 1;
dataSize += record.EventData.Length;
}
sw.Stop();
var speed = dataSize / sw.Elapsed.TotalSeconds;
var messageSpeed = (int)(msgCount / sw.Elapsed.TotalSeconds);
context.Log.Info("{0} msgPerSec or {1}", messageSpeed, FormatEvil.SpeedInBytes(speed));
PerfUtils.LogTeamCityGraphData(string.Format("EN_{0}_msgPerSec", maxCount), messageSpeed);
PerfUtils.LogTeamCityGraphData(string.Format("EN_{0}_bytesPerSec", maxCount), (int)speed);
return(true);
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
int batchSize = 10000;
string streamId = "batch";
string streamData = "Batch test";
if (args.Length > 0)
{
int.TryParse(args[0], out batchSize);
}
if (args.Length > 1)
{
streamId = args[1];
}
if (args.Length > 2)
{
streamData = args.Skip(2).Aggregate("", (x, y) => x + " " + y);
}
var global = Stopwatch.StartNew();
context.Client.EventStores.WriteEventsInLargeBatch(streamId, Enumerable.Repeat((Encoding.UTF8.GetBytes(streamData)), batchSize));
context.Log.Info("{0} per second", batchSize / global.Elapsed.TotalSeconds);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Key), (int)(batchSize / global.Elapsed.TotalSeconds));
return(true);
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
int threadCount = 5;
int batchSize = 10000;
int repeatForEachThread = 1;
string streamId = "batch";
string streamData = "Batch test";
if (args.Length > 0)
{
int.TryParse(args[0], out threadCount);
}
if (args.Length > 1)
{
int.TryParse(args[1], out batchSize);
}
if (args.Length > 2)
{
int.TryParse(args[2], out repeatForEachThread);
}
var global = Stopwatch.StartNew();
long total = 0;
long count = 0;
var threads = new List <Task>();
for (int t = 0; t < threadCount; t++)
{
var task = Task.Factory.StartNew(() =>
{
var watch = Stopwatch.StartNew();
for (int i = 0; i < repeatForEachThread; i++)
{
context.Client.Platform.WriteEventsInLargeBatch(streamId, Enumerable.Repeat(new RecordForStaging(Encoding.UTF8.GetBytes(streamData)), batchSize));
}
Interlocked.Add(ref total, watch.Elapsed.Ticks);
Interlocked.Add(ref count, batchSize * repeatForEachThread);
}, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness);
threads.Add(task);
}
Task.WaitAll(threads.ToArray());
//context.Completed();
context.Log.Info("{0} per second", count / global.Elapsed.TotalSeconds);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Key), (int)(count / global.Elapsed.TotalSeconds));
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
context.IsAsync();
long total = 0;
int count = 0;
var threads = new List <Task>();
int threadCount = 5;
var size = 1000;
if (args.Length > 0)
{
int.TryParse(args[0], out threadCount);
}
if (args.Length > 1)
{
int.TryParse(args[1], out size);
}
var global = Stopwatch.StartNew();
for (int t = 0; t < threadCount; t++)
{
var task = Task.Factory.StartNew(() =>
{
var watch = Stopwatch.StartNew();
for (int i = 0; i < size; i++)
{
context.Client.JsonClient.Post <ClientDto.WriteEvent>("/stream", new ClientDto.WriteEvent()
{
Data = Encoding.UTF8.GetBytes("This is some test message to load the server"),
Stream = "name",
ExpectedVersion = -1
});
//client.Get<ClientDto.WriteEvent>("/stream/name");
}
Interlocked.Add(ref total, watch.Elapsed.Ticks);
Interlocked.Add(ref count, size);
}, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness);
threads.Add(task);
}
Task.WaitAll(threads.ToArray());
context.Completed();
context.Log.Info("{0} per second", count / global.Elapsed.TotalSeconds);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Key), (int)(count / global.Elapsed.TotalSeconds));
return(true);
}
HashSet <string> ImportBatch(CommandProcessorContext context, string streamId, int batchCount, int batchSize)
{
var result = new HashSet <string>();
int totalBytes = 0;
var watch = Stopwatch.StartNew();
for (int i = 0; i < batchCount; i++)
{
string message = string.Format(singleThreadMessageTemplate, i);
context.Client.EventStores.WriteEventsInLargeBatch("",
Enumerable.Range(0, batchSize).Select(
x =>
{
try
{
var bytes = Encoding.UTF8.GetBytes(streamId + string.Format(message, x));
totalBytes += bytes.Length;
return(bytes);
}
catch (Exception ex)
{
context.Log.Error(ex.Message);
throw ex;
}
}));
for (int j = 0; j < batchSize; j++)
{
result.Add(string.Format(message, j));
}
}
var totalMs = watch.ElapsedMilliseconds;
var byteSize = batchSize * batchCount > 0 ? totalBytes / (batchCount * batchSize) : 0;
var key = string.Format("WB-{0}-{1}-{2}-{3}-bytesPerSec", 1, batchCount, batchSize, byteSize);
var bytesPerSec = totalMs > 0 ? (totalBytes * 1000D / totalMs) : 0;
context.Log.Debug("Throughput: {0}", FormatEvil.SpeedInBytes(bytesPerSec));
PerfUtils.LogTeamCityGraphData(key, (int)bytesPerSec);
return(result);
}
void TestPointer(CommandProcessorContext context, IEventPointer pointer, int repeat, string type)
{
var stop = Stopwatch.StartNew();
pointer.Write(0);
for (var i = 0; i < repeat; i++)
{
pointer.Write(i);
}
var writesPerSecond = repeat / stop.Elapsed.TotalSeconds;
context.Log.Debug("Throughput {1}: {0}", Math.Round(writesPerSecond), type);
context.Log.Debug("Elapsed sec {1}: {0}", stop.Elapsed.TotalSeconds, type);
var key = string.Format("{0}-{1}-{2}", Key, type, repeat);
PerfUtils.LogTeamCityGraphData(key + "-writesPerSec", (int)writesPerSecond);
}
private void WriteFlood(CommandProcessorContext context, int clientsCnt, int requestsCnt, int payloadSize)
{
context.IsAsync();
var dataSize = Math.Max(0, payloadSize - 100);
var metadataSize = Math.Min(100, payloadSize);
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
var doneEvent = new ManualResetEventSlim(false);
var succ = 0;
var fail = 0;
var all = 0;
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
var autoEvent = new AutoResetEvent(false);
var eventStreamId = "es" + Guid.NewGuid();
var received = 0;
var client = context.Client.CreateTcpConnection(
context,
(conn, pkg) =>
{
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
Interlocked.Increment(ref received);
var dto = pkg.Data.Deserialize <TcpClientMessageDto.WriteEventsCompleted>();
if (dto.Result == TcpClientMessageDto.OperationResult.Success)
{
if (Interlocked.Increment(ref succ) % 1000 == 0)
{
Console.Write(".");
}
}
else
{
if (Interlocked.Increment(ref fail) % 1000 == 0)
{
Console.Write("#");
}
}
if (Interlocked.Increment(ref all) == requestsCnt)
{
context.Success();
doneEvent.Set();
}
autoEvent.Set();
},
connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
clients.Add(client);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var write = new TcpClientMessageDto.WriteEvents(
eventStreamId,
ExpectedVersion.Any,
new[]
{
new TcpClientMessageDto.NewEvent(Guid.NewGuid().ToByteArray(),
"TakeSomeSpaceEvent",
0, 0,
Common.Utils.Helper.UTF8NoBom.GetBytes("DATA" + new string('*', dataSize)),
Common.Utils.Helper.UTF8NoBom.GetBytes("METADATA" + new string('$', metadataSize)))
},
false);
var package = new TcpPackage(TcpCommand.WriteEvents, Guid.NewGuid(), write.Serialize());
client.EnqueueSend(package.AsByteArray());
autoEvent.WaitOne();
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
clients.ForEach(client => client.Close());
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("Completed. Successes: {0}, failures: {1}", succ, fail);
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec, latency: {3:0.00} ms).",
all, sw.ElapsedMilliseconds, reqPerSec, (sw.Elapsed.TotalMilliseconds + 0.0) / requestsCnt);
PerfUtils.LogData(Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), (int)(100.0 * fail / (fail + succ)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)Math.Round(sw.Elapsed.TotalMilliseconds / requestsCnt));
if (succ != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
private void WriteFlood(CommandProcessorContext context, WriteFloodStats stats, int clientsCnt, long requestsCnt, int streamsCnt,
int size, int batchSize, string streamNamePrefix, RequestMonitor monitor)
{
context.IsAsync();
var doneEvent = new ManualResetEventSlim(false);
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
long last = 0;
string[] streams = Enumerable.Range(0, streamsCnt).Select(x =>
string.IsNullOrWhiteSpace(streamNamePrefix)
? Guid.NewGuid().ToString()
: $"{streamNamePrefix}-{x}"
).ToArray();
context.Log.Information("Writing streams randomly between {first} and {last}",
streams.FirstOrDefault(),
streams.LastOrDefault());
Console.WriteLine($"Last stream: {streams.LastOrDefault()}");
stats.StartTime = DateTime.UtcNow;
var sw2 = new Stopwatch();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
long sent = 0;
long received = 0;
var rnd = new Random();
var client = context._tcpTestClient.CreateTcpConnection(
context,
(conn, pkg) => {
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <WriteEventsCompleted>();
monitor.EndOperation(pkg.CorrelationId);
switch (dto.Result)
{
case OperationResult.Success:
Interlocked.Add(ref stats.Succ, batchSize);
if (stats.Succ - last > 1000)
{
last = stats.Succ;
Console.Write(".");
}
break;
case OperationResult.PrepareTimeout:
Interlocked.Increment(ref stats.PrepTimeout);
break;
case OperationResult.CommitTimeout:
Interlocked.Increment(ref stats.CommitTimeout);
break;
case OperationResult.ForwardTimeout:
Interlocked.Increment(ref stats.ForwardTimeout);
break;
case OperationResult.WrongExpectedVersion:
Interlocked.Increment(ref stats.WrongExpVersion);
break;
case OperationResult.StreamDeleted:
Interlocked.Increment(ref stats.StreamDeleted);
break;
default:
throw new ArgumentOutOfRangeException();
}
if (dto.Result != OperationResult.Success)
{
if (Interlocked.Increment(ref stats.Fail) % 1000 == 0)
{
Console.Write('#');
}
}
Interlocked.Increment(ref received);
var localAll = Interlocked.Add(ref stats.All, batchSize);
if (localAll % 100000 == 0)
{
stats.Elapsed = sw2.Elapsed;
stats.Rate = 1000.0 * 100000 / stats.Elapsed.TotalMilliseconds;
sw2.Restart();
context.Log.Debug(
"\nDONE TOTAL {writes} WRITES IN {elapsed} ({rate:0.0}/s) [S:{success}, F:{failures} (WEV:{wrongExpectedVersion}, " +
"P:{prepareTimeout}, C:{commitTimeout}, F:{forwardTimeout}, D:{streamDeleted})].",
localAll, stats.Elapsed, stats.Rate, stats.Succ, stats.Fail,
stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
stats.WriteStatsToFile(context.StatsLogger);
}
if (localAll >= requestsCnt)
{
context.Success();
doneEvent.Set();
}
},
connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
clients.Add(client);
threads.Add(new Thread(() => {
for (int j = 0; j < count; ++j)
{
var events = new NewEvent[batchSize];
for (int q = 0; q < batchSize; q++)
{
events[q] = new NewEvent(Guid.NewGuid().ToByteArray(),
"TakeSomeSpaceEvent",
1, 0,
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"DATA\" : \"" + new string('*', size) + "\"}"),
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"METADATA\" : \"" + new string('$', 100) + "\"}"));
}
var corrid = Guid.NewGuid();
var write = new WriteEvents(
streams[rnd.Next(streamsCnt)],
ExpectedVersion.Any,
events,
false);
var package = new TcpPackage(TcpCommand.WriteEvents, corrid, write.Serialize());
monitor.StartOperation(corrid);
client.EnqueueSend(package.AsByteArray());
var localSent = Interlocked.Increment(ref sent);
while (localSent - Interlocked.Read(ref received) >
context._tcpTestClient.Options.WriteWindow / clientsCnt)
{
Thread.Sleep(1);
}
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
sw2.Start();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
clients.ForEach(client => client.Close());
stats.WriteStatsToFile(context.StatsLogger);
context.Log.Information(
"Completed. Successes: {success}, failures: {failures} (WRONG VERSION: {wrongExpectedVersion}, P: {prepareTimeout}, " +
"C: {commitTimeout}, F: {forwardTimeout}, D: {streamDeleted})",
stats.Succ, stats.Fail, stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
var reqPerSec = (stats.All + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", stats.All,
sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", stats.Succ), PerfUtils.Col("failures", stats.Fail)));
var failuresRate = (int)(100 * stats.Fail / (stats.Fail + stats.Succ));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt, size),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt,
streamsCnt, size), failuresRate);
monitor.GetMeasurementDetails();
if (Interlocked.Read(ref stats.Succ) != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
var data = "test-data";
string metadata = null;
bool isJson = false;
string login = null;
string pass = null;
if (args.Length > 0)
{
if (args.Length < 3 || args.Length > 7 || args.Length == 6)
{
return(false);
}
eventStreamId = args[0];
expectedVersion = args[1].ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
data = args[2];
if (args.Length >= 4)
{
metadata = args[3];
}
if (args.Length >= 5)
{
isJson = bool.Parse(args[4]);
}
if (args.Length >= 7)
{
login = args[5];
pass = args[6];
}
}
context.IsAsync();
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}, L{1}]: Writing...", conn.RemoteEndPoint, conn.LocalEndPoint);
var writeDto = new TcpClientMessageDto.WriteEvents(
eventStreamId,
expectedVersion,
new[]
{
new TcpClientMessageDto.NewEvent(Guid.NewGuid().ToByteArray(),
"TakeSomeSpaceEvent",
isJson ? 1: 0, 0,
Helper.UTF8NoBom.GetBytes(data),
Helper.UTF8NoBom.GetBytes(metadata ?? string.Empty))
},
false);
var package = new TcpPackage(TcpCommand.WriteEvents,
login == null ? TcpFlags.None : TcpFlags.Authenticated,
Guid.NewGuid(),
login,
pass,
writeDto.Serialize()).AsByteArray();
sw.Start();
conn.EnqueueSend(package);
},
handlePackage: (conn, pkg) =>
{
sw.Stop();
context.Log.Info("Write request took: {0}.", sw.Elapsed);
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.WriteEventsCompleted>();
if (dto.Result == TcpClientMessageDto.OperationResult.Success)
{
context.Log.Info("Successfully written.");
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)sw.ElapsedMilliseconds);
context.Success();
}
else
{
context.Log.Info("Error while writing: {0} ({1}).", dto.Message, dto.Result);
context.Fail();
}
conn.Close();
},
connectionClosed: (connection, error) => context.Fail(reason: "Connection was closed prematurely."));
context.WaitForCompletion();
return(true);
}
private void ReadFlood(CommandProcessorContext context, string eventStreamId, int clientsCnt, long requestsCnt,
bool resolveLinkTos, bool requireMaster)
{
context.IsAsync();
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
var doneEvent = new ManualResetEventSlim(false);
var sw2 = new Stopwatch();
long succ = 0;
long fail = 0;
long all = 0;
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
long received = 0;
long sent = 0;
var client = context.Client.CreateTcpConnection(
context,
(conn, pkg) =>
{
if (pkg.Command != TcpCommand.ReadEventCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.ReadEventCompleted>();
_monitor.EndOperation(pkg.CorrelationId);
if (dto.Result == TcpClientMessageDto.ReadEventCompleted.ReadEventResult.Success)
{
if (Interlocked.Increment(ref succ) % 1000 == 0)
{
Console.Write(".");
}
}
else
{
if (Interlocked.Increment(ref fail) % 1000 == 0)
{
Console.Write("#");
}
}
Interlocked.Increment(ref received);
var localAll = Interlocked.Increment(ref all);
if (localAll % 100000 == 0)
{
var elapsed = sw2.Elapsed;
sw2.Restart();
context.Log.Trace("\nDONE TOTAL {0} READS IN {1} ({2:0.0}/s).", localAll, elapsed, 1000.0 * 100000 / elapsed.TotalMilliseconds);
}
if (localAll == requestsCnt)
{
context.Success();
doneEvent.Set();
}
},
connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
clients.Add(client);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var corrId = Guid.NewGuid();
var read = new TcpClientMessageDto.ReadEvent(eventStreamId, 0, resolveLinkTos, requireMaster);
var package = new TcpPackage(TcpCommand.ReadEvent, corrId, read.Serialize());
_monitor.StartOperation(corrId);
client.EnqueueSend(package.AsByteArray());
var localSent = Interlocked.Increment(ref sent);
while (localSent - Interlocked.Read(ref received) > context.Client.Options.ReadWindow / clientsCnt)
{
Thread.Sleep(1);
}
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
sw2.Start();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
clients.ForEach(client => client.Close());
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("Completed. READS succ: {0}, fail: {1}.", Interlocked.Read(ref succ), Interlocked.Read(ref fail));
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);
_monitor.GetMeasurementDetails();
PerfUtils.LogData(Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("readsCnt", all)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
if (succ != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
private void WriteFlood(CommandProcessorContext context, int clientsCnt, long requestsCnt, int streamsCnt)
{
context.IsAsync();
var doneEvent = new AutoResetEvent(false);
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
long succ = 0;
long fail = 0;
long prepTimeout = 0;
long commitTimeout = 0;
long forwardTimeout = 0;
long wrongExpVersion = 0;
long streamDeleted = 0;
long all = 0;
var streams = Enumerable.Range(0, streamsCnt).Select(x => Guid.NewGuid().ToString()).ToArray();
var sw2 = new Stopwatch();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
int sent = 0;
int received = 0;
var rnd = new Random();
var client = context.Client.CreateTcpConnection(
context,
(conn, pkg) =>
{
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>();
switch ((OperationErrorCode)dto.ErrorCode)
{
case OperationErrorCode.Success:
Interlocked.Increment(ref succ);
break;
case OperationErrorCode.PrepareTimeout:
Interlocked.Increment(ref prepTimeout);
break;
case OperationErrorCode.CommitTimeout:
Interlocked.Increment(ref commitTimeout);
break;
case OperationErrorCode.ForwardTimeout:
Interlocked.Increment(ref forwardTimeout);
break;
case OperationErrorCode.WrongExpectedVersion:
Interlocked.Increment(ref wrongExpVersion);
break;
case OperationErrorCode.StreamDeleted:
Interlocked.Increment(ref streamDeleted);
break;
default:
throw new ArgumentOutOfRangeException();
}
if (dto.ErrorCode != (int)OperationErrorCode.Success)
{
Interlocked.Increment(ref fail);
}
Interlocked.Increment(ref received);
var localAll = Interlocked.Increment(ref all);
if (localAll % 1000 == 0)
{
Console.Write(".");
}
if (localAll % 100000 == 0)
{
var elapsed = sw2.Elapsed;
sw2.Restart();
context.Log.Trace("\nDONE TOTAL {0} WRITES IN {1} ({2:0.0}/s).",
localAll,
elapsed,
1000.0 * 100000 / elapsed.TotalMilliseconds);
}
if (localAll == requestsCnt)
{
doneEvent.Set();
}
},
connectionClosed: (conn, err) =>
{
if (all < requestsCnt)
{
context.Fail(null, "Socket was closed, but not all requests were completed.");
}
else
{
context.Success();
}
});
clients.Add(client);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var corrid = Guid.NewGuid();
var write = new ClientMessageDto.WriteEvents(
Guid.Empty,
streams[rnd.Next(streamsCnt)],
ExpectedVersion.Any,
new[] {
new ClientMessageDto.Event(Guid.NewGuid(),
"TakeSomeSpaceEvent",
Encoding.UTF8.GetBytes("DATA" + new string('*', 256)),
Encoding.UTF8.GetBytes("METADATA" + new string('$', 100)))
});
var package = new TcpPackage(TcpCommand.WriteEvents, corrid, write.Serialize());
client.EnqueueSend(package.AsByteArray());
Interlocked.Increment(ref sent);
while (sent - received > context.Client.Options.WriteWindow)
{
Thread.Sleep(1);
}
}
}));
}
var sw = Stopwatch.StartNew();
sw2.Start();
foreach (var thread in threads)
{
thread.IsBackground = true;
thread.Start();
}
doneEvent.WaitOne();
sw.Stop();
foreach (var client in clients)
{
client.Close();
}
context.Log.Info("Completed. Successes: {0}, failures: {1} (WRONG VERSION: {2}, P: {3}, C: {4}, F: {5}, D: {6})",
succ,
fail,
wrongExpVersion,
prepTimeout,
commitTimeout,
forwardTimeout,
streamDeleted);
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).",
all,
sw.ElapsedMilliseconds,
reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt),
100 * fail / (fail + succ));
context.Success();
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var fromNumber = 0;
if (args.Length > 0)
{
if (args.Length > 2)
{
return(false);
}
eventStreamId = args[0];
if (args.Length == 2)
{
fromNumber = int.Parse(args[1]);
}
}
context.IsAsync();
var corrid = Guid.NewGuid();
var readDto = new ClientMessageDto.ReadEvent(corrid, eventStreamId, fromNumber);
var package = new TcpPackage(TcpCommand.ReadEvent, corrid, readDto.Serialize());
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}]: Reading...", conn.EffectiveEndPoint);
sw.Start();
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) =>
{
if (pkg.Command != TcpCommand.ReadEventCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
sw.Stop();
var dto = pkg.Data.Deserialize <ClientMessageDto.ReadEventCompleted>();
context.Log.Info("READ events from <{0}>:\n\n"
+ "\tCorrelationId: {1}\n"
+ "\tEventStreamId: {2}\n"
+ "\tEventNumber: {3}\n"
+ "\tReadResult: {4}\n"
+ "\tEventType: {5}\n"
+ "\tData: {6}\n"
+ "\tMetadata: {7}\n",
eventStreamId,
dto.CorrelationId,
dto.EventStreamId,
dto.EventNumber,
(SingleReadResult)dto.Result,
dto.EventType,
Encoding.UTF8.GetString(dto.Data ?? new byte[0]),
Encoding.UTF8.GetString(dto.Metadata ?? new byte[0]));
context.Log.Info("Read request took: {0}.", sw.Elapsed);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)sw.ElapsedMilliseconds);
conn.Close();
context.Success();
},
connectionClosed: (connection, error) =>
{
if (error == SocketError.Success)
{
context.Success();
}
else
{
context.Fail();
}
});
context.WaitForCompletion();
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
int eventsCnt = 10;
if (args.Length > 0)
{
if (args.Length > 3)
{
return(false);
}
eventStreamId = args[0];
if (args.Length > 1)
{
expectedVersion = args[1].ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
}
if (args.Length > 2)
{
eventsCnt = int.Parse(args[1]);
}
}
context.IsAsync();
var sw = new Stopwatch();
var stage = Stage.AcquiringTransactionId;
long transactionId = -1;
var writtenEvents = 0;
var corrid = Guid.NewGuid();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}]: Starting transaction...", conn.EffectiveEndPoint);
sw.Start();
var tranStart = new ClientMessageDto.TransactionStart(eventStreamId, expectedVersion);
var package = new TcpPackage(TcpCommand.TransactionStart, corrid, tranStart.Serialize());
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) =>
{
switch (stage)
{
case Stage.AcquiringTransactionId:
{
if (pkg.Command != TcpCommand.TransactionStartCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <ClientMessageDto.TransactionStartCompleted>();
if ((OperationErrorCode)dto.ErrorCode != OperationErrorCode.Success)
{
var msg = string.Format("Error while starting transaction: {0} ({1}).", dto.Error, (OperationErrorCode)dto.ErrorCode);
context.Log.Info(msg);
context.Fail(reason: msg);
}
else
{
context.Log.Info("Successfully started transaction. TransactionId: {0}.", dto.TransactionId);
context.Log.Info("Now sending transactional events...", dto.TransactionId);
transactionId = dto.TransactionId;
stage = Stage.Writing;
for (int i = 0; i < eventsCnt; ++i)
{
var writeDto = new ClientMessageDto.TransactionWrite(transactionId,
eventStreamId,
new[]
{
new ClientMessageDto.Event(Guid.NewGuid(),
"TakeSomeSpaceEvent",
Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()),
Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()))
});
var package = new TcpPackage(TcpCommand.TransactionWrite, corrid, writeDto.Serialize());
conn.EnqueueSend(package.AsByteArray());
}
}
break;
}
case Stage.Writing:
{
if (pkg.Command != TcpCommand.TransactionWriteCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <ClientMessageDto.TransactionWriteCompleted>();
if ((OperationErrorCode)dto.ErrorCode != OperationErrorCode.Success)
{
var msg = string.Format("Error while writing transactional event: {0} ({1}).", dto.Error, (OperationErrorCode)dto.ErrorCode);
context.Log.Info(msg);
context.Fail(reason: msg);
}
else
{
writtenEvents += 1;
if (writtenEvents == eventsCnt)
{
context.Log.Info("Written all events. Committing...");
stage = Stage.Committing;
var commitDto = new ClientMessageDto.TransactionCommit(transactionId, eventStreamId);
var package = new TcpPackage(TcpCommand.TransactionCommit, corrid, commitDto.Serialize());
conn.EnqueueSend(package.AsByteArray());
}
}
break;
}
case Stage.Committing:
{
if (pkg.Command != TcpCommand.TransactionCommitCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
sw.Stop();
var dto = pkg.Data.Deserialize <ClientMessageDto.TransactionCommitCompleted>();
if ((OperationErrorCode)dto.ErrorCode != OperationErrorCode.Success)
{
var msg = string.Format("Error while committing transaction: {0} ({1}).", dto.Error, (OperationErrorCode)dto.ErrorCode);
context.Log.Info(msg);
context.Log.Info("Transaction took: {0}.", sw.Elapsed);
context.Fail(reason: msg);
}
else
{
context.Log.Info("Successfully committed transaction [{0}]!", dto.TransactionId);
context.Log.Info("Transaction took: {0}.", sw.Elapsed);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)sw.ElapsedMilliseconds);
context.Success();
}
conn.Close();
break;
}
default:
throw new ArgumentOutOfRangeException();
}
},
connectionClosed: (connection, error) =>
{
if (error == SocketError.Success && stage == Stage.Done)
{
context.Success();
}
else
{
context.Fail();
}
});
context.WaitForCompletion();
return(true);
}
private void WriteFlood(CommandProcessorContext context, int clientsCnt, int requestsCnt)
{
context.IsAsync();
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
var autoResetEvent = new AutoResetEvent(false);
var succ = 0;
var fail = 0;
var all = 0;
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
var autoEvent = new AutoResetEvent(false);
var eventStreamId = "es" + Guid.NewGuid();
var client = context.Client.CreateTcpConnection(
context,
(conn, pkg) =>
{
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>();
if (dto.ErrorCode == (int)OperationErrorCode.Success)
{
if (Interlocked.Increment(ref succ) % 1000 == 0)
{
Console.Write(".");
}
}
else
{
Interlocked.Increment(ref fail);
}
if (Interlocked.Increment(ref all) == requestsCnt)
{
autoResetEvent.Set();
}
autoEvent.Set();
},
connectionClosed: (conn, err) =>
{
if (all < requestsCnt)
{
context.Fail(null, "Socket was closed, but not all requests were completed.");
}
else
{
context.Success();
}
});
clients.Add(client);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var write = new ClientMessageDto.WriteEvents(
eventStreamId,
ExpectedVersion.Any,
new[] {
new ClientMessageDto.Event(Guid.NewGuid(),
"TakeSomeSpaceEvent",
Encoding.UTF8.GetBytes("DATA" + new string('*', 256)),
Encoding.UTF8.GetBytes("METADATA" + new string('$', 100)))
});
var package = new TcpPackage(TcpCommand.WriteEvents, Guid.NewGuid(), write.Serialize());
client.EnqueueSend(package.AsByteArray());
autoEvent.WaitOne();
}
}));
}
var sw = Stopwatch.StartNew();
foreach (var thread in threads)
{
thread.IsBackground = true;
thread.Start();
}
autoResetEvent.WaitOne();
sw.Stop();
foreach (var client in clients)
{
client.Close();
}
context.Log.Info("Completed. Successes: {0}, failures: {1}", succ, fail);
var reqPerSec = (requestsCnt + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).",
all,
sw.ElapsedMilliseconds,
reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt),
100 * fail / (fail + succ));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword),
(int)(sw.ElapsedMilliseconds / requestsCnt));
context.Success();
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
var data = GenerateTestData();
string metadata = null;
if (args.Length > 0)
{
if (args.Length < 3 || args.Length > 4)
{
return(false);
}
eventStreamId = args[0];
expectedVersion = args[1].ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
data = args[2];
if (args.Length == 4)
{
metadata = args[3];
}
}
context.IsAsync();
var writeDto = new TcpClientMessageDto.WriteEvents(
eventStreamId,
expectedVersion,
new[]
{
new TcpClientMessageDto.NewEvent(Guid.NewGuid().ToByteArray(),
"JsonDataEvent",
1, 0,
Helper.UTF8NoBom.GetBytes(data),
Helper.UTF8NoBom.GetBytes(metadata ?? string.Empty))
},
false);
var package = new TcpPackage(TcpCommand.WriteEvents, Guid.NewGuid(), writeDto.Serialize());
var sw = new Stopwatch();
bool dataReceived = false;
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}, L{1}]: Writing...", conn.RemoteEndPoint, conn.LocalEndPoint);
sw.Start();
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) =>
{
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
dataReceived = true;
sw.Stop();
var dto = pkg.Data.Deserialize <TcpClientMessageDto.WriteEventsCompleted>();
if (dto.Result == TcpClientMessageDto.OperationResult.Success)
{
context.Log.Info("Successfully written. EventId: {0}.", package.CorrelationId);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)Math.Round(sw.Elapsed.TotalMilliseconds));
}
else
{
context.Log.Info("Error while writing: {0} ({1}).", dto.Message, dto.Result);
}
context.Log.Info("Write request took: {0}.", sw.Elapsed);
conn.Close();
context.Success();
},
connectionClosed: (connection, error) =>
{
if (dataReceived && error == SocketError.Success)
{
context.Success();
}
else
{
context.Fail();
}
});
context.WaitForCompletion();
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
const string data = "test-data";
var eventStreamId = "test-stream";
var writeCount = 10;
var expectedVersion = ExpectedVersion.Any;
if (args.Length > 0)
{
if (args.Length > 3)
{
return(false);
}
writeCount = int.Parse(args[0]);
if (args.Length >= 2)
{
eventStreamId = args[1];
}
if (args.Length >= 3)
{
expectedVersion = args[2].Trim().ToUpper() == "ANY"
? ExpectedVersion.Any
: int.Parse(args[2].Trim());
}
}
context.IsAsync();
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn => {
context.Log.Information("[{remoteEndPoint}, L{localEndPoint}]: Writing...", conn.RemoteEndPoint,
conn.LocalEndPoint);
var writeDto = new TcpClientMessageDto.WriteEvents(
eventStreamId,
expectedVersion,
Enumerable.Range(0, writeCount).Select(x => new TcpClientMessageDto.NewEvent(
Guid.NewGuid().ToByteArray(),
"type",
0, 0,
Helper.UTF8NoBom.GetBytes(data),
new byte[0])).ToArray(),
false);
var package = new TcpPackage(TcpCommand.WriteEvents, Guid.NewGuid(), writeDto.Serialize())
.AsByteArray();
sw.Start();
conn.EnqueueSend(package);
},
handlePackage: (conn, pkg) => {
sw.Stop();
context.Log.Information("Write request took: {elapsed}.", sw.Elapsed);
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.WriteEventsCompleted>();
if (dto.Result == TcpClientMessageDto.OperationResult.Success)
{
context.Log.Information("Successfully written {writeCount} events.", writeCount);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword),
(int)Math.Round(sw.Elapsed.TotalMilliseconds));
context.Success();
}
else
{
context.Log.Information("Error while writing: {e}.", dto.Result);
context.Fail();
}
conn.Close();
},
connectionClosed: (connection, error) => context.Fail(reason: "Connection was closed prematurely."));
context.WaitForCompletion();
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var fromNumber = 0;
const bool resolveLinkTos = false;
var requireLeader = false;
if (args.Length > 0)
{
if (args.Length > 3)
{
return(false);
}
eventStreamId = args[0];
if (args.Length >= 2)
{
fromNumber = int.Parse(args[1]);
}
if (args.Length >= 3)
{
requireLeader = bool.Parse(args[2]);
}
}
context.IsAsync();
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn => {
context.Log.Information("[{remoteEndPoint}, L{localEndPoint}]: Reading...", conn.RemoteEndPoint,
conn.LocalEndPoint);
var readDto =
new TcpClientMessageDto.ReadEvent(eventStreamId, fromNumber, resolveLinkTos, requireLeader);
var package =
new TcpPackage(TcpCommand.ReadEvent, Guid.NewGuid(), readDto.Serialize()).AsByteArray();
sw.Start();
conn.EnqueueSend(package);
},
handlePackage: (conn, pkg) => {
sw.Stop();
context.Log.Information("Read request took: {elapsed}.", sw.Elapsed);
if (pkg.Command != TcpCommand.ReadEventCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.ReadEventCompleted>();
context.Log.Information("READ events from <{stream}>:\n\n"
+ "\tEventStreamId: {stream}\n"
+ "\tEventNumber: {eventNumber}\n"
+ "\tReadResult: {readResult}\n"
+ "\tEventType: {eventType}\n"
+ "\tData: {data}\n"
+ "\tMetadata: {metadata}\n",
eventStreamId,
eventStreamId,
dto.Event.Event.EventNumber,
(ReadEventResult)dto.Result,
dto.Event.Event.EventType,
Helper.UTF8NoBom.GetString(dto.Event.Event.Data ?? new byte[0]),
Helper.UTF8NoBom.GetString(dto.Event.Event.Metadata ?? new byte[0]));
if (dto.Result == TcpClientMessageDto.ReadEventCompleted.ReadEventResult.Success)
{
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword),
(int)Math.Round(sw.Elapsed.TotalMilliseconds));
context.Success();
}
else
{
context.Fail();
}
conn.Close();
},
connectionClosed: (connection, error) => context.Fail(reason: "Connection was closed prematurely."));
return(true);
}
private void Flood(CommandProcessorContext context,
string eventStreamId,
int clientsCnt,
int minPerSecond,
int maxPerSecond,
int runTimeMinutes)
{
context.IsAsync();
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
var doneEvent = new ManualResetEvent(false);
var done = false;
var succ = 0;
var fail = 0;
var requestsCnt = 0;
int sent = 0;
int received = 0;
var watchLockRoot = new object();
var sw = Stopwatch.StartNew();
for (int i = 0; i < clientsCnt; i++)
{
var esId = eventStreamId ?? "Stream-" + Thread.CurrentThread.ManagedThreadId % 3;
var client = context.Client.CreateTcpConnection(
context,
(conn, pkg) => {
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.WriteEventsCompleted>();
if (dto.Result == TcpClientMessageDto.OperationResult.Success)
{
var succDone = Interlocked.Increment(ref succ);
if (succDone % maxPerSecond == 0)
{
Console.Write(".");
}
Interlocked.Increment(ref requestsCnt);
}
else
{
Interlocked.Increment(ref fail);
}
Interlocked.Increment(ref received);
},
connectionClosed: (conn, err) => {
if (!done)
{
context.Fail(reason: "Socket was closed, but not all requests were completed.");
}
else
{
context.Success();
}
});
clients.Add(client);
threads.Add(new Thread(() => {
var sentCount = 0;
var sleepTime = 0;
var dataSizeCoefficient = 1;
var currentMinute = -1;
while (true)
{
TimeSpan elapsed;
lock (watchLockRoot)
elapsed = sw.Elapsed;
if (elapsed.TotalMinutes > runTimeMinutes)
{
done = true;
doneEvent.Set();
break;
}
if (sentCount == 0)
{
int elapsedMinutesInt = (int)elapsed.TotalMinutes;
lock (_randomLockRoot) {
sentCount = minPerSecond == maxPerSecond
? maxPerSecond
: _random.Next(minPerSecond, maxPerSecond);
dataSizeCoefficient = _random.Next(8, 256);
}
if (currentMinute != elapsedMinutesInt)
{
currentMinute = elapsedMinutesInt;
context.Log.Information(
"\nElapsed {elapsed} of {runTime} minutes, sent {sent}; next block coef. {dataSizeCoefficient}",
elapsedMinutesInt,
runTimeMinutes,
sent,
dataSizeCoefficient);
}
sleepTime = 1000 / sentCount;
}
var dataSize = dataSizeCoefficient * 8;
var write = new TcpClientMessageDto.WriteEvents(
esId,
ExpectedVersion.Any,
new[] {
new TcpClientMessageDto.NewEvent(
Guid.NewGuid().ToByteArray(),
"TakeSomeSpaceEvent",
0, 0,
Helper.UTF8NoBom.GetBytes(
"DATA" + dataSize.ToString(" 00000 ") + new string('*', dataSize)),
Helper.UTF8NoBom.GetBytes("METADATA" + new string('$', 100)))
},
false);
var package = new TcpPackage(TcpCommand.WriteEvents, Guid.NewGuid(), write.Serialize());
client.EnqueueSend(package.AsByteArray());
Interlocked.Increment(ref sent);
Thread.Sleep(sleepTime);
sentCount -= 1;
while (sent - received > context.Client.Options.WriteWindow / clientsCnt)
{
Thread.Sleep(1);
}
}
}));
}
foreach (var thread in threads)
{
thread.IsBackground = true;
thread.Start();
}
doneEvent.WaitOne();
sw.Stop();
foreach (var client in clients)
{
client.Close();
}
context.Log.Information("Completed. Successes: {success}, failures: {failures}", succ, fail);
var reqPerSec = (requestsCnt + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).",
requestsCnt,
sw.ElapsedMilliseconds,
reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail))
);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt),
100 * fail / (fail + succ));
context.Success();
}
private void WriteFlood(CommandProcessorContext context, int clientsCnt, long requestsCnt, int streamsCnt,
int size)
{
context.IsAsync();
var doneEvent = new ManualResetEventSlim(false);
var clients = new List <IEventStoreConnection>();
var threads = new List <Thread>();
long succ = 0;
var streams = Enumerable.Range(0, streamsCnt).Select(x => Guid.NewGuid().ToString()).ToArray();
var sw2 = new Stopwatch();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
var rnd = new Random();
var settings = ConnectionSettings.Create()
.UseConsoleLogger()
.PerformOnAnyNode()
.LimitReconnectionsTo(10)
.LimitRetriesForOperationTo(10)
.LimitOperationsQueueTo(10000)
.LimitConcurrentOperationsTo(context.Client.Options.WriteWindow / clientsCnt)
.FailOnNoServerResponse();
var client = EventStoreConnection.Create(settings,
new Uri($"tcp://{context.Client.TcpEndpoint.GetHost()}:{context.Client.TcpEndpoint.GetPort()}"));
clients.Add(client);
threads.Add(new Thread(_ => {
client.ErrorOccurred += (s, e) => context.Fail(e.Exception, "Error on connection");
client.ConnectAsync().Wait();
for (int j = 0; j < count; ++j)
{
var task = client.AppendToStreamAsync(streams[rnd.Next(streamsCnt)],
ExpectedVersion.Any,
new EventData(Guid.NewGuid(),
"TakeSomeSpaceEvent",
false,
Common.Utils.Helper.UTF8NoBom.GetBytes("DATA" + new string('*', size)),
Common.Utils.Helper.UTF8NoBom.GetBytes("METADATA" + new string('$', 100))));
task.ContinueWith(x => {
if (x.IsFaulted)
{
context.Fail(x.Exception.InnerException, "Error on writing operation.");
return;
}
var localAll = Interlocked.Increment(ref succ);
if (localAll % 1000 == 0)
{
Console.Write('.');
}
if (localAll % 100000 == 0)
{
var elapsed = sw2.Elapsed;
sw2.Restart();
context.Log.Debug("\nDONE TOTAL {writes} WRITES IN {elapsed} ({rate:0.0}/s).",
localAll,
elapsed,
1000.0 * 100000 / elapsed.TotalMilliseconds);
}
if (localAll == requestsCnt)
{
context.Success();
doneEvent.Set();
}
});
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
sw2.Start();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
clients.ForEach(x => x.Close());
sw.Stop();
context.Log.Information("Completed. Successes: {success}.", succ);
var reqPerSec = (succ + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", succ,
sw.ElapsedMilliseconds, reqPerSec);
var fail = requestsCnt - succ;
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));
var failuresRate = (int)(100 * fail / (fail + succ));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt, size),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt,
streamsCnt, size), failuresRate);
if (Interlocked.Read(ref succ) != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
int eventsCnt = 10;
if (args.Length > 0)
{
if (args.Length > 3)
{
return(false);
}
eventStreamId = args[0];
if (args.Length > 1)
{
expectedVersion = args[1].ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
}
if (args.Length > 2)
{
eventsCnt = int.Parse(args[1]);
}
}
context.IsAsync();
var sw = new Stopwatch();
var stage = Stage.AcquiringTransactionId;
long transactionId = -1;
var writtenEvents = 0;
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn => {
context.Log.Info("[{remoteEndPoint}, L{localEndPoint}]: Starting transaction...",
conn.RemoteEndPoint, conn.LocalEndPoint);
sw.Start();
var tranStart = new TcpClientMessageDto.TransactionStart(eventStreamId, expectedVersion, false);
var package = new TcpPackage(TcpCommand.TransactionStart, Guid.NewGuid(), tranStart.Serialize());
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) => {
switch (stage)
{
case Stage.AcquiringTransactionId: {
if (pkg.Command != TcpCommand.TransactionStartCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.TransactionStartCompleted>();
if (dto.Result != TcpClientMessageDto.OperationResult.Success)
{
var msg = string.Format("Error while starting transaction: {0} ({1}).", dto.Message,
dto.Result);
context.Log.Info("Error while starting transaction: {message} ({e}).", dto.Message,
dto.Result);
context.Fail(reason: msg);
}
else
{
context.Log.Info("Successfully started transaction. TransactionId: {transactionId}.",
dto.TransactionId);
context.Log.Info("Now sending transactional events. TransactionId: {transactionId}",
dto.TransactionId);
transactionId = dto.TransactionId;
stage = Stage.Writing;
for (int i = 0; i < eventsCnt; ++i)
{
var writeDto = new TcpClientMessageDto.TransactionWrite(
transactionId,
new[] {
new TcpClientMessageDto.NewEvent(Guid.NewGuid().ToByteArray(),
"TakeSomeSpaceEvent",
0, 0,
Common.Utils.Helper.UTF8NoBom.GetBytes(Guid.NewGuid().ToString()),
Common.Utils.Helper.UTF8NoBom.GetBytes(Guid.NewGuid().ToString()))
},
false);
var package = new TcpPackage(TcpCommand.TransactionWrite, Guid.NewGuid(),
writeDto.Serialize());
conn.EnqueueSend(package.AsByteArray());
}
}
break;
}
case Stage.Writing: {
if (pkg.Command != TcpCommand.TransactionWriteCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <TcpClientMessageDto.TransactionWriteCompleted>();
if (dto.Result != TcpClientMessageDto.OperationResult.Success)
{
context.Log.Info("Error while writing transactional event: {message} ({e}).",
dto.Message, dto.Result);
var msg = String.Format("Error while writing transactional event: {0} ({1}).",
dto.Message, dto.Result);
context.Fail(reason: msg);
}
else
{
writtenEvents += 1;
if (writtenEvents == eventsCnt)
{
context.Log.Info("Written all events. Committing...");
stage = Stage.Committing;
var commitDto = new TcpClientMessageDto.TransactionCommit(transactionId, false);
var package = new TcpPackage(TcpCommand.TransactionCommit, Guid.NewGuid(),
commitDto.Serialize());
conn.EnqueueSend(package.AsByteArray());
}
}
break;
}
case Stage.Committing: {
if (pkg.Command != TcpCommand.TransactionCommitCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
sw.Stop();
var dto = pkg.Data.Deserialize <TcpClientMessageDto.TransactionCommitCompleted>();
if (dto.Result != TcpClientMessageDto.OperationResult.Success)
{
var msg = string.Format("Error while committing transaction: {0} ({1}).", dto.Message,
dto.Result);
context.Log.Info("Error while committing transaction: {message} ({e}).", dto.Message,
dto.Result);
context.Log.Info("Transaction took: {elapsed}.", sw.Elapsed);
context.Fail(reason: msg);
}
else
{
context.Log.Info("Successfully committed transaction [{transactionId}]!",
dto.TransactionId);
context.Log.Info("Transaction took: {elapsed}.", sw.Elapsed);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword),
(int)Math.Round(sw.Elapsed.TotalMilliseconds));
context.Success();
}
conn.Close();
break;
}
default:
throw new ArgumentOutOfRangeException();
}
},
connectionClosed: (connection, error) => context.Fail(reason: "Connection was closed prematurely."));
context.WaitForCompletion();
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
if (args.Length > 0)
{
if (args.Length > 2)
{
return(false);
}
eventStreamId = args[0];
if (args.Length == 2)
{
expectedVersion = args[1].Trim().ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
}
}
context.IsAsync();
var sw = new Stopwatch();
context._tcpTestClient.CreateTcpConnection(
context,
connectionEstablished: conn => {
context.Log.Information(
"[{remoteEndPoint}, L{localEndPoint}]: Trying to delete event stream '{stream}'...",
conn.RemoteEndPoint, conn.LocalEndPoint, eventStreamId);
var corrid = Guid.NewGuid();
var deleteDto = new DeleteStream(eventStreamId, expectedVersion, false, true);
var package = new TcpPackage(TcpCommand.DeleteStream, corrid, deleteDto.Serialize()).AsByteArray();
sw.Start();
conn.EnqueueSend(package);
},
handlePackage: (conn, pkg) => {
sw.Stop();
context.Log.Information("Delete request took: {elapsed}.", sw.Elapsed);
if (pkg.Command != TcpCommand.DeleteStreamCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
var dto = pkg.Data.Deserialize <DeleteStreamCompleted>();
if (dto.Result == OperationResult.Success)
{
context.Log.Information("DELETED event stream {stream}.", eventStreamId);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword),
(int)Math.Round(sw.Elapsed.TotalMilliseconds));
context.Success();
}
else
{
context.Log.Information("DELETION FAILED for event stream {stream}: {message} ({e}).", eventStreamId,
dto.Message, dto.Result);
context.Fail();
}
conn.Close();
},
connectionClosed: (connection, error) => context.Fail(reason: "Connection was closed prematurely."));
context.WaitForCompletion();
return(true);
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
string metadata = null;
if (args.Length > 0)
{
if (args.Length > 2)
{
return(false);
}
eventStreamId = args[0];
if (args.Length > 1)
{
metadata = args[1];
}
}
context.IsAsync();
var corrid = Guid.NewGuid();
var createStreamDto = new ClientMessageDto.CreateStream(
corrid,
eventStreamId,
Encoding.UTF8.GetBytes(metadata ?? string.Format("{{\"StreamName\": \"{0}\"}}", eventStreamId)));
var package = new TcpPackage(TcpCommand.CreateStream, corrid, createStreamDto.Serialize());
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}]: Trying to create stream '{1}'...", conn.EffectiveEndPoint, eventStreamId);
sw.Start();
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) =>
{
if (pkg.Command != TcpCommand.CreateStreamCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
sw.Stop();
var dto = pkg.Data.Deserialize <ClientMessageDto.CreateStreamCompleted>();
if ((OperationErrorCode)dto.ErrorCode == OperationErrorCode.Success)
{
context.Log.Info("Successfully created stream '{0}'.", dto.EventStreamId);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)sw.ElapsedMilliseconds);
}
else
{
context.Log.Info("Error while creating stream {0}: {1} ({2}).",
eventStreamId,
dto.Error,
(OperationErrorCode)dto.ErrorCode);
}
context.Log.Info("Create stream request took: {0}.", sw.Elapsed);
conn.Close();
context.Success();
},
connectionClosed: (connection, error) =>
{
if (error == SocketError.Success)
{
context.Success();
}
else
{
context.Fail();
}
});
context.WaitForCompletion();
return(true);
}
private async Task WriteFlood(CommandProcessorContext context, WriteFloodStats stats, int clientsCnt, long requestsCnt, int streamsCnt,
int size, int batchSize, string streamPrefix, RequestMonitor monitor)
{
context.IsAsync();
var doneEvent = new ManualResetEventSlim(false);
var clients = new List <IEventStoreConnection>();
long last = 0;
var streams = Enumerable.Range(0, streamsCnt).Select(x =>
string.IsNullOrWhiteSpace(streamPrefix)
? Guid.NewGuid().ToString()
: $"{streamPrefix}-{x}").ToArray();
context.Log.Information("Writing streams randomly between {first} and {last}",
streams.FirstOrDefault(),
streams.LastOrDefault());
var start = new TaskCompletionSource();
stats.StartTime = DateTime.UtcNow;
var sw2 = new Stopwatch();
var capacity = 2000 / clientsCnt;
var clientTasks = new List <Task>();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
var client = context._clientApiTestClient.CreateConnection();
await client.ConnectAsync();
clientTasks.Add(RunClient(client, count));
}
async Task RunClient(IEventStoreConnection client, long count)
{
var rnd = new Random();
List <Task> pending = new List <Task>(capacity);
await start.Task;
for (int j = 0; j < count; ++j)
{
var events = new EventData[batchSize];
for (int q = 0; q < batchSize; q++)
{
events[q] = new EventData(Guid.NewGuid(),
"TakeSomeSpaceEvent", false,
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"DATA\" : \"" + new string('*', size) + "\"}"),
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"METADATA\" : \"" + new string('$', 100) + "\"}"));
}
var corrid = Guid.NewGuid();
monitor.StartOperation(corrid);
pending.Add(client.AppendToStreamAsync(streams[rnd.Next(streamsCnt)], ExpectedVersion.Any, events)
.ContinueWith(t => {
monitor.EndOperation(corrid);
if (t.IsCompletedSuccessfully)
{
Interlocked.Add(ref stats.Succ, batchSize);
}
else
{
if (Interlocked.Increment(ref stats.Fail) % 1000 == 0)
{
Console.Write("#");
}
if (t.Exception != null)
{
var exception = t.Exception.Flatten();
switch (exception.InnerException)
{
case WrongExpectedVersionException _:
Interlocked.Increment(ref stats.WrongExpVersion);
break;
case StreamDeletedException _:
Interlocked.Increment(ref stats.StreamDeleted);
break;
case OperationTimedOutException _:
Interlocked.Increment(ref stats.CommitTimeout);
break;
}
}
}
Interlocked.Add(ref stats.Succ, batchSize);
if (stats.Succ - last > 1000)
{
last = stats.Succ;
Console.Write(".");
}
var localAll = Interlocked.Add(ref stats.All, batchSize);
if (localAll % 100000 == 0)
{
stats.Elapsed = sw2.Elapsed;
stats.Rate = 1000.0 * 100000 / stats.Elapsed.TotalMilliseconds;
sw2.Restart();
context.Log.Debug(
"\nDONE TOTAL {writes} WRITES IN {elapsed} ({rate:0.0}/s) [S:{success}, F:{failures} (WEV:{wrongExpectedVersion}, P:{prepareTimeout}, C:{commitTimeout}, F:{forwardTimeout}, D:{streamDeleted})].",
localAll, stats.Elapsed, stats.Rate,
stats.Succ, stats.Fail,
stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
stats.WriteStatsToFile(context.StatsLogger);
}
if (localAll >= requestsCnt)
{
context.Success();
doneEvent.Set();
}
}));
if (pending.Count == capacity)
{
await Task.WhenAny(pending).ConfigureAwait(false);
while (pending.Count > 0 && Task.WhenAny(pending).IsCompleted)
{
pending.RemoveAll(x => x.IsCompleted);
if (stats.Succ - last > 1000)
{
Console.Write(".");
last = stats.Succ;
}
}
}
}
if (pending.Count > 0)
{
await Task.WhenAll(pending);
}
}
var sw = Stopwatch.StartNew();
sw2.Start();
start.SetResult();
await Task.WhenAll(clientTasks);
sw.Stop();
clients.ForEach(client => client.Close());
context.Log.Information(
"Completed. Successes: {success}, failures: {failures} (WRONG VERSION: {wrongExpectedVersion}, P: {prepareTimeout}, C: {commitTimeout}, F: {forwardTimeout}, D: {streamDeleted})",
stats.Succ, stats.Fail,
stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
stats.WriteStatsToFile(context.StatsLogger);
var reqPerSec = (stats.All + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", stats.All,
sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", stats.Succ), PerfUtils.Col("failures", stats.Fail)));
var failuresRate = (int)(100 * stats.Fail / (stats.Fail + stats.Succ));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt,
size),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt,
streamsCnt, size), failuresRate);
monitor.GetMeasurementDetails();
if (Interlocked.Read(ref stats.Succ) != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
public bool Execute(CommandProcessorContext context, string[] args)
{
var eventStreamId = "test-stream";
var expectedVersion = ExpectedVersion.Any;
var data = "test-data";
string metadata = null;
if (args.Length > 0)
{
if (args.Length < 3 || args.Length > 4)
{
return(false);
}
eventStreamId = args[0];
expectedVersion = args[1].ToUpper() == "ANY" ? ExpectedVersion.Any : int.Parse(args[1]);
data = args[2];
if (args.Length == 4)
{
metadata = args[3];
}
}
context.IsAsync();
var writeDto = new ClientMessageDto.WriteEvents(
Guid.Empty,
eventStreamId,
expectedVersion,
new[]
{
new ClientMessageDto.Event(Guid.NewGuid(),
"TakeSomeSpaceEvent",
Encoding.UTF8.GetBytes(data),
Encoding.UTF8.GetBytes(metadata ?? string.Empty))
});
var package = new TcpPackage(TcpCommand.WriteEvents, writeDto.Serialize());
var sw = new Stopwatch();
context.Client.CreateTcpConnection(
context,
connectionEstablished: conn =>
{
context.Log.Info("[{0}]: Writing...", conn.EffectiveEndPoint);
sw.Start();
conn.EnqueueSend(package.AsByteArray());
},
handlePackage: (conn, pkg) =>
{
if (pkg.Command != TcpCommand.WriteEventsCompleted)
{
context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
return;
}
sw.Stop();
var dto = pkg.Data.Deserialize <ClientMessageDto.WriteEventsCompleted>();
if ((OperationErrorCode)dto.ErrorCode == OperationErrorCode.Success)
{
context.Log.Info("Successfully written. EventId: {0}.", dto.CorrelationId);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)sw.ElapsedMilliseconds);
}
else
{
context.Log.Info("Error while writing: {0} ({1}). EventId: {2}.",
dto.Error,
(OperationErrorCode)dto.ErrorCode,
dto.CorrelationId);
}
context.Log.Info("Write request took: {0}.", sw.Elapsed);
conn.Close();
context.Success();
},
connectionClosed: (connection, error) =>
{
if (error == SocketError.Success)
{
context.Success();
}
else
{
context.Fail();
}
});
context.WaitForCompletion();
return(true);
}
private void PingFloodWaiting(CommandProcessorContext context, int clientsCnt, long requestsCnt)
{
context.IsAsync();
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
var doneEvent = new ManualResetEventSlim(false);
var clientsDone = 0;
long all = 0;
for (int i = 0; i < clientsCnt; i++)
{
var autoResetEvent = new AutoResetEvent(false);
var client = context.Client.CreateTcpConnection(
context,
(_, __) =>
{
Interlocked.Increment(ref all);
autoResetEvent.Set();
},
connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
clients.Add(client);
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var package = new TcpPackage(TcpCommand.Ping, Guid.NewGuid(), null);
client.EnqueueSend(package.AsByteArray());
autoResetEvent.WaitOne();
}
if (Interlocked.Increment(ref clientsDone) == clientsCnt)
{
context.Success();
doneEvent.Set();
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
clients.ForEach(x => x.Close());
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)Math.Round(sw.Elapsed.TotalMilliseconds / all));
if (Interlocked.Read(ref all) == requestsCnt)
{
context.Success();
}
else
{
context.Fail();
}
}
private void WriteFlood(CommandProcessorContext context, int clientsCnt, int requestsCnt)
{
context.IsAsync();
var threads = new List <Thread>();
var doneEvent = new ManualResetEventSlim(false);
var succ = 0;
var fail = 0;
var all = 0;
var sw = Stopwatch.StartNew();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
threads.Add(new Thread(() =>
{
var autoEvent = new AutoResetEvent(false);
var eventStreamId = "es" + Guid.NewGuid();
var client = new HttpAsyncClient();
var url = context.Client.HttpEndpoint.ToHttpUrl("/streams/{0}", eventStreamId);
Action <HttpResponse> succHandler = response =>
{
if (response.HttpStatusCode == HttpStatusCode.Created)
{
if (Interlocked.Increment(ref succ) % 1000 == 0)
{
Console.Write(".");
}
}
else
{
if (Interlocked.Increment(ref fail) % 10 == 0)
{
context.Log.Info("ANOTHER 10th WRITE FAILED. [{0}] - [{1}]", response.HttpStatusCode, response.StatusDescription);
}
}
if (Interlocked.Increment(ref all) == requestsCnt)
{
doneEvent.Set();
}
autoEvent.Set();
};
for (int j = 0; j < count; ++j)
{
var write = new[] { new HttpClientMessageDto.ClientEventText(Guid.NewGuid(),
"type",
"DATA" + new string('*', 256),
"METADATA" + new string('$', 100)) };
var request = Codec.Xml.To(write);
client.Post(url,
request,
Codec.Xml.ContentType,
TimeSpan.FromMilliseconds(10000),
succHandler,
exc =>
{
context.Log.ErrorException(exc, "Error during POST.");
Interlocked.Increment(ref fail);
if (Interlocked.Increment(ref all) == requestsCnt)
{
doneEvent.Set();
}
autoEvent.Set();
});
autoEvent.WaitOne();
}
})
{
IsBackground = true
});
}
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("Completed. Successes: {0}, failures: {1}", succ, fail);
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), (int)(100.0 * fail / (fail + succ)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-latency-ms", Keyword), (int)Math.Round(sw.Elapsed.TotalMilliseconds / requestsCnt));
if (succ != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
private void Flood(CommandProcessorContext context,
int clientsCnt,
int minPerSecond,
int maxPerSecond,
int runTimeMinutes)
{
context.IsAsync();
var threads = new List <Thread>();
var autoResetEvent = new AutoResetEvent(false);
var succ = 0;
var fail = 0;
var requestsCnt = 0;
var watchLockRoot = new object();
var sw = Stopwatch.StartNew();
for (int i = 0; i < clientsCnt; i++)
{
int sent = 0;
int received = 0;
threads.Add(new Thread(() =>
{
var client = new HttpAsyncClient();
Action <HttpResponse> succHandler = response =>
{
var succDone = Interlocked.Increment(ref succ);
if (succDone % maxPerSecond == 0)
{
Console.Write(".");
}
Interlocked.Increment(ref requestsCnt);
Interlocked.Increment(ref received);
};
var sentCount = 0;
var sleepTime = 0;
var currentMinute = -1;
while (true)
{
TimeSpan elapsed;
lock (watchLockRoot)
elapsed = sw.Elapsed;
if (elapsed.TotalMinutes > runTimeMinutes)
{
autoResetEvent.Set();
break;
}
if (sentCount == 0)
{
int elapsedMinutesInt = (int)elapsed.TotalMinutes;
lock (_randomLockRoot)
{
sentCount = minPerSecond == maxPerSecond
? maxPerSecond
: _random.Next(minPerSecond, maxPerSecond);
}
if (currentMinute != elapsedMinutesInt)
{
currentMinute = elapsedMinutesInt;
context.Log.Info(Environment.NewLine + "Elapsed {0} of {1} minutes, sent {2}",
elapsedMinutesInt,
runTimeMinutes,
sent);
}
sleepTime = 1000 / sentCount;
}
var url = context.Client.HttpEndpoint.ToHttpUrl("/ping");
client.Get(url,
succHandler,
exc => context.Log.ErrorException(exc, "Error during GET."));
Interlocked.Increment(ref sent);
Thread.Sleep(sleepTime);
sentCount -= 1;
while (sent - received > context.Client.Options.PingWindow)
{
Thread.Sleep(1);
}
}
}));
}
foreach (var thread in threads)
{
thread.IsBackground = true;
thread.Start();
}
autoResetEvent.WaitOne();
sw.Stop();
context.Log.Info("Completed. Successes: {0}, failures: {1}", succ, fail);
var reqPerSec = (requestsCnt + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).",
requestsCnt,
sw.ElapsedMilliseconds,
reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail))
);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt),
(int)((fail / (succ + 0.0)) * 100));
context.Success();
}
private void PingFlood(CommandProcessorContext context, int clientsCnt, long requestsCnt)
{
context.IsAsync();
var doneEvent = new ManualResetEventSlim(false);
var clients = new List <TcpTypedConnection <byte[]> >();
var threads = new List <Thread>();
long all = 0;
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
long received = 0;
long sent = 0;
var client = context.Client.CreateTcpConnection(
context,
(conn, msg) =>
{
Interlocked.Increment(ref received);
var pongs = Interlocked.Increment(ref all);
if (pongs % 10000 == 0)
{
Console.Write('.');
}
if (pongs == requestsCnt)
{
context.Success();
doneEvent.Set();
}
},
connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
clients.Add(client);
threads.Add(new Thread(() =>
{
for (int j = 0; j < count; ++j)
{
var package = new TcpPackage(TcpCommand.Ping, Guid.NewGuid(), Payload);
client.EnqueueSend(package.AsByteArray());
var localSent = Interlocked.Increment(ref sent);
while (localSent - Interlocked.Read(ref received) > context.Client.Options.PingWindow / clientsCnt)
{
Thread.Sleep(1);
}
}
})
{
IsBackground = true
});
}
var sw = Stopwatch.StartNew();
threads.ForEach(thread => thread.Start());
doneEvent.Wait();
sw.Stop();
clients.ForEach(client => client.Close());
var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
if (Interlocked.Read(ref all) == requestsCnt)
{
context.Success();
}
else
{
context.Fail();
}
}
private void Flood(CommandProcessorContext context,
string eventStreamId,
int clientsCnt,
int minPerSecond,
int maxPerSecond,
int runTimeMinutes,
int dataSize)
{
context.IsAsync();
var threads = new List <Thread>();
var doneEvent = new ManualResetEvent(false);
var succ = 0;
var fail = 0;
var requestsCnt = 0;
int sent = 0;
int received = 0;
var watchLockRoot = new object();
var sw = Stopwatch.StartNew();
for (int i = 0; i < clientsCnt; i++)
{
threads.Add(new Thread(() =>
{
var esId = eventStreamId ?? "Stream-" + Thread.CurrentThread.ManagedThreadId % 3;
var client = new HttpAsyncClient();
Action <HttpResponse> succHandler = response =>
{
if (response.HttpStatusCode == HttpStatusCode.Created)
{
var succDone = Interlocked.Increment(ref succ);
if (succDone % maxPerSecond == 0)
{
Console.Write(".");
}
}
else
{
if (Interlocked.Increment(ref fail) % 10 == 0)
{
context.Log.Info("ANOTHER 10th WRITE FAILED. [{0}] - [{1}]", response.HttpStatusCode, response.StatusDescription);
}
}
Interlocked.Increment(ref requestsCnt);
Interlocked.Increment(ref received);
};
var sentCount = 0;
var sleepTime = 0;
var dataSizeCoefficient = 1;
var currentMinute = -1;
while (true)
{
TimeSpan elapsed;
lock (watchLockRoot)
elapsed = sw.Elapsed;
if (elapsed.TotalMinutes > runTimeMinutes)
{
doneEvent.Set();
break;
}
if (sentCount == 0)
{
int elapsedMinutesInt = (int)elapsed.TotalMinutes;
lock (_randomLockRoot)
{
sentCount = minPerSecond == maxPerSecond
? maxPerSecond : _random.Next(minPerSecond, maxPerSecond);
dataSizeCoefficient = _random.Next(8, 256);
}
if (currentMinute != elapsedMinutesInt)
{
currentMinute = elapsedMinutesInt;
context.Log.Info("\nElapsed {0} of {1} minutes, sent {2}; next block coef. {3}",
elapsedMinutesInt,
runTimeMinutes,
sent,
dataSizeCoefficient);
}
sleepTime = 1000 / sentCount;
}
var url = context.Client.HttpEndpoint.ToHttpUrl("/streams/{0}", esId);
var dataResultingSize = dataSizeCoefficient * dataSize;
var write = new[] { new HttpClientMessageDto.ClientEventText(Guid.NewGuid(),
"type",
"DATA" + dataResultingSize.ToString(" 00000 ") + new string('*', dataResultingSize),
"METADATA" + new string('$', 100)) };
var request = Codec.Xml.To(write);
client.Post(url,
request,
Codec.Xml.ContentType,
TimeSpan.FromMilliseconds(10000),
succHandler,
exc =>
{
Interlocked.Increment(ref fail);
Interlocked.Increment(ref requestsCnt);
context.Log.ErrorException(exc, "Error during POST.");
});
Interlocked.Increment(ref sent);
Thread.Sleep(sleepTime);
sentCount -= 1;
while (sent - received > context.Client.Options.WriteWindow / clientsCnt)
{
Thread.Sleep(1);
}
}
}));
}
foreach (var thread in threads)
{
thread.IsBackground = true;
thread.Start();
}
doneEvent.WaitOne();
sw.Stop();
context.Log.Info("Completed. Successes: {0}, failures: {1}", succ, fail);
var reqPerSec = (requestsCnt + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).",
requestsCnt,
sw.ElapsedMilliseconds,
reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail))
);
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-{3}-{4}-reqPerSec",
Keyword,
clientsCnt,
minPerSecond,
maxPerSecond,
runTimeMinutes),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt),
100 * fail / (fail + succ));
context.Success();
}
public bool Execute(CommandProcessorContext context, CancellationToken token, string[] args)
{
var batchCount = 15;
var batchSize = 10000;
var projectionCount = 10;
var readerCount = 10;
if (args.Length > 0)
{
int.TryParse(args[0], out batchCount);
}
if (args.Length > 1)
{
int.TryParse(args[1], out batchSize);
}
if (args.Length > 2)
{
int.TryParse(args[2], out projectionCount);
}
if (args.Length > 3)
{
int.TryParse(args[3], out readerCount);
}
context.Log.Debug("batchCount: {0}", batchCount);
context.Log.Debug("batchSize: {0}", batchSize);
context.Log.Debug("projectionCount: {0}", projectionCount);
context.Log.Debug("readerCount: {0}", readerCount);
var startEvt = new ManualResetEventSlim(false);
_failures = 0;
var streamId = "SmartAppTest-" + Guid.NewGuid();
var totalSw = Stopwatch.StartNew();
// write half of data to the stream
var writeStat = new Stat();
WriteEvents(streamId, batchCount / 2, batchSize, context, writeStat);
var wrBytesPerSec = writeStat.ElapsedMsec > 0 ? (writeStat.Count * 1000D / writeStat.ElapsedMsec) : 0;
context.Log.Debug("Events write throughput: {0}", FormatEvil.SpeedInBytes(wrBytesPerSec));
var threads = new List <Thread>();
using (var cts = new CancellationTokenSource())
using (var linked = CancellationTokenSource.CreateLinkedTokenSource(token, cts.Token))
{
// Upload some random data to an event stream in batches (a thread and large batches)
writeStat = new Stat();
var writerThread = CreateEventWriterThread(streamId, batchCount / 2, batchSize, context, writeStat, startEvt, linked.Token);
writerThread.Start();
// run a set of projections in parallel for this event stream (1 thread per projection)
var projStat = new Stat[projectionCount];
for (var i = 0; i < projectionCount; i++)
{
projStat[i] = new Stat();
threads.Add(CreateProjectionThread(streamId, i, context, startEvt, linked.Token, projStat[i]));
threads.Last().Start();
}
// poll projected views with multiple concurrent readers
var projectionIndex = 0;
var readerStat = new Stat[readerCount];
for (var i = 0; i < readerCount; i++)
{
readerStat[i] = new Stat();
threads.Add(CreateViewReaderThread(i, projectionIndex, context, startEvt, linked.Token, readerStat[i]));
threads.Last().Start();
projectionIndex++;
if (projectionIndex >= projectionCount)
{
projectionIndex = 0;
}
}
// Start all thread
startEvt.Set();
// Wait until second half of data will be written
writerThread.Join();
// Cancel rest threads
cts.Cancel();
foreach (var thread in threads)
{
thread.Join();
}
totalSw.Stop();
// Projections stat
wrBytesPerSec = writeStat.ElapsedMsec > 0 ? (writeStat.Count * 1000D / writeStat.ElapsedMsec) : 0;
context.Log.Debug("Events write throughput under load: {0}", FormatEvil.SpeedInBytes(wrBytesPerSec));
var elapsedMsec = projStat.Max(s => s.ElapsedMsec);
var totalBytes = projStat.Sum(s => s.Count);
var bytesPerSec = elapsedMsec > 0 ? (totalBytes * 1000D / elapsedMsec) : 0;
context.Log.Debug("Events read throughput: {0}", FormatEvil.SpeedInBytes(bytesPerSec));
elapsedMsec = readerStat.Max(s => s.ElapsedMsec);
var totalReads = readerStat.Sum(s => s.Count);
var readsPerSec = elapsedMsec > 0 ? (totalReads * 1000D / elapsedMsec) : 0;
context.Log.Debug("Views object read rate: {0} instance/sec", FormatEvil.ToHumanReadable(readsPerSec));
context.Log.Info("Total time: {0}", totalSw.Elapsed);
var key = string.Format("SAB-WR-{0}-{1}-bytesPerSec", batchCount, batchSize);
PerfUtils.LogTeamCityGraphData(key, (int)wrBytesPerSec);
PerfUtils.LogTeamCityGraphData("SAB-RE-bytesPerSec", (int)bytesPerSec);
PerfUtils.LogTeamCityGraphData("SAB-RV-objectsPerSec", (int)readsPerSec);
}
return(true);
}