private async Task WriteFlood(CommandProcessorContext context, WriteFloodStats stats, int clientsCnt, long requestsCnt, int streamsCnt,
int size, int batchSize, string streamPrefix, RequestMonitor monitor)
{
context.IsAsync();
var doneEvent = new ManualResetEventSlim(false);
var clients = new List <IEventStoreConnection>();
long last = 0;
var streams = Enumerable.Range(0, streamsCnt).Select(x =>
string.IsNullOrWhiteSpace(streamPrefix)
? Guid.NewGuid().ToString()
: $"{streamPrefix}-{x}").ToArray();
context.Log.Information("Writing streams randomly between {first} and {last}",
streams.FirstOrDefault(),
streams.LastOrDefault());
var start = new TaskCompletionSource();
stats.StartTime = DateTime.UtcNow;
var sw2 = new Stopwatch();
var capacity = 2000 / clientsCnt;
var clientTasks = new List <Task>();
for (int i = 0; i < clientsCnt; i++)
{
var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
var client = context._clientApiTestClient.CreateConnection();
await client.ConnectAsync();
clientTasks.Add(RunClient(client, count));
}
async Task RunClient(IEventStoreConnection client, long count)
{
var rnd = new Random();
List <Task> pending = new List <Task>(capacity);
await start.Task;
for (int j = 0; j < count; ++j)
{
var events = new EventData[batchSize];
for (int q = 0; q < batchSize; q++)
{
events[q] = new EventData(Guid.NewGuid(),
"TakeSomeSpaceEvent", false,
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"DATA\" : \"" + new string('*', size) + "\"}"),
Common.Utils.Helper.UTF8NoBom.GetBytes(
"{ \"METADATA\" : \"" + new string('$', 100) + "\"}"));
}
var corrid = Guid.NewGuid();
monitor.StartOperation(corrid);
pending.Add(client.AppendToStreamAsync(streams[rnd.Next(streamsCnt)], ExpectedVersion.Any, events)
.ContinueWith(t => {
monitor.EndOperation(corrid);
if (t.IsCompletedSuccessfully)
{
Interlocked.Add(ref stats.Succ, batchSize);
}
else
{
if (Interlocked.Increment(ref stats.Fail) % 1000 == 0)
{
Console.Write("#");
}
if (t.Exception != null)
{
var exception = t.Exception.Flatten();
switch (exception.InnerException)
{
case WrongExpectedVersionException _:
Interlocked.Increment(ref stats.WrongExpVersion);
break;
case StreamDeletedException _:
Interlocked.Increment(ref stats.StreamDeleted);
break;
case OperationTimedOutException _:
Interlocked.Increment(ref stats.CommitTimeout);
break;
}
}
}
Interlocked.Add(ref stats.Succ, batchSize);
if (stats.Succ - last > 1000)
{
last = stats.Succ;
Console.Write(".");
}
var localAll = Interlocked.Add(ref stats.All, batchSize);
if (localAll % 100000 == 0)
{
stats.Elapsed = sw2.Elapsed;
stats.Rate = 1000.0 * 100000 / stats.Elapsed.TotalMilliseconds;
sw2.Restart();
context.Log.Debug(
"\nDONE TOTAL {writes} WRITES IN {elapsed} ({rate:0.0}/s) [S:{success}, F:{failures} (WEV:{wrongExpectedVersion}, P:{prepareTimeout}, C:{commitTimeout}, F:{forwardTimeout}, D:{streamDeleted})].",
localAll, stats.Elapsed, stats.Rate,
stats.Succ, stats.Fail,
stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
stats.WriteStatsToFile(context.StatsLogger);
}
if (localAll >= requestsCnt)
{
context.Success();
doneEvent.Set();
}
}));
if (pending.Count == capacity)
{
await Task.WhenAny(pending).ConfigureAwait(false);
while (pending.Count > 0 && Task.WhenAny(pending).IsCompleted)
{
pending.RemoveAll(x => x.IsCompleted);
if (stats.Succ - last > 1000)
{
Console.Write(".");
last = stats.Succ;
}
}
}
}
if (pending.Count > 0)
{
await Task.WhenAll(pending);
}
}
var sw = Stopwatch.StartNew();
sw2.Start();
start.SetResult();
await Task.WhenAll(clientTasks);
sw.Stop();
clients.ForEach(client => client.Close());
context.Log.Information(
"Completed. Successes: {success}, failures: {failures} (WRONG VERSION: {wrongExpectedVersion}, P: {prepareTimeout}, C: {commitTimeout}, F: {forwardTimeout}, D: {streamDeleted})",
stats.Succ, stats.Fail,
stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
stats.WriteStatsToFile(context.StatsLogger);
var reqPerSec = (stats.All + 0.0) / sw.ElapsedMilliseconds * 1000;
context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", stats.All,
sw.ElapsedMilliseconds, reqPerSec);
PerfUtils.LogData(
Keyword,
PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
PerfUtils.Col("requestsCnt", requestsCnt),
PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
PerfUtils.Row(PerfUtils.Col("successes", stats.Succ), PerfUtils.Col("failures", stats.Fail)));
var failuresRate = (int)(100 * stats.Fail / (stats.Fail + stats.Succ));
PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt,
size),
(int)reqPerSec);
PerfUtils.LogTeamCityGraphData(
string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt,
streamsCnt, size), failuresRate);
monitor.GetMeasurementDetails();
if (Interlocked.Read(ref stats.Succ) != requestsCnt)
{
context.Fail(reason: "There were errors or not all requests completed.");
}
else
{
context.Success();
}
}
public void AddRequestMonitor(RequestMonitor requestMonitor)
{
requestMonitors.Add(requestMonitor);
}
internal BackupClient(string destinationHostNameOrIp, int destinationPort, string originHostNameOrIp, LogProvider logProvider, StoreId storeId, long timeout, ResponseUnpacker unpacker, ByteCounterMonitor byteCounterMonitor, RequestMonitor requestMonitor, LogEntryReader <ReadableClosablePositionAwareChannel> reader) : base(destinationHostNameOrIp, destinationPort, originHostNameOrIp, logProvider, storeId, FRAME_LENGTH, timeout, Client.DEFAULT_MAX_NUMBER_OF_CONCURRENT_CHANNELS_PER_CLIENT, FRAME_LENGTH, unpacker, byteCounterMonitor, requestMonitor, reader)
{
}
public Client(string destinationHostNameOrIp, int destinationPort, string originHostNameOrIp, LogProvider logProvider, StoreId storeId, int frameLength, long readTimeout, int maxConcurrentChannels, int chunkSize, ResponseUnpacker responseUnpacker, ByteCounterMonitor byteCounterMonitor, RequestMonitor requestMonitor, LogEntryReader <ReadableClosablePositionAwareChannel> entryReader)
{
this._entryReader = entryReader;
Debug.Assert(byteCounterMonitor != null);
Debug.Assert(requestMonitor != null);
this._byteCounterMonitor = byteCounterMonitor;
this._requestMonitor = requestMonitor;
assertChunkSizeIsWithinFrameSize(chunkSize, frameLength);
this._msgLog = logProvider.getLog(this.GetType());
this._storeId = storeId;
this._frameLength = frameLength;
this._readTimeout = readTimeout;
// ResourcePool no longer controls max concurrent channels. Use this value for the pool size
this._maxUnusedChannels = maxConcurrentChannels;
this._comExceptionHandler = NoOpComExceptionHandler;
if (destinationHostNameOrIp.Equals("0.0.0.0"))
{
// So it turns out that on Windows, connecting to 0.0.0.0 when specifying
// an origin address will not succeed. But since we know we are
// connecting to ourselves, and that we are listening on everything,
// replacing with localhost is the proper thing to do.
this._destination = new InetSocketAddress(LocalAddress, destinationPort);
}
else
{
// An explicit destination address is always correct
this._destination = new InetSocketAddress(destinationHostNameOrIp, destinationPort);
}
if (string.ReferenceEquals(originHostNameOrIp, null) || originHostNameOrIp.Equals("0.0.0.0"))
{
_origin = null;
}
else
{
_origin = new InetSocketAddress(originHostNameOrIp, 0);
}
ProtocolVersion protocolVersion = ProtocolVersion;
this._protocol = CreateProtocol(chunkSize, protocolVersion.ApplicationProtocol);
this._responseUnpacker = responseUnpacker;
_msgLog.info(this.GetType().Name + " communication channel created towards " + _destination);
}
public Server(T requestTarget, Configuration config, LogProvider logProvider, int frameLength, ProtocolVersion protocolVersion, TxChecksumVerifier txVerifier, Clock clock, ByteCounterMonitor byteCounterMonitor, RequestMonitor requestMonitor)
{
this._requestTarget = requestTarget;
this._config = config;
this._frameLength = frameLength;
this._applicationProtocolVersion = protocolVersion.ApplicationProtocol;
this._logProvider = logProvider;
this._msgLog = this._logProvider.getLog(this.GetType());
this._txVerifier = txVerifier;
this._byteCounterMonitor = byteCounterMonitor;
this._requestMonitor = requestMonitor;
this._connectedSlaveChannels = new IdleChannelReaper(this, logProvider, clock, config.OldChannelThreshold);
this._chunkSize = config.ChunkSize;
assertChunkSizeIsWithinFrameSize(_chunkSize, frameLength);
}
public SlaveClient(InstanceId machineId, string destinationHostNameOrIp, int destinationPort, string originHostNameOrIp, LogProvider logProvider, StoreId storeId, int maxConcurrentChannels, int chunkSize, ByteCounterMonitor byteCounterMonitor, RequestMonitor requestMonitor, LogEntryReader <ReadableClosablePositionAwareChannel> entryReader) : base(destinationHostNameOrIp, destinationPort, originHostNameOrIp, logProvider, storeId, Protocol.DEFAULT_FRAME_LENGTH, HaSettings.read_timeout.apply(from->null).toMillis(), maxConcurrentChannels, chunkSize, NO_OP_RESPONSE_UNPACKER, byteCounterMonitor, requestMonitor, entryReader)