Ejemplo n.º 1
0
        private void WriteFlood(CommandProcessorContext context, WriteFloodStats stats, int clientsCnt, long requestsCnt, int streamsCnt,
                                int size, int batchSize, string streamNamePrefix, RequestMonitor monitor)
        {
            context.IsAsync();

            var doneEvent = new ManualResetEventSlim(false);
            var clients   = new List <TcpTypedConnection <byte[]> >();
            var threads   = new List <Thread>();

            long last = 0;

            string[] streams = Enumerable.Range(0, streamsCnt).Select(x =>
                                                                      string.IsNullOrWhiteSpace(streamNamePrefix)
                                        ? Guid.NewGuid().ToString()
                                        : $"{streamNamePrefix}-{x}"
                                                                      ).ToArray();

            context.Log.Information("Writing streams randomly between {first} and {last}",
                                    streams.FirstOrDefault(),
                                    streams.LastOrDefault());

            Console.WriteLine($"Last stream: {streams.LastOrDefault()}");
            stats.StartTime = DateTime.UtcNow;
            var sw2 = new Stopwatch();

            for (int i = 0; i < clientsCnt; i++)
            {
                var  count    = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
                long sent     = 0;
                long received = 0;
                var  rnd      = new Random();
                var  client   = context._tcpTestClient.CreateTcpConnection(
                    context,
                    (conn, pkg) => {
                    if (pkg.Command != TcpCommand.WriteEventsCompleted)
                    {
                        context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
                        return;
                    }

                    var dto = pkg.Data.Deserialize <WriteEventsCompleted>();
                    monitor.EndOperation(pkg.CorrelationId);
                    switch (dto.Result)
                    {
                    case OperationResult.Success:
                        Interlocked.Add(ref stats.Succ, batchSize);
                        if (stats.Succ - last > 1000)
                        {
                            last = stats.Succ;
                            Console.Write(".");
                        }

                        break;

                    case OperationResult.PrepareTimeout:
                        Interlocked.Increment(ref stats.PrepTimeout);
                        break;

                    case OperationResult.CommitTimeout:
                        Interlocked.Increment(ref stats.CommitTimeout);
                        break;

                    case OperationResult.ForwardTimeout:
                        Interlocked.Increment(ref stats.ForwardTimeout);
                        break;

                    case OperationResult.WrongExpectedVersion:
                        Interlocked.Increment(ref stats.WrongExpVersion);
                        break;

                    case OperationResult.StreamDeleted:
                        Interlocked.Increment(ref stats.StreamDeleted);
                        break;

                    default:
                        throw new ArgumentOutOfRangeException();
                    }

                    if (dto.Result != OperationResult.Success)
                    {
                        if (Interlocked.Increment(ref stats.Fail) % 1000 == 0)
                        {
                            Console.Write('#');
                        }
                    }
                    Interlocked.Increment(ref received);
                    var localAll = Interlocked.Add(ref stats.All, batchSize);
                    if (localAll % 100000 == 0)
                    {
                        stats.Elapsed = sw2.Elapsed;
                        stats.Rate    = 1000.0 * 100000 / stats.Elapsed.TotalMilliseconds;
                        sw2.Restart();
                        context.Log.Debug(
                            "\nDONE TOTAL {writes} WRITES IN {elapsed} ({rate:0.0}/s) [S:{success}, F:{failures} (WEV:{wrongExpectedVersion}, " +
                            "P:{prepareTimeout}, C:{commitTimeout}, F:{forwardTimeout}, D:{streamDeleted})].",
                            localAll, stats.Elapsed, stats.Rate, stats.Succ, stats.Fail,
                            stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);
                        stats.WriteStatsToFile(context.StatsLogger);
                    }

                    if (localAll >= requestsCnt)
                    {
                        context.Success();
                        doneEvent.Set();
                    }
                },
                    connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
                clients.Add(client);

                threads.Add(new Thread(() => {
                    for (int j = 0; j < count; ++j)
                    {
                        var events = new NewEvent[batchSize];
                        for (int q = 0; q < batchSize; q++)
                        {
                            events[q] = new NewEvent(Guid.NewGuid().ToByteArray(),
                                                     "TakeSomeSpaceEvent",
                                                     1, 0,
                                                     Common.Utils.Helper.UTF8NoBom.GetBytes(
                                                         "{ \"DATA\" : \"" + new string('*', size) + "\"}"),
                                                     Common.Utils.Helper.UTF8NoBom.GetBytes(
                                                         "{ \"METADATA\" : \"" + new string('$', 100) + "\"}"));
                        }

                        var corrid = Guid.NewGuid();
                        var write  = new WriteEvents(
                            streams[rnd.Next(streamsCnt)],
                            ExpectedVersion.Any,
                            events,
                            false);
                        var package = new TcpPackage(TcpCommand.WriteEvents, corrid, write.Serialize());
                        monitor.StartOperation(corrid);
                        client.EnqueueSend(package.AsByteArray());

                        var localSent = Interlocked.Increment(ref sent);
                        while (localSent - Interlocked.Read(ref received) >
                               context._tcpTestClient.Options.WriteWindow / clientsCnt)
                        {
                            Thread.Sleep(1);
                        }
                    }
                })
                {
                    IsBackground = true
                });
            }

            var sw = Stopwatch.StartNew();

            sw2.Start();
            threads.ForEach(thread => thread.Start());
            doneEvent.Wait();
            sw.Stop();
            clients.ForEach(client => client.Close());

            stats.WriteStatsToFile(context.StatsLogger);
            context.Log.Information(
                "Completed. Successes: {success}, failures: {failures} (WRONG VERSION: {wrongExpectedVersion}, P: {prepareTimeout}, " +
                "C: {commitTimeout}, F: {forwardTimeout}, D: {streamDeleted})",
                stats.Succ, stats.Fail, stats.WrongExpVersion, stats.PrepTimeout, stats.CommitTimeout, stats.ForwardTimeout, stats.StreamDeleted);

            var reqPerSec = (stats.All + 0.0) / sw.ElapsedMilliseconds * 1000;

            context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", stats.All,
                                    sw.ElapsedMilliseconds, reqPerSec);

            PerfUtils.LogData(
                Keyword,
                PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
                              PerfUtils.Col("requestsCnt", requestsCnt),
                              PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
                PerfUtils.Row(PerfUtils.Col("successes", stats.Succ), PerfUtils.Col("failures", stats.Fail)));

            var failuresRate = (int)(100 * stats.Fail / (stats.Fail + stats.Succ));

            PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
                                           (int)reqPerSec);
            PerfUtils.LogTeamCityGraphData(
                string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
            PerfUtils.LogTeamCityGraphData(
                string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt, size),
                (int)reqPerSec);
            PerfUtils.LogTeamCityGraphData(
                string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt,
                              streamsCnt, size), failuresRate);
            monitor.GetMeasurementDetails();
            if (Interlocked.Read(ref stats.Succ) != requestsCnt)
            {
                context.Fail(reason: "There were errors or not all requests completed.");
            }
            else
            {
                context.Success();
            }
        }
Ejemplo n.º 2
0
        private void ReadFlood(CommandProcessorContext context, string streamPrefix, int clientsCnt, long requestsCnt, int streamsCnt,
                               bool resolveLinkTos, bool requireLeader, RequestMonitor monitor)
        {
            context.IsAsync();

            string[] streams = streamsCnt == 1
                                ? new[] { streamPrefix }
                                : Enumerable.Range(0, streamsCnt).Select(x => $"{streamPrefix}-{x}").ToArray();

            context.Log.Information("Reading streams {first} through to {last}",
                                    streams.FirstOrDefault(),
                                    streams.LastOrDefault());

            var  clients   = new List <TcpTypedConnection <byte[]> >();
            var  threads   = new List <Thread>();
            var  doneEvent = new ManualResetEventSlim(false);
            var  sw2       = new Stopwatch();
            long succ      = 0;
            long fail      = 0;
            long all       = 0;

            for (int i = 0; i < clientsCnt; i++)
            {
                var  count    = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
                long received = 0;
                long sent     = 0;
                var  client   = context._tcpTestClient.CreateTcpConnection(
                    context,
                    (conn, pkg) => {
                    if (pkg.Command != TcpCommand.ReadEventCompleted)
                    {
                        context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
                        return;
                    }

                    var dto = pkg.Data.Deserialize <ReadEventCompleted>();
                    monitor.EndOperation(pkg.CorrelationId);
                    if (dto.Result == ReadEventCompleted.Types.ReadEventResult.Success)
                    {
                        if (Interlocked.Increment(ref succ) % 1000 == 0)
                        {
                            Console.Write(".");
                        }
                    }
                    else
                    {
                        if (Interlocked.Increment(ref fail) % 1000 == 0)
                        {
                            Console.Write("#");
                        }
                    }

                    Interlocked.Increment(ref received);
                    var localAll = Interlocked.Increment(ref all);
                    if (localAll % 100000 == 0)
                    {
                        var elapsed = sw2.Elapsed;
                        sw2.Restart();
                        context.Log.Verbose("\nDONE TOTAL {reads} READS IN {elapsed} ({rate:0.0}/s).", localAll,
                                            elapsed, 1000.0 * 100000 / elapsed.TotalMilliseconds);
                    }

                    if (localAll == requestsCnt)
                    {
                        context.Success();
                        doneEvent.Set();
                    }
                },
                    connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
                clients.Add(client);

                var clientNum = i;
                threads.Add(new Thread(() => {
                    int streamIndex = (streamsCnt / clientsCnt) * clientNum;
                    context.Log.Information("Reader #{clientNum} performing {count} reads on {streamsCnt} streams starting at stream index {streamIndex}",
                                            clientNum, count, streamsCnt, streamIndex);
                    for (int j = 0; j < count; ++j)
                    {
                        var corrId = Guid.NewGuid();

                        var eventStreamId = streams[streamIndex++];
                        if (streamIndex >= streamsCnt)
                        {
                            streamIndex = 0;
                        }
                        var read    = new ReadEvent(eventStreamId, 0, resolveLinkTos, requireLeader);
                        var package = new TcpPackage(TcpCommand.ReadEvent, corrId, read.Serialize());
                        monitor.StartOperation(corrId);
                        client.EnqueueSend(package.AsByteArray());

                        var localSent = Interlocked.Increment(ref sent);
                        while (localSent - Interlocked.Read(ref received) >
                               context._tcpTestClient.Options.ReadWindow / clientsCnt)
                        {
                            Thread.Sleep(1);
                        }
                    }
                })
                {
                    IsBackground = true
                });
            }

            var sw = Stopwatch.StartNew();

            sw2.Start();
            threads.ForEach(thread => thread.Start());
            doneEvent.Wait();
            sw.Stop();
            clients.ForEach(client => client.Close());

            var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;

            context.Log.Information("Completed. READS succ: {success}, fail: {failures}.", Interlocked.Read(ref succ),
                                    Interlocked.Read(ref fail));
            context.Log.Information("{requests} requests completed in {elapsed}ms ({rate:0.00} reqs per sec).", all,
                                    sw.ElapsedMilliseconds, reqPerSec);
            monitor.GetMeasurementDetails();
            PerfUtils.LogData(Keyword,
                              PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
                                            PerfUtils.Col("requestsCnt", requestsCnt),
                                            PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
                              PerfUtils.Row(PerfUtils.Col("readsCnt", all)));
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt),
                                           (int)reqPerSec);

            if (succ != requestsCnt)
            {
                context.Fail(reason: "There were errors or not all requests completed.");
            }
            else
            {
                context.Success();
            }
        }
        private void ReadFlood(CommandProcessorContext context, string eventStreamId, int clientsCnt, long requestsCnt,
                               bool resolveLinkTos, bool requireMaster)
        {
            context.IsAsync();

            var  clients   = new List <TcpTypedConnection <byte[]> >();
            var  threads   = new List <Thread>();
            var  doneEvent = new ManualResetEventSlim(false);
            var  sw2       = new Stopwatch();
            long succ      = 0;
            long fail      = 0;
            long all       = 0;

            for (int i = 0; i < clientsCnt; i++)
            {
                var  count    = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
                long received = 0;
                long sent     = 0;
                var  client   = context.Client.CreateTcpConnection(
                    context,
                    (conn, pkg) =>
                {
                    if (pkg.Command != TcpCommand.ReadEventCompleted)
                    {
                        context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
                        return;
                    }

                    var dto = pkg.Data.Deserialize <TcpClientMessageDto.ReadEventCompleted>();
                    _monitor.EndOperation(pkg.CorrelationId);
                    if (dto.Result == TcpClientMessageDto.ReadEventCompleted.ReadEventResult.Success)
                    {
                        if (Interlocked.Increment(ref succ) % 1000 == 0)
                        {
                            Console.Write(".");
                        }
                    }
                    else
                    {
                        if (Interlocked.Increment(ref fail) % 1000 == 0)
                        {
                            Console.Write("#");
                        }
                    }

                    Interlocked.Increment(ref received);
                    var localAll = Interlocked.Increment(ref all);
                    if (localAll % 100000 == 0)
                    {
                        var elapsed = sw2.Elapsed;
                        sw2.Restart();
                        context.Log.Trace("\nDONE TOTAL {0} READS IN {1} ({2:0.0}/s).", localAll, elapsed, 1000.0 * 100000 / elapsed.TotalMilliseconds);
                    }
                    if (localAll == requestsCnt)
                    {
                        context.Success();
                        doneEvent.Set();
                    }
                },
                    connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
                clients.Add(client);

                threads.Add(new Thread(() =>
                {
                    for (int j = 0; j < count; ++j)
                    {
                        var corrId  = Guid.NewGuid();
                        var read    = new TcpClientMessageDto.ReadEvent(eventStreamId, 0, resolveLinkTos, requireMaster);
                        var package = new TcpPackage(TcpCommand.ReadEvent, corrId, read.Serialize());
                        _monitor.StartOperation(corrId);
                        client.EnqueueSend(package.AsByteArray());

                        var localSent = Interlocked.Increment(ref sent);
                        while (localSent - Interlocked.Read(ref received) > context.Client.Options.ReadWindow / clientsCnt)
                        {
                            Thread.Sleep(1);
                        }
                    }
                })
                {
                    IsBackground = true
                });
            }

            var sw = Stopwatch.StartNew();

            sw2.Start();
            threads.ForEach(thread => thread.Start());
            doneEvent.Wait();
            sw.Stop();
            clients.ForEach(client => client.Close());

            var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;

            context.Log.Info("Completed. READS succ: {0}, fail: {1}.", Interlocked.Read(ref succ), Interlocked.Read(ref fail));
            context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);
            _monitor.GetMeasurementDetails();
            PerfUtils.LogData(Keyword,
                              PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
                                            PerfUtils.Col("requestsCnt", requestsCnt),
                                            PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
                              PerfUtils.Row(PerfUtils.Col("readsCnt", all)));
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);

            if (succ != requestsCnt)
            {
                context.Fail(reason: "There were errors or not all requests completed.");
            }
            else
            {
                context.Success();
            }
        }
Ejemplo n.º 4
0
        private void WriteFlood(CommandProcessorContext context, int clientsCnt, long requestsCnt, int streamsCnt, int size)
        {
            context.IsAsync();

            var doneEvent = new ManualResetEventSlim(false);
            var clients = new List<TcpTypedConnection<byte[]>>();
            var threads = new List<Thread>();

            long succ = 0;
            long fail = 0;
            long prepTimeout = 0;
            long commitTimeout = 0;
            long forwardTimeout = 0;
            long wrongExpVersion = 0;
            long streamDeleted = 0;
            long all = 0;

            var streams = Enumerable.Range(0, streamsCnt).Select(x => Guid.NewGuid().ToString()).ToArray();
            //var streams = Enumerable.Range(0, streamsCnt).Select(x => string.Format("stream-{0}", x)).ToArray();
            var sw2 = new Stopwatch();
            for (int i = 0; i < clientsCnt; i++)
            {
                var count = requestsCnt / clientsCnt + ((i == clientsCnt - 1) ? requestsCnt % clientsCnt : 0);
                long sent = 0;
                long received = 0;
                var rnd = new Random();
                var client = context.Client.CreateTcpConnection(
                    context,
                    (conn, pkg) =>
                    {
                        if (pkg.Command != TcpCommand.WriteEventsCompleted)
                        {
                            context.Fail(reason: string.Format("Unexpected TCP package: {0}.", pkg.Command));
                            return;
                        }

                        var dto = pkg.Data.Deserialize<TcpClientMessageDto.WriteEventsCompleted>();
                        _monitor.EndOperation(pkg.CorrelationId);
                        switch(dto.Result)
                        {
                            case TcpClientMessageDto.OperationResult.Success:
                                if (Interlocked.Increment(ref succ) % 1000 == 0) Console.Write('.');
                                break;
                            case TcpClientMessageDto.OperationResult.PrepareTimeout:
                                Interlocked.Increment(ref prepTimeout);
                                break;
                            case TcpClientMessageDto.OperationResult.CommitTimeout:
                                Interlocked.Increment(ref commitTimeout);
                                break;
                            case TcpClientMessageDto.OperationResult.ForwardTimeout:
                                Interlocked.Increment(ref forwardTimeout);
                                break;
                            case TcpClientMessageDto.OperationResult.WrongExpectedVersion:
                                Interlocked.Increment(ref wrongExpVersion);
                                break;
                            case TcpClientMessageDto.OperationResult.StreamDeleted:
                                Interlocked.Increment(ref streamDeleted);
                                break;
                            default:
                                throw new ArgumentOutOfRangeException();
                        }
                        if (dto.Result != TcpClientMessageDto.OperationResult.Success)
                            if (Interlocked.Increment(ref fail)%1000 == 0) 
                                Console.Write('#');
                        Interlocked.Increment(ref received);
                        var localAll = Interlocked.Increment(ref all);
                        if (localAll % 100000 == 0)
                        {
                            var elapsed = sw2.Elapsed;
                            sw2.Restart();
                            context.Log.Trace("\nDONE TOTAL {0} WRITES IN {1} ({2:0.0}/s) [S:{3}, F:{4} (WEV:{5}, P:{6}, C:{7}, F:{8}, D:{9})].",
                                              localAll, elapsed, 1000.0*100000/elapsed.TotalMilliseconds,
                                              succ, fail,
                                              wrongExpVersion, prepTimeout, commitTimeout, forwardTimeout, streamDeleted);
                        }
                        if (localAll == requestsCnt)
                        {
                            context.Success();
                            doneEvent.Set();
                        }
                    },
                    connectionClosed: (conn, err) => context.Fail(reason: "Connection was closed prematurely."));
                clients.Add(client);

                threads.Add(new Thread(() =>
                {
                    for (int j = 0; j < count; ++j)
                    {
                        var corrid = Guid.NewGuid();
                        var write = new TcpClientMessageDto.WriteEvents(
                            streams[rnd.Next(streamsCnt)],
                            ExpectedVersion.Any,
                            new[]
                            {
                                new TcpClientMessageDto.NewEvent(Guid.NewGuid().ToByteArray(),
                                                                 "TakeSomeSpaceEvent",
                                                                 1,0,
                                                                 Common.Utils.Helper.UTF8NoBom.GetBytes("{ \"DATA\" : \"" + new string('*', size) + "\"}"),
                                                                 Common.Utils.Helper.UTF8NoBom.GetBytes("{ \"METADATA\" : \"" + new string('$', 100) + "\"}"))
                            },
                            false);
                        var package = new TcpPackage(TcpCommand.WriteEvents, corrid, write.Serialize());
                        _monitor.StartOperation(corrid);
                        client.EnqueueSend(package.AsByteArray());

                        var localSent = Interlocked.Increment(ref sent);
                        while (localSent - Interlocked.Read(ref received) > context.Client.Options.WriteWindow/clientsCnt)
                        {
                            Thread.Sleep(1);
                        }
                    }
                }) { IsBackground = true });
            }

            var sw = Stopwatch.StartNew();
            sw2.Start();
            threads.ForEach(thread => thread.Start());
            doneEvent.Wait();
            sw.Stop();
            clients.ForEach(client => client.Close());

            context.Log.Info("Completed. Successes: {0}, failures: {1} (WRONG VERSION: {2}, P: {3}, C: {4}, F: {5}, D: {6})",
                             succ, fail,
                             wrongExpVersion, prepTimeout, commitTimeout, forwardTimeout, streamDeleted);

            var reqPerSec = (all + 0.0) / sw.ElapsedMilliseconds * 1000;
            context.Log.Info("{0} requests completed in {1}ms ({2:0.00} reqs per sec).", all, sw.ElapsedMilliseconds, reqPerSec);

            PerfUtils.LogData(
                Keyword,
                PerfUtils.Row(PerfUtils.Col("clientsCnt", clientsCnt),
                              PerfUtils.Col("requestsCnt", requestsCnt),
                              PerfUtils.Col("ElapsedMilliseconds", sw.ElapsedMilliseconds)),
                PerfUtils.Row(PerfUtils.Col("successes", succ), PerfUtils.Col("failures", fail)));

            var failuresRate = (int) (100 * fail / (fail + succ));
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-reqPerSec", Keyword, clientsCnt, requestsCnt), (int)reqPerSec);
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-{1}-{2}-failureSuccessRate", Keyword, clientsCnt, requestsCnt), failuresRate);
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-c{1}-r{2}-st{3}-s{4}-reqPerSec", Keyword, clientsCnt, requestsCnt, streamsCnt, size), (int)reqPerSec);
            PerfUtils.LogTeamCityGraphData(string.Format("{0}-c{1}-r{2}-st{3}-s{4}-failureSuccessRate", Keyword, clientsCnt, requestsCnt, streamsCnt, size), failuresRate);
            _monitor.GetMeasurementDetails();
            if (Interlocked.Read(ref succ) != requestsCnt)
                context.Fail(reason: "There were errors or not all requests completed.");
            else
                context.Success();
        }