Exemplo n.º 1
0
        public void TestSerializationConfig()
        {
            var           config = new SerializationConfig();
            ISerializer   ser    = new StringSerializer();
            IDeserializer deser  = new StringDeserializer();

            var t1 = Tuple.Create(ser, ser);
            var t2 = Tuple.Create(ser, ser);

            Assert.That(t1, Is.EqualTo(t2));

            var s = config.GetSerializersForTopic("topicnotfound");

            Assert.That(s,
                        Is.EqualTo(Tuple.Create(ByteArraySerialization.DefaultSerializer, ByteArraySerialization.DefaultSerializer)));

            var d = config.GetDeserializersForTopic("topicnotfound");

            Assert.That(d,
                        Is.EqualTo(Tuple.Create(ByteArraySerialization.DefaultDeserializer, ByteArraySerialization.DefaultDeserializer)));

            config.SetSerializersForTopic("topicnotfound", ser, ser);
            config.SetDeserializersForTopic("topicnotfound", deser, deser);

            s = config.GetSerializersForTopic("topicnotfound");
            Assert.That(s, Is.EqualTo(Tuple.Create(ser, ser)));

            d = config.GetDeserializersForTopic("topicnotfound");
            Assert.That(d, Is.EqualTo(Tuple.Create(deser, deser)));

            var config2 = new SerializationConfig(config);

            s = config2.GetSerializersForTopic("topicnotfound");
            Assert.That(s, Is.EqualTo(Tuple.Create(ser, ser)));

            d = config2.GetDeserializersForTopic("topicnotfound");
            Assert.That(d, Is.EqualTo(Tuple.Create(deser, deser)));

            config2.SetSerializersForTopic("topicnotfound", null, ser);
            config2.SetDeserializersForTopic("topicnotfound", null, deser);

            s = config2.GetSerializersForTopic("topicnotfound");
            Assert.That(s, Is.EqualTo(Tuple.Create(ByteArraySerialization.DefaultSerializer, ser)));

            d = config2.GetDeserializersForTopic("topicnotfound");
            Assert.That(d, Is.EqualTo(Tuple.Create(ByteArraySerialization.DefaultDeserializer, deser)));

            config2.SetSerializersForTopic("topicnotfound", ser, null);
            config2.SetDeserializersForTopic("topicnotfound", deser, null);

            s = config2.GetSerializersForTopic("topicnotfound");
            Assert.That(s, Is.EqualTo(Tuple.Create(ser, ByteArraySerialization.DefaultSerializer)));

            d = config2.GetDeserializersForTopic("topicnotfound");
            Assert.That(d, Is.EqualTo(Tuple.Create(deser, ByteArraySerialization.DefaultDeserializer)));
        }
Exemplo n.º 2
0
        public void TestSerializeProduceRequest()
        {
            var produce = new ProduceRequest
            {
                Timeout      = 1223,
                RequiredAcks = 1,
                TopicsData   = new[]
                {
                    new TopicData <PartitionData>
                    {
                        TopicName      = "barbu",
                        PartitionsData = new[]
                        {
                            new PartitionData
                            {
                                Partition        = 22,
                                CompressionCodec = CompressionCodec.None,
                                Messages         = new[]
                                {
                                    new Message {
                                        Value = TheValue
                                    }
                                },
                            }
                        }
                    },
                }
            };
            var config = new SerializationConfig();

            config.SetSerializersForTopic("barbu", new StringSerializer(), new StringSerializer());
            config.SetDeserializersForTopic("barbu", new StringDeserializer(), new StringDeserializer());
            using (var serialized = produce.Serialize(new ReusableMemoryStream(null), 321, ClientId, config))
            {
                CheckHeader(Basics.ApiKey.ProduceRequest, 0, 321, TheClientId, serialized);
                Assert.AreEqual(produce.RequiredAcks, BigEndianConverter.ReadInt16(serialized));
                Assert.AreEqual(produce.Timeout, BigEndianConverter.ReadInt32(serialized));
                Assert.AreEqual(1, BigEndianConverter.ReadInt32(serialized)); // 1 topic data
                Assert.AreEqual("barbu", Basics.DeserializeString(serialized));
                Assert.AreEqual(1, BigEndianConverter.ReadInt32(serialized)); // 1 partition data
                Assert.AreEqual(22, BigEndianConverter.ReadInt32(serialized));
                var msgs = FetchPartitionResponse.DeserializeMessageSet(serialized, config.GetDeserializersForTopic("barbu"));
                Assert.AreEqual(1, msgs.Count);
                //Assert.AreEqual(TheValue, Encoding.UTF8.GetString(msgs[0].Message.Value));
                Assert.AreEqual(TheValue, msgs[0].Message.Value as string);
            }
        }
Exemplo n.º 3
0
        private static void Main(string[] args)
        {
            Mode mode          = Mode.Profile;
            bool mix           = false;
            var  configuration = new Configuration
            {
                ConsumeBatchSize = 100
            };

            // Ugly command line parsing
            string curOpt = "";

            try
            {
                bool seeds  = false;
                bool topics = false;
                for (int i = 0; i < args.Length; ++i)
                {
                    curOpt = args[i];
                    switch (args[i])
                    {
                    case "--global":
                        configuration.BatchStrategy = BatchStrategy.Global;
                        break;

                    case "--mix":
                        mix = true;
                        break;

                    case "--stress":
                        mode = Mode.Stress;
                        break;

                    case "--hard":
                        mode = Mode.StressHard;
                        break;

                    case "--discard":
                        configuration.ErrorStrategy = ErrorStrategy.Discard;
                        break;

                    case "--retry":
                        configuration.ErrorStrategy = ErrorStrategy.Retry;
                        break;

                    case "--gzip":
                        configuration.CompressionCodec = CompressionCodec.Gzip;
                        break;

                    case "--snappy":
                        configuration.CompressionCodec = CompressionCodec.Snappy;
                        break;

                    case "--no-ack":
                        configuration.RequiredAcks = RequiredAcks.None;
                        break;

                    case "--all-sync-ack":
                        configuration.RequiredAcks = RequiredAcks.AllInSyncReplicas;
                        break;

                    case "--ttl":
                        configuration.MessageTtl = TimeSpan.FromSeconds(int.Parse(args[++i]));
                        break;

                    case "--batch":
                        configuration.ProduceBatchSize = int.Parse(args[++i]);
                        break;

                    case "--time":
                        configuration.ProduceBufferingTime = TimeSpan.FromMilliseconds(int.Parse(args[++i]));
                        break;

                    case "--max-messages":
                        configuration.MaxBufferedMessages = int.Parse(args[++i]);
                        break;

                    case "--topics":
                        topics  = true;
                        _topics = args[++i].Split(',');
                        break;

                    case "--seeds":
                        seeds = true;
                        configuration.Seeds = args[++i];
                        break;

                    case "--clientid":
                        configuration.ClientId = args[++i];
                        break;

                    case "--concurrency":
                        configuration.MaximumConcurrency = int.Parse(args[++i]);
                        break;

                    case "--send-buffer":
                        configuration.SendBufferSize = int.Parse(args[++i]);
                        break;

                    case "--receive-buffer":
                        configuration.ReceiveBufferSize = int.Parse(args[++i]);
                        break;

                    case "--timeout":
                        configuration.RequestTimeoutMs = int.Parse(args[++i]);
                        break;

                    case "--min-bytes":
                        configuration.FetchMinBytes = int.Parse(args[++i]);
                        break;

                    case "--max-wait":
                        configuration.FetchMaxWaitTime = int.Parse(args[++i]);
                        break;

                    case "--max-bytes":
                        configuration.FetchMessageMaxBytes = int.Parse(args[++i]);
                        break;

                    case "--delay":
                        configuration.NumberOfMessagesBeforeRoundRobin = int.Parse(args[++i]);
                        break;

                    case "--consume":
                    {
                        _consumeFrom = long.Parse(args[++i]);
                        var p = args[++i].Split(';');
                        _partitions = new int[p.Length][];
                        for (int j = 0; j < _partitions.Length; ++j)
                        {
                            _partitions[j] = p[j].Split(',').Select(int.Parse).ToArray();
                        }
                    }
                    break;
                    }
                }
                // Minimal error management
                if (args.Length < 1 || !seeds || !topics)
                {
                    throw new ArgumentException();
                }
            }
            catch
            {
                // Minimal error management
                Console.WriteLine("Syntax error in option {0}", curOpt);
                Usage();
                Environment.Exit(-1);
            }

            var serializer          = new StringSerializer();
            var deserializer        = new StringDeserializer();
            var serializationConfig = new SerializationConfig()
            {
                SerializeOnProduce = true
            };

            foreach (var topic in _topics)
            {
                serializationConfig.SetSerializersForTopic(topic, serializer, serializer);
                serializationConfig.SetDeserializersForTopic(topic, deserializer, deserializer);
            }
            configuration.SerializationConfig = serializationConfig;

            var cluster =
                new ClusterClient(configuration, new ConsoleLogger());

            if (_partitions == null)
            {
                var task = Start(mode, cluster);
                Console.ReadKey();
                _running = false;
                Console.ReadKey();
                task.Wait();
            }
            else
            {
                int i = 0;
                foreach (var topic in _topics)
                {
                    var capturedTopic = topic;
                    cluster.Messages.Where(kr => kr.Topic == capturedTopic).Sample(TimeSpan.FromMilliseconds(15))
                    .Subscribe(kr => Console.WriteLine("{0}/{1} {2}: {3}", kr.Topic, kr.Partition, kr.Offset, kr.Value as string));
                    foreach (var p in _partitions[i])
                    {
                        cluster.Consume(topic, p, _consumeFrom);
                    }
                    ++i;
                }

                Task task = null;
                if (mix)
                {
                    task = Start(mode, cluster);
                }

                Console.ReadKey();
                i = 0;
                foreach (var topic in _topics)
                {
                    foreach (var p in _partitions[i])
                    {
                        if (p < 0)
                        {
                            cluster.StopConsume(topic);
                        }
                        else
                        {
                            cluster.StopConsume(topic, p);
                        }
                    }
                    ++i;
                }
                if (task != null)
                {
                    _running = false;
                    task.Wait();
                }
            }

            Console.WriteLine(cluster.Statistics);
            Console.ReadKey();
            cluster.Dispose();
        }