Example #1
0
        public static void Setup()
        {
            var client = BaseAction.Client;

            if (client.IndexExists(BaseAction.DefaultIndex).Exists)
            {
                return;
            }

            var projects = NestTestData.Data;
            var people   = NestTestData.Session.List <Person>(1000).Get();

            client.CreateIndex(BaseAction.DefaultIndex, c => c
                               .NumberOfReplicas(0)
                               .NumberOfShards(1)
                               .AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
                               .AddMapping <Person>(m => m.MapFromAttributes())
                               );


            var bulkParameters = new SimpleBulkParameters()
            {
                Refresh = true
            };

            client.IndexMany(projects, bulkParameters);
            client.IndexMany(people, bulkParameters);
            client.Refresh(new[] { BaseAction.DefaultIndex });
        }
Example #2
0
        public static void Setup()
        {
            var client = ElasticsearchConfiguration.Client;

            if (client.IndexExists(ElasticsearchConfiguration.DefaultIndex).Exists)
                return;

            var projects = NestTestData.Data;
            var people = NestTestData.People;

            client.DeleteIndex(ElasticsearchConfiguration.DefaultIndex);

            client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
                .NumberOfReplicas(0)
                .NumberOfShards(1)
                .AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
                .AddMapping<Person>(m => m.MapFromAttributes())
            );
            client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
                .NumberOfReplicas(0)
                .NumberOfShards(1)
                .AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
                .AddMapping<Person>(m => m.MapFromAttributes())
            );
            var bulkParameters = new SimpleBulkParameters() { Refresh = true };
            client.IndexMany(projects, bulkParameters);
            client.IndexMany(people, bulkParameters);
        }
Example #3
0
        protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
        {
            // refresh = false is default on elasticsearch's side.
            var bulkParms = new SimpleBulkParameters()
            {
                Refresh = false
            };

            var msgGenerator        = new MessageGenerator();
            var tasks               = new List <Task>();
            var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);

            Interlocked.Exchange(ref NumSent, 0);
            foreach (var messages in partitionedMessages)
            {
                var t = client.IndexMany(messages, indexName, bulkParms);

                Interlocked.Add(ref NumSent, bufferSize);
                if (NumSent % 10000 == 0)
                {
                    Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
                }
            }
            //Task.WaitAll(tasks.ToArray());
        }
Example #4
0
        public static void Setup()
        {
            var client = ElasticsearchConfiguration.Client;

            if (client.IndexExists(ElasticsearchConfiguration.DefaultIndex).Exists)
            {
                return;
            }

            var projects = NestTestData.Data;
            var people   = NestTestData.People;

            client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c
                               .NumberOfReplicas(0)
                               .NumberOfShards(1)
                               .AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
                               .AddMapping <Person>(m => m.MapFromAttributes())
                               );
            client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c
                               .NumberOfReplicas(0)
                               .NumberOfShards(1)
                               .AddMapping <ElasticSearchProject>(m => m.MapFromAttributes())
                               .AddMapping <Person>(m => m.MapFromAttributes())
                               );

            var bulkParameters = new SimpleBulkParameters()
            {
                Refresh = true
            };

            client.IndexMany(projects, bulkParameters);
            client.IndexMany(people, bulkParameters);
            client.Refresh(new[] { ElasticsearchConfiguration.DefaultIndex, ElasticsearchConfiguration.DefaultIndex + "_clone" });
        }
        protected void BulkIndexData()
        {
            var client = CreateClient();
            if (client.IsValid)
            {
                var projects = NestTestData.Data;
                var cloneIndex = Test.Default.DefaultIndex + "_clone";
                var bulkParameters = new SimpleBulkParameters() { Refresh = true };
                client.Index(projects, bulkParameters);
                client.Index(projects, cloneIndex, bulkParameters);

            }
        }
Example #6
0
        private static void ResetType <T>(IElasticClient client, IEnumerable <T> objects) where T : class
        {
            var cloneIndex     = Test.Default.DefaultIndex + "_clone";
            var bulkParameters = new SimpleBulkParameters()
            {
                Refresh = true
            };

            client.MapFromAttributes <T>();
            client.MapFromAttributes <T>(cloneIndex);

            client.IndexMany(objects, bulkParameters);
            client.IndexMany(objects, cloneIndex, bulkParameters);
        }
Example #7
0
        protected void BulkIndexData()
        {
            var client = CreateClient();

            if (client.IsValid)
            {
                var projects       = NestTestData.Data;
                var cloneIndex     = Test.Default.DefaultIndex + "_clone";
                var bulkParameters = new SimpleBulkParameters()
                {
                    Refresh = true
                };
                client.Index(projects, bulkParameters);
                client.Index(projects, cloneIndex, bulkParameters);
            }
        }
        protected void ResetIndexes()
        {
            var client = CreateClient();

            if (client.IsValid)
            {
                var projects       = NestTestData.Data;
                var cloneIndex     = Test.Default.DefaultIndex + "_clone";
                var bulkParameters = new SimpleBulkParameters()
                {
                    Refresh = true
                };
                client.DeleteMapping <ElasticSearchProject>();
                client.DeleteMapping <ElasticSearchProject>(cloneIndex);
                client.Index(projects, bulkParameters);
                client.Index(projects, cloneIndex, bulkParameters);
            }
        }
Example #9
0
        protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
        {
            // refresh = false is default on elasticsearch's side.
            var bulkParms = new SimpleBulkParameters() { Refresh = false };

            var msgGenerator = new MessageGenerator();
            var tasks = new List<Task>();
            var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
            Interlocked.Exchange(ref NumSent, 0);
            foreach (var messages in partitionedMessages)
            {
                var t = client.IndexMany(messages, indexName, bulkParms);

                Interlocked.Add(ref NumSent, bufferSize);
                if (NumSent % 10000 == 0)
                {
                    Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
                }
            }
            //Task.WaitAll(tasks.ToArray());
        }
Example #10
0
        public static void Setup()
        {
            var client = BaseAction.Client;

            if (client.IndexExists(BaseAction.DefaultIndex).Exists)
                return;

            var projects = NestTestData.Data;
            var people = NestTestData.Session.List<Person>(1000).Get();

            client.CreateIndex(BaseAction.DefaultIndex, c => c
                .NumberOfReplicas(0)
                .NumberOfShards(1)
                .AddMapping<ElasticSearchProject>(m => m.MapFromAttributes())
                .AddMapping<Person>(m => m.MapFromAttributes())
            );

            var bulkParameters = new SimpleBulkParameters() { Refresh = true };
            client.IndexMany(projects, bulkParameters);
            client.IndexMany(people, bulkParameters);
            client.Refresh(new[] { BaseAction.DefaultIndex });
        }
Example #11
0
        protected void ResetIndexes()
        {
            var client = CreateClient();
            if (client.IsValid)
            {
                var projects = NestTestData.Data;
                var cloneIndex = Test.Default.DefaultIndex + "_clone";
                var bulkParameters = new SimpleBulkParameters() { Refresh = true };
                client.DeleteMapping<ElasticSearchProject>();
                client.DeleteMapping<ElasticSearchProject>(cloneIndex);
                client.Map<ElasticSearchProject>();
                client.Map<ElasticSearchProject>(cloneIndex);
                client.Index(projects, bulkParameters);
                client.Index(projects, cloneIndex, bulkParameters);

            }
        }
Example #12
-1
        public void Run(string indexName, int port, int numMessages, int bufferSize)
        {
            var bulkParms = new SimpleBulkParameters() { Refresh = false };

            var settings = new ConnectionSettings("localhost", port)
                .SetDefaultIndex(indexName);

            var client = new ElasticClient(settings, new ThriftConnection(settings));

            Connect(client, settings);

            IList<Message> msgBuffer = new List<Message>(bufferSize);

            var msgGenerator = new MessageGenerator();

            foreach (var msg in msgGenerator.Generate(numMessages))
            {
                msgBuffer.Add(msg);

                // Flush buffer once max size reached
                if (msgBuffer.Count >= bufferSize)
                {
                    client.IndexMany(msgBuffer, indexName, bulkParms);
                    msgBuffer.Clear();

                    Interlocked.Add(ref NumSent, bufferSize);

                    // Output how many messages sent so far
                    if (NumSent % 10000 == 0)
                    {
                        Console.WriteLine("Sent {0:0,0} messages over Thrift", NumSent);
                    }
                }
            }
        }