protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List <Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); client.CreateIndex(indexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s.Add("refresh_interval", "-1")) .AddMapping <Message>(p => p.MapFromAttributes()) ); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); client.UpdateSettings(u => u .Index(indexName) .RefreshInterval("1s") ); }
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize) { //settings from http://benchmarks.elasticsearch.org/ var msgGenerator = new MessageGenerator(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); this.Client.CreateIndex(this.IndexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s .Add("refresh_interval", "30s") .Add("index.store.type", "mmapfs") .Add("index.store.throttle.type", "none") .Add("indices.store.throttle.type", "none") .Add("index.number_of_shards", 6) .Add("index.number_of_replicas", 0) .Add("indices.memory.index_buffer_size", "10%") .Add("index.translog.flush_threshold_size", "4g") .Add("index.translog.flush_threshold_ops", 500000) .Add("index.merge.scheduler.max_thread_count", 3) .Add("index.merge.scheduler.max_merge_count", 6) ) .AddMapping <Message>(p => p.MapFromAttributes()) ); var sw = Stopwatch.StartNew(); Interlocked.Exchange(ref NumSent, 0); Task <IBulkResponse>[] array = partitionedMessages .Select(messages => Client.IndexManyAsync(messages, this.IndexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ", NumSent, tt.Result.Took, this.Type); return(tt.Result); }) ) .ToArray(); Task.WaitAll(array); sw.Stop(); Console.WriteLine(); Client.UpdateSettings(u => u .Index(this.IndexName) .RefreshInterval("1s") ); return(new IndexResults { Elapsed = sw.ElapsedMilliseconds, EsTimings = array.Select(a => a.Result.Took).ToList() }); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List <Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); }
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); this.Client.CreateIndex(this.IndexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s.Add("refresh_interval", "-1")) .AddMapping <Message>(p => p.MapFromAttributes()) ); var sw = Stopwatch.StartNew(); Interlocked.Exchange(ref NumSent, 0); Task <IBulkResponse>[] array = partitionedMessages .Select(messages => Client.IndexManyAsync(messages, this.IndexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ", NumSent, tt.Result.Took, this.Type); return(tt.Result); }) ) .ToArray(); Task.WaitAll(array); sw.Stop(); Console.WriteLine(); Client.UpdateSettings(u => u .Index(this.IndexName) .RefreshInterval("1s") ); return(new IndexResults { Elapsed = sw.ElapsedMilliseconds, EsTimings = array.Select(a => a.Result.Took).ToList() }); }
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize) { //settings from http://benchmarks.elasticsearch.org/ var msgGenerator = new MessageGenerator(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); this.Client.CreateIndex(this.IndexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s .Add("refresh_interval", "30s") .Add("index.store.type", "mmapfs") .Add("index.store.throttle.type", "none") .Add("indices.store.throttle.type", "none") .Add("index.number_of_shards", 6) .Add("index.number_of_replicas", 0) .Add("indices.memory.index_buffer_size", "10%") .Add("index.translog.flush_threshold_size", "4g") .Add("index.translog.flush_threshold_ops", 500000) .Add("index.merge.scheduler.max_thread_count", 3) .Add("index.merge.scheduler.max_merge_count", 6) ) .AddMapping<Message>(p=>p.MapFromAttributes()) ); var sw = Stopwatch.StartNew(); Interlocked.Exchange(ref NumSent, 0); Task<IBulkResponse>[] array = partitionedMessages .Select(messages => Client.IndexManyAsync(messages, this.IndexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ", NumSent, tt.Result.Took, this.Type); return tt.Result; }) ) .ToArray(); Task.WaitAll(array); sw.Stop(); Console.WriteLine(); Client.UpdateSettings(u => u .Index(this.IndexName) .RefreshInterval("1s") ); return new IndexResults { Elapsed = sw.ElapsedMilliseconds, EsTimings = array.Select(a=>a.Result.Took).ToList() }; }
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); this.Client.CreateIndex(this.IndexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s.Add("refresh_interval", "-1")) .AddMapping<Message>(p=>p.MapFromAttributes()) ); var sw = Stopwatch.StartNew(); Interlocked.Exchange(ref NumSent, 0); Task<IBulkResponse>[] array = partitionedMessages .Select(messages => Client.IndexManyAsync(messages, this.IndexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ", NumSent, tt.Result.Took, this.Type); return tt.Result; }) ) .ToArray(); Task.WaitAll(array); sw.Stop(); Console.WriteLine(); Client.UpdateSettings(u => u .Index(this.IndexName) .RefreshInterval("1s") ); return new IndexResults { Elapsed = sw.ElapsedMilliseconds, EsTimings = array.Select(a=>a.Result.Took).ToList() }; }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); client.CreateIndex(indexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s.Add("refresh_interval", "-1")) .AddMapping<Message>(p=>p.MapFromAttributes()) ); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); client.UpdateSettings(u => u .Index(indexName) .RefreshInterval("1s") ); }