protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List <Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
client.CreateIndex(indexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s.Add("refresh_interval", "-1"))
.AddMapping <Message>(p => p.MapFromAttributes())
);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
client.UpdateSettings(u => u
.Index(indexName)
.RefreshInterval("1s")
);
}
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize)
{
//settings from http://benchmarks.elasticsearch.org/
var msgGenerator = new MessageGenerator();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
this.Client.CreateIndex(this.IndexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s
.Add("refresh_interval", "30s")
.Add("index.store.type", "mmapfs")
.Add("index.store.throttle.type", "none")
.Add("indices.store.throttle.type", "none")
.Add("index.number_of_shards", 6)
.Add("index.number_of_replicas", 0)
.Add("indices.memory.index_buffer_size", "10%")
.Add("index.translog.flush_threshold_size", "4g")
.Add("index.translog.flush_threshold_ops", 500000)
.Add("index.merge.scheduler.max_thread_count", 3)
.Add("index.merge.scheduler.max_merge_count", 6)
)
.AddMapping <Message>(p => p.MapFromAttributes())
);
var sw = Stopwatch.StartNew();
Interlocked.Exchange(ref NumSent, 0);
Task <IBulkResponse>[] array = partitionedMessages
.Select(messages => Client.IndexManyAsync(messages, this.IndexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ",
NumSent, tt.Result.Took, this.Type);
return(tt.Result);
})
)
.ToArray();
Task.WaitAll(array);
sw.Stop();
Console.WriteLine();
Client.UpdateSettings(u => u
.Index(this.IndexName)
.RefreshInterval("1s")
);
return(new IndexResults
{
Elapsed = sw.ElapsedMilliseconds,
EsTimings = array.Select(a => a.Result.Took).ToList()
});
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List <Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
}
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
this.Client.CreateIndex(this.IndexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s.Add("refresh_interval", "-1"))
.AddMapping <Message>(p => p.MapFromAttributes())
);
var sw = Stopwatch.StartNew();
Interlocked.Exchange(ref NumSent, 0);
Task <IBulkResponse>[] array = partitionedMessages
.Select(messages => Client.IndexManyAsync(messages, this.IndexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ",
NumSent, tt.Result.Took, this.Type);
return(tt.Result);
})
)
.ToArray();
Task.WaitAll(array);
sw.Stop();
Console.WriteLine();
Client.UpdateSettings(u => u
.Index(this.IndexName)
.RefreshInterval("1s")
);
return(new IndexResults
{
Elapsed = sw.ElapsedMilliseconds,
EsTimings = array.Select(a => a.Result.Took).ToList()
});
}
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize)
{
//settings from http://benchmarks.elasticsearch.org/
var msgGenerator = new MessageGenerator();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
this.Client.CreateIndex(this.IndexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s
.Add("refresh_interval", "30s")
.Add("index.store.type", "mmapfs")
.Add("index.store.throttle.type", "none")
.Add("indices.store.throttle.type", "none")
.Add("index.number_of_shards", 6)
.Add("index.number_of_replicas", 0)
.Add("indices.memory.index_buffer_size", "10%")
.Add("index.translog.flush_threshold_size", "4g")
.Add("index.translog.flush_threshold_ops", 500000)
.Add("index.merge.scheduler.max_thread_count", 3)
.Add("index.merge.scheduler.max_merge_count", 6)
)
.AddMapping<Message>(p=>p.MapFromAttributes())
);
var sw = Stopwatch.StartNew();
Interlocked.Exchange(ref NumSent, 0);
Task<IBulkResponse>[] array = partitionedMessages
.Select(messages => Client.IndexManyAsync(messages, this.IndexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ",
NumSent, tt.Result.Took, this.Type);
return tt.Result;
})
)
.ToArray();
Task.WaitAll(array);
sw.Stop();
Console.WriteLine();
Client.UpdateSettings(u => u
.Index(this.IndexName)
.RefreshInterval("1s")
);
return new IndexResults
{
Elapsed = sw.ElapsedMilliseconds,
EsTimings = array.Select(a=>a.Result.Took).ToList()
};
}
protected IndexResults GenerateAndIndex(int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
this.Client.CreateIndex(this.IndexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s.Add("refresh_interval", "-1"))
.AddMapping<Message>(p=>p.MapFromAttributes())
);
var sw = Stopwatch.StartNew();
Interlocked.Exchange(ref NumSent, 0);
Task<IBulkResponse>[] array = partitionedMessages
.Select(messages => Client.IndexManyAsync(messages, this.IndexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.Write("\r{2}: {0:0,0} msgs es-time: {1} ",
NumSent, tt.Result.Took, this.Type);
return tt.Result;
})
)
.ToArray();
Task.WaitAll(array);
sw.Stop();
Console.WriteLine();
Client.UpdateSettings(u => u
.Index(this.IndexName)
.RefreshInterval("1s")
);
return new IndexResults
{
Elapsed = sw.ElapsedMilliseconds,
EsTimings = array.Select(a=>a.Result.Took).ToList()
};
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
client.CreateIndex(indexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s.Add("refresh_interval", "-1"))
.AddMapping<Message>(p=>p.MapFromAttributes())
);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
client.UpdateSettings(u => u
.Index(indexName)
.RefreshInterval("1s")
);
}