protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { // refresh = false is default on elasticsearch's side. var bulkParms = new SimpleBulkParameters() { Refresh = false }; var msgGenerator = new MessageGenerator(); var tasks = new List <Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexMany(messages, indexName, bulkParms); Interlocked.Add(ref NumSent, bufferSize); if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName); } } //Task.WaitAll(tasks.ToArray()); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { // refresh = false is default on elasticsearch's side. var bulkParms = new SimpleBulkParameters() { Refresh = false }; var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexMany(messages, indexName, bulkParms); Interlocked.Add(ref NumSent, bufferSize); if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName); } } //Task.WaitAll(tasks.ToArray()); }
public void Run(string indexName, int port, int numMessages, int bufferSize) { var bulkParms = new SimpleBulkParameters() { Refresh = false }; var settings = new ConnectionSettings("localhost", port) .SetDefaultIndex(indexName); var client = new ElasticClient(settings, new ThriftConnection(settings)); Connect(client, settings); IList<Message> msgBuffer = new List<Message>(bufferSize); var msgGenerator = new MessageGenerator(); foreach (var msg in msgGenerator.Generate(numMessages)) { msgBuffer.Add(msg); // Flush buffer once max size reached if (msgBuffer.Count >= bufferSize) { client.IndexMany(msgBuffer, indexName, bulkParms); msgBuffer.Clear(); Interlocked.Add(ref NumSent, bufferSize); // Output how many messages sent so far if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages over Thrift", NumSent); } } } }