protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
// refresh = false is default on elasticsearch's side.
var bulkParms = new SimpleBulkParameters()
{
Refresh = false
};
var msgGenerator = new MessageGenerator();
var tasks = new List <Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexMany(messages, indexName, bulkParms);
Interlocked.Add(ref NumSent, bufferSize);
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
}
}
//Task.WaitAll(tasks.ToArray());
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
// refresh = false is default on elasticsearch's side.
var bulkParms = new SimpleBulkParameters() { Refresh = false };
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexMany(messages, indexName, bulkParms);
Interlocked.Add(ref NumSent, bufferSize);
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
}
}
//Task.WaitAll(tasks.ToArray());
}
public void Run(string indexName, int port, int numMessages, int bufferSize)
{
var bulkParms = new SimpleBulkParameters() { Refresh = false };
var settings = new ConnectionSettings("localhost", port)
.SetDefaultIndex(indexName);
var client = new ElasticClient(settings, new ThriftConnection(settings));
Connect(client, settings);
IList<Message> msgBuffer = new List<Message>(bufferSize);
var msgGenerator = new MessageGenerator();
foreach (var msg in msgGenerator.Generate(numMessages))
{
msgBuffer.Add(msg);
// Flush buffer once max size reached
if (msgBuffer.Count >= bufferSize)
{
client.IndexMany(msgBuffer, indexName, bulkParms);
msgBuffer.Clear();
Interlocked.Add(ref NumSent, bufferSize);
// Output how many messages sent so far
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages over Thrift", NumSent);
}
}
}
}
