public async Task<IBulkResponse> ReIndexBulk(IEnumerable<BeerDto> beers, string index)
{
//string url = WebConfigurationManager.AppSettings["elasticsearch"];
//var node = new Uri(url);
var settings = new ConnectionSettings(_node, defaultIndex: index);
var client = new ElasticClient(settings);
await client.MapAsync<BeerDto>(d => d.Properties(p => p.String(s => s.Name(n => n.Name).Analyzer("autocomplete"))));
return await client.IndexManyAsync(beers);
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
// refresh = false is default on elasticsearch's side.
var bulkParms = new SimpleBulkParameters() { Refresh = false };
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName, bulkParms);
tasks.Add(t);
Interlocked.Add(ref NumSent, bufferSize);
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
}
}
Task.WaitAll(tasks.ToArray());
}
/// <summary>
/// Depends on hn_full_11-07-2010.xml which you can download from:
/// http://api.ihackernews.com/torrents/hn_full_11-07-2010.zip.torrent
///
/// When run from debug make sure to change the default debug arguments.
/// <param name="args">Full filepath to hn_full_11-07-2010.xml</param>
static void Main(string[] args)
{
var filePath = args.First();
var elasticSettings = new ConnectionSettings("127.0.0.1.", 9200)
.SetDefaultIndex("mpdreamz")
.SetMaximumAsyncConnections(50);
var client = new ElasticClient(elasticSettings);
ConnectionStatus connectionStatus;
if (!client.TryConnect(out connectionStatus))
{
Console.Error.WriteLine("Could not connect to {0}:\r\n{1}",
elasticSettings.Host, connectionStatus.Error.OriginalException.Message);
Console.Read();
return;
}
var reader = new XmlTextReader(filePath);
Post post = new Post();
PostMetaData meta = new PostMetaData();
int processed = 0, dropped = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
var postQueue = new List<Post>();
try
{
while (reader.Read())
{
var name = reader.Name;
if (reader.NodeType == XmlNodeType.Element)
{
if (name == "HackerNews")
continue;
if (name == "ID")
post.Id = reader.ReadElementContentAsInt();
else if (name == "ParentID")
post.ParentId = reader.ReadElementContentAsInt();
else if (name == "Url")
post.Url = reader.ReadElementContentAsString();
else if (name == "Title")
post.Title = reader.ReadElementContentAsString();
else if (name == "Text")
post.Text = reader.ReadElementContentAsString();
else if (name == "Username")
meta.Username = reader.ReadElementContentAsString();
else if (name == "Points")
meta.Points = reader.ReadElementContentAsInt();
else if (name == "Type")
meta.Type = reader.ReadElementContentAsInt();
else if (name == "Timestamp")
meta.Created = reader.ReadElementContentAsDateTime();
else if (name == "CommentCount")
meta.CommentsCount = reader.ReadElementContentAsInt();
}
if (reader.NodeType == XmlNodeType.EndElement
&& name == "row")
{
post.Meta = meta;
postQueue.Add(post);
if (postQueue.Count() == 1000)
{
var t = client.IndexManyAsync(postQueue);
t.ContinueWith(c =>
{
var result = c.Result;
if (!result.Success)
dropped++;
});
processed += postQueue.Count();
postQueue = new List<Post>();
}
Console.Write("\rProcessed:{0}, Dropped:{2} in {1}", processed, sw.Elapsed, dropped);
post = new Post();
meta = new PostMetaData();
}
}
if (postQueue.Count() > 0)
{
var task = client.IndexManyAsync(postQueue).ContinueWith(t =>
{
var c = t.Result;
if (!c.Success)
Interlocked.Increment(ref dropped);
return t;
});
Interlocked.Add(ref processed, postQueue.Count());
postQueue = new List<Post>();
}
sw.Stop();
Console.WriteLine("\nDone!", sw.Elapsed);
Console.WriteLine("{0} docs in {1} => {2} docs/s", processed, sw.Elapsed, processed / sw.Elapsed.TotalSeconds);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
client.CreateIndex(indexName, c => c
.NumberOfReplicas(0)
.NumberOfShards(1)
.Settings(s => s.Add("refresh_interval", "-1"))
.AddMapping<Message>(p=>p.MapFromAttributes())
);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexManyAsync(messages, indexName)
.ContinueWith(tt =>
{
Interlocked.Add(ref NumSent, bufferSize);
Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took);
})
;
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
client.UpdateSettings(u => u
.Index(indexName)
.RefreshInterval("1s")
);
}