public async Task<IBulkResponse> ReIndexBulk(IEnumerable<BeerDto> beers, string index) { //string url = WebConfigurationManager.AppSettings["elasticsearch"]; //var node = new Uri(url); var settings = new ConnectionSettings(_node, defaultIndex: index); var client = new ElasticClient(settings); await client.MapAsync<BeerDto>(d => d.Properties(p => p.String(s => s.Name(n => n.Name).Analyzer("autocomplete")))); return await client.IndexManyAsync(beers); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { // refresh = false is default on elasticsearch's side. var bulkParms = new SimpleBulkParameters() { Refresh = false }; var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName, bulkParms); tasks.Add(t); Interlocked.Add(ref NumSent, bufferSize); if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName); } } Task.WaitAll(tasks.ToArray()); }
/// <summary> /// Depends on hn_full_11-07-2010.xml which you can download from: /// http://api.ihackernews.com/torrents/hn_full_11-07-2010.zip.torrent /// /// When run from debug make sure to change the default debug arguments. /// <param name="args">Full filepath to hn_full_11-07-2010.xml</param> static void Main(string[] args) { var filePath = args.First(); var elasticSettings = new ConnectionSettings("127.0.0.1.", 9200) .SetDefaultIndex("mpdreamz") .SetMaximumAsyncConnections(50); var client = new ElasticClient(elasticSettings); ConnectionStatus connectionStatus; if (!client.TryConnect(out connectionStatus)) { Console.Error.WriteLine("Could not connect to {0}:\r\n{1}", elasticSettings.Host, connectionStatus.Error.OriginalException.Message); Console.Read(); return; } var reader = new XmlTextReader(filePath); Post post = new Post(); PostMetaData meta = new PostMetaData(); int processed = 0, dropped = 0; Stopwatch sw = new Stopwatch(); sw.Start(); var postQueue = new List<Post>(); try { while (reader.Read()) { var name = reader.Name; if (reader.NodeType == XmlNodeType.Element) { if (name == "HackerNews") continue; if (name == "ID") post.Id = reader.ReadElementContentAsInt(); else if (name == "ParentID") post.ParentId = reader.ReadElementContentAsInt(); else if (name == "Url") post.Url = reader.ReadElementContentAsString(); else if (name == "Title") post.Title = reader.ReadElementContentAsString(); else if (name == "Text") post.Text = reader.ReadElementContentAsString(); else if (name == "Username") meta.Username = reader.ReadElementContentAsString(); else if (name == "Points") meta.Points = reader.ReadElementContentAsInt(); else if (name == "Type") meta.Type = reader.ReadElementContentAsInt(); else if (name == "Timestamp") meta.Created = reader.ReadElementContentAsDateTime(); else if (name == "CommentCount") meta.CommentsCount = reader.ReadElementContentAsInt(); } if (reader.NodeType == XmlNodeType.EndElement && name == "row") { post.Meta = meta; postQueue.Add(post); if (postQueue.Count() == 1000) { var t = client.IndexManyAsync(postQueue); t.ContinueWith(c => { var result = c.Result; if (!result.Success) dropped++; }); processed += postQueue.Count(); postQueue = new List<Post>(); } Console.Write("\rProcessed:{0}, Dropped:{2} in {1}", processed, sw.Elapsed, dropped); post = new Post(); meta = new PostMetaData(); } } if (postQueue.Count() > 0) { var task = client.IndexManyAsync(postQueue).ContinueWith(t => { var c = t.Result; if (!c.Success) Interlocked.Increment(ref dropped); return t; }); Interlocked.Add(ref processed, postQueue.Count()); postQueue = new List<Post>(); } sw.Stop(); Console.WriteLine("\nDone!", sw.Elapsed); Console.WriteLine("{0} docs in {1} => {2} docs/s", processed, sw.Elapsed, processed / sw.Elapsed.TotalSeconds); } catch (Exception e) { Console.WriteLine(e.Message); } }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); client.CreateIndex(indexName, c => c .NumberOfReplicas(0) .NumberOfShards(1) .Settings(s => s.Add("refresh_interval", "-1")) .AddMapping<Message>(p=>p.MapFromAttributes()) ); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexManyAsync(messages, indexName) .ContinueWith(tt => { Interlocked.Add(ref NumSent, bufferSize); Console.WriteLine("Sent {0:0,0} messages to {1}, {2}", NumSent, indexName, tt.Result.Took); }) ; tasks.Add(t); } Task.WaitAll(tasks.ToArray()); client.UpdateSettings(u => u .Index(indexName) .RefreshInterval("1s") ); }