public async Task Bulk_document_crud_must_process_all_documents_correctly() { var es = new ElasticsearchClient(); await es.DeleteIndexAsync("test"); var cir = await es.CreateIndexAsync("test"); Assert.IsTrue(cir.IsSuccess); Assert.IsTrue(cir.Index == "test"); var documents = new ElasticsearchDocument[] { new ElasticsearchDocument("1", new JsonObject().Add("name", "YM"), "test"), new ElasticsearchDocument("2", new JsonObject().Add("name", "SV"), "test") }; var br = await es.BulkIndexDocumentsAsync(documents); Assert.IsTrue(br.IsSuccess); var bulk = new BulkRequest(true).Add(documents.Select(e => new BulkRequestItem("index", e.Id, e.Index, e.Type, e.Source))); br = await es.BulkDocumentsAsync(bulk); Assert.IsTrue(br.IsSuccess); var gdr = await es.GetDocumentAsync(new GetDocumentRequest("1", "test")); Assert.IsTrue(gdr.IsSuccess); Assert.IsTrue(gdr.Document.Id == "1"); Assert.IsTrue(gdr.Document.Index == "test"); Assert.IsTrue(gdr.Document.Version == 2); bulk = new BulkRequest(true).Add(documents.Select(e => new BulkRequestItem("delete", e.Id, e.Index, e.Type, null))); br = await es.BulkDocumentsAsync(bulk); Assert.IsTrue(br.IsSuccess); gdr = await es.GetDocumentAsync(new GetDocumentRequest("1", "test")); Assert.IsFalse(gdr.IsSuccess); Assert.IsTrue(gdr.Document.Id == "1"); Assert.IsTrue(gdr.Document.Index == "test"); }
public async Task Test() { var http = new HttpClient(); //string json = await http.GetStringAsync("http://proteus-prd-uno-esdata.associatedpress.com:9200/appl/doc/_search?q=(type:text AND NOT headline:AP* AND NOT headline:BC\\-* AND NOT headline:DIS*)&sort=arrivaldatetime:desc&_source=headline,arrivaldatetime&size=10000"); string json = File.ReadAllText(@"C:\Users\Yuri Metelkin\Desktop\scroll.json"); //var headlines = JsonObject.Parse(json) // .Property<JsonObject>("hits") // .Property<JsonArray>("hits") // .Select(e => e.Get<JsonObject>().Property<JsonObject>("_source").Property<string>("headline")) // .Distinct() // .ToArray(); var suggestions = new Dictionary <string, SuggestData>(); var es = new ElasticsearchClient("http://proteus-prd-uno-esdata.associatedpress.com:9200"); var request = new SearchRequest(JsonObject.Parse(json)) .SetIndex("appl-breaking") .SetType("doc"); var response = await es.ScrollAsync(new ScrollRequest(request)); while (!response.IsEmpty && suggestions.Count < 10000) { var docs = new List <ElasticsearchDocument>(); var sources = response .Hits .Hits .Select(hit => hit.Source) .ToArray(); foreach (var source in sources) { GetClassifiers(source, suggestions); GetHeadlines(source, suggestions); } response = await es.ScrollAsync(new ScrollRequest(response.ScrollId)); } es = new ElasticsearchClient(); int from = 0; while (from < suggestions.Count) { var docs = new List <ElasticsearchDocument>(); foreach (var kvp in suggestions.Skip(from).Take(1000)) { var doc = new JsonObject() .Add("text", kvp.Value.Text) .Add("suggest", new JsonObject() .Add("input", kvp.Value.Inputs) .Add("weight", kvp.Value.Occurances)); docs.Add(new ElasticsearchDocument(kvp.Key, doc, "suggest")); } await es.BulkIndexDocumentsAsync(docs); from += 1000; } }