public async Task Bulk_document_crud_must_process_all_documents_correctly()
{
var es = new ElasticsearchClient();
await es.DeleteIndexAsync("test");
var cir = await es.CreateIndexAsync("test");
Assert.IsTrue(cir.IsSuccess);
Assert.IsTrue(cir.Index == "test");
var documents = new ElasticsearchDocument[] {
new ElasticsearchDocument("1", new JsonObject().Add("name", "YM"), "test"),
new ElasticsearchDocument("2", new JsonObject().Add("name", "SV"), "test")
};
var br = await es.BulkIndexDocumentsAsync(documents);
Assert.IsTrue(br.IsSuccess);
var bulk = new BulkRequest(true).Add(documents.Select(e => new BulkRequestItem("index", e.Id, e.Index, e.Type, e.Source)));
br = await es.BulkDocumentsAsync(bulk);
Assert.IsTrue(br.IsSuccess);
var gdr = await es.GetDocumentAsync(new GetDocumentRequest("1", "test"));
Assert.IsTrue(gdr.IsSuccess);
Assert.IsTrue(gdr.Document.Id == "1");
Assert.IsTrue(gdr.Document.Index == "test");
Assert.IsTrue(gdr.Document.Version == 2);
bulk = new BulkRequest(true).Add(documents.Select(e => new BulkRequestItem("delete", e.Id, e.Index, e.Type, null)));
br = await es.BulkDocumentsAsync(bulk);
Assert.IsTrue(br.IsSuccess);
gdr = await es.GetDocumentAsync(new GetDocumentRequest("1", "test"));
Assert.IsFalse(gdr.IsSuccess);
Assert.IsTrue(gdr.Document.Id == "1");
Assert.IsTrue(gdr.Document.Index == "test");
}
public async Task Test()
{
var http = new HttpClient();
//string json = await http.GetStringAsync("http://proteus-prd-uno-esdata.associatedpress.com:9200/appl/doc/_search?q=(type:text AND NOT headline:AP* AND NOT headline:BC\\-* AND NOT headline:DIS*)&sort=arrivaldatetime:desc&_source=headline,arrivaldatetime&size=10000");
string json = File.ReadAllText(@"C:\Users\Yuri Metelkin\Desktop\scroll.json");
//var headlines = JsonObject.Parse(json)
// .Property<JsonObject>("hits")
// .Property<JsonArray>("hits")
// .Select(e => e.Get<JsonObject>().Property<JsonObject>("_source").Property<string>("headline"))
// .Distinct()
// .ToArray();
var suggestions = new Dictionary <string, SuggestData>();
var es = new ElasticsearchClient("http://proteus-prd-uno-esdata.associatedpress.com:9200");
var request = new SearchRequest(JsonObject.Parse(json))
.SetIndex("appl-breaking")
.SetType("doc");
var response = await es.ScrollAsync(new ScrollRequest(request));
while (!response.IsEmpty && suggestions.Count < 10000)
{
var docs = new List <ElasticsearchDocument>();
var sources = response
.Hits
.Hits
.Select(hit => hit.Source)
.ToArray();
foreach (var source in sources)
{
GetClassifiers(source, suggestions);
GetHeadlines(source, suggestions);
}
response = await es.ScrollAsync(new ScrollRequest(response.ScrollId));
}
es = new ElasticsearchClient();
int from = 0;
while (from < suggestions.Count)
{
var docs = new List <ElasticsearchDocument>();
foreach (var kvp in suggestions.Skip(from).Take(1000))
{
var doc = new JsonObject()
.Add("text", kvp.Value.Text)
.Add("suggest", new JsonObject()
.Add("input", kvp.Value.Inputs)
.Add("weight", kvp.Value.Occurances));
docs.Add(new ElasticsearchDocument(kvp.Key, doc, "suggest"));
}
await es.BulkIndexDocumentsAsync(docs);
from += 1000;
}
}