public void TestBulkForFramdThrift()
{
var client = new ElasticSearchClient("localhost");
var fields = new Dictionary <string, object>();
fields.Add("name", "jack");
fields.Add("age", 25);
var index = "index_bulk_framed";
try
{
client.DeleteIndex(index);
client.CreateIndex(index);
}
catch (Exception e)
{
Console.WriteLine(e);
}
var jsondata = JsonSerializer.Get(fields);
var result = client.Bulk(new List <BulkObject>()
{
new BulkObject()
{
Id = "1", Index = index, Type = "type", JsonData = jsondata
},
new BulkObject()
{
Id = "2", Index = index, Type = "type", JsonData = jsondata
},
new BulkObject()
{
Id = "3", Index = index, Type = "type", JsonData = jsondata
},
new BulkObject()
{
Id = "4", Index = index, Type = "type", JsonData = jsondata
},
new BulkObject()
{
Id = "5", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
new BulkObject()
{
Id = "6", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
new BulkObject()
{
Id = "7", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
new BulkObject()
{
Id = "8", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
});
Assert.AreEqual(true, result.Success);
}
public void TestBulkIndexWithParentId()
{
var client = new ElasticSearchClient("localhost");
var fields = new Dictionary <string, object>();
fields.Add("name", "jack");
fields.Add("age", 25);
var index = "index_12312312311";
try
{
client.DeleteIndex(index);
client.CreateIndex(index);
}
catch (Exception e)
{
Console.WriteLine(e);
}
var jsondata = JsonSerializer.Get(fields);
var result = client.Bulk(new List <BulkObject>()
{
new BulkObject()
{
Id = "1", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
new BulkObject()
{
Id = "2", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
},
new BulkObject()
{
Id = "3", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
}
});
Assert.AreEqual(true, result.Success);
client.Refresh();
var c = client.Count(index, "type", "age:25");
Assert.AreEqual(3, c);
result = client.Delete(index, "type", new string[] { "1", "2", "3" }, "1");
Assert.AreEqual(true, result.Success);
client.Refresh();
c = client.Count(index, "type", "age:25");
Assert.AreEqual(0, c);
client.DeleteIndex(index);
}
private void IndexTransfer(string index, string toIndex, int from, int limit, int bulkSize, ElasticSearchClient srcClient, ElasticSearchClient descClient, bool complicatedSource, bool resolveTenant, bool showLog)
{
var docs = srcClient.Search(index, "*", from, limit, "_id:asc");
WriteLog("Search:{0},{1},{2}", index, from, limit);
int i = 0;
var bulkObjects = new List <BulkObject>();
if (complicatedSource)
{
//complicated object
if (!string.IsNullOrEmpty(docs.Response))
{
var obj = JObject.Parse(docs.Response);
var hits = obj["hits"]["hits"];
foreach (var hit in hits)
{
var source = ((Newtonsoft.Json.Linq.JObject)(hit["_source"])).ToString().Replace("\r\n", string.Empty); // hit["_source"].Value<string>();
var _type = hit["_type"].Value <string>();
var _id = hit["_id"].Value <string>();
if (resolveTenant)
{
//methond for setting
//if (index.StartsWith("setting") || index.StartsWith("labs.setting") || index.StartsWith("demo.setting"))
{
var fields = ElasticSearch.Client.Utils.JsonSerializer.Get <Dictionary <string, object> >(source);
fields = ParseFieldValue(fields);
source = ElasticSearch.Client.Utils.JsonSerializer.Get(fields);
}
}
i++;
if (showLog)
{
WriteLog("curl -XPOST http://localhost:9200/{0}/{1}/{2} -d'{3}'", toIndex, _type, _id, source);
}
bulkObjects.Add(new BulkObject(toIndex, _type, _id, source));
if (i > bulkSize)
{
descClient.Bulk(bulkObjects);
bulkObjects.Clear();
i = 0;
WriteLog("Buik Commit.");
}
}
}
}
else
{
foreach (var variable in docs.GetHits().Hits)
{
#region logging
// WriteLog("\tIndex:{0}", variable.Index);
// WriteLog("\tType:{0}", variable.Type);
// WriteLog("\tId:{0}", variable.Id);
Dictionary <string, object> fields = variable.Fields;
// WriteLog("\tTotalFieldsCount:{0}", fields.Count);
#endregion
#region
// foreach (var VARIABLE in fields)
// {
//WriteLog(string.Format("\t\t{0}:{1}", VARIABLE.Key, VARIABLE.Value));
// }
#endregion
#region multi-tenant
if (resolveTenant)
{
if (index.StartsWith("setting") || index.StartsWith("labs.setting") || index.StartsWith("demo.setting"))
{
fields = ParseFieldValue(fields);
// fields["_tenantid"] = fields["__TENANTID"];
// fields.Remove("__TENANTID");
// fields.Remove("__TYPEID");
}
}
#endregion
#region BulkInsert
i++;
bulkObjects.Add(new BulkObject(toIndex, variable.Type, variable.Id, fields));
if (i > bulkSize)
{
descClient.Bulk(bulkObjects);
bulkObjects.Clear();
i = 0;
WriteLog("Buik Commit.");
}
#endregion
}
}
#region final cleanup
if (i > 0)
{
descClient.Bulk(bulkObjects);
WriteLog("Final Cleanup,{0}.", bulkObjects.Count);
bulkObjects.Clear();
}
#endregion
}