Пример #1
0
        public void TestBulkForFramdThrift()
        {
            var client = new ElasticSearchClient("localhost");

            var fields = new Dictionary <string, object>();

            fields.Add("name", "jack");
            fields.Add("age", 25);
            var index = "index_bulk_framed";

            try
            {
                client.DeleteIndex(index);
                client.CreateIndex(index);
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
            }
            var jsondata = JsonSerializer.Get(fields);

            var result = client.Bulk(new List <BulkObject>()
            {
                new BulkObject()
                {
                    Id = "1", Index = index, Type = "type", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "2", Index = index, Type = "type", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "3", Index = index, Type = "type", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "4", Index = index, Type = "type", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "5", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "6", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "7", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "8", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
            });

            Assert.AreEqual(true, result.Success);
        }
Пример #2
0
        public void TestBulkIndexWithParentId()
        {
            var client = new ElasticSearchClient("localhost");

            var fields = new Dictionary <string, object>();

            fields.Add("name", "jack");
            fields.Add("age", 25);
            var index = "index_12312312311";

            try
            {
                client.DeleteIndex(index);
                client.CreateIndex(index);
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
            }
            var jsondata = JsonSerializer.Get(fields);

            var result = client.Bulk(new List <BulkObject>()
            {
                new BulkObject()
                {
                    Id = "1", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "2", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                },
                new BulkObject()
                {
                    Id = "3", Index = index, Type = "type", ParentId = "1", JsonData = jsondata
                }
            });

            Assert.AreEqual(true, result.Success);
            client.Refresh();
            var c = client.Count(index, "type", "age:25");

            Assert.AreEqual(3, c);
            result = client.Delete(index, "type", new string[] { "1", "2", "3" }, "1");
            Assert.AreEqual(true, result.Success);
            client.Refresh();
            c = client.Count(index, "type", "age:25");
            Assert.AreEqual(0, c);
            client.DeleteIndex(index);
        }
Пример #3
0
        private void IndexTransfer(string index, string toIndex, int from, int limit, int bulkSize, ElasticSearchClient srcClient, ElasticSearchClient descClient, bool complicatedSource, bool resolveTenant, bool showLog)
        {
            var docs = srcClient.Search(index, "*", from, limit, "_id:asc");

            WriteLog("Search:{0},{1},{2}", index, from, limit);
            int i           = 0;
            var bulkObjects = new List <BulkObject>();

            if (complicatedSource)
            {
                //complicated object
                if (!string.IsNullOrEmpty(docs.Response))
                {
                    var obj  = JObject.Parse(docs.Response);
                    var hits = obj["hits"]["hits"];
                    foreach (var hit in hits)
                    {
                        var source = ((Newtonsoft.Json.Linq.JObject)(hit["_source"])).ToString().Replace("\r\n", string.Empty);                       // hit["_source"].Value<string>();
                        var _type  = hit["_type"].Value <string>();
                        var _id    = hit["_id"].Value <string>();

                        if (resolveTenant)
                        {
                            //methond for setting
                            //if (index.StartsWith("setting") || index.StartsWith("labs.setting") || index.StartsWith("demo.setting"))
                            {
                                var fields = ElasticSearch.Client.Utils.JsonSerializer.Get <Dictionary <string, object> >(source);

                                fields = ParseFieldValue(fields);
                                source = ElasticSearch.Client.Utils.JsonSerializer.Get(fields);
                            }
                        }


                        i++;

                        if (showLog)
                        {
                            WriteLog("curl -XPOST http://localhost:9200/{0}/{1}/{2} -d'{3}'", toIndex, _type, _id, source);
                        }

                        bulkObjects.Add(new BulkObject(toIndex, _type, _id, source));
                        if (i > bulkSize)
                        {
                            descClient.Bulk(bulkObjects);
                            bulkObjects.Clear();
                            i = 0;
                            WriteLog("Buik Commit.");
                        }
                    }
                }
            }
            else
            {
                foreach (var variable in docs.GetHits().Hits)
                {
                    #region logging

                    //				WriteLog("\tIndex:{0}", variable.Index);
                    //				WriteLog("\tType:{0}", variable.Type);
                    //				WriteLog("\tId:{0}", variable.Id);
                    Dictionary <string, object> fields = variable.Fields;
                    //				WriteLog("\tTotalFieldsCount:{0}", fields.Count);

                    #endregion

                    #region

                    //					                                    foreach (var VARIABLE in fields)
                    //					                                    {
                    //WriteLog(string.Format("\t\t{0}:{1}", VARIABLE.Key, VARIABLE.Value));
                    //					                                    }

                    #endregion

                    #region multi-tenant
                    if (resolveTenant)
                    {
                        if (index.StartsWith("setting") || index.StartsWith("labs.setting") || index.StartsWith("demo.setting"))
                        {
                            fields = ParseFieldValue(fields);
//						fields["_tenantid"] = fields["__TENANTID"];
//						fields.Remove("__TENANTID");
//						fields.Remove("__TYPEID");
                        }
                    }

                    #endregion

                    #region BulkInsert

                    i++;
                    bulkObjects.Add(new BulkObject(toIndex, variable.Type, variable.Id, fields));


                    if (i > bulkSize)
                    {
                        descClient.Bulk(bulkObjects);
                        bulkObjects.Clear();
                        i = 0;
                        WriteLog("Buik Commit.");
                    }

                    #endregion
                }
            }

            #region final cleanup

            if (i > 0)
            {
                descClient.Bulk(bulkObjects);
                WriteLog("Final Cleanup,{0}.", bulkObjects.Count);
                bulkObjects.Clear();
            }

            #endregion
        }