public static void BulkSave(IEnumerable <OsobaES> osoby) { var result = _esClient.IndexMany <OsobaES>(osoby); if (result.Errors) { var a = result.DebugInformation; Util.Consts.Logger.Error($"Error when bulkSaving osoby to ES: {a}"); } }
public static void LoadData() { var nodes = new List <Uri> { new Uri("http://*****:*****@"C:\cities_canada-usa.csv")) { if (!reader.EndOfStream) { reader.ReadLine(); } var documents = new List <City>(); while (!reader.EndOfStream) { var line = reader.ReadLine(); var values = line.Split(';'); //0 - id //1 - name //3 - alt_name //4 - lat //5 - lon //8 - country_code documents.Add(new City { Id = int.Parse(values[0]), Name = values[1], AltName = values[3], Country = values[8], Location = new Location(double.Parse(values[4]), double.Parse(values[5])), FullText = string.Concat(values[1], ", ", values[8]) }); } var result = client.IndexMany(documents, "city"); } }
/// <summary> /// returns true if some error ocurred /// </summary> /// <param name="dotace"></param> /// <returns></returns> public bool BulkSave(List <Dotace> dotace) { foreach (var d in dotace) { // fill in sums d.CalculateTotals(); d.CalculateCerpaniYears(); } var result = _esClient.IndexMany <Dotace>(dotace); if (result.Errors) { var a = result.DebugInformation; Util.Consts.Logger.Error($"Error when bulkSaving dotace to ES: {a}"); } return(result.Errors); }
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize) { // refresh = false is default on elasticsearch's side. var bulkParms = new SimpleBulkParameters() { Refresh = false }; var msgGenerator = new MessageGenerator(); var tasks = new List<Task>(); var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize); Interlocked.Exchange(ref NumSent, 0); foreach (var messages in partitionedMessages) { var t = client.IndexMany(messages, indexName, bulkParms); Interlocked.Add(ref NumSent, bufferSize); if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName); } } //Task.WaitAll(tasks.ToArray()); }
private IBulkResponse IndexNews(IEnumerable <NewsArticle> newsArticles) { if (newsArticles == null || !newsArticles.Any()) { return(null); } // Connecting to Elasticsearch string protocol = Settings.GetSetting("ElasticSearch.Protocol", "http"); string host = Settings.GetSetting("ElasticSearch.Host", "elastic.local"); string port = Settings.GetSetting("ElasticSearch.Port", "9200"); var node = new Uri(string.Format("{0}://{1}:{2}", protocol, host, port)); var settings = new Nest.ConnectionSettings(node); var client = new Nest.ElasticClient(settings); // Reindexing items var indexName = Settings.GetSetting("ElasticSearch.ArticlesIndex", "articles-index"); var indexerResponse = client.IndexMany(newsArticles, indexName); return(indexerResponse); }
public ActionResult ReIndexAll() { var documents = db.DocumentModels.ToList(); var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"]; var searchBoxUri = new Uri(uriString); var settings = new ConnectionSettings(searchBoxUri); settings.SetDefaultIndex(indexName); var client = new ElasticClient(settings); // delete index if exists at startup if (client.IndexExists(indexName).Exists) { client.DeleteIndex(indexName); } // Create a new "sample" index with default settings //client.CreateIndex("sample", new IndexSettings()); ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName); iCreateIndexReq.IndexSettings = new IndexSettings(); iCreateIndexReq.IndexSettings.NumberOfReplicas = 10; //client.CreateIndex(iCreateIndexReq); var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10)); //client.CreateIndex() // Index all documents client.IndexMany<DocumentModel>(documents); //client.Index() ViewBag.Message = "Reindexing all database is complete!"; return RedirectToAction("Index"); }
private void IndexProds(ElasticClient client,DateTime? benchDate,Func<IQueryable<ProductEntity>,YintaiHangzhouContext,IQueryable<ProductEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int successCount = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.Products.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition!=null) { linq = whereCondition(linq, db); } var prods = from p in linq join s in db.Stores on p.Store_Id equals s.Id join b in db.Brands on p.Brand_Id equals b.Id join t in db.Tags on p.Tag_Id equals t.Id let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 1 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) let specials = from psp in db.SpecialTopicProductRelations where psp.Product_Id == p.Id join sp in db.SpecialTopics on psp.SpecialTopic_Id equals sp.Id select new ESSpecialTopic { Id = sp.Id, Name = sp.Name, Description = sp.Description } let promotions = from ppr in db.Promotion2Product where ppr.ProdId == p.Id join pro in db.Promotions on ppr.ProId equals pro.Id select new ESPromotion { Id = pro.Id, Name = pro.Name, Description = pro.Description, CreatedDate = p.CreatedDate, StartDate = pro.StartDate, EndDate = pro.EndDate, Status = pro.Status } let section = (from section in db.Sections where section.BrandId == p.Brand_Id && section.StoreId == p.Store_Id select new ESSection(){ ContactPerson = section.ContactPerson, ContactPhone = section.ContactPhone, Id = section.Id, Location = section.Location, Name = section.Name, Status = section.Status }) select new ESProduct() { Id = p.Id, Name = p.Name, Description = p.Description, CreatedDate = p.CreatedDate, Price = p.Price, RecommendedReason = p.RecommendedReason, Status = p.Status, CreateUserId = p.CreatedUser, SortOrder = p.SortOrder, Tag = new ESTag() { Id = t.Id, Name = t.Name, Description = t.Description }, Store = new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel }, Brand = new ESBrand() { Id = b.Id, Name = b.Name, Description = b.Description, EngName = b.EnglishName }, Resource = resource, SpecialTopic = specials, Promotion = promotions, Is4Sale = p.Is4Sale ?? false, UnitPrice = p.UnitPrice, FavoriteCount = p.FavoriteCount, InvolvedCount = p.InvolvedCount, ShareCount = p.ShareCount, RecommendUserId = p.RecommendUser, Section=section.FirstOrDefault(), UpcCode = p.SkuCode }; int totalCount = prods.Count(); client.MapFromAttributes<ESProduct>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} products in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexUser(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Users where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) let resource = from r in db.Resources where r.SourceId == p.Id && r.SourceType == (int)SourceType.CustomerPortrait select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height } select new ESUser() { Id = p.Id, Status = p.Status, Thumnail = resource.FirstOrDefault(), Nickie = p.Nickname, Level = p.UserLevel }; int totalCount = prods.Count(); client.MapFromAttributes<ESUser>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} users in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexStorePromotion(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = db.StorePromotions.Where(r=> r.CreateDate >= benchDate || r.UpdateDate >= benchDate) .GroupJoin(db.PointOrderRules.Where(r=>r.Status!=(int)DataStatus.Deleted),o=>o.Id,i=>i.StorePromotionId, (o,i)=>new {S=o,R = i}); int totalCount = prods.Count(); client.MapFromAttributes<ESStorePromotion>(); while (cursor < totalCount) { var linq = from l in prods.OrderByDescending(p => p.S.Id).Skip(cursor).Take(size).ToList() select new ESStorePromotion().FromEntity<ESStorePromotion>(l.S, s => { s.ExchangeRule = JsonConvert.SerializeObject(l.R.Select(r=>new { rangefrom=r.RangeFrom, rangeto = r.RangeTo, ratio = r.Ratio })); }); var result = client.IndexMany(linq); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} store promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexPros(ElasticClient client, DateTime? benchDate, Func<IQueryable<PromotionEntity>, YintaiHangzhouContext, IQueryable<PromotionEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.Promotions.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition != null) { linq = whereCondition(linq, db); } var prods = from p in linq join s in db.Stores on p.Store_Id equals s.Id let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 2 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESPromotion() { Id = p.Id, Name = p.Name, Description = p.Description, CreatedDate = p.CreatedDate, StartDate = p.StartDate, EndDate = p.EndDate, FavoriteCount = p.FavoriteCount, IsTop = p.IsTop, Status = p.Status, CreateUserId = p.CreatedUser, Store = new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel }, Resource = resource, ShowInList = p.IsMain.HasValue ? p.IsMain.Value : true, PublicCode = p.PublicProCode, InvolvedCount = p.InvolvedCount, IsProdBindable = p.IsProdBindable ?? false, LikeCount = p.LikeCount, PublicationLimit = p.PublicationLimit ?? -1, ShareCount = p.ShareCount }; int totalCount = prods.Count(); client.MapFromAttributes<ESPromotion>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach(var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}",item.Id)); } } else successCount+=result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related promotions "); IndexProds(client, null, (p,db) => { return p.Where(prod => (from pro in db.Promotions from ppr in db.Promotion2Product where ppr.ProId == pro.Id && (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate) && ppr.ProdId == prod.Id select ppr.ProdId).Any()); }); } }
private void IndexBrand(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Brands where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select new ESBrand() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, Group = p.Group }; int totalCount = prods.Count(); client.MapFromAttributes<ESBrand>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} brands in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); //update related source if any affected if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related brands "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from r in db.Brands where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.Id == prod.Brand_Id select r.Id).Any()); }); } }
private void IndexSpecialTopic(ElasticClient client, DateTime? benchDate, Func<IQueryable<SpecialTopicEntity>, YintaiHangzhouContext, IQueryable<SpecialTopicEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.SpecialTopics.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition != null) { linq = whereCondition(linq, db); } var prods = from p in linq let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 9 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESSpecialTopic() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, CreatedDate = p.CreatedDate, CreateUser = p.CreatedUser, Resource = resource, Type = p.Type, TargetValue = p.TargetValue }; int totalCount = prods.Count(); client.MapFromAttributes<ESSpecialTopic>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} special topics in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related specialtopic "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from pro in db.SpecialTopics from ppr in db.SpecialTopicProductRelations where ppr.SpecialTopic_Id == pro.Id && (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate) && ppr.Product_Id == prod.Id select ppr.Product_Id).Any()); }); } }
private void IndexTag(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Tags where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select new ESTag() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, SortOrder = p.SortOrder }; int totalCount = prods.Count(); client.MapFromAttributes<ESTag>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} tags in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexStore(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from s in db.Stores let resource = (from r in db.Resources where r.SourceId == s.Id && r.SourceType == (int)SourceType.StoreLogo select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) where (s.CreatedDate >= benchDate || s.UpdatedDate >= benchDate) select new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel, Status = s.Status, Resource = resource }; int totalCount = prods.Count(); client.MapFromAttributes<ESStore>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} stores in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexBanner(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Banners join pro in db.Promotions on p.SourceId equals pro.Id where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) && p.SourceType == 2 let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 11 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESBanner() { Id = p.Id, SortOrder = p.SortOrder, CreatedDate = p.CreatedDate, Status = p.Status, SourceType = p.SourceType, Promotion = new ESPromotion() { Id = pro.Id }, Resource = resource }; int totalCount = prods.Count(); client.MapFromAttributes<ESBanner>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} banners in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexHotwork(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var words = from p in db.HotWords where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select p; var prods = from p in words.ToList() select new ESHotword() { Id = p.Id, SortOrder = p.SortOrder, Status = p.Status, Type = p.Type, Word = p.Type==1?p.Word:JsonConvert.DeserializeObject<dynamic>(p.Word).name, BrandId = p.Type==1?0:JsonConvert.DeserializeObject<dynamic>(p.Word).id }; int totalCount = prods.Count(); client.MapFromAttributes<ESHotword>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} hotwords in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexResource(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from r in db.Resources where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) select new ESResource() { Id = r.Id, Status= r.Status, Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height, SourceId = r.SourceId, SourceType = r.SourceType }; int totalCount = prods.Count(); client.MapFromAttributes<ESResource>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} resources in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); //update related source if any affected if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related resources "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.Product select r.Id).Any()); }); //index related promotions log.Info("index promotions affected by related resources "); IndexPros(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.Promotion select r.Id).Any()); }); //index related specialtopics log.Info("index specialtopics affected by related resources "); IndexSpecialTopic(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.SpecialTopic select r.Id).Any()); }); } }
public void Run(string indexName, int port, int numMessages, int bufferSize) { var bulkParms = new SimpleBulkParameters() { Refresh = false }; var settings = new ConnectionSettings("localhost", port) .SetDefaultIndex(indexName); var client = new ElasticClient(settings, new ThriftConnection(settings)); Connect(client, settings); IList<Message> msgBuffer = new List<Message>(bufferSize); var msgGenerator = new MessageGenerator(); foreach (var msg in msgGenerator.Generate(numMessages)) { msgBuffer.Add(msg); // Flush buffer once max size reached if (msgBuffer.Count >= bufferSize) { client.IndexMany(msgBuffer, indexName, bulkParms); msgBuffer.Clear(); Interlocked.Add(ref NumSent, bufferSize); // Output how many messages sent so far if (NumSent % 10000 == 0) { Console.WriteLine("Sent {0:0,0} messages over Thrift", NumSent); } } } }
public ActionResult ReIndexAll() { var documents = db.Documents.ToList(); var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"]; var searchBoxUri = new Uri(uriString); var settings = new ConnectionSettings(searchBoxUri); settings.SetDefaultIndex("sample"); var client = new ElasticClient(settings); // delete index if exists at startup if (client.IndexExists("sample").Exists) { client.DeleteIndex("sample"); } // Create a new "sample" index with default settings client.CreateIndex("sample", new IndexSettings()); // Index all documents client.IndexMany<Document>(documents); ViewBag.Message = "Reindexing all database is complete!"; return RedirectToAction("Index"); }