private void IndexStorePromotion(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = db.StorePromotions.Where(r=> r.CreateDate >= benchDate || r.UpdateDate >= benchDate) .GroupJoin(db.PointOrderRules.Where(r=>r.Status!=(int)DataStatus.Deleted),o=>o.Id,i=>i.StorePromotionId, (o,i)=>new {S=o,R = i}); int totalCount = prods.Count(); client.MapFromAttributes<ESStorePromotion>(); while (cursor < totalCount) { var linq = from l in prods.OrderByDescending(p => p.S.Id).Skip(cursor).Take(size).ToList() select new ESStorePromotion().FromEntity<ESStorePromotion>(l.S, s => { s.ExchangeRule = JsonConvert.SerializeObject(l.R.Select(r=>new { rangefrom=r.RangeFrom, rangeto = r.RangeTo, ratio = r.Ratio })); }); var result = client.IndexMany(linq); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} store promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexProds(ElasticClient client,DateTime? benchDate,Func<IQueryable<ProductEntity>,YintaiHangzhouContext,IQueryable<ProductEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int successCount = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.Products.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition!=null) { linq = whereCondition(linq, db); } var prods = from p in linq join s in db.Stores on p.Store_Id equals s.Id join b in db.Brands on p.Brand_Id equals b.Id join t in db.Tags on p.Tag_Id equals t.Id let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 1 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) let specials = from psp in db.SpecialTopicProductRelations where psp.Product_Id == p.Id join sp in db.SpecialTopics on psp.SpecialTopic_Id equals sp.Id select new ESSpecialTopic { Id = sp.Id, Name = sp.Name, Description = sp.Description } let promotions = from ppr in db.Promotion2Product where ppr.ProdId == p.Id join pro in db.Promotions on ppr.ProId equals pro.Id select new ESPromotion { Id = pro.Id, Name = pro.Name, Description = pro.Description, CreatedDate = p.CreatedDate, StartDate = pro.StartDate, EndDate = pro.EndDate, Status = pro.Status } let section = (from section in db.Sections where section.BrandId == p.Brand_Id && section.StoreId == p.Store_Id select new ESSection(){ ContactPerson = section.ContactPerson, ContactPhone = section.ContactPhone, Id = section.Id, Location = section.Location, Name = section.Name, Status = section.Status }) select new ESProduct() { Id = p.Id, Name = p.Name, Description = p.Description, CreatedDate = p.CreatedDate, Price = p.Price, RecommendedReason = p.RecommendedReason, Status = p.Status, CreateUserId = p.CreatedUser, SortOrder = p.SortOrder, Tag = new ESTag() { Id = t.Id, Name = t.Name, Description = t.Description }, Store = new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel }, Brand = new ESBrand() { Id = b.Id, Name = b.Name, Description = b.Description, EngName = b.EnglishName }, Resource = resource, SpecialTopic = specials, Promotion = promotions, Is4Sale = p.Is4Sale ?? false, UnitPrice = p.UnitPrice, FavoriteCount = p.FavoriteCount, InvolvedCount = p.InvolvedCount, ShareCount = p.ShareCount, RecommendUserId = p.RecommendUser, Section=section.FirstOrDefault(), UpcCode = p.SkuCode }; int totalCount = prods.Count(); client.MapFromAttributes<ESProduct>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} products in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexPros(ElasticClient client, DateTime? benchDate, Func<IQueryable<PromotionEntity>, YintaiHangzhouContext, IQueryable<PromotionEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.Promotions.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition != null) { linq = whereCondition(linq, db); } var prods = from p in linq join s in db.Stores on p.Store_Id equals s.Id let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 2 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESPromotion() { Id = p.Id, Name = p.Name, Description = p.Description, CreatedDate = p.CreatedDate, StartDate = p.StartDate, EndDate = p.EndDate, FavoriteCount = p.FavoriteCount, IsTop = p.IsTop, Status = p.Status, CreateUserId = p.CreatedUser, Store = new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel }, Resource = resource, ShowInList = p.IsMain.HasValue ? p.IsMain.Value : true, PublicCode = p.PublicProCode, InvolvedCount = p.InvolvedCount, IsProdBindable = p.IsProdBindable ?? false, LikeCount = p.LikeCount, PublicationLimit = p.PublicationLimit ?? -1, ShareCount = p.ShareCount }; int totalCount = prods.Count(); client.MapFromAttributes<ESPromotion>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach(var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}",item.Id)); } } else successCount+=result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related promotions "); IndexProds(client, null, (p,db) => { return p.Where(prod => (from pro in db.Promotions from ppr in db.Promotion2Product where ppr.ProId == pro.Id && (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate) && ppr.ProdId == prod.Id select ppr.ProdId).Any()); }); } }
private void IndexSpecialTopic(ElasticClient client, DateTime? benchDate, Func<IQueryable<SpecialTopicEntity>, YintaiHangzhouContext, IQueryable<SpecialTopicEntity>> whereCondition) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var linq = db.SpecialTopics.AsQueryable(); if (benchDate.HasValue) linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value); else if (whereCondition != null) { linq = whereCondition(linq, db); } var prods = from p in linq let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 9 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESSpecialTopic() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, CreatedDate = p.CreatedDate, CreateUser = p.CreatedUser, Resource = resource, Type = p.Type, TargetValue = p.TargetValue }; int totalCount = prods.Count(); client.MapFromAttributes<ESSpecialTopic>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} special topics in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related specialtopic "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from pro in db.SpecialTopics from ppr in db.SpecialTopicProductRelations where ppr.SpecialTopic_Id == pro.Id && (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate) && ppr.Product_Id == prod.Id select ppr.Product_Id).Any()); }); } }
private void IndexStore(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from s in db.Stores let resource = (from r in db.Resources where r.SourceId == s.Id && r.SourceType == (int)SourceType.StoreLogo select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) where (s.CreatedDate >= benchDate || s.UpdatedDate >= benchDate) select new ESStore() { Id = s.Id, Name = s.Name, Description = s.Description, Address = s.Location, Location = new Location { Lon = s.Longitude, Lat = s.Latitude }, GpsAlt = s.GpsAlt, GpsLat = s.GpsLat, GpsLng = s.GpsLng, Tel = s.Tel, Status = s.Status, Resource = resource }; int totalCount = prods.Count(); client.MapFromAttributes<ESStore>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} stores in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexBrand(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Brands where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select new ESBrand() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, Group = p.Group }; int totalCount = prods.Count(); client.MapFromAttributes<ESBrand>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} brands in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); //update related source if any affected if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related brands "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from r in db.Brands where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.Id == prod.Brand_Id select r.Id).Any()); }); } }
private void IndexTag(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Tags where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select new ESTag() { Id = p.Id, Name = p.Name, Description = p.Description, Status = p.Status, SortOrder = p.SortOrder }; int totalCount = prods.Count(); client.MapFromAttributes<ESTag>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} tags in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexBanner(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Banners join pro in db.Promotions on p.SourceId equals pro.Id where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) && p.SourceType == 2 let resource = (from r in db.Resources where r.SourceId == p.Id && r.SourceType == 11 select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height }) select new ESBanner() { Id = p.Id, SortOrder = p.SortOrder, CreatedDate = p.CreatedDate, Status = p.Status, SourceType = p.SourceType, Promotion = new ESPromotion() { Id = pro.Id }, Resource = resource }; int totalCount = prods.Count(); client.MapFromAttributes<ESBanner>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} banners in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexHotwork(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var words = from p in db.HotWords where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) select p; var prods = from p in words.ToList() select new ESHotword() { Id = p.Id, SortOrder = p.SortOrder, Status = p.Status, Type = p.Type, Word = p.Type==1?p.Word:JsonConvert.DeserializeObject<dynamic>(p.Word).name, BrandId = p.Type==1?0:JsonConvert.DeserializeObject<dynamic>(p.Word).id }; int totalCount = prods.Count(); client.MapFromAttributes<ESHotword>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} hotwords in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexUser(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from p in db.Users where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate) let resource = from r in db.Resources where r.SourceId == p.Id && r.SourceType == (int)SourceType.CustomerPortrait select new ESResource() { Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height } select new ESUser() { Id = p.Id, Status = p.Status, Thumnail = resource.FirstOrDefault(), Nickie = p.Nickname, Level = p.UserLevel }; int totalCount = prods.Count(); client.MapFromAttributes<ESUser>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} users in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); }
private void IndexResource(ElasticClient client, DateTime benchDate) { ILog log = LogManager.GetLogger(this.GetType()); int cursor = 0; int size = JobConfig.DEFAULT_PAGE_SIZE; int successCount = 0; Stopwatch sw = new Stopwatch(); sw.Start(); using (var db = new YintaiHangzhouContext("YintaiHangzhouContext")) { var prods = from r in db.Resources where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) select new ESResource() { Id = r.Id, Status= r.Status, Domain = r.Domain, Name = r.Name, SortOrder = r.SortOrder, IsDefault = r.IsDefault, Type = r.Type, Width = r.Width, Height = r.Height, SourceId = r.SourceId, SourceType = r.SourceType }; int totalCount = prods.Count(); client.MapFromAttributes<ESResource>(); while (cursor < totalCount) { var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size)); if (!result.IsValid) { foreach (var item in result.Items) { if (item.OK) successCount++; else log.Info(string.Format("id index failed:{0}", item.Id)); } } else successCount += result.Items.Count(); cursor += size; } } sw.Stop(); log.Info(string.Format("{0} resources in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds)); //update related source if any affected if (successCount > 0 && CascadPush) { //index related products log.Info("index products affected by related resources "); IndexProds(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.Product select r.Id).Any()); }); //index related promotions log.Info("index promotions affected by related resources "); IndexPros(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.Promotion select r.Id).Any()); }); //index related specialtopics log.Info("index specialtopics affected by related resources "); IndexSpecialTopic(client, null, (p, db) => { return p.Where(prod => (from r in db.Resources where r.SourceId == prod.Id && (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate) && r.SourceType == (int)SourceType.SpecialTopic select r.Id).Any()); }); } }