public static void BulkSave(IEnumerable <OsobaES> osoby)
{
var result = _esClient.IndexMany <OsobaES>(osoby);
if (result.Errors)
{
var a = result.DebugInformation;
Util.Consts.Logger.Error($"Error when bulkSaving osoby to ES: {a}");
}
}
public static void LoadData()
{
var nodes = new List <Uri> {
new Uri("http://*****:*****@"C:\cities_canada-usa.csv"))
{
if (!reader.EndOfStream)
{
reader.ReadLine();
}
var documents = new List <City>();
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(';');
//0 - id
//1 - name
//3 - alt_name
//4 - lat
//5 - lon
//8 - country_code
documents.Add(new City
{
Id = int.Parse(values[0]),
Name = values[1],
AltName = values[3],
Country = values[8],
Location = new Location(double.Parse(values[4]), double.Parse(values[5])),
FullText = string.Concat(values[1], ", ", values[8])
});
}
var result = client.IndexMany(documents, "city");
}
}
/// <summary>
/// returns true if some error ocurred
/// </summary>
/// <param name="dotace"></param>
/// <returns></returns>
public bool BulkSave(List <Dotace> dotace)
{
foreach (var d in dotace)
{
// fill in sums
d.CalculateTotals();
d.CalculateCerpaniYears();
}
var result = _esClient.IndexMany <Dotace>(dotace);
if (result.Errors)
{
var a = result.DebugInformation;
Util.Consts.Logger.Error($"Error when bulkSaving dotace to ES: {a}");
}
return(result.Errors);
}
protected static void GenerateAndIndex(ElasticClient client, string indexName, int numMessages, int bufferSize)
{
// refresh = false is default on elasticsearch's side.
var bulkParms = new SimpleBulkParameters() { Refresh = false };
var msgGenerator = new MessageGenerator();
var tasks = new List<Task>();
var partitionedMessages = msgGenerator.Generate(numMessages).Partition(bufferSize);
Interlocked.Exchange(ref NumSent, 0);
foreach (var messages in partitionedMessages)
{
var t = client.IndexMany(messages, indexName, bulkParms);
Interlocked.Add(ref NumSent, bufferSize);
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages to {1}", NumSent, indexName);
}
}
//Task.WaitAll(tasks.ToArray());
}
private IBulkResponse IndexNews(IEnumerable <NewsArticle> newsArticles)
{
if (newsArticles == null || !newsArticles.Any())
{
return(null);
}
// Connecting to Elasticsearch
string protocol = Settings.GetSetting("ElasticSearch.Protocol", "http");
string host = Settings.GetSetting("ElasticSearch.Host", "elastic.local");
string port = Settings.GetSetting("ElasticSearch.Port", "9200");
var node = new Uri(string.Format("{0}://{1}:{2}", protocol, host, port));
var settings = new Nest.ConnectionSettings(node);
var client = new Nest.ElasticClient(settings);
// Reindexing items
var indexName = Settings.GetSetting("ElasticSearch.ArticlesIndex", "articles-index");
var indexerResponse = client.IndexMany(newsArticles, indexName);
return(indexerResponse);
}
public ActionResult ReIndexAll()
{
var documents = db.DocumentModels.ToList();
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex(indexName);
var client = new ElasticClient(settings);
// delete index if exists at startup
if (client.IndexExists(indexName).Exists)
{
client.DeleteIndex(indexName);
}
// Create a new "sample" index with default settings
//client.CreateIndex("sample", new IndexSettings());
ICreateIndexRequest iCreateIndexReq = new CreateIndexRequest(indexName);
iCreateIndexReq.IndexSettings = new IndexSettings();
iCreateIndexReq.IndexSettings.NumberOfReplicas = 10;
//client.CreateIndex(iCreateIndexReq);
var resCreate = client.CreateIndex(indexName, s => s.AddMapping<DocumentModel>(f => f.MapFromAttributes()).NumberOfReplicas(1).NumberOfShards(10));
//client.CreateIndex()
// Index all documents
client.IndexMany<DocumentModel>(documents);
//client.Index()
ViewBag.Message = "Reindexing all database is complete!";
return RedirectToAction("Index");
}
private void IndexProds(ElasticClient client,DateTime? benchDate,Func<IQueryable<ProductEntity>,YintaiHangzhouContext,IQueryable<ProductEntity>> whereCondition)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int successCount = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var linq = db.Products.AsQueryable();
if (benchDate.HasValue)
linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value);
else if (whereCondition!=null)
{
linq = whereCondition(linq, db);
}
var prods = from p in linq
join s in db.Stores on p.Store_Id equals s.Id
join b in db.Brands on p.Brand_Id equals b.Id
join t in db.Tags on p.Tag_Id equals t.Id
let resource = (from r in db.Resources
where r.SourceId == p.Id
&& r.SourceType == 1
select new ESResource()
{
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
})
let specials = from psp in db.SpecialTopicProductRelations
where psp.Product_Id == p.Id
join sp in db.SpecialTopics on psp.SpecialTopic_Id equals sp.Id
select new ESSpecialTopic
{
Id = sp.Id,
Name = sp.Name,
Description = sp.Description
}
let promotions = from ppr in db.Promotion2Product
where ppr.ProdId == p.Id
join pro in db.Promotions on ppr.ProId equals pro.Id
select new ESPromotion {
Id = pro.Id,
Name = pro.Name,
Description = pro.Description,
CreatedDate = p.CreatedDate,
StartDate = pro.StartDate,
EndDate = pro.EndDate,
Status = pro.Status
}
let section = (from section in db.Sections
where section.BrandId == p.Brand_Id && section.StoreId == p.Store_Id
select new ESSection(){
ContactPerson = section.ContactPerson,
ContactPhone = section.ContactPhone,
Id = section.Id,
Location = section.Location,
Name = section.Name,
Status = section.Status
})
select new ESProduct()
{
Id = p.Id,
Name = p.Name,
Description = p.Description,
CreatedDate = p.CreatedDate,
Price = p.Price,
RecommendedReason = p.RecommendedReason,
Status = p.Status,
CreateUserId = p.CreatedUser,
SortOrder = p.SortOrder,
Tag = new ESTag()
{
Id = t.Id,
Name = t.Name,
Description = t.Description
},
Store = new ESStore()
{
Id = s.Id,
Name = s.Name,
Description = s.Description,
Address = s.Location,
Location = new Location
{
Lon = s.Longitude,
Lat = s.Latitude
},
GpsAlt = s.GpsAlt,
GpsLat = s.GpsLat,
GpsLng = s.GpsLng,
Tel = s.Tel
},
Brand = new ESBrand()
{
Id = b.Id,
Name = b.Name,
Description = b.Description,
EngName = b.EnglishName
},
Resource = resource,
SpecialTopic = specials,
Promotion = promotions,
Is4Sale = p.Is4Sale ?? false,
UnitPrice = p.UnitPrice,
FavoriteCount = p.FavoriteCount,
InvolvedCount = p.InvolvedCount,
ShareCount = p.ShareCount,
RecommendUserId = p.RecommendUser,
Section=section.FirstOrDefault(),
UpcCode = p.SkuCode
};
int totalCount = prods.Count();
client.MapFromAttributes<ESProduct>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} products in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexUser(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from p in db.Users
where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate)
let resource = from r in db.Resources
where r.SourceId == p.Id && r.SourceType == (int)SourceType.CustomerPortrait
select new ESResource()
{
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
}
select new ESUser()
{
Id = p.Id,
Status = p.Status,
Thumnail = resource.FirstOrDefault(),
Nickie = p.Nickname,
Level = p.UserLevel
};
int totalCount = prods.Count();
client.MapFromAttributes<ESUser>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} users in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexStorePromotion(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = db.StorePromotions.Where(r=> r.CreateDate >= benchDate || r.UpdateDate >= benchDate)
.GroupJoin(db.PointOrderRules.Where(r=>r.Status!=(int)DataStatus.Deleted),o=>o.Id,i=>i.StorePromotionId,
(o,i)=>new {S=o,R = i});
int totalCount = prods.Count();
client.MapFromAttributes<ESStorePromotion>();
while (cursor < totalCount)
{
var linq = from l in prods.OrderByDescending(p => p.S.Id).Skip(cursor).Take(size).ToList()
select new ESStorePromotion().FromEntity<ESStorePromotion>(l.S, s => {
s.ExchangeRule = JsonConvert.SerializeObject(l.R.Select(r=>new {
rangefrom=r.RangeFrom,
rangeto = r.RangeTo,
ratio = r.Ratio
}));
});
var result = client.IndexMany(linq);
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} store promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexPros(ElasticClient client, DateTime? benchDate, Func<IQueryable<PromotionEntity>, YintaiHangzhouContext, IQueryable<PromotionEntity>> whereCondition)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var linq = db.Promotions.AsQueryable();
if (benchDate.HasValue)
linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value);
else if (whereCondition != null)
{
linq = whereCondition(linq, db);
}
var prods = from p in linq
join s in db.Stores on p.Store_Id equals s.Id
let resource = (from r in db.Resources
where r.SourceId == p.Id
&& r.SourceType == 2
select new ESResource() {
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
})
select new ESPromotion()
{
Id = p.Id,
Name = p.Name,
Description = p.Description,
CreatedDate = p.CreatedDate,
StartDate = p.StartDate,
EndDate = p.EndDate,
FavoriteCount = p.FavoriteCount,
IsTop = p.IsTop,
Status = p.Status,
CreateUserId = p.CreatedUser,
Store = new ESStore()
{
Id = s.Id,
Name = s.Name,
Description = s.Description,
Address = s.Location,
Location = new Location
{
Lon = s.Longitude,
Lat = s.Latitude
},
GpsAlt = s.GpsAlt,
GpsLat = s.GpsLat,
GpsLng = s.GpsLng,
Tel = s.Tel
},
Resource = resource,
ShowInList = p.IsMain.HasValue ? p.IsMain.Value : true,
PublicCode = p.PublicProCode,
InvolvedCount = p.InvolvedCount,
IsProdBindable = p.IsProdBindable ?? false,
LikeCount = p.LikeCount,
PublicationLimit = p.PublicationLimit ?? -1,
ShareCount = p.ShareCount
};
int totalCount = prods.Count();
client.MapFromAttributes<ESPromotion>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach(var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}",item.Id));
}
} else
successCount+=result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} promotions in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
if (successCount > 0 && CascadPush)
{
//index related products
log.Info("index products affected by related promotions ");
IndexProds(client, null,
(p,db) => {
return p.Where(prod => (from pro in db.Promotions
from ppr in db.Promotion2Product
where ppr.ProId == pro.Id
&& (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate)
&& ppr.ProdId == prod.Id
select ppr.ProdId).Any());
});
}
}
private void IndexBrand(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from p in db.Brands
where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate)
select new ESBrand()
{
Id = p.Id,
Name = p.Name,
Description = p.Description,
Status = p.Status,
Group = p.Group
};
int totalCount = prods.Count();
client.MapFromAttributes<ESBrand>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} brands in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
//update related source if any affected
if (successCount > 0 && CascadPush)
{
//index related products
log.Info("index products affected by related brands ");
IndexProds(client, null,
(p, db) =>
{
return p.Where(prod => (from r in db.Brands
where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate)
&& r.Id == prod.Brand_Id
select r.Id).Any());
});
}
}
private void IndexSpecialTopic(ElasticClient client, DateTime? benchDate, Func<IQueryable<SpecialTopicEntity>, YintaiHangzhouContext, IQueryable<SpecialTopicEntity>> whereCondition)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var linq = db.SpecialTopics.AsQueryable();
if (benchDate.HasValue)
linq = linq.Where(p => p.CreatedDate >= benchDate.Value || p.UpdatedDate >= benchDate.Value);
else if (whereCondition != null)
{
linq = whereCondition(linq, db);
}
var prods = from p in linq
let resource = (from r in db.Resources
where r.SourceId == p.Id
&& r.SourceType == 9
select new ESResource()
{
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
})
select new ESSpecialTopic()
{
Id = p.Id,
Name = p.Name,
Description = p.Description,
Status = p.Status,
CreatedDate = p.CreatedDate,
CreateUser = p.CreatedUser,
Resource = resource,
Type = p.Type,
TargetValue = p.TargetValue
};
int totalCount = prods.Count();
client.MapFromAttributes<ESSpecialTopic>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} special topics in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
if (successCount > 0 && CascadPush)
{
//index related products
log.Info("index products affected by related specialtopic ");
IndexProds(client, null,
(p, db) =>
{
return p.Where(prod => (from pro in db.SpecialTopics
from ppr in db.SpecialTopicProductRelations
where ppr.SpecialTopic_Id == pro.Id
&& (pro.CreatedDate >= benchDate || pro.UpdatedDate >= benchDate)
&& ppr.Product_Id == prod.Id
select ppr.Product_Id).Any());
});
}
}
private void IndexTag(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from p in db.Tags
where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate)
select new ESTag()
{
Id = p.Id,
Name = p.Name,
Description = p.Description,
Status = p.Status,
SortOrder = p.SortOrder
};
int totalCount = prods.Count();
client.MapFromAttributes<ESTag>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} tags in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexStore(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from s in db.Stores
let resource = (from r in db.Resources
where r.SourceId == s.Id
&& r.SourceType == (int)SourceType.StoreLogo
select new ESResource()
{
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
})
where (s.CreatedDate >= benchDate || s.UpdatedDate >= benchDate)
select new ESStore()
{
Id = s.Id,
Name = s.Name,
Description = s.Description,
Address = s.Location,
Location = new Location
{
Lon = s.Longitude,
Lat = s.Latitude
},
GpsAlt = s.GpsAlt,
GpsLat = s.GpsLat,
GpsLng = s.GpsLng,
Tel = s.Tel,
Status = s.Status,
Resource = resource
};
int totalCount = prods.Count();
client.MapFromAttributes<ESStore>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} stores in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexBanner(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from p in db.Banners
join pro in db.Promotions on p.SourceId equals pro.Id
where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate)
&& p.SourceType == 2
let resource = (from r in db.Resources
where r.SourceId == p.Id
&& r.SourceType == 11
select new ESResource()
{
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height
})
select new ESBanner()
{
Id = p.Id,
SortOrder = p.SortOrder,
CreatedDate = p.CreatedDate,
Status = p.Status,
SourceType = p.SourceType,
Promotion = new ESPromotion()
{
Id = pro.Id
},
Resource = resource
};
int totalCount = prods.Count();
client.MapFromAttributes<ESBanner>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} banners in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexHotwork(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var words = from p in db.HotWords
where (p.CreatedDate >= benchDate || p.UpdatedDate >= benchDate)
select p;
var prods = from p in words.ToList()
select new ESHotword()
{
Id = p.Id,
SortOrder = p.SortOrder,
Status = p.Status,
Type = p.Type,
Word = p.Type==1?p.Word:JsonConvert.DeserializeObject<dynamic>(p.Word).name,
BrandId = p.Type==1?0:JsonConvert.DeserializeObject<dynamic>(p.Word).id
};
int totalCount = prods.Count();
client.MapFromAttributes<ESHotword>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} hotwords in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
}
private void IndexResource(ElasticClient client, DateTime benchDate)
{
ILog log = LogManager.GetLogger(this.GetType());
int cursor = 0;
int size = JobConfig.DEFAULT_PAGE_SIZE;
int successCount = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (var db = new YintaiHangzhouContext("YintaiHangzhouContext"))
{
var prods = from r in db.Resources
where (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate)
select new ESResource()
{
Id = r.Id,
Status= r.Status,
Domain = r.Domain,
Name = r.Name,
SortOrder = r.SortOrder,
IsDefault = r.IsDefault,
Type = r.Type,
Width = r.Width,
Height = r.Height,
SourceId = r.SourceId,
SourceType = r.SourceType
};
int totalCount = prods.Count();
client.MapFromAttributes<ESResource>();
while (cursor < totalCount)
{
var result = client.IndexMany(prods.OrderByDescending(p => p.Id).Skip(cursor).Take(size));
if (!result.IsValid)
{
foreach (var item in result.Items)
{
if (item.OK)
successCount++;
else
log.Info(string.Format("id index failed:{0}", item.Id));
}
}
else
successCount += result.Items.Count();
cursor += size;
}
}
sw.Stop();
log.Info(string.Format("{0} resources in {1} => {2} docs/s", successCount, sw.Elapsed, successCount / sw.Elapsed.TotalSeconds));
//update related source if any affected
if (successCount > 0 && CascadPush)
{
//index related products
log.Info("index products affected by related resources ");
IndexProds(client, null,
(p, db) =>
{
return p.Where(prod => (from r in db.Resources
where r.SourceId == prod.Id
&& (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate)
&& r.SourceType == (int)SourceType.Product
select r.Id).Any());
});
//index related promotions
log.Info("index promotions affected by related resources ");
IndexPros(client, null,
(p, db) =>
{
return p.Where(prod => (from r in db.Resources
where r.SourceId == prod.Id
&& (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate)
&& r.SourceType == (int)SourceType.Promotion
select r.Id).Any());
});
//index related specialtopics
log.Info("index specialtopics affected by related resources ");
IndexSpecialTopic(client, null,
(p, db) =>
{
return p.Where(prod => (from r in db.Resources
where r.SourceId == prod.Id
&& (r.CreatedDate >= benchDate || r.UpdatedDate >= benchDate)
&& r.SourceType == (int)SourceType.SpecialTopic
select r.Id).Any());
});
}
}
public void Run(string indexName, int port, int numMessages, int bufferSize)
{
var bulkParms = new SimpleBulkParameters() { Refresh = false };
var settings = new ConnectionSettings("localhost", port)
.SetDefaultIndex(indexName);
var client = new ElasticClient(settings, new ThriftConnection(settings));
Connect(client, settings);
IList<Message> msgBuffer = new List<Message>(bufferSize);
var msgGenerator = new MessageGenerator();
foreach (var msg in msgGenerator.Generate(numMessages))
{
msgBuffer.Add(msg);
// Flush buffer once max size reached
if (msgBuffer.Count >= bufferSize)
{
client.IndexMany(msgBuffer, indexName, bulkParms);
msgBuffer.Clear();
Interlocked.Add(ref NumSent, bufferSize);
// Output how many messages sent so far
if (NumSent % 10000 == 0)
{
Console.WriteLine("Sent {0:0,0} messages over Thrift", NumSent);
}
}
}
}
public ActionResult ReIndexAll()
{
var documents = db.Documents.ToList();
var uriString = ConfigurationManager.AppSettings["SEARCHBOX_URL"];
var searchBoxUri = new Uri(uriString);
var settings = new ConnectionSettings(searchBoxUri);
settings.SetDefaultIndex("sample");
var client = new ElasticClient(settings);
// delete index if exists at startup
if (client.IndexExists("sample").Exists)
{
client.DeleteIndex("sample");
}
// Create a new "sample" index with default settings
client.CreateIndex("sample", new IndexSettings());
// Index all documents
client.IndexMany<Document>(documents);
ViewBag.Message = "Reindexing all database is complete!";
return RedirectToAction("Index");
}