public IUser Store(IUser user)
{
if (!WriteUsersEnabled)
{
throw new Exception("not enabled");
}
lock (this) {
user.Id = UserSerializer.GetId(user);
if (user.CreateTime.Year <= 1900)
{
user.CreateTime = DateTime.Now.ToUniversalTime();
}
user.UpdateTime = DateTime.Now.ToUniversalTime();
var json = UserSerializer.GetJson(user, "store");
var url = GetBaseUrl() + user.Id + "?refresh=true";
var result = EsClient.ExecuteCommand(url, json);
if (null == result)
{
throw new Exception("invalid storage operation", EsClient.LastError);
}
var j = result.jsonify();
var version = j.num("_version");
user.Version = version;
_cache[user.Login] = user;
return(user);
}
}
public AutoCompleteResponse SuggestCustomer(string query, int page, int pageSize)
{
var response = EsClient.Search <Customer>(s => s.Index("customer")
.Suggest(su =>
su.Completion("suggestions", c =>
c.Field(t =>
t.EmailSuggest).Prefix(query).Fuzzy(f =>
f.Fuzziness(Fuzziness.Auto)
)
)
)
);
var suggestions = response.Suggest["suggestions"];
AutoCompleteResponse autoCompleteResponse = new AutoCompleteResponse
{
text = suggestions[0].Text,
customerDetails = new List <CustomerDetails>()
};
foreach (var option in suggestions[0].Options)
{
autoCompleteResponse.customerDetails.Add(new CustomerDetails
{
Email = option.Text,
Details = option.Source
});
}
return(autoCompleteResponse);
}
public void Initailze_Client(string server = null, int?port = null, string index = null, int?searchRange = null, string session = null)
{
if (server != null)
{
Server = server;
}
if (port != null)
{
Port = (int)port;
}
if (index != null)
{
Index = index;
}
if (searchRange != null)
{
SearchRange = (int)searchRange;
}
if (session != null)
{
Session = session;
}
esClient = new EsClient();
esClient.Server = Server;
esClient.Port = Port;
esClient.Index = Index;
}
public PaymentController(EsClient eSClient)
{
_esClient = eSClient;
var esUrl = Environment.GetEnvironmentVariable("ELASTICSEARCH_URL");
_indexingService = RestService.For <IIndexingService>(esUrl);//http://localhost:52708
}
/// <summary>
/// 批量保存
/// </summary>
/// <param name="entitys">参数</param>
public virtual void Save(List <T> entitys)
{
if (entitys == null || entitys.Count == 0)
{
return;
}
EsClient.IndexMany(entitys, CurrentIndex);
EsClient.Indices.Refresh(CurrentIndex);
}
public void MarkSent(string id)
{
EsClient.ExecuteCommand(BaseUrl() + "/" + id + "/_update?refresh=true", new {
doc = new {
WasSent = true,
SentTime = DateTime.Now.ToUniversalTime()
}
}.stringify());
}
public async Task Get()
{
string query = "shap";
EsClient client = new EsClient();
StringResponse searchResponse = await client.Client.SearchAsync <StringResponse>("product_search", PostData.Serializable(new
{
query = new
{
@bool = new
{
should = new
{
@bool = new
{
must = new
{
match_phrase_prefix = new
{
name = new
{
query = $"{query}",
slop = 3,
max_expansions = 10
}
}
},
should = new
{
multi_match = new
{
query = $"{query}",
fields = @"[""name"",""url"", ""image""]",
fuzziness = "auto",
@operator = "or"
}
}
}
},
filter = new
{
term = new
{
type = "product"
}
}
}
},
size = 50,
_source = @"[""name"", ""price"", ""sku""]",
}));
var jsonDoc = JsonDocument.Parse(searchResponse.Body).RootElement.GetProperty("hits").GetProperty("hits");
var count = jsonDoc.EnumerateArray().Count();
Debug.WriteLine($"Get Count: {count}");
}
public void IndexCheck()
{
string index = "product_search";
EsClient client = new EsClient();
bool success;
success = client.Client.Indices.Exists <StringResponse>(index).Success;
Assert.True(success.Equals(true));
}
public async Task GetAllEsProductsV_2()
{
EsClient client = new EsClient();
StringResponse searchResponse = await client.Client.SearchAsync <StringResponse>("product_search", PostData.Serializable(new
{
from = 0,
size = 10000,
query = new
{
match = new
{
type = "product"
}
}
}));
string responseJson = searchResponse.Body;
Assert.NotNull(responseJson);
Assert.NotEmpty(responseJson);
List <ProductModel> products = new List <ProductModel>();
using JsonDocument document = JsonDocument.Parse(responseJson);
Assert.NotNull(document);
JsonElement root = document.RootElement;
JsonElement sourceElement = root.GetProperty("hits").GetProperty("hits");
foreach (JsonElement source in sourceElement.EnumerateArray())
{
if (source.TryGetProperty("_source", out JsonElement rec))
{
ProductModel product = new ProductModel
{
Type = rec.GetProperty("type").GetString(),
Sku = rec.GetProperty("sku").GetString(),
Image = rec.GetProperty("image").GetString(),
Url = rec.GetProperty("url").GetString(),
Price = rec.GetProperty("price").GetDecimal(),
Name = rec.GetProperty("name").GetString(),
Description = rec.GetProperty("description").GetString()
};
products.Add(product);
}
}
Assert.NotNull(products);
Assert.IsType <List <ProductModel> >(products);
Assert.NotEmpty(products);
}
public SearchService(OrangeContext orangeContext,
GoodsService goodsService,
SpecService specService,
EsClient elasticSearch
)
{
_orangeContext = orangeContext;
_goodsService = goodsService;
_specService = specService;
_elasticSearch = elasticSearch;
}
public InitESIndexWorker(ILogger <InitESIndexWorker> logger,
RabbitMQClient rabbitMQ,
EsClient elasticSearch,
SearchService searchService,
InitDataServcie initDataServcie
)
{
_logger = logger;
_rabbitMQ = rabbitMQ;
_elasticSearch = elasticSearch;
_searchService = searchService;
_initDataServcie = initDataServcie;
}
/// <summary>
/// 增量更新
/// </summary>
/// <param name="id">Id</param>
/// <param name="incrementModifyParams">增量参数：key-字段,value-修改的值</param>
/// <returns></returns>
public virtual UpdateResponse <T> IncrementModify(string id, Dictionary <string, object> incrementModifyParams)
{
var r = new UpdateResponse <T>();
if (incrementModifyParams == null)
{
return(r);
}
var updatePath = new DocumentPath <T>(id);
r = EsClient.Update <T, object>(updatePath, u => u.Doc(incrementModifyParams).Refresh(Elasticsearch.Net.Refresh.True));
return(r);
}
private void Store(PostMessage message)
{
var result = EsClient.ExecuteCommand(BaseUrl() + "/" + message.Id + "?refresh=true", message.stringify());
if (null == result)
{
Logg.Error(new {
error_in_es = EsClient.LastError.Message,
urls = EsClient.Urls.ToArray()
}.stringify());
throw EsClient.LastError;
}
}
public IUser GetUser(string login)
{
if (!WriteUsersEnabled)
{
return(null);
}
lock (this) {
CheckCache();
if (EsClient.InvalidConnection)
{
return(null);
}
if (_cache.ContainsKey(login))
{
return(_cache[login]);
}
string json;
if (login.StartsWith("!certid:"))
{
var post = new { query = new { match = new { publickey = login.Replace("!certid:", "") } } }.stringify();
json = EsClient.ExecuteCommand(GetBaseUrl() + "_search", post);
}
else
{
var id = UserSerializer.GetId(login);
json = EsClient.ExecuteCommand(GetBaseUrl() + id);
}
if (null == json)
{
return(null);
}
var j = json.jsonify();
var found = j.bul("found") || j.num("*.total") > 0;
if (found)
{
var src = j.map("*._source");
var version = j.resolvenum("_version", "__version");
var user = UserSerializer.CreateFromJson(src);
user.Id = j.resolvestr("_id", "__id");
user.Version = version;
_cache[login] = user;
}
else
{
_cache[login] = null;
}
return(_cache[login]);
}
}
public IEnumerable <IUser> SearchUsers(UserSearchQuery query)
{
var q = string.IsNullOrWhiteSpace(query.Query)? "login:*" : query.Query;
if (!string.IsNullOrWhiteSpace(query.Login))
{
q += " AND login:"******" AND name:" + query.Name;
}
if (!query.Groups)
{
q += " AND NOT isgroup:true";
}
if (!query.Users)
{
q += " AND isgroup:true";
}
if (!string.IsNullOrWhiteSpace(query.Domain))
{
q += " AND domain:" + query.Domain;
}
var esquery = new {
query = new { query_string = new { query = q } }
};
var json = EsClient.ExecuteCommand(GetBaseUrl() + "_search", esquery.stringify());
if (null == json)
{
yield break;
}
var j = json.jsonify();
var hits = j.arr("hits.hits");
foreach (var hit in hits)
{
var src = hit.map("*._source");
var version = hit.resolvenum("_version", "__version");
var user = UserSerializer.CreateFromJson(src);
user.Id = hit.resolvestr("_id", "__id");
user.Version = version;
_cache[user.Login] = user;
yield return(user);
}
}
public async Task GetAllEsProductsV_4()
{
EsClient client = new EsClient();
StringResponse searchResponse = await client.Client.SearchAsync <StringResponse>("product_search", PostData.Serializable(new
{
from = 0,
size = 10000,
query = new
{
match = new
{
type = "product"
}
}
}));
var responseJson = JsonDocument.Parse(searchResponse.Body).RootElement.GetProperty("hits").GetProperty("hits");
List <ProductModel> products = new List <ProductModel>();
Parallel.ForEach(responseJson.EnumerateArray(), source =>
{
if (source.TryGetProperty("_source", out JsonElement rec))
{
ProductModel product = new ProductModel
{
Type = rec.GetProperty("type").GetString(),
Sku = rec.GetProperty("sku").GetString(),
Image = rec.GetProperty("image").GetString(),
Url = rec.GetProperty("url").GetString(),
Price = rec.GetProperty("price").GetDecimal(),
Name = rec.GetProperty("name").GetString(),
Description = rec.GetProperty("description").GetString()
};
products.Add(product);
}
});
Assert.NotNull(products);
Assert.IsType <List <ProductModel> >(products);
Assert.NotEmpty(products);
Debug.WriteLine($"V4 Count: {products.Count}");
}
public PostMessage GetMessage(string id)
{
var result = EsClient.ExecuteCommand("/" + Index + "/" + Type + "/" + id);
if (null == result)
{
throw EsClient.LastError;
}
var j = result.jsonify();
if (!j.bul("found"))
{
return(null);
}
var pm = CreateFromJson(j);
return(pm);
}
/// <summary>
/// 批量删除
/// </summary>
/// <param name="ids">Id</param>
public virtual void Delete(string[] ids)
{
var esResult = EsClient.MultiGet(m => m.GetMany <T>(ids).Index(CurrentIndex));
if (esResult.Hits.Count <= 0)
{
return;
}
foreach (var esResultHits in esResult.Hits)
{
if (!esResultHits.Found)
{
continue;
}
EsClient.Delete <T>(esResultHits.Id, f => f.Index(CurrentIndex));
}
EsClient.Indices.Refresh(CurrentIndex);
}
internal async Task <JsonElement> GetAllEsProductsV_3A()
{
EsClient client = new EsClient();
StringResponse searchResponse = await client.Client.SearchAsync <StringResponse>("product_search", PostData.Serializable(new
{
from = 0,
size = 10000,
query = new
{
match = new
{
type = "product"
}
}
}));
return(JsonDocument.Parse(searchResponse.Body).RootElement.GetProperty("hits").GetProperty("hits"));
}
private void CheckCache(bool forced = false)
{
if (!_initialized)
{
Initialize();
_initialized = true;
}
if (!forced && EsClient.InvalidConnection)
{
if (EsClient.LastPing.AddMilliseconds(PingRate) > DateTime.Now)
{
return;
}
}
if (!forced && LastCheck.AddMilliseconds(CacheRate) > DateTime.Now)
{
return;
}
var currentETag = ETag;
var currentVersion = Version;
var url = GetBaseUrl() + "_search?search_type=count";
var query = leasequery;
var json = EsClient.ExecuteCommand(url, query);
if (null == json)
{
ETag = null;
Version = DateTime.MinValue;
LastCheck = DateTime.MinValue;
}
else
{
var j = json.jsonify();
ETag = j.str("aggregations._version.value");
Version = j.date("aggregations._timestamp.value_as_string");
}
if (ETag != currentETag || Version != currentVersion)
{
_cache.Clear();
}
}
public IEnumerable <PostMessage> SearchMessages(object query)
{
var q = "";
if (query is string)
{
q = (string)query;
if (!q.StartsWith("{"))
{
q = new {
query = new {
query_string = new {
query = q
}
}
}.stringify();
}
}
else
{
q = query.stringify();
}
var result = EsClient.ExecuteCommand(BaseUrl() + "/_search", q);
if (null == result)
{
throw EsClient.LastError;
}
var j = result.jsonify();
var hits = j.arr("hits.hits");
if (null == hits)
{
yield break;
}
foreach (var hit in hits)
{
yield return(CreateFromJson(hit));
}
}
public bool UploadData()
{
IEsDbOperation esDbOperation = new EsDbOperation();
int customerTotalCount = esDbOperation.GetCustomersCount();
int size = 500, index = 0;
while (index <= customerTotalCount)
{
List <KcaCustomer> customers = esDbOperation.GetCustomers(size, index);
List <Customer> esCustomer = new List <Customer>();
customers.ForEach(x => esCustomer.Add(new Customer
{
Email = x.Email,
EmailSuggest = x.Email,
FirstName = x.FirstName,
FirstNameSuggest = x.FirstName,
LastName = x.LastName,
LastNameSuggest = x.LastName,
Landline = x.Landline,
Id = x.Id,
CustomerId = x.Id.ToString(),
school = new School
{
schoolid = x.School.Id,
schoolname = x.School.Name,
schooladdress1 = x.School.Address,
postcode = x.School.Suburb.Postcode.Code,
suburb = x.School.Suburb.Name,
state = x.School.Suburb.Postcode.State.Name
}
}));
EsClient.BulkAsync(b => b.Index("customer").IndexMany(esCustomer));
index += size;
}
return(true);
}
/// <summary>
/// 获取一条数据
/// </summary>
/// <param name="id">Id</param>
/// <returns></returns>
public virtual T Get(string id)
{
var esResult = EsClient.Get <T>(id, i => i.Index(CurrentIndex));
return(esResult.Found ? esResult.Source : null);
}
public IndexingController(EsClient eSClient)
{
_esClient = eSClient;
}
/// <summary>
/// 单条保存
/// </summary>
/// <param name="t">参数</param>
/// <returns></returns>
public virtual IndexResponse Save(T t)
{
var r = EsClient.Index(t, i => i.Index(CurrentIndex).Refresh(Elasticsearch.Net.Refresh.True));
return(r);
}
public LogActionFilter(ILogger <LogActionFilter> logger,
EsClient esClient)
{
this._logger = logger;
_esClient = esClient;
}
public DbLevel()
{
esClient = new EsClient();
}
public static string GetWebsiteIdIfAlreadyCrawled(string url)
{
var existingWebsite = EsClient.GetWebsitesByUrl(url);
return(existingWebsite.Count > 0 ? existingWebsite.First().Id : null);
}
public ExceptionFilter(ILogger <ExceptionFilter> logger,
EsClient esClient)
{
this._logger = logger;
_esClient = esClient;
}
public static bool IsWebsiteRecentlyIndexed(string url)
{
return(EsClient.GetWebsitesByUrl(url).Count > 0);
}
