public static DataSearchResult SearchData(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true, bool withHighlighting = false, bool exactNumOfResults = false) { Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); var query = Lib.Searching.Tools.FixInvalidQuery(queryString, queryShorcuts, queryOperators); var res = _searchData(ds, query, page, pageSize, sort, excludeBigProperties, withHighlighting, exactNumOfResults); sw.Stop(); if (!res.IsValid) { throw DataSetException.GetExc( ds.DatasetId, ApiResponseStatus.InvalidSearchQuery.error.number, ApiResponseStatus.InvalidSearchQuery.error.description, queryString ); } if (res.Total > 0) { var expConverter = new Newtonsoft.Json.Converters.ExpandoObjectConverter(); return(new DataSearchResult() { ElapsedTime = sw.Elapsed, Q = queryString, IsValid = true, Total = res.Total, Result = res.Hits .Select(m => Newtonsoft.Json.JsonConvert.SerializeObject(m.Source)) .Select(s => (dynamic)Newtonsoft.Json.JsonConvert.DeserializeObject <System.Dynamic.ExpandoObject>(s, expConverter)), Page = page, PageSize = pageSize, DataSet = ds, ElasticResultsRaw = res, }); } else { return new DataSearchResult() { ElapsedTime = sw.Elapsed, Q = queryString, IsValid = true, Total = 0, Result = new dynamic[] { }, Page = page, PageSize = pageSize, DataSet = ds, ElasticResultsRaw = res, } }; }
public static DatasetMultiResult GeneralSearch(string query, IEnumerable <External.DataSets.DataSet> datasets = null, int page = 1, int pageSize = 20, string sort = null) { DatasetMultiResult res = new DatasetMultiResult() { Query = query, DataSource = "DatasetMultiResult.GeneralSearch" }; if (string.IsNullOrEmpty(query)) { return(res); } if (!Lib.Searching.Tools.ValidateQuery(query)) { res.Exceptions.Add(new System.Exception($"Invalid Query: {query}")); return(res); } if (datasets == null) { datasets = Lib.Data.External.DataSets.DataSetDB.ProductionDataSets.Get(); } ParallelOptions po = new ParallelOptions(); po.MaxDegreeOfParallelism = System.Diagnostics.Debugger.IsAttached ? 1 : po.MaxDegreeOfParallelism; Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); Parallel.ForEach(datasets, po, ds => { try { var rds = ds.SearchData(query, page, pageSize, sort); if (rds.IsValid) { res.Results.Add(rds); } } catch (External.DataSets.DataSetException e) { res.Exceptions.Add(e); } catch (System.Exception e) { res.Exceptions.Add(e); //HlidacStatu.Util.Consts.Logger.Warning("DatasetMultiResult GeneralSearch for query" + query, e); } }); sw.Stop(); res.ElapsedTime = sw.Elapsed; return(res); }
/// <summary> /// returns Osoba.NameId[] /// </summary> /// <param name="text"></param> /// <returns></returns> public static string[] FindCitations(string text) { var stopw = new Devmasters.DT.StopWatchEx(); stopw.Start(); string[] sText = Stems(text); stopw.Stop(); //Console.WriteLine($"stemmer {stopw.ExactElapsedMiliseconds} "); stopw.Restart(); List <string> found = new List <string>(); foreach (var kv in PoliticiStems) { string zkratka = kv.Item1; string[] politik = kv.Item2; for (int i = 0; i < sText.Length - (politik.Length - 1); i++) { bool same = true; for (int j = 0; j < politik.Length; j++) { if (sText[i + j] == politik[j]) { same = same & true; } else { same = false; break; } } if (same) { if (!found.Contains(zkratka)) { found.Add(zkratka); } break; } } } stopw.Stop(); //Console.WriteLine($"location {stopw.ExactElapsedMiliseconds} "); return(found.ToArray()); }
public static OsobaEsSearchResult FulltextSearch(string query, int page, int pageSize, int?status = null) { string modifQ = Lib.Searching.SimpleQueryCreator .GetSimpleQuery(query, new Searching.Rules.IRule[] { new Searching.Rules.RemoveAllOperators() }) .FullQuery(); if (string.IsNullOrWhiteSpace(modifQ)) { return(new OsobaEsSearchResult() { OrigQuery = query, Total = 0, IsValid = true, ElasticResults = null, ElapsedTime = TimeSpan.Zero }); } page = page - 1 < 0 ? 0 : page - 1; var sw = new Devmasters.DT.StopWatchEx(); sw.Start(); ISearchResponse <OsobaES> res = null; try { if (status.HasValue) { res = _esClient .Search <OsobaES>(s => s .Size(pageSize) .From(page * pageSize) .Query(_query => _query .Bool(_bool => _bool .Must(_must => _must .Fuzzy(_fuzzy => _fuzzy .Field(_field => _field.FullName) .Value(modifQ) .Fuzziness(Fuzziness.EditDistance(2)) ) && _must.Term(_field => _field.Status, status.Value) ) .Should( _boostWomen => _boostWomen .Match(_match => _match .Field(_field => _field.FullName) .Query(modifQ) ), _boostExact => _boostExact .Match(_match => _match .Field("fullName.lower") .Query(modifQ) ), _boostAscii => _boostAscii .Match(_match => _match .Field("fullName.lowerascii") .Query(modifQ) ) ) ) ) .TrackTotalHits(true) ); } else { res = _esClient //.MultiSearch<OsobaES>(s => s .Search <OsobaES>(s => s .Size(pageSize) .From(page * pageSize) .Query(_query => _query .MultiMatch(c => c .Fields(f => f .Field(p => p.FullName) .Field("fullName.lower", 2) .Field("fullName.lowerascii", 1.5) ) .Type(TextQueryType.MostFields) .Fuzziness(Fuzziness.EditDistance(2)) .Query(modifQ) ) ) .TrackTotalHits(true) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "OsobaES", "error", query, null); if (res != null && res.ServerError != null) { ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:" + query + " query:" + modifQ , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "OsobaES", res.IsValid ? "valid" : "invalid", query, null); if (res.IsValid == false) { ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:" + query + " query:" + query ); } var search = new OsobaEsSearchResult { OrigQuery = query, Total = res?.Total ?? 0, IsValid = res?.IsValid ?? false, ElasticResults = res, ElapsedTime = sw.Elapsed }; return(search); }
private Lib.Data.Logs.ProfilZadavateleDownload _processReqProfiluZadavatel(VZ.ProfilZadavatele profil, DateTime from, DateTime to) { string xmlUrlTemp = profil.Url; if (profil.Url?.EndsWith("/") == true) { xmlUrlTemp = xmlUrlTemp + "XMLdataVZ?od={0:ddMMyyy}&do={1:ddMMyyyy}"; } else { xmlUrlTemp = xmlUrlTemp + "/XMLdataVZ?od={0:ddMMyyy}&do={1:ddMMyyyy}"; } var xml = ""; Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); var surl = string.Format(xmlUrlTemp, from, to); var ReqLog = new Lib.Data.Logs.ProfilZadavateleDownload() { Date = DateTime.Now, ProfileId = profil.Id, RequestedUrl = surl }; try { sem.WaitOne(); using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(surl)) { //net.TimeInMsBetweenTries = 20*1000; //net.Tries = 1; net.Timeout = 60 * 1000; xml = net.GetContent().Text; ReqLog.HttpValid = true; } } catch (Devmasters.Net.HttpClient.UrlContentException ex) { ReqLog.HttpValid = false; ReqLog.HttpError = ex.ToString(); if (ex.InnerException != null && ex.InnerException.GetType() == typeof(System.Net.WebException)) { var wex = (System.Net.WebException)ex.InnerException; ReqLog.HttpError = wex.ToString(); if (wex.Status == WebExceptionStatus.ProtocolError && wex.Response != null) { ReqLog.HttpErrorCode = (int)(((HttpWebResponse)wex.Response).StatusCode); } } ReqLog.Save(); profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError; profil.LastAccess = DateTime.Now; profil.Save(); return(ReqLog); } catch (System.Net.WebException wex) { ReqLog.HttpValid = false; ReqLog.HttpError = wex.ToString(); if (wex.Status == WebExceptionStatus.ProtocolError && wex.Response != null) { ReqLog.HttpErrorCode = (int)(((HttpWebResponse)wex.Response).StatusCode); } ReqLog.Save(); profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError; profil.LastAccess = DateTime.Now; profil.Save(); return(ReqLog); } catch (Exception e) { ReqLog.HttpValid = false; ReqLog.HttpError = e.ToString(); ReqLog.Save(); profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.HttpError; profil.LastAccess = DateTime.Now; profil.Save(); return(ReqLog); } finally { sem.Release(); sw.Stop(); ReqLog.ResponseMs = sw.ElapsedMilliseconds; } Lib.Data.External.ProfilZadavatelu.ProfilStructure prof = null; try { prof = ParserXml(xml); ReqLog.XmlValid = true; } catch (Exception e) { ReqLog.XmlValid = false; ReqLog.XmlError = e.ToString(); ReqLog.XmlInvalidContent = xml; ReqLog.Save(); profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.XmlError; profil.LastAccess = DateTime.Now; profil.Save(); return(ReqLog); } if (prof != null) { var cli = Lib.ES.Manager.GetESClient_VerejneZakazkyNaProfiluRaw(); foreach (var zak in prof.zakazka) { Lib.Data.External.ProfilZadavatelu.ZakazkaRaw myZak = new Lib.Data.External.ProfilZadavatelu.ZakazkaRaw(zak, profil); myZak.Save(); } ReqLog.Save(); profil.LastAccessResult = VZ.ProfilZadavatele.LastAccessResults.OK; profil.LastAccess = DateTime.Now; profil.Save(); } return(ReqLog); }
public static SmlouvaSearchResult SimpleSearch(string query, int page, int pageSize, OrderResult order, AggregationContainerDescriptor <Lib.Data.Smlouva> anyAggregation = null, bool?platnyZaznam = null, bool includeNeplatne = false, bool logError = true, bool fixQuery = true, bool withHighlighting = false, bool exactNumOfResults = false) { var result = new SmlouvaSearchResult() { Page = page, PageSize = pageSize, OrigQuery = query, Q = query, Order = ((int)order).ToString() }; if (string.IsNullOrEmpty(query)) { result.ElasticResults = null; result.IsValid = false; result.Total = 0; return(result); } Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); if (fixQuery) { query = Searching.Tools.FixInvalidQuery(query, irules, Searching.Tools.DefaultQueryOperators); result.Q = query; } if (platnyZaznam.HasValue) { query = Lib.Searching.Tools.ModifyQueryAND(query, "platnyZaznam:" + platnyZaznam.Value.ToString().ToLower()); } ISearchResponse <Lib.Data.Smlouva> res = _coreSearch(GetSimpleQuery(query), page, pageSize, order, anyAggregation, platnyZaznam, includeNeplatne, logError, withHighlighting, exactNumOfResults); Data.Audit.Add(Data.Audit.Operations.Search, "", "", "Smlouva", res.IsValid ? "valid" : "invalid", query, null); if (res.IsValid == false && logError) { Lib.ES.Manager.LogQueryError <Lib.Data.Smlouva>(res, query); } sw.Stop(); result.ElapsedTime = sw.Elapsed; try { result.Total = res?.Total ?? 0; } catch (Exception) { result.Total = 0; } result.IsValid = res?.IsValid ?? false; result.ElasticResults = res; return(result); }
public static VerejnaZakazkaSearchData SimpleSearch( VerejnaZakazkaSearchData search, AggregationContainerDescriptor <VerejnaZakazka> anyAggregation = null, bool logError = true, bool fixQuery = true, ElasticClient client = null, bool withHighlighting = false) { if (client == null) { client = HlidacStatu.Lib.ES.Manager.GetESClient_VZ(); } string query = search.Q ?? ""; int page = search.Page - 1; if (page < 0) { page = 0; } AggregationContainerDescriptor <VerejnaZakazka> baseAggrDesc = null; baseAggrDesc = anyAggregation == null ? null //new AggregationContainerDescriptor<VerejnaZakazka>().Sum("sumKc", m => m.Field(f => f.Castka)) : anyAggregation; Func <AggregationContainerDescriptor <VerejnaZakazka>, AggregationContainerDescriptor <VerejnaZakazka> > aggrFunc = (aggr) => { return(baseAggrDesc); }; Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); if (fixQuery) { search.OrigQuery = query; query = Lib.Searching.Tools.FixInvalidQuery(query, queryShorcuts, queryOperators); } search.Q = query; ISearchResponse <VerejnaZakazka> res = null; try { res = client .Search <VerejnaZakazka>(s => s .Size(search.PageSize) .Source(so => so.Excludes(ex => ex.Field("dokumenty.plainText"))) .From(page * search.PageSize) .Query(q => GetSimpleQuery(search)) .Sort(ss => GetSort(search.Order)) .Aggregations(aggrFunc) .Highlight(h => Lib.Searching.Tools.GetHighlight <VerejnaZakazka>(withHighlighting)) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); if (withHighlighting && res.Shards != null && res.Shards.Failed > 0) //if some error, do it again without highlighting { res = client .Search <VerejnaZakazka>(s => s .Size(search.PageSize) .Source(so => so.Excludes(ex => ex.Field("dokumenty.plainText"))) .From(page * search.PageSize) .Query(q => GetSimpleQuery(search)) .Sort(ss => GetSort(search.Order)) .Aggregations(aggrFunc) .Highlight(h => Lib.Searching.Tools.GetHighlight <VerejnaZakazka>(false)) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "VerejnaZakazka", "error", search.Q, null); if (res != null && res.ServerError != null) { Lib.ES.Manager.LogQueryError <VerejnaZakazka>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults.ToString() , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "VerejnaZakazka", res.IsValid ? "valid" : "invalid", search.Q, null); if (res.IsValid == false && logError) { Lib.ES.Manager.LogQueryError <VerejnaZakazka>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults?.ToString() ); } search.Total = res?.Total ?? 0; search.IsValid = res?.IsValid ?? false; search.ElasticResults = res; search.ElapsedTime = sw.Elapsed; return(search); }
private static MultiResult GeneralSearch(Elastic.Apm.Api.ITransaction apmtran, string query, int page = 1, int pageSize = 10, bool showBeta = false, string order = null) { MultiResult res = new MultiResult() { Query = query }; if (string.IsNullOrEmpty(query)) { return(res); } if (!Lib.Searching.Tools.ValidateQuery(query)) { res.Smlouvy = new Searching.SmlouvaSearchResult(); res.Smlouvy.Q = query; res.Smlouvy.IsValid = false; return(res); } var totalsw = new Devmasters.DT.StopWatchEx(); totalsw.Start(); ParallelOptions po = new ParallelOptions(); //po.MaxDegreeOfParallelism = 20; po.MaxDegreeOfParallelism = System.Diagnostics.Debugger.IsAttached ? 1 : po.MaxDegreeOfParallelism; Parallel.Invoke(po, () => { Elastic.Apm.Api.ISpan sp = null; try { apmtran.CaptureSpan("Smlouvy", "search", () => { res.Smlouvy = HlidacStatu.Lib.Data.Smlouva.Search.SimpleSearch(query, 1, pageSize, order, anyAggregation: new Nest.AggregationContainerDescriptor <HlidacStatu.Lib.Data.Smlouva>().Sum("sumKc", m => m.Field(f => f.CalculatedPriceWithVATinCZK)) ); }); } catch (System.Exception e) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for Smlouvy query" + query, e); } finally { sp?.End(); } }, () => { try { Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); res.Firmy = Firma.Search.SimpleSearch(query, 0, 50); sw.Stop(); res.Firmy.ElapsedTime = sw.Elapsed; } catch (System.Exception e) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for Firmy query" + query, e); } }, () => { try { res.VZ = VZ.VerejnaZakazka.Searching.SimpleSearch(query, null, 1, pageSize, order); } catch (System.Exception e) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for Verejne zakazky query" + query, e); } }, () => { try { Devmasters.DT.StopWatchEx sw = new Devmasters.DT.StopWatchEx(); sw.Start(); res.Osoby = Osoba.Search.SimpleSearch(query, 1, 10, Osoba.Search.OrderResult.Relevance); sw.Stop(); res.Osoby.ElapsedTime = sw.Elapsed; } catch (System.Exception e) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for Osoba query" + query, e); } }, () => { try { var iqu = new Searching.InsolvenceSearchResult { Q = query, PageSize = pageSize, Order = order }; res.Insolvence = iqu; //if (showBeta) res.Insolvence = Insolvence.Insolvence.SimpleSearch(new Searching.InsolvenceSearchResult { Q = query, PageSize = pageSize, Order = order }); } catch (System.Exception e) { Util.Consts.Logger.Error("MultiResult GeneralSearch for insolvence query" + query, e); } }, () => { try { var dotaceService = new Dotace.DotaceService(); var iqu = new Searching.DotaceSearchResult { Q = query, PageSize = pageSize, Order = order }; res.Dotace = dotaceService.SimpleSearch( new Searching.DotaceSearchResult { Q = query, PageSize = pageSize, Order = order }, anyAggregation: new Nest.AggregationContainerDescriptor <Lib.Data.Dotace.Dotace>().Sum("souhrn", s => s.Field(f => f.DotaceCelkem)) ); } catch (System.Exception e) { Util.Consts.Logger.Error("MultiResult GeneralSearch for insolvence query" + query, e); } }, () => { try { apmtran.CaptureSpan("Dataset GeneralSearch", "search", () => { res.Datasets = Lib.Data.Search.DatasetMultiResult.GeneralSearch(query, null, 1, 5); if (res.Datasets.Exceptions.Count > 0) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for DatasetMulti query " + query, res.Datasets.GetExceptions()); } }); } catch (System.Exception e) { HlidacStatu.Util.Consts.Logger.Error("MultiResult GeneralSearch for DatasetMulti query " + query, e); } finally { } } ); //TODO too slow, temporarily disabled totalsw.Stop(); res.TotalSearchTime = totalsw.Elapsed; return(res); }
public static InsolvenceSearchResult SimpleSearch(InsolvenceSearchResult search, bool withHighlighting = false, AggregationContainerDescriptor <Lib.Data.Insolvence.Rizeni> anyAggregation = null, bool exactNumOfResults = false) { var client = Manager.GetESClient_Insolvence(); var page = search.Page - 1 < 0 ? 0 : search.Page - 1; var sw = new Devmasters.DT.StopWatchEx(); sw.Start(); search.OrigQuery = search.Q; search.Q = Lib.Searching.Tools.FixInvalidQuery(search.Q ?? "", queryShorcuts, queryOperators); var sq = GetSimpleQuery(search); ISearchResponse <Rizeni> res = null; try { res = client .Search <Rizeni>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText"))) .Query(q => sq) //.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending())) .Sort(ss => GetSort(search.Order)) .Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(withHighlighting)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); if (withHighlighting && res.Shards != null && res.Shards.Failed > 0) //if some error, do it again without highlighting { res = client .Search <Rizeni>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText"))) .Query(q => sq) //.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending())) .Sort(ss => GetSort(search.Order)) .Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(false)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "Insolvence", "error", search.Q, null); if (res != null && res.ServerError != null) { Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults.ToString() , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "Insolvence", res.IsValid ? "valid" : "invalid", search.Q, null); if (res.IsValid == false) { Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults?.ToString() ); } search.Total = res?.Total ?? 0; search.IsValid = res?.IsValid ?? false; search.ElasticResults = res; search.ElapsedTime = sw.Elapsed; return(search); }