public static InsolvenceSearchResult SimpleSearch(InsolvenceSearchResult search, bool withHighlighting = false, AggregationContainerDescriptor <Lib.Data.Insolvence.Rizeni> anyAggregation = null, bool exactNumOfResults = false) { var client = Manager.GetESClient_Insolvence(); var page = search.Page - 1 < 0 ? 0 : search.Page - 1; var sw = new Devmasters.DT.StopWatchEx(); sw.Start(); search.OrigQuery = search.Q; search.Q = Lib.Searching.Tools.FixInvalidQuery(search.Q ?? "", queryShorcuts, queryOperators); var sq = GetSimpleQuery(search); ISearchResponse <Rizeni> res = null; try { res = client .Search <Rizeni>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText"))) .Query(q => sq) //.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending())) .Sort(ss => GetSort(search.Order)) .Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(withHighlighting)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); if (withHighlighting && res.Shards != null && res.Shards.Failed > 0) //if some error, do it again without highlighting { res = client .Search <Rizeni>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText"))) .Query(q => sq) //.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending())) .Sort(ss => GetSort(search.Order)) .Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(false)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "Insolvence", "error", search.Q, null); if (res != null && res.ServerError != null) { Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults.ToString() , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "Insolvence", res.IsValid ? "valid" : "invalid", search.Q, null); if (res.IsValid == false) { Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.ElasticResults?.ToString() ); } search.Total = res?.Total ?? 0; search.IsValid = res?.IsValid ?? false; search.ElasticResults = res; search.ElapsedTime = sw.Elapsed; return(search); }
public virtual string AddData(Registration reg, string user, bool skipallowWriteAccess = false) { if (reg.jsonSchema == null) { throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetJsonSchemaMissing); } Registration oldReg = null; oldReg = DataSet.CachedDatasets.Get(reg.datasetId)?.Registration(); if (oldReg == null) { Audit.Add <Registration>(Audit.Operations.Create, user, reg, null); } else { Audit.Add <Registration>(Audit.Operations.Update, user, reg, oldReg); } var addDataResult = base.AddData(reg, reg.datasetId, reg.createdBy, validateSchema: false, skipallowWriteAccess: true); DataSet.CachedDatasets.Delete(reg.datasetId); //check orderList if (reg.orderList?.Length > 0) { //get mapping var ds = CachedDatasets.Get(addDataResult); var txtProps = ds.GetTextMappingList(); var allProps = ds.GetMappingList(); if (allProps.Where(m => !DataSet.DefaultDatasetProperties.Keys.Contains(m)).Count() > 0) //0=mapping not available , (no record in db) { bool changedOrderList = false; //find missing and remove it List <int> toRemove = new List <int>(); for (int i = 0; i < reg.orderList.GetLength(0); i++) { string oProp = reg.orderList[i, 1] .Replace(DataSearchResult.OrderAsc, "") .Replace(DataSearchResult.OrderDesc, "") .Replace(DataSearchResult.OrderAscUrl, "") .Replace(DataSearchResult.OrderDescUrl, "") .Trim(); if (oProp.EndsWith(".keyword")) { oProp = System.Text.RegularExpressions.Regex.Replace(oProp, "\\.keyword$", ""); } if (allProps.Contains(oProp) == false) { //neni na seznamu properties, pridej do seznamu k smazani toRemove.Add(i); } } if (toRemove.Count > 0) { foreach (var i in toRemove.OrderByDescending(n => n)) { reg.orderList = Devmasters.Collections.ArrayExt.TrimArray <string>(i, null, reg.orderList); changedOrderList = true; } } for (int i = 0; i < reg.orderList.GetLength(0); i++) { string oProp = reg.orderList[i, 1] .Replace(DataSearchResult.OrderAsc, "") .Replace(DataSearchResult.OrderAscUrl, "") .Replace(DataSearchResult.OrderDesc, "") .Replace(DataSearchResult.OrderDescUrl, "") .Trim(); if (txtProps.Contains(oProp)) { //pridej keyword na konec reg.orderList[i, 1] = reg.orderList[i, 1].Replace(oProp, oProp + ".keyword"); changedOrderList = true; } } if (changedOrderList) { addDataResult = base.AddData(reg, reg.datasetId, reg.createdBy); } } } DataSet.CachedDatasets.Set(reg.datasetId, null); return(addDataResult); }
private List <AuditEntry> OnBeforeSaveChanges() { ChangeTracker.DetectChanges(); var auditEntries = new List <AuditEntry>(); foreach (var entry in ChangeTracker.Entries()) { if (entry.Entity is Audit || entry.State == EntityState.Detached || entry.State == EntityState.Unchanged) { continue; } var auditEntry = new AuditEntry(entry); auditEntry.TableName = entry.Metadata.GetTableName(); auditEntries.Add(auditEntry); foreach (var property in entry.Properties) { if (property.IsTemporary) { // value will be generated by the database, get the value after saving auditEntry.TemporaryProperties.Add(property); continue; } string propertyName = property.Metadata.Name; if (SkipProperty(property)) { continue; } if (IsKeyValue(property)) { auditEntry.KeyValues[propertyName] = property.CurrentValue; continue; } switch (entry.State) { case EntityState.Added: auditEntry.NewValues[propertyName] = property.CurrentValue; break; case EntityState.Deleted: auditEntry.OldValues[propertyName] = property.OriginalValue; break; case EntityState.Modified: if (property.IsModified) { auditEntry.OldValues[propertyName] = property.OriginalValue; auditEntry.NewValues[propertyName] = property.CurrentValue; } break; } } } // Save audit entities that have all the modifications foreach (var auditEntry in auditEntries.Where(_ => !_.HasTemporaryProperties)) { Audit.Add(auditEntry.ToAudit(GetUserIdFromContext())); } // keep a list of entries where the value of some properties are unknown at this step return(auditEntries.Where(_ => _.HasTemporaryProperties).ToList()); }
public void InstallFiles(int packageId, string tempDir) { using (DisposableTimer.DebugDuration <Installer>( () => "Installing package files for package id " + packageId + " into temp folder " + tempDir, () => "Package file installation complete for package id " + packageId)) { //retrieve the manifest to continue installation var insPack = InstalledPackage.GetById(packageId); //TODO: Depending on some files, some files should be installed differently. //i.e. if stylsheets should probably be installed via business logic, media items should probably use the media IFileSystem! // Move files //string virtualBasePath = System.Web.HttpContext.Current.Request.ApplicationPath; string basePath = System.Web.Hosting.HostingEnvironment.ApplicationPhysicalPath; try { foreach (XmlNode n in Config.DocumentElement.SelectNodes("//file")) { var destPath = GetFileName(basePath, XmlHelper.GetNodeValue(n.SelectSingleNode("orgPath"))); var sourceFile = GetFileName(tempDir, XmlHelper.GetNodeValue(n.SelectSingleNode("guid"))); var destFile = GetFileName(destPath, XmlHelper.GetNodeValue(n.SelectSingleNode("orgName"))); // Create the destination directory if it doesn't exist if (Directory.Exists(destPath) == false) { Directory.CreateDirectory(destPath); } //If a file with this name exists, delete it else if (File.Exists(destFile)) { File.Delete(destFile); } // Copy the file // SJ: Note - this used to do a move but some packages included the same file to be // copied to multiple locations like so: // // <file> // <guid>my-icon.png</guid> // <orgPath>/umbraco/Images/</orgPath> // <orgName>my-icon.png</orgName> // </file> // <file> // <guid>my-icon.png</guid> // <orgPath>/App_Plugins/MyPlugin/Images</orgPath> // <orgName>my-icon.png</orgName> // </file> // // Since this file unzips as a flat list of files, moving the file the first time means // that when you try to do that a second time, it would result in a FileNotFoundException File.Copy(sourceFile, destFile); //PPH log file install insPack.Data.Files.Add(XmlHelper.GetNodeValue(n.SelectSingleNode("orgPath")) + "/" + XmlHelper.GetNodeValue(n.SelectSingleNode("orgName"))); } // Once we're done copying, remove all the files foreach (XmlNode n in Config.DocumentElement.SelectNodes("//file")) { var sourceFile = GetFileName(tempDir, XmlHelper.GetNodeValue(n.SelectSingleNode("guid"))); if (File.Exists(sourceFile)) { File.Delete(sourceFile); } } } catch (Exception exception) { LogHelper.Error <Installer>("Package install error", exception); throw; } // log that a user has install files if (_currentUserId > -1) { Audit.Add(AuditTypes.PackagerInstall, string.Format("Package '{0}' installed. Package guid: {1}", insPack.Data.Name, insPack.Data.PackageGuid), _currentUserId, -1); } insPack.Save(); } }
public void LogPolicyViolation() { var identity = _userQrySvc.GetIdentityContext(); Audit.Add(new AuditEntry(identity.IdentityId, identity.GivenName, AuditCategory.Administration, PasswordAudit.UnsuccessfulPasswordChangeAuditFormat)); }
public static OsobaEsSearchResult FulltextSearch(string query, int page, int pageSize, int?status = null) { string modifQ = Lib.Searching.SimpleQueryCreator .GetSimpleQuery(query, new Searching.Rules.IRule[] { new Searching.Rules.RemoveAllOperators() }) .FullQuery(); if (string.IsNullOrWhiteSpace(modifQ)) { return(new OsobaEsSearchResult() { OrigQuery = query, Total = 0, IsValid = true, ElasticResults = null, ElapsedTime = TimeSpan.Zero }); } page = page - 1 < 0 ? 0 : page - 1; var sw = new Devmasters.DT.StopWatchEx(); sw.Start(); ISearchResponse <OsobaES> res = null; try { int fuzzyDistance = 1; if (status.HasValue) { res = _esClient .Search <OsobaES>(s => s .Size(pageSize) .From(page * pageSize) .Query(_query => _query .Bool(_bool => _bool .Must(_must => _must .Fuzzy(_fuzzy => _fuzzy .Field(_field => _field.FullName) .Value(modifQ) .Fuzziness(Fuzziness.EditDistance(fuzzyDistance)) ) && _must.Term(_field => _field.Status, status.Value) ) .Should( _boostWomen => _boostWomen .Match(_match => _match .Field(_field => _field.FullName) .Query(modifQ) .Operator(Operator.And) ), _boostExact => _boostExact .Match(_match => _match .Field("fullName.lower") .Query(modifQ) .Operator(Operator.And) ), _boostAscii => _boostAscii .Match(_match => _match .Field("fullName.lowerascii") .Query(modifQ) .Operator(Operator.And) ) ) ) ) .TrackTotalHits(true) ); } else { res = _esClient //.MultiSearch<OsobaES>(s => s .Search <OsobaES>(s => s .Size(pageSize) .From(page * pageSize) .Query(_query => _query .MultiMatch(c => c .Fields(f => f .Field(p => p.FullName) .Field("fullName.lower", 2) .Field("fullName.lowerascii", 1.5) ) .Type(TextQueryType.MostFields) .Fuzziness(Fuzziness.EditDistance(fuzzyDistance)) .Query(modifQ) .Operator(Operator.And) ) ) .TrackTotalHits(true) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "OsobaES", "error", query, null); if (res != null && res.ServerError != null) { ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:" + query + " query:" + modifQ , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "OsobaES", res.IsValid ? "valid" : "invalid", query, null); if (res.IsValid == false) { ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:" + query + " query:" + query ); } var search = new OsobaEsSearchResult { OrigQuery = query, Total = res?.Total ?? 0, IsValid = res?.IsValid ?? false, ElasticResults = res, ElapsedTime = sw.Elapsed }; return(search); }
public DotaceSearchResult SimpleSearch(DotaceSearchResult search, bool withHighlighting = false, AggregationContainerDescriptor <Dotace> anyAggregation = null) { var page = search.Page - 1 < 0 ? 0 : search.Page - 1; var sw = new StopWatchEx(); sw.Start(); search.OrigQuery = search.Q; search.Q = Lib.Searching.Tools.FixInvalidQuery(search.Q ?? "", queryShorcuts, queryOperators); ISearchResponse <Dotace> res = null; try { res = _esClient .Search <Dotace>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Query(q => GetSimpleQuery(search)) .Sort(ss => GetSort(Convert.ToInt32(search.Order))) .Highlight(h => Lib.Searching.Tools.GetHighlight <Dotace>(withHighlighting)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults ? true : (bool?)null) ); if (res.IsValid && withHighlighting && res.Shards.Failed > 0) //if some error, do it again without highlighting { res = _esClient .Search <Dotace>(s => s .Size(search.PageSize) .ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All) .From(page * search.PageSize) .Query(q => GetSimpleQuery(search)) .Sort(ss => GetSort(Convert.ToInt32(search.Order))) .Highlight(h => Lib.Searching.Tools.GetHighlight <Dotace>(false)) .Aggregations(aggr => anyAggregation) .TrackTotalHits(search.ExactNumOfResults ? true : (bool?)null) ); } } catch (Exception e) { Audit.Add(Audit.Operations.Search, "", "", "Dotace", "error", search.Q, null); if (res != null && res.ServerError != null) { ES.Manager.LogQueryError <Dotace>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.Result.ToString() , ex: e); } else { HlidacStatu.Util.Consts.Logger.Error("", e); } throw; } sw.Stop(); Audit.Add(Audit.Operations.Search, "", "", "Dotace", res.IsValid ? "valid" : "invalid", search.Q, null); if (res.IsValid == false) { ES.Manager.LogQueryError <Dotace>(res, "Exception, Orig query:" + search.OrigQuery + " query:" + search.Q + "\n\n res:" + search.Result?.ToString() ); } search.Total = res?.Total ?? 0; search.IsValid = res?.IsValid ?? false; search.ElasticResults = res; search.ElapsedTime = sw.Elapsed; return(search); }
public static ISearchResponse <object> _searchData(DataSet ds, string queryString, int page, int pageSize, string sort = null, bool excludeBigProperties = true, bool withHighlighting = false, bool exactNumOfResults = false) { SortDescriptor <object> sortD = new SortDescriptor <object>(); if (sort == "0") { sort = null; } if (!string.IsNullOrEmpty(sort)) { if (sort.EndsWith(DataSearchResult.OrderDesc) || sort.ToLower().EndsWith(DataSearchResult.OrderDescUrl)) { sort = sort.Replace(DataSearchResult.OrderDesc, "").Replace(DataSearchResult.OrderDescUrl, "").Trim(); sortD = sortD.Field(sort, SortOrder.Descending); } else { sort = sort.Replace(DataSearchResult.OrderAsc, "").Replace(DataSearchResult.OrderAscUrl, "").Trim(); sortD = sortD.Field(sort, SortOrder.Ascending); } } Nest.ElasticClient client = Lib.ES.Manager.GetESClient(ds.DatasetId, idxType: ES.Manager.IndexType.DataSource); QueryContainer qc = GetSimpleQuery(ds, queryString); //QueryContainer qc = null; //if (queryString == null) // qc = new QueryContainerDescriptor<object>().MatchNone(); //else if (string.IsNullOrEmpty(queryString)) // qc = new QueryContainerDescriptor<object>().MatchAll(); //else //{ // qc = new QueryContainerDescriptor<object>() // .QueryString(qs => qs // .Query(queryString) // .DefaultOperator(Operator.And) // ); //} page = page - 1; if (page < 0) { page = 0; } if (page * pageSize > Lib.Searching.Tools.MaxResultWindow) { page = (Lib.Searching.Tools.MaxResultWindow / pageSize) - 1; } //exclude big properties from result var maps = excludeBigProperties ? ds.GetMappingList("DocumentPlainText").ToArray() : new string[] { }; var res = client .Search <object>(s => s .Size(pageSize) .Source(ss => ss.Excludes(ex => ex.Fields(maps))) .From(page * pageSize) .Query(q => qc) .Sort(ss => sortD) .Highlight(h => Lib.Searching.Tools.GetHighlight <Object>(withHighlighting)) .TrackTotalHits(exactNumOfResults || page * pageSize == 0 ? true : (bool?)null) ); //fix Highlighting for large texts if (withHighlighting && res.Shards != null && res.Shards.Failed > 0) //if some error, do it again without highlighting { res = client .Search <object>(s => s .Size(pageSize) .Source(ss => ss.Excludes(ex => ex.Fields(maps))) .From(page * pageSize) .Query(q => qc) .Sort(ss => sortD) .Highlight(h => Lib.Searching.Tools.GetHighlight <Object>(false)) .TrackTotalHits(exactNumOfResults || page * pageSize == 0 ? true : (bool?)null) ); } Audit.Add(Audit.Operations.Search, "", "", "Dataset." + ds.DatasetId, res.IsValid ? "valid" : "invalid", queryString, null); return(res); }