public static InsolvenceSearchResult SimpleSearch(InsolvenceSearchResult search,
bool withHighlighting = false,
AggregationContainerDescriptor <Lib.Data.Insolvence.Rizeni> anyAggregation = null, bool exactNumOfResults = false)
{
var client = Manager.GetESClient_Insolvence();
var page = search.Page - 1 < 0 ? 0 : search.Page - 1;
var sw = new Devmasters.DT.StopWatchEx();
sw.Start();
search.OrigQuery = search.Q;
search.Q = Lib.Searching.Tools.FixInvalidQuery(search.Q ?? "", queryShorcuts, queryOperators);
var sq = GetSimpleQuery(search);
ISearchResponse <Rizeni> res = null;
try
{
res = client
.Search <Rizeni>(s => s
.Size(search.PageSize)
.ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All)
.From(page * search.PageSize)
.Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText")))
.Query(q => sq)
//.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending()))
.Sort(ss => GetSort(search.Order))
.Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(withHighlighting))
.Aggregations(aggr => anyAggregation)
.TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null)
);
if (withHighlighting && res.Shards != null && res.Shards.Failed > 0) //if some error, do it again without highlighting
{
res = client
.Search <Rizeni>(s => s
.Size(search.PageSize)
.ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All)
.From(page * search.PageSize)
.Source(sr => sr.Excludes(r => r.Fields("dokumenty.plainText")))
.Query(q => sq)
//.Sort(ss => new SortDescriptor<Rizeni>().Field(m => m.Field(f => f.PosledniZmena).Descending()))
.Sort(ss => GetSort(search.Order))
.Highlight(h => Lib.Searching.Tools.GetHighlight <Rizeni>(false))
.Aggregations(aggr => anyAggregation)
.TrackTotalHits(search.ExactNumOfResults || page * search.PageSize == 0 ? true : (bool?)null)
);
}
}
catch (Exception e)
{
Audit.Add(Audit.Operations.Search, "", "", "Insolvence", "error", search.Q, null);
if (res != null && res.ServerError != null)
{
Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:"
+ search.OrigQuery + " query:"
+ search.Q
+ "\n\n res:" + search.ElasticResults.ToString()
, ex: e);
}
else
{
HlidacStatu.Util.Consts.Logger.Error("", e);
}
throw;
}
sw.Stop();
Audit.Add(Audit.Operations.Search, "", "", "Insolvence", res.IsValid ? "valid" : "invalid", search.Q, null);
if (res.IsValid == false)
{
Manager.LogQueryError <Rizeni>(res, "Exception, Orig query:"
+ search.OrigQuery + " query:"
+ search.Q
+ "\n\n res:" + search.ElasticResults?.ToString()
);
}
search.Total = res?.Total ?? 0;
search.IsValid = res?.IsValid ?? false;
search.ElasticResults = res;
search.ElapsedTime = sw.Elapsed;
return(search);
}
public virtual string AddData(Registration reg, string user, bool skipallowWriteAccess = false)
{
if (reg.jsonSchema == null)
{
throw new DataSetException(this.datasetId, ApiResponseStatus.DatasetJsonSchemaMissing);
}
Registration oldReg = null;
oldReg = DataSet.CachedDatasets.Get(reg.datasetId)?.Registration();
if (oldReg == null)
{
Audit.Add <Registration>(Audit.Operations.Create, user, reg, null);
}
else
{
Audit.Add <Registration>(Audit.Operations.Update, user, reg, oldReg);
}
var addDataResult = base.AddData(reg, reg.datasetId, reg.createdBy, validateSchema: false, skipallowWriteAccess: true);
DataSet.CachedDatasets.Delete(reg.datasetId);
//check orderList
if (reg.orderList?.Length > 0)
{
//get mapping
var ds = CachedDatasets.Get(addDataResult);
var txtProps = ds.GetTextMappingList();
var allProps = ds.GetMappingList();
if (allProps.Where(m => !DataSet.DefaultDatasetProperties.Keys.Contains(m)).Count() > 0) //0=mapping not available , (no record in db)
{
bool changedOrderList = false;
//find missing and remove it
List <int> toRemove = new List <int>();
for (int i = 0; i < reg.orderList.GetLength(0); i++)
{
string oProp = reg.orderList[i, 1]
.Replace(DataSearchResult.OrderAsc, "")
.Replace(DataSearchResult.OrderDesc, "")
.Replace(DataSearchResult.OrderAscUrl, "")
.Replace(DataSearchResult.OrderDescUrl, "")
.Trim();
if (oProp.EndsWith(".keyword"))
{
oProp = System.Text.RegularExpressions.Regex.Replace(oProp, "\\.keyword$", "");
}
if (allProps.Contains(oProp) == false)
{
//neni na seznamu properties, pridej do seznamu k smazani
toRemove.Add(i);
}
}
if (toRemove.Count > 0)
{
foreach (var i in toRemove.OrderByDescending(n => n))
{
reg.orderList = Devmasters.Collections.ArrayExt.TrimArray <string>(i, null, reg.orderList);
changedOrderList = true;
}
}
for (int i = 0; i < reg.orderList.GetLength(0); i++)
{
string oProp = reg.orderList[i, 1]
.Replace(DataSearchResult.OrderAsc, "")
.Replace(DataSearchResult.OrderAscUrl, "")
.Replace(DataSearchResult.OrderDesc, "")
.Replace(DataSearchResult.OrderDescUrl, "")
.Trim();
if (txtProps.Contains(oProp))
{
//pridej keyword na konec
reg.orderList[i, 1] = reg.orderList[i, 1].Replace(oProp, oProp + ".keyword");
changedOrderList = true;
}
}
if (changedOrderList)
{
addDataResult = base.AddData(reg, reg.datasetId, reg.createdBy);
}
}
}
DataSet.CachedDatasets.Set(reg.datasetId, null);
return(addDataResult);
}
private List <AuditEntry> OnBeforeSaveChanges()
{
ChangeTracker.DetectChanges();
var auditEntries = new List <AuditEntry>();
foreach (var entry in ChangeTracker.Entries())
{
if (entry.Entity is Audit || entry.State == EntityState.Detached || entry.State == EntityState.Unchanged)
{
continue;
}
var auditEntry = new AuditEntry(entry);
auditEntry.TableName = entry.Metadata.GetTableName();
auditEntries.Add(auditEntry);
foreach (var property in entry.Properties)
{
if (property.IsTemporary)
{
// value will be generated by the database, get the value after saving
auditEntry.TemporaryProperties.Add(property);
continue;
}
string propertyName = property.Metadata.Name;
if (SkipProperty(property))
{
continue;
}
if (IsKeyValue(property))
{
auditEntry.KeyValues[propertyName] = property.CurrentValue;
continue;
}
switch (entry.State)
{
case EntityState.Added:
auditEntry.NewValues[propertyName] = property.CurrentValue;
break;
case EntityState.Deleted:
auditEntry.OldValues[propertyName] = property.OriginalValue;
break;
case EntityState.Modified:
if (property.IsModified)
{
auditEntry.OldValues[propertyName] = property.OriginalValue;
auditEntry.NewValues[propertyName] = property.CurrentValue;
}
break;
}
}
}
// Save audit entities that have all the modifications
foreach (var auditEntry in auditEntries.Where(_ => !_.HasTemporaryProperties))
{
Audit.Add(auditEntry.ToAudit(GetUserIdFromContext()));
}
// keep a list of entries where the value of some properties are unknown at this step
return(auditEntries.Where(_ => _.HasTemporaryProperties).ToList());
}
public void InstallFiles(int packageId, string tempDir)
{
using (DisposableTimer.DebugDuration <Installer>(
() => "Installing package files for package id " + packageId + " into temp folder " + tempDir,
() => "Package file installation complete for package id " + packageId))
{
//retrieve the manifest to continue installation
var insPack = InstalledPackage.GetById(packageId);
//TODO: Depending on some files, some files should be installed differently.
//i.e. if stylsheets should probably be installed via business logic, media items should probably use the media IFileSystem!
// Move files
//string virtualBasePath = System.Web.HttpContext.Current.Request.ApplicationPath;
string basePath = System.Web.Hosting.HostingEnvironment.ApplicationPhysicalPath;
try
{
foreach (XmlNode n in Config.DocumentElement.SelectNodes("//file"))
{
var destPath = GetFileName(basePath, XmlHelper.GetNodeValue(n.SelectSingleNode("orgPath")));
var sourceFile = GetFileName(tempDir, XmlHelper.GetNodeValue(n.SelectSingleNode("guid")));
var destFile = GetFileName(destPath, XmlHelper.GetNodeValue(n.SelectSingleNode("orgName")));
// Create the destination directory if it doesn't exist
if (Directory.Exists(destPath) == false)
{
Directory.CreateDirectory(destPath);
}
//If a file with this name exists, delete it
else if (File.Exists(destFile))
{
File.Delete(destFile);
}
// Copy the file
// SJ: Note - this used to do a move but some packages included the same file to be
// copied to multiple locations like so:
//
// <file>
// <guid>my-icon.png</guid>
// <orgPath>/umbraco/Images/</orgPath>
// <orgName>my-icon.png</orgName>
// </file>
// <file>
// <guid>my-icon.png</guid>
// <orgPath>/App_Plugins/MyPlugin/Images</orgPath>
// <orgName>my-icon.png</orgName>
// </file>
//
// Since this file unzips as a flat list of files, moving the file the first time means
// that when you try to do that a second time, it would result in a FileNotFoundException
File.Copy(sourceFile, destFile);
//PPH log file install
insPack.Data.Files.Add(XmlHelper.GetNodeValue(n.SelectSingleNode("orgPath")) + "/" + XmlHelper.GetNodeValue(n.SelectSingleNode("orgName")));
}
// Once we're done copying, remove all the files
foreach (XmlNode n in Config.DocumentElement.SelectNodes("//file"))
{
var sourceFile = GetFileName(tempDir, XmlHelper.GetNodeValue(n.SelectSingleNode("guid")));
if (File.Exists(sourceFile))
{
File.Delete(sourceFile);
}
}
}
catch (Exception exception)
{
LogHelper.Error <Installer>("Package install error", exception);
throw;
}
// log that a user has install files
if (_currentUserId > -1)
{
Audit.Add(AuditTypes.PackagerInstall,
string.Format("Package '{0}' installed. Package guid: {1}", insPack.Data.Name, insPack.Data.PackageGuid),
_currentUserId, -1);
}
insPack.Save();
}
}
public void LogPolicyViolation()
{
var identity = _userQrySvc.GetIdentityContext();
Audit.Add(new AuditEntry(identity.IdentityId, identity.GivenName, AuditCategory.Administration, PasswordAudit.UnsuccessfulPasswordChangeAuditFormat));
}
public static OsobaEsSearchResult FulltextSearch(string query, int page, int pageSize, int?status = null)
{
string modifQ = Lib.Searching.SimpleQueryCreator
.GetSimpleQuery(query, new Searching.Rules.IRule[] { new Searching.Rules.RemoveAllOperators() })
.FullQuery();
if (string.IsNullOrWhiteSpace(modifQ))
{
return(new OsobaEsSearchResult()
{
OrigQuery = query,
Total = 0,
IsValid = true,
ElasticResults = null,
ElapsedTime = TimeSpan.Zero
});
}
page = page - 1 < 0 ? 0 : page - 1;
var sw = new Devmasters.DT.StopWatchEx();
sw.Start();
ISearchResponse <OsobaES> res = null;
try
{
int fuzzyDistance = 1;
if (status.HasValue)
{
res = _esClient
.Search <OsobaES>(s => s
.Size(pageSize)
.From(page * pageSize)
.Query(_query => _query
.Bool(_bool => _bool
.Must(_must => _must
.Fuzzy(_fuzzy => _fuzzy
.Field(_field => _field.FullName)
.Value(modifQ)
.Fuzziness(Fuzziness.EditDistance(fuzzyDistance))
) &&
_must.Term(_field => _field.Status, status.Value)
)
.Should(
_boostWomen => _boostWomen
.Match(_match => _match
.Field(_field => _field.FullName)
.Query(modifQ)
.Operator(Operator.And)
),
_boostExact => _boostExact
.Match(_match => _match
.Field("fullName.lower")
.Query(modifQ)
.Operator(Operator.And)
),
_boostAscii => _boostAscii
.Match(_match => _match
.Field("fullName.lowerascii")
.Query(modifQ)
.Operator(Operator.And)
)
)
)
)
.TrackTotalHits(true)
);
}
else
{
res = _esClient //.MultiSearch<OsobaES>(s => s
.Search <OsobaES>(s => s
.Size(pageSize)
.From(page * pageSize)
.Query(_query => _query
.MultiMatch(c => c
.Fields(f => f
.Field(p => p.FullName)
.Field("fullName.lower", 2)
.Field("fullName.lowerascii", 1.5)
)
.Type(TextQueryType.MostFields)
.Fuzziness(Fuzziness.EditDistance(fuzzyDistance))
.Query(modifQ)
.Operator(Operator.And)
)
)
.TrackTotalHits(true)
);
}
}
catch (Exception e)
{
Audit.Add(Audit.Operations.Search, "", "", "OsobaES", "error", query, null);
if (res != null && res.ServerError != null)
{
ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:"
+ query + " query:"
+ modifQ
, ex: e);
}
else
{
HlidacStatu.Util.Consts.Logger.Error("", e);
}
throw;
}
sw.Stop();
Audit.Add(Audit.Operations.Search, "", "", "OsobaES", res.IsValid ? "valid" : "invalid", query, null);
if (res.IsValid == false)
{
ES.Manager.LogQueryError <OsobaES>(res, "Exception, Orig query:"
+ query + " query:"
+ query
);
}
var search = new OsobaEsSearchResult
{
OrigQuery = query,
Total = res?.Total ?? 0,
IsValid = res?.IsValid ?? false,
ElasticResults = res,
ElapsedTime = sw.Elapsed
};
return(search);
}
public DotaceSearchResult SimpleSearch(DotaceSearchResult search,
bool withHighlighting = false,
AggregationContainerDescriptor <Dotace> anyAggregation = null)
{
var page = search.Page - 1 < 0 ? 0 : search.Page - 1;
var sw = new StopWatchEx();
sw.Start();
search.OrigQuery = search.Q;
search.Q = Lib.Searching.Tools.FixInvalidQuery(search.Q ?? "", queryShorcuts, queryOperators);
ISearchResponse <Dotace> res = null;
try
{
res = _esClient
.Search <Dotace>(s => s
.Size(search.PageSize)
.ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All)
.From(page * search.PageSize)
.Query(q => GetSimpleQuery(search))
.Sort(ss => GetSort(Convert.ToInt32(search.Order)))
.Highlight(h => Lib.Searching.Tools.GetHighlight <Dotace>(withHighlighting))
.Aggregations(aggr => anyAggregation)
.TrackTotalHits(search.ExactNumOfResults ? true : (bool?)null)
);
if (res.IsValid && withHighlighting && res.Shards.Failed > 0) //if some error, do it again without highlighting
{
res = _esClient
.Search <Dotace>(s => s
.Size(search.PageSize)
.ExpandWildcards(Elasticsearch.Net.ExpandWildcards.All)
.From(page * search.PageSize)
.Query(q => GetSimpleQuery(search))
.Sort(ss => GetSort(Convert.ToInt32(search.Order)))
.Highlight(h => Lib.Searching.Tools.GetHighlight <Dotace>(false))
.Aggregations(aggr => anyAggregation)
.TrackTotalHits(search.ExactNumOfResults ? true : (bool?)null)
);
}
}
catch (Exception e)
{
Audit.Add(Audit.Operations.Search, "", "", "Dotace", "error", search.Q, null);
if (res != null && res.ServerError != null)
{
ES.Manager.LogQueryError <Dotace>(res, "Exception, Orig query:"
+ search.OrigQuery + " query:"
+ search.Q
+ "\n\n res:" + search.Result.ToString()
, ex: e);
}
else
{
HlidacStatu.Util.Consts.Logger.Error("", e);
}
throw;
}
sw.Stop();
Audit.Add(Audit.Operations.Search, "", "", "Dotace", res.IsValid ? "valid" : "invalid", search.Q, null);
if (res.IsValid == false)
{
ES.Manager.LogQueryError <Dotace>(res, "Exception, Orig query:"
+ search.OrigQuery + " query:"
+ search.Q
+ "\n\n res:" + search.Result?.ToString()
);
}
search.Total = res?.Total ?? 0;
search.IsValid = res?.IsValid ?? false;
search.ElasticResults = res;
search.ElapsedTime = sw.Elapsed;
return(search);
}
public static ISearchResponse <object> _searchData(DataSet ds, string queryString, int page, int pageSize, string sort = null,
bool excludeBigProperties = true, bool withHighlighting = false, bool exactNumOfResults = false)
{
SortDescriptor <object> sortD = new SortDescriptor <object>();
if (sort == "0")
{
sort = null;
}
if (!string.IsNullOrEmpty(sort))
{
if (sort.EndsWith(DataSearchResult.OrderDesc) || sort.ToLower().EndsWith(DataSearchResult.OrderDescUrl))
{
sort = sort.Replace(DataSearchResult.OrderDesc, "").Replace(DataSearchResult.OrderDescUrl, "").Trim();
sortD = sortD.Field(sort, SortOrder.Descending);
}
else
{
sort = sort.Replace(DataSearchResult.OrderAsc, "").Replace(DataSearchResult.OrderAscUrl, "").Trim();
sortD = sortD.Field(sort, SortOrder.Ascending);
}
}
Nest.ElasticClient client = Lib.ES.Manager.GetESClient(ds.DatasetId, idxType: ES.Manager.IndexType.DataSource);
QueryContainer qc = GetSimpleQuery(ds, queryString);
//QueryContainer qc = null;
//if (queryString == null)
// qc = new QueryContainerDescriptor<object>().MatchNone();
//else if (string.IsNullOrEmpty(queryString))
// qc = new QueryContainerDescriptor<object>().MatchAll();
//else
//{
// qc = new QueryContainerDescriptor<object>()
// .QueryString(qs => qs
// .Query(queryString)
// .DefaultOperator(Operator.And)
// );
//}
page = page - 1;
if (page < 0)
{
page = 0;
}
if (page * pageSize > Lib.Searching.Tools.MaxResultWindow)
{
page = (Lib.Searching.Tools.MaxResultWindow / pageSize) - 1;
}
//exclude big properties from result
var maps = excludeBigProperties ? ds.GetMappingList("DocumentPlainText").ToArray() : new string[] { };
var res = client
.Search <object>(s => s
.Size(pageSize)
.Source(ss => ss.Excludes(ex => ex.Fields(maps)))
.From(page * pageSize)
.Query(q => qc)
.Sort(ss => sortD)
.Highlight(h => Lib.Searching.Tools.GetHighlight <Object>(withHighlighting))
.TrackTotalHits(exactNumOfResults || page * pageSize == 0 ? true : (bool?)null)
);
//fix Highlighting for large texts
if (withHighlighting &&
res.Shards != null &&
res.Shards.Failed > 0) //if some error, do it again without highlighting
{
res = client
.Search <object>(s => s
.Size(pageSize)
.Source(ss => ss.Excludes(ex => ex.Fields(maps)))
.From(page * pageSize)
.Query(q => qc)
.Sort(ss => sortD)
.Highlight(h => Lib.Searching.Tools.GetHighlight <Object>(false))
.TrackTotalHits(exactNumOfResults || page * pageSize == 0 ? true : (bool?)null)
);
}
Audit.Add(Audit.Operations.Search, "", "", "Dataset." + ds.DatasetId, res.IsValid ? "valid" : "invalid", queryString, null);
return(res);
}
