public virtual IDataItems GetAll(DataSourceContext context, Select selectQuery) { if (selectQuery == null) { return(GetAll(context)); } SelectQueryDefinition def = BuildQuery(context, selectQuery); OpenContentController ctrl = new OpenContentController(context.PortalId); SearchResults docs = LuceneController.Instance.Search(OpenContentInfo.GetScope(GetModuleId(context), context.Collection), def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex); int total = docs.TotalResults; var dataList = new List <IDataItem>(); foreach (string item in docs.ids) { var content = ctrl.GetContent(int.Parse(item)); if (content != null) { dataList.Add(CreateDefaultDataItem(content)); } else { App.Services.Logger.Debug($"OpenContentDataSource.GetAll() ContentItem not found [{item}]"); } } return(new DefaultDataItems() { Items = dataList, Total = total, DebugInfo = def.Filter + " - " + def.Query + " - " + def.Sort }); }
internal SearchResults Search(SelectQueryDefinition def) { if (!Store.ValidateIndexFolder()) { IndexAll(); return(new SearchResults()); } Func <Document, LuceneIndexItem> resultMapper = LuceneMappingUtils.CreateLuceneItem; var luceneResults = Store.Search(def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex, resultMapper); return(luceneResults); }
private static SelectQueryDefinition BuildQuery(DataSourceContext context, Select selectQuery) { SelectQueryDefinition def = new SelectQueryDefinition(); def.Build(selectQuery); if (LogContext.IsLogActive) { var logKey = "Lucene query"; LogContext.Log(context.ActiveModuleId, logKey, "Filter", def.Filter.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "Query", def.Query.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "Sort", def.Sort.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "PageIndex", def.PageIndex); LogContext.Log(context.ActiveModuleId, logKey, "PageSize", def.PageSize); } return(def); }
public virtual IDataItems GetAll(DataSourceContext context, Select select) { if (select == null) { return(GetAll(context)); } else { OpenContentController ctrl = new OpenContentController(); SelectQueryDefinition def = new SelectQueryDefinition(); def.Build(@select); if (LogContext.IsLogActive) { var logKey = "Lucene query"; LogContext.Log(context.ActiveModuleId, logKey, "Filter", def.Filter.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "Query", def.Query.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "Sort", def.Sort.ToString()); LogContext.Log(context.ActiveModuleId, logKey, "PageIndex", def.PageIndex); LogContext.Log(context.ActiveModuleId, logKey, "PageSize", def.PageSize); } SearchResults docs = LuceneController.Instance.Search(GetModuleId(context).ToString(), def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex); int total = docs.TotalResults; var dataList = new List <IDataItem>(); foreach (string item in docs.ids) { var content = ctrl.GetContent(int.Parse(item)); if (content != null) { dataList.Add(CreateDefaultDataItem(content)); } else { Log.Logger.DebugFormat("OpenContentDataSource.GetAll() ContentItem not found [{0}]", item); } } return(new DefaultDataItems() { Items = dataList, Total = total, DebugInfo = def.Filter.ToString() + " - " + def.Query.ToString() + " - " + def.Sort.ToString() }); } }
public HttpResponseMessage List(RequestDTO req) { try { Log.Logger.DebugFormat("OpenFiles.JplistApiController.List() called with request [{0}].", req.ToJson()); FieldConfig indexConfig = FilesRepository.GetIndexConfig(PortalSettings.PortalId); bool addWorkFlow = PortalSettings.UserMode != PortalSettings.Mode.Edit; QueryBuilder queryBuilder = new QueryBuilder(indexConfig); queryBuilder.BuildFilter(PortalSettings.PortalId, req.folder, addWorkFlow, PortalSettings.UserInfo.Social.Roles); JplistQueryBuilder.MergeJpListQuery(FilesRepository.GetIndexConfig(PortalSettings), queryBuilder.Select, req.StatusLst, DnnLanguageUtils.GetCurrentCultureCode()); string curFolder = NormalizePath(req.folder); foreach (var item in queryBuilder.Select.Query.FilterRules.Where(f => f.Field == LuceneMappingUtils.FolderField)) { curFolder = NormalizePath(item.Value.AsString); item.Value = new StringRuleValue(NormalizePath(item.Value.AsString)); //any file of current folder } var def = new SelectQueryDefinition(); def.Build(queryBuilder.Select); var docs = LuceneController.Instance.Search(def); int total = docs.TotalResults; var ratio = string.IsNullOrEmpty(req.imageRatio) ? new Ratio(100, 100) : new Ratio(req.imageRatio); Log.Logger.DebugFormat("OpenFiles.JplistApiController.List() Searched for [{0}], found [{1}] items", def.Filter.ToString() + " / " + def.Query.ToString(), total); //if (LogContext.IsLogActive) //{ // var logKey = "Query"; // LogContext.Log(ActiveModule.ModuleID, logKey, "select", queryBuilder.Select); // LogContext.Log(ActiveModule.ModuleID, logKey, "debuginfo", dsItems.DebugInfo); // LogContext.Log(ActiveModule.ModuleID, logKey, "model", model); // model["Logs"] = JToken.FromObject(LogContext.Current.ModuleLogs(ActiveModule.ModuleID)); //} var fileManager = FileManager.Instance; var data = new List <FileDTO>(); var breadcrumbs = new List <IFolderInfo>(); if (req.withSubFolder) { //hier blijken we resultaten toe te voegen die niet uit lucene komen breadcrumbs = AddFolders(NormalizePath(req.folder), curFolder, fileManager, data, ratio); } foreach (var doc in docs.ids) { IFileInfo f = fileManager.GetFile(doc.FileId); if (f == null) { //file seems to have been deleted LuceneController.Instance.Delete(LuceneMappingUtils.CreateLuceneItem(doc.PortalId, doc.FileId)); total -= 1; } else { if (f.FileName == "_folder.jpg") { continue; // skip } dynamic title = null; var custom = GetCustomFileDataAsDynamic(f); if (custom != null && custom.meta != null) { try { title = Normalize.DynamicValue(custom.meta.title, ""); } catch (Exception ex) { Log.Logger.Debug("OpenFiles.JplistApiController.List() Failed to get title.", ex); } } data.Add(new FileDTO() { Id = f.FileId, Name = Normalize.DynamicValue(title, f.FileName), FileName = f.FileName, CreatedOnDate = f.CreatedOnDate, LastModifiedOnDate = f.LastModifiedOnDate, FolderName = f.Folder, Url = fileManager.GetUrl(f), IsImage = fileManager.IsImageFile(f), ImageUrl = ImageHelper.GetImageUrl(f, ratio), Custom = custom, IconUrl = GetFileIconUrl(f.Extension), IsEditable = IsEditable, EditUrl = IsEditable ? GetFileEditUrl(f) : "" }); } } var res = new ResultExtDTO <FileDTO>() { data = new ResultDataDTO <FileDTO>() { items = data, breadcrumbs = breadcrumbs.Select(f => new ResultBreadcrumbDTO { name = f.FolderName, path = f.FolderPath.Trim('/') }) }, count = total }; return(Request.CreateResponse(HttpStatusCode.OK, res)); } catch (Exception exc) { Log.Logger.Error(exc); return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, exc)); } }
public override IDataItems GetAll(DataSourceContext context, Select selectQuery) { if (context.Index && selectQuery != null) { SelectQueryDefinition def = BuildQuery(context, selectQuery); SearchResults docs = LuceneController.Instance.Search(LUCENE_SCOPE, def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex); int total = docs.TotalResults; var dataList = new List <IDataItem>(); foreach (string item in docs.ids) { var user = UserController.GetUserById(context.PortalId, int.Parse(item)); if (user != null) { dataList.Add(ToData(user)); } else { Log.Logger.Debug($"DnnUsersDataSource.GetAll() ContentItem not found [{item}]"); } } return(new DefaultDataItems() { Items = dataList, Total = total, DebugInfo = def.Filter + " - " + def.Query + " - " + def.Sort }); } else { int pageIndex = 0; int pageSize = 1000; int total = 0; IEnumerable <UserInfo> users; if (selectQuery != null) { pageIndex = selectQuery.PageIndex; pageSize = selectQuery.PageSize; var ruleDisplayName = selectQuery.Query.FilterRules.FirstOrDefault(f => f.Field == "DisplayName"); var ruleRoles = selectQuery.Query.FilterRules.FirstOrDefault(f => f.Field == "Roles"); if (ruleDisplayName != null) { string displayName = ruleDisplayName.Value.AsString + "%"; users = UserController.GetUsersByDisplayName(context.PortalId, displayName, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>(); } else { users = UserController.GetUsers(context.PortalId, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>(); total = users.Count(); } if (ruleRoles != null) { var roleNames = ruleRoles.MultiValue.Select(r => r.AsString).ToList(); users = users.Where(u => u.Roles.Intersect(roleNames).Any()); } } else { users = UserController.GetUsers(context.PortalId, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>(); } users = users.Where(u => !u.IsInRole("Administrators")); //users = users.Skip(pageIndex * pageSize).Take(pageSize); var dataList = new List <IDataItem>(); foreach (var user in users) { dataList.Add(ToData(user)); } return(new DefaultDataItems() { Items = dataList, Total = total, //DebugInfo = }); } }
/// <summary> /// Adds the given JToken to the specified Document. /// </summary> /// <param name="doc"> /// The Document to add to. /// </param> /// <param name="prefix"> /// The prefix to use for field names. /// </param> /// <param name="token"> /// The JToken to add. /// </param> /// <param name="fieldconfig"></param> private static void Add(Document doc, string prefix, JToken token, FieldConfig fieldconfig) { if (token is JObject) { AddProperties(doc, prefix, token as JObject, fieldconfig); } else if (token is JArray) { var itemsConfig = fieldconfig?.Items; if (fieldconfig != null && fieldconfig.Index && itemsConfig == null) { throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json."); } AddArray(doc, prefix, token as JArray, itemsConfig); } else if (token is JValue) { JValue value = token as JValue; bool index = false; bool sort = false; if (fieldconfig != null) { index = fieldconfig.Index; sort = fieldconfig.Sort; if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String) { DateTime d; if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d)) { value = new JValue(d); } } } switch (value.Type) //todo: simple date gets detected as string { case JTokenType.Boolean: if (index || sort) { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0)); } break; case JTokenType.Date: if (index || sort) { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks)); //doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); /* * if (field != null ){ * if (field.IndexType == "datetime") * { * doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); * } * else if (field.IndexType == "date") * { * doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED)); * } * else if (field.IndexType == "time") * { * doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED)); * } * } * else * { * doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED)); * } */ } break; case JTokenType.Float: if (index || sort) { if (value.Value is float) { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value)); } else { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value))); } } break; case JTokenType.Guid: if (index || sort) { doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED)); } break; case JTokenType.Integer: if (index || sort) { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value))); } break; case JTokenType.Null: break; case JTokenType.String: if (fieldconfig != null && fieldconfig.IndexType == "key") { doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED)); } else if (fieldconfig != null && fieldconfig.IndexType == "html") { if (index) { doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED)); } if (sort) { doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED)); } } else if (fieldconfig != null && fieldconfig.IndexType == "file") { var val = value.Value.ToString(); if (!string.IsNullOrEmpty(val)) { var fileIndexer = FileIndexerManager.GetFileIndexer(val); if (fileIndexer != null) { var content = fileIndexer.GetContent(val); if (index) { doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED)); } if (sort) { doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); } } } } else { var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString()); val = val.Replace('-', ' '); // concider '-' as a space val = val.Replace(',', ' '); // concider ',' as a space //var val = LuceneUtils.CleanupText(value.Value.ToString()); if (index) { doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED)); } if (sort) { doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED)); } } break; case JTokenType.TimeSpan: if (index || sort) { doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks)); } break; case JTokenType.Uri: if (index || sort) { doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED)); } break; default: Debug.Fail("Unsupported JValue type: " + value.Type); break; } } else { Debug.Fail("Unsupported JToken: " + token); } }