public virtual IDataItems GetAll(DataSourceContext context, Select selectQuery)
{
if (selectQuery == null)
{
return(GetAll(context));
}
SelectQueryDefinition def = BuildQuery(context, selectQuery);
OpenContentController ctrl = new OpenContentController(context.PortalId);
SearchResults docs = LuceneController.Instance.Search(OpenContentInfo.GetScope(GetModuleId(context), context.Collection), def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex);
int total = docs.TotalResults;
var dataList = new List <IDataItem>();
foreach (string item in docs.ids)
{
var content = ctrl.GetContent(int.Parse(item));
if (content != null)
{
dataList.Add(CreateDefaultDataItem(content));
}
else
{
App.Services.Logger.Debug($"OpenContentDataSource.GetAll() ContentItem not found [{item}]");
}
}
return(new DefaultDataItems()
{
Items = dataList,
Total = total,
DebugInfo = def.Filter + " - " + def.Query + " - " + def.Sort
});
}
internal SearchResults Search(SelectQueryDefinition def)
{
if (!Store.ValidateIndexFolder())
{
IndexAll();
return(new SearchResults());
}
Func <Document, LuceneIndexItem> resultMapper = LuceneMappingUtils.CreateLuceneItem;
var luceneResults = Store.Search(def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex, resultMapper);
return(luceneResults);
}
private static SelectQueryDefinition BuildQuery(DataSourceContext context, Select selectQuery)
{
SelectQueryDefinition def = new SelectQueryDefinition();
def.Build(selectQuery);
if (LogContext.IsLogActive)
{
var logKey = "Lucene query";
LogContext.Log(context.ActiveModuleId, logKey, "Filter", def.Filter.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "Query", def.Query.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "Sort", def.Sort.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "PageIndex", def.PageIndex);
LogContext.Log(context.ActiveModuleId, logKey, "PageSize", def.PageSize);
}
return(def);
}
public virtual IDataItems GetAll(DataSourceContext context, Select select)
{
if (select == null)
{
return(GetAll(context));
}
else
{
OpenContentController ctrl = new OpenContentController();
SelectQueryDefinition def = new SelectQueryDefinition();
def.Build(@select);
if (LogContext.IsLogActive)
{
var logKey = "Lucene query";
LogContext.Log(context.ActiveModuleId, logKey, "Filter", def.Filter.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "Query", def.Query.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "Sort", def.Sort.ToString());
LogContext.Log(context.ActiveModuleId, logKey, "PageIndex", def.PageIndex);
LogContext.Log(context.ActiveModuleId, logKey, "PageSize", def.PageSize);
}
SearchResults docs = LuceneController.Instance.Search(GetModuleId(context).ToString(), def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex);
int total = docs.TotalResults;
var dataList = new List <IDataItem>();
foreach (string item in docs.ids)
{
var content = ctrl.GetContent(int.Parse(item));
if (content != null)
{
dataList.Add(CreateDefaultDataItem(content));
}
else
{
Log.Logger.DebugFormat("OpenContentDataSource.GetAll() ContentItem not found [{0}]", item);
}
}
return(new DefaultDataItems()
{
Items = dataList,
Total = total,
DebugInfo = def.Filter.ToString() + " - " + def.Query.ToString() + " - " + def.Sort.ToString()
});
}
}
public HttpResponseMessage List(RequestDTO req)
{
try
{
Log.Logger.DebugFormat("OpenFiles.JplistApiController.List() called with request [{0}].", req.ToJson());
FieldConfig indexConfig = FilesRepository.GetIndexConfig(PortalSettings.PortalId);
bool addWorkFlow = PortalSettings.UserMode != PortalSettings.Mode.Edit;
QueryBuilder queryBuilder = new QueryBuilder(indexConfig);
queryBuilder.BuildFilter(PortalSettings.PortalId, req.folder, addWorkFlow, PortalSettings.UserInfo.Social.Roles);
JplistQueryBuilder.MergeJpListQuery(FilesRepository.GetIndexConfig(PortalSettings), queryBuilder.Select, req.StatusLst, DnnLanguageUtils.GetCurrentCultureCode());
string curFolder = NormalizePath(req.folder);
foreach (var item in queryBuilder.Select.Query.FilterRules.Where(f => f.Field == LuceneMappingUtils.FolderField))
{
curFolder = NormalizePath(item.Value.AsString);
item.Value = new StringRuleValue(NormalizePath(item.Value.AsString)); //any file of current folder
}
var def = new SelectQueryDefinition();
def.Build(queryBuilder.Select);
var docs = LuceneController.Instance.Search(def);
int total = docs.TotalResults;
var ratio = string.IsNullOrEmpty(req.imageRatio) ? new Ratio(100, 100) : new Ratio(req.imageRatio);
Log.Logger.DebugFormat("OpenFiles.JplistApiController.List() Searched for [{0}], found [{1}] items", def.Filter.ToString() + " / " + def.Query.ToString(), total);
//if (LogContext.IsLogActive)
//{
// var logKey = "Query";
// LogContext.Log(ActiveModule.ModuleID, logKey, "select", queryBuilder.Select);
// LogContext.Log(ActiveModule.ModuleID, logKey, "debuginfo", dsItems.DebugInfo);
// LogContext.Log(ActiveModule.ModuleID, logKey, "model", model);
// model["Logs"] = JToken.FromObject(LogContext.Current.ModuleLogs(ActiveModule.ModuleID));
//}
var fileManager = FileManager.Instance;
var data = new List <FileDTO>();
var breadcrumbs = new List <IFolderInfo>();
if (req.withSubFolder)
{
//hier blijken we resultaten toe te voegen die niet uit lucene komen
breadcrumbs = AddFolders(NormalizePath(req.folder), curFolder, fileManager, data, ratio);
}
foreach (var doc in docs.ids)
{
IFileInfo f = fileManager.GetFile(doc.FileId);
if (f == null)
{
//file seems to have been deleted
LuceneController.Instance.Delete(LuceneMappingUtils.CreateLuceneItem(doc.PortalId, doc.FileId));
total -= 1;
}
else
{
if (f.FileName == "_folder.jpg")
{
continue; // skip
}
dynamic title = null;
var custom = GetCustomFileDataAsDynamic(f);
if (custom != null && custom.meta != null)
{
try
{
title = Normalize.DynamicValue(custom.meta.title, "");
}
catch (Exception ex)
{
Log.Logger.Debug("OpenFiles.JplistApiController.List() Failed to get title.", ex);
}
}
data.Add(new FileDTO()
{
Id = f.FileId,
Name = Normalize.DynamicValue(title, f.FileName),
FileName = f.FileName,
CreatedOnDate = f.CreatedOnDate,
LastModifiedOnDate = f.LastModifiedOnDate,
FolderName = f.Folder,
Url = fileManager.GetUrl(f),
IsImage = fileManager.IsImageFile(f),
ImageUrl = ImageHelper.GetImageUrl(f, ratio),
Custom = custom,
IconUrl = GetFileIconUrl(f.Extension),
IsEditable = IsEditable,
EditUrl = IsEditable ? GetFileEditUrl(f) : ""
});
}
}
var res = new ResultExtDTO <FileDTO>()
{
data = new ResultDataDTO <FileDTO>()
{
items = data,
breadcrumbs = breadcrumbs.Select(f => new ResultBreadcrumbDTO
{
name = f.FolderName,
path = f.FolderPath.Trim('/')
})
},
count = total
};
return(Request.CreateResponse(HttpStatusCode.OK, res));
}
catch (Exception exc)
{
Log.Logger.Error(exc);
return(Request.CreateErrorResponse(HttpStatusCode.InternalServerError, exc));
}
}
public override IDataItems GetAll(DataSourceContext context, Select selectQuery)
{
if (context.Index && selectQuery != null)
{
SelectQueryDefinition def = BuildQuery(context, selectQuery);
SearchResults docs = LuceneController.Instance.Search(LUCENE_SCOPE, def.Filter, def.Query, def.Sort, def.PageSize, def.PageIndex);
int total = docs.TotalResults;
var dataList = new List <IDataItem>();
foreach (string item in docs.ids)
{
var user = UserController.GetUserById(context.PortalId, int.Parse(item));
if (user != null)
{
dataList.Add(ToData(user));
}
else
{
Log.Logger.Debug($"DnnUsersDataSource.GetAll() ContentItem not found [{item}]");
}
}
return(new DefaultDataItems()
{
Items = dataList,
Total = total,
DebugInfo = def.Filter + " - " + def.Query + " - " + def.Sort
});
}
else
{
int pageIndex = 0;
int pageSize = 1000;
int total = 0;
IEnumerable <UserInfo> users;
if (selectQuery != null)
{
pageIndex = selectQuery.PageIndex;
pageSize = selectQuery.PageSize;
var ruleDisplayName = selectQuery.Query.FilterRules.FirstOrDefault(f => f.Field == "DisplayName");
var ruleRoles = selectQuery.Query.FilterRules.FirstOrDefault(f => f.Field == "Roles");
if (ruleDisplayName != null)
{
string displayName = ruleDisplayName.Value.AsString + "%";
users = UserController.GetUsersByDisplayName(context.PortalId, displayName, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>();
}
else
{
users = UserController.GetUsers(context.PortalId, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>();
total = users.Count();
}
if (ruleRoles != null)
{
var roleNames = ruleRoles.MultiValue.Select(r => r.AsString).ToList();
users = users.Where(u => u.Roles.Intersect(roleNames).Any());
}
}
else
{
users = UserController.GetUsers(context.PortalId, pageIndex, pageSize, ref total, true, false).Cast <UserInfo>();
}
users = users.Where(u => !u.IsInRole("Administrators"));
//users = users.Skip(pageIndex * pageSize).Take(pageSize);
var dataList = new List <IDataItem>();
foreach (var user in users)
{
dataList.Add(ToData(user));
}
return(new DefaultDataItems()
{
Items = dataList,
Total = total,
//DebugInfo =
});
}
}
/// <summary>
/// Adds the given JToken to the specified Document.
/// </summary>
/// <param name="doc">
/// The Document to add to.
/// </param>
/// <param name="prefix">
/// The prefix to use for field names.
/// </param>
/// <param name="token">
/// The JToken to add.
/// </param>
/// <param name="fieldconfig"></param>
private static void Add(Document doc, string prefix, JToken token, FieldConfig fieldconfig)
{
if (token is JObject)
{
AddProperties(doc, prefix, token as JObject, fieldconfig);
}
else if (token is JArray)
{
var itemsConfig = fieldconfig?.Items;
if (fieldconfig != null && fieldconfig.Index && itemsConfig == null)
{
throw new Exception($"Error indexing Array field {prefix}. No 'Items' section defined in index.json. Please fix your index.json.");
}
AddArray(doc, prefix, token as JArray, itemsConfig);
}
else if (token is JValue)
{
JValue value = token as JValue;
bool index = false;
bool sort = false;
if (fieldconfig != null)
{
index = fieldconfig.Index;
sort = fieldconfig.Sort;
if (fieldconfig.IndexType == "datetime" && value.Type == JTokenType.String)
{
DateTime d;
if (DateTime.TryParse(value.Value.ToString(), null, System.Globalization.DateTimeStyles.RoundtripKind, out d))
{
value = new JValue(d);
}
}
}
switch (value.Type) //todo: simple date gets detected as string
{
case JTokenType.Boolean:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetIntValue((bool)value.Value ? 1 : 0));
}
break;
case JTokenType.Date:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((DateTime)value.Value).Ticks));
//doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
/*
* if (field != null ){
* if (field.IndexType == "datetime")
* {
* doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
* }
* else if (field.IndexType == "date")
* {
* doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.DAY), Field.Store.NO, Field.Index.NOT_ANALYZED));
* }
* else if (field.IndexType == "time")
* {
* doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND).Substring(8), Field.Store.NO, Field.Index.NOT_ANALYZED));
* }
* }
* else
* {
* doc.Add(new Field(prefix, DateTools.DateToString((DateTime)value.Value, DateTools.Resolution.SECOND), Field.Store.NO, Field.Index.NOT_ANALYZED));
* }
*/
}
break;
case JTokenType.Float:
if (index || sort)
{
if (value.Value is float)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)value.Value));
}
else
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToDouble(value.Value)));
}
}
break;
case JTokenType.Guid:
if (index || sort)
{
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
break;
case JTokenType.Integer:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetFloatValue((float)Convert.ToInt64(value.Value)));
}
break;
case JTokenType.Null:
break;
case JTokenType.String:
if (fieldconfig != null && fieldconfig.IndexType == "key")
{
doc.Add(new Field(prefix, QueryParser.Escape(value.Value.ToString()), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
else if (fieldconfig != null && fieldconfig.IndexType == "html")
{
if (index)
{
doc.Add(new Field(prefix, CleanHtml(value.Value.ToString(), true), Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, CleanHtml(Truncate(value.Value.ToString(), 100), true), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
else if (fieldconfig != null && fieldconfig.IndexType == "file")
{
var val = value.Value.ToString();
if (!string.IsNullOrEmpty(val))
{
var fileIndexer = FileIndexerManager.GetFileIndexer(val);
if (fileIndexer != null)
{
var content = fileIndexer.GetContent(val);
if (index)
{
doc.Add(new Field(prefix, content, Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, Truncate(content, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
}
}
else
{
var val = SelectQueryDefinition.RemoveDiacritics(value.Value.ToString());
val = val.Replace('-', ' '); // concider '-' as a space
val = val.Replace(',', ' '); // concider ',' as a space
//var val = LuceneUtils.CleanupText(value.Value.ToString());
if (index)
{
doc.Add(new Field(prefix, val, Field.Store.NO, Field.Index.ANALYZED));
}
if (sort)
{
doc.Add(new Field("@" + prefix, Truncate(val, 100), Field.Store.NO, Field.Index.NOT_ANALYZED));
}
}
break;
case JTokenType.TimeSpan:
if (index || sort)
{
doc.Add(new NumericField(prefix, Field.Store.NO, true).SetLongValue(((TimeSpan)value.Value).Ticks));
}
break;
case JTokenType.Uri:
if (index || sort)
{
doc.Add(new Field(prefix, value.Value.ToString(), Field.Store.NO, Field.Index.ANALYZED));
}
break;
default:
Debug.Fail("Unsupported JValue type: " + value.Type);
break;
}
}
else
{
Debug.Fail("Unsupported JToken: " + token);
}
}
