public IndexResult RemoveTestCaseIndex(TestCaseDTO testCase)
{
if (testCase.TestCaseID == null)
{
return(new IndexResult());
}
var entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
var doc = entityIndex.FindDocumentByName <EntityDocument>(EntityDocument.CreateName(testCase.TestCaseID));
if (doc == null)
{
_log.Error(string.Format("CANNOT PROCESS UPDATE '{0}DeletedMessage' FOR ENTITY #{1} - '{2}'. ENTITY WAS NOT ADDED DURING PROFILE INITIALIZATION OR ENTITY CREATION !!!", testCase.EntityTypeName, testCase.TestCaseID.GetValueOrDefault(), testCase.Name));
return(new IndexResult());
}
var indexResult = entityIndex.Rebuild(CreateEmptyEntityDocument(doc.DocNumber, EntityDocument.CreateName(testCase.TestCaseID)));
_log.Debug(string.Format("Removed {0} #{1} - '{2}'", testCase.EntityTypeName, testCase.TestCaseID.GetValueOrDefault(), testCase.Name));
if (doc.DocNumber >= 0)
{
IDocumentIndex entityProjectIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityProject);
entityProjectIndex.Update(doc.DocNumber, string.Empty);
IDocumentIndex entityTypeIndices = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityType);
entityTypeIndices.Update(doc.DocNumber, string.Empty);
}
return(indexResult);
}
public IndexResult RemoveCommentIndex(CommentDTO comment)
{
if (comment.CommentID == null || comment.GeneralID == null)
{
return(new IndexResult());
}
var entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
var commentEntity = entityIndex.FindDocumentByName <EntityDocument>(EntityDocument.CreateName(comment.GeneralID));
if (commentEntity == null)
{
_log.Error(string.Format("CANNOT PROCESS 'CommentRemovedMessage' FOR COMMENT#{0} FOR ENTITY #{1} - '{2}'. ENTITY WAS NOT ADDED DURING PROFILE INITIALIZATION OR ENTITY CREATION !!!", comment.CommentID.GetValueOrDefault(), comment.GeneralID.GetValueOrDefault(), comment.GeneralName));
return(new IndexResult());
}
var commentIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Comment);
var indexResult = commentIndex.Rebuild(CreateEmptyCommentDocument(comment.CommentID.Value), true);
_log.Debug(string.Format("Removed comment #{0} from #{1} - '{2}'", comment.CommentID.GetValueOrDefault(), comment.GeneralID.GetValueOrDefault(), comment.GeneralName));
if (indexResult.DocNumber >= 0)
{
IDocumentIndex commentProjectIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.CommentProject);
commentProjectIndex.Update(indexResult.DocNumber, string.Empty);
IDocumentIndex commentSquadIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.CommentSquad);
commentSquadIndex.Update(indexResult.DocNumber, string.Empty);
IDocumentIndex commentEntityType = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.CommentEntityType);
commentEntityType.Update(indexResult.DocNumber, string.Empty);
}
return(indexResult);
}
public IndexResult AddGeneralIndex(GeneralDTO general)
{
if (general.GeneralID == null)
{
return(new IndexResult());
}
if (Exists <EntityDocument>(general.GeneralID.Value, DocumentIndexTypeToken.Entity))
{
if (_profile.Initialized)
{
_log.WarnFormat("CANNOT PROCESS '{0}CreatedMessage' FOR ENTITY #{1} - '{2}'. ENTITY HAS ALREADY BEEN ADDED ON PROFILE INITIALIZATION OR ENTITY CREATION !!!", general.EntityTypeName, general.GeneralID.GetValueOrDefault(), general.Name);
}
else
{
_log.ErrorFormat("CANNOT PROCESS '{0}CreatedMessage' FOR ENTITY #{1} - '{2}'. ENTITY HAS ALREADY BEEN ADDED !!!", general.EntityTypeName, general.GeneralID.GetValueOrDefault(), general.Name);
}
return(new IndexResult());
}
IDocumentIndex entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
EntityDocument document = _documentFactory.CreateGeneral(general);
IndexResult indexResult = document == null ? new IndexResult() : entityIndex.Index(document, false);
if (indexResult.DocNumber != -1)
{
IDocumentIndex projectContextIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityProject);
projectContextIndex.Index(indexResult.DocNumber, _documentIdFactory.EncodeProjectId(general.ParentProjectID));
Maybe <string> maybeEntityTypeName = _entityTypeProvider.GetEntityTypeName(general.EntityTypeID);
string entityTypeName = maybeEntityTypeName.FailIfNothing(() => new ApplicationException("Entity type name was not found {0}".Fmt(general.EntityTypeID)));
IDocumentIndex entityTypeIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityType);
entityTypeIndex.Index(indexResult.DocNumber, entityTypeName);
}
_log.Debug(string.Format("Added {0} #{1} - '{2}':{3}", general.EntityTypeName, general.GeneralID.GetValueOrDefault(), general.Name, indexResult.WordsAdded.Any() ? string.Format(" added words - {0};", string.Join(",", indexResult.WordsAdded.Keys)) : " NO WORDS ADDED;"));
return(indexResult);
}
private Tuple <IEnumerable <EntityDocument>, int> FindEntities(Lazy <WAHBitArray> plan, Page page, out int entitiesTotalCount)
{
IDocumentIndex entityIndexes = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
IEnumerable <EntityDocument> found = entityIndexes.Find <EntityDocument>(plan, page.Number, page.Size, 0, out entitiesTotalCount);
var lastDocument = entityIndexes.GetLastDocument <EntityDocument>();
return(Tuple.Create(found, lastDocument != null ? int.Parse(lastDocument.FileName) : 0));
}
public void Index(ContentItem contentItem, IDocumentIndex documentIndex) {
var indexContentContext = new IndexContentContext(contentItem, documentIndex);
// dispatch to handlers to retrieve index information
Handlers.Invoke(handler => handler.Indexing(indexContentContext), Logger);
Handlers.Invoke(handler => handler.Indexed(indexContentContext), Logger);
}
private void BornOrRessurectIfDead()
{
if (_documentIndex == null || !_documentIndex.IsAlive)
{
_logger.DebugFormat("[{0}] Create index because {1}", _documentIndexType.TypeToken, _documentIndex == null ? "there is no current index" : "current index is not alive");
_documentIndex = _documentIndexFactory();
}
}
public Maybe <QueryPlan> Build(QueryData data, DocumentIndexTypeToken projectContextType, DocumentIndexTypeToken _, DocumentIndexTypeToken __)
{
var query = _indexDataFactory.CreateImpedimentData(false, data.LoggedUserId, data.LoggedUserId);
IDocumentIndex impedimentContextIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, DocumentIndexTypeToken.Impediment);
var impedimentPlan = impedimentContextIndex.BuildExecutionPlan(new ParsedQuery(words: query), _profile.Initialized);
var projectContextPlan = _plansBuilder.BuildProjectContextPlan(data.ProjectIds, data.IncludeNoProject, projectContextType);
return(projectContextPlan.And(Maybe.Return(impedimentPlan)));
}
public override void PopulateDocumentIndex(IDocumentIndex document, out string description)
{
document.Add("title", InstanceName).Analyze().Store();
document.Add("meta_title", MetaTitle).Analyze();
document.Add("meta_keywords", MetaKeywords).Analyze();
document.Add("meta_description", MetaDescription).Analyze();
document.Add("body", BodyContent).Analyze().Store();
description = BodyContent;
}
public IndexResult UpdateGeneralIndex(GeneralDTO general, ICollection <GeneralField> changedFields)
{
if (general.GeneralID == null)
{
return(new IndexResult());
}
IEnumerable <IDocumentIndex> indexes = _documentIndexProvider.GetOrCreateDocumentIndexes(_pluginContext.AccountName, DocumentIndexTypeToken.Entity, DocumentIndexTypeToken.EntityProject, DocumentIndexTypeToken.EntityType);
if (!indexes.Any(i => changedFields.Any(f => i.Type.IsBelongedToIndexFields(f) || i.Type.IsBelongedToDocumentFields(f))))
{
return(new IndexResult());
}
IDocumentIndex entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
var document = entityIndex.FindDocumentByName <EntityDocument>(EntityDocument.CreateName(general.GeneralID));
if (document == null)
{
_log.Error(string.Format("CANNOT PROCESS UPDATE '{0}UpdatedMessage' FOR ENTITY #{1} - '{2}'. ENTITY WAS NOT ADDED DURING PROFILE INITIALIZATION OR ENTITY CREATION !!!", general.EntityTypeName, general.GeneralID.GetValueOrDefault(), general.Name));
return(new IndexResult());
}
if (changedFields.Any(f => entityIndex.Type.IsBelongedToDocumentFields(f)))
{
document.ProjectId = _documentIdFactory.CreateProjectId(general.ParentProjectID.GetValueOrDefault());
document.EntityTypeId = _documentIdFactory.CreateEntityTypeId(general.EntityTypeID.GetValueOrDefault());
entityIndex.SaveDocument(document, false);
_log.Debug(string.Format("Updated {0} #{1} - '{2}':{3}{4}", general.EntityTypeName, general.GeneralID.GetValueOrDefault(), general.Name, changedFields.Contains(GeneralField.ParentProjectID) ? string.Format(" Project - {0};", string.Join(",", general.ParentProjectName)) : string.Empty, changedFields.Contains(GeneralField.EntityTypeID) ? string.Format(" EntityType - {0};", string.Join(",", general.EntityTypeName)) : string.Empty));
}
if (changedFields.Any(f => entityIndex.Type.IsBelongedToIndexFields(f)))
{
var text = _textOperations.Prepare(string.Format("{0} {1} ", general.Name, general.Description ?? string.Empty));
var indexResult = entityIndex.Update(document.FileName, text);
_log.Debug(string.Format("Updated {0} #{1} - '{2}':{3}{4}", general.EntityTypeName,
general.GeneralID.GetValueOrDefault(), general.Name,
indexResult.WordsAdded.Any() ? string.Format(" added words - {0};", string.Join(",", indexResult.WordsAdded.Keys)) : " NO WORDS ADDED;",
indexResult.WordsRemoved.Any() ? string.Format(" removed words - {0};", string.Join(",", indexResult.WordsRemoved)) : " NO WORDS REMOVED;"));
}
IDocumentIndex entityProjectIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityProject);
if (changedFields.Any(f => entityProjectIndex.Type.IsBelongedToIndexFields(f)) && document.DocNumber >= 0)
{
entityProjectIndex.Update(document.DocNumber, general.ParentProjectID.HasValue ? _documentIdFactory.EncodeProjectId(general.ParentProjectID.Value) : string.Empty);
_localBus.SendLocal(new GeneralProjectChangedLocalMessage
{
GeneralId = general.GeneralID.Value,
ProjectId = general.ParentProjectID
});
}
IDocumentIndex entityTypeIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.EntityType);
if (changedFields.Any(f => entityTypeIndex.Type.IsBelongedToIndexFields(f)) && document.DocNumber >= 0)
{
entityTypeIndex.Index(document.DocNumber, general.EntityTypeName);
}
return(new IndexResult {
DocNumber = document.DocNumber
});
}
public DocumentResult(Document doc)
{
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
this.lexicon = FactoryLexicon.GetLexicon();
this.DocID = doc.DocID;
this.File = doc.File;
this.Title = doc.Title;
this.WordQuantity = doc.WordQuantity;
}
RankFunctionPivotedLengthNormVSM()
{
this.engConf = EngineConfiguration.Instance;
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
this.s = engConf.SNormalizationfactor;
this.totalDocQuantity = docIndex.GetQuantity();
this.avdl = docIndex.GetAverageDocumentLenght();
}
RankFunctionBM25_Okapi()
{
this.engConf = EngineConfiguration.Instance;
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
this.b = engConf.BNormalizationfactor;
this.k1 = engConf.BM25OkapiK1factor;
this.k3 = engConf.BM25OkapiK3factor;
this.totalDocQuantity = docIndex.GetQuantity();
this.avdl = docIndex.GetAverageDocumentLenght();
}
private Maybe <QueryPlan> BuildNoSquadEntityProjectContextPlan(QueryData data, DocumentIndexTypeToken project, DocumentIndexTypeToken entityType)
{
var projectContextPlan = BuildProjectContextPlan(data.ProjectIds, data.IncludeNoProject, project);
if (!projectContextPlan.HasValue)
{
return(Maybe.Nothing);
}
string noSquadEntityTypeIdsQuery = String.Join(" ", _entityTypeProvider.NoSquadEntityTypeNames);
IDocumentIndex entityTypeIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, entityType);
QueryPlan noSquadEntityPlan = entityTypeIndex.BuildExecutionPlan(new ParsedQuery(words: noSquadEntityTypeIdsQuery), _profile.Initialized);
return(Maybe.Return(QueryPlan.And(noSquadEntityPlan, projectContextPlan.Value)));
}
private Maybe <Lazy <WAHBitArray> > BuildNoSquadEntityProjectContextPlan(QueryData data, DocumentIndexTypeToken project, DocumentIndexTypeToken entityType)
{
var projectContextPlan = BuildProjectContextPlan(data.ProjectIds, data.IncludeNoProject, project);
if (!projectContextPlan.HasValue)
{
return(Maybe.Nothing);
}
string noSquadEntityTypeIdsQuery = String.Join(" ", _entityTypeProvider.NoSquadEntityTypeNames);
IDocumentIndex entityTypeIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_accountName, entityType);
Lazy <WAHBitArray> noSquadEntityPlan = entityTypeIndex.BuildExecutionPlan(noSquadEntityTypeIdsQuery, _profile.Initialized);
return(Maybe.Return(Lazy.Create(() => noSquadEntityPlan.Value.And(projectContextPlan.Value.Value))));
}
public DocumentResult(int docID)
{
Document docTmp;
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
this.lexicon = FactoryLexicon.GetLexicon();
docTmp = docIndex.Search(docID);
this.DocID = DocID;
this.File = docTmp.File;
this.Title = docTmp.Title;
this.WordQuantity = docTmp.WordQuantity;
}
public Maybe <QueryPlan> BuildProjectContextPlan(IEnumerable <int> projectIds, bool includeNoProject, DocumentIndexTypeToken projectIndexTypeToken)
{
ProjectIndexData projectIdsIndexData = projectIds != null ? new ProjectIndexData(projectIds.Cast <int?>().ToList()) : ProjectIndexData.Empty;
ProjectIndexData nullProjectIndexData = includeNoProject ? new ProjectIndexData(new int?[] { null }) : ProjectIndexData.Empty;
var result = ProjectIndexData.Sum(projectIdsIndexData, nullProjectIndexData);
string query = result.ToString();
if (string.IsNullOrEmpty(query))
{
return(Maybe.Nothing);
}
IDocumentIndex projectContextIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, projectIndexTypeToken);
return(projectContextIndex.BuildExecutionPlan(new ParsedQuery(words: query), _profile.Initialized));
}
private IDocumentIndex DoGetOrCreateDocumentIndex(IPluginContext context, DocumentIndexTypeToken documentIndexTypeToken)
{
ConcurrentDictionary <string, Lazy <IDocumentIndex> > storage = _documentIndexes[documentIndexTypeToken];
var index = storage.GetOrAdd(context.AccountName.Value, key => Lazy.Create(() =>
{
IDocumentIndex documentIndex = CreateDocumentIndex(context, documentIndexTypeToken);
var otherVersions = documentIndex.Type.GetVersions(context.AccountName, _documentIndexSetup).Except(new[] { documentIndex.Type.Version });
foreach (int version in otherVersions)
{
IDocumentIndex versionedDocumentIndex = CreateDocumentIndex(context, documentIndexTypeToken, version);
versionedDocumentIndex.Shutdown(new DocumentIndexShutdownSetup(forceShutdown: true, cleanStorage: true));
}
return(documentIndex);
}));
return(index.Value);
}
public IndexResult UpdateCommentIndex(CommentDTO comment, ICollection <CommentField> changedFields, Maybe <int?> projectId, Maybe <int?> squadId)
{
if (comment.CommentID == null || comment.GeneralID == null)
{
return(new IndexResult());
}
var entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Entity);
var commentEntityDocument = entityIndex.FindDocumentByName <EntityDocument>(EntityDocument.CreateName(comment.GeneralID));
if (commentEntityDocument == null)
{
_log.Error(string.Format("CANNOT PROCESS 'CommentUpdatedMessage' FOR COMMENT#{0} FOR ENTITY #{1} - '{2}'. ENTITY WAS NOT ADDED DURING PROFILE INITIALIZATION OR ENTITY CREATION !!!", comment.CommentID.GetValueOrDefault(), comment.GeneralID.GetValueOrDefault(), comment.GeneralName));
return(new IndexResult());
}
var commentIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.Comment);
var commentDocument = commentIndex.FindDocumentByName <hOOt.Document>(EntityDocument.CreateName(comment.CommentID));
if (commentDocument == null)
{
_log.Error(string.Format("CANNOT PROCESS 'CommentUpdatedMessage' FOR COMMENT#{0} FOR ENTITY #{1} - '{2}'. COMMENT WAS NOT ADDED DURING PROFILE INITIALIZATION OR ENTITY CREATION !!!", comment.CommentID.GetValueOrDefault(), comment.GeneralID.GetValueOrDefault(), comment.GeneralName));
return(new IndexResult());
}
if (commentDocument.DocNumber >= 0)
{
if (projectId.HasValue)
{
IDocumentIndex commentProjectIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.CommentProject);
int? projectIdValue = projectId.Value;
commentProjectIndex.Update(commentDocument.DocNumber, _documentIdFactory.EncodeProjectId(projectIdValue));
}
if (squadId.HasValue)
{
IDocumentIndex commentSquadIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext.AccountName, DocumentIndexTypeToken.CommentSquad);
int? squadIdValue = squadId.Value;
commentSquadIndex.Update(commentDocument.DocNumber, _documentIdFactory.EncodeSquadId(squadIdValue));
}
if (changedFields.Any(f => commentIndex.Type.IsBelongedToIndexFields(f)))
{
var text = _textOperations.Prepare(string.Format("{0}", comment.Description ?? string.Empty));
var indexResult = commentIndex.Update(commentDocument.FileName, text);
_log.Debug(string.Format("Updated comment #{0} for #{1} - '{2}':{3}{4}", comment.CommentID.GetValueOrDefault(), comment.GeneralID.GetValueOrDefault(), comment.GeneralName, indexResult.WordsAdded.Any() ? string.Format(" added words - {0};", string.Join(",", indexResult.WordsAdded.Keys)) : " NO WORDS ADDED;", indexResult.WordsRemoved.Any() ? string.Format(" removed words - {0};", string.Join(",", indexResult.WordsRemoved)) : " NO WORDS REMOVED;"));
return(indexResult);
}
}
return(new IndexResult());
}
private Maybe <Lazy <WAHBitArray> > CreateEntityStatePlan(IEnumerable <int> entityStateIds)
{
if (entityStateIds == null)
{
return(Maybe.Nothing);
}
if (!entityStateIds.Any())
{
return(Maybe.Nothing);
}
var queryBuf = new List <string>();
IDocumentIndex entityStateIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_accountName, DocumentIndexTypeToken.EntityState);
queryBuf.AddRange(entityStateIds.Select(entityStateId => _documentIdFactory.EncodeEntityStateId(entityStateId)));
string query = string.Join(" ", queryBuf.ToArray());
return(entityStateIndex.BuildExecutionPlan(query, _profile.Initialized));
}
private Maybe <QueryPlan> CreateEntityStatePlan(IEnumerable <int> entityStateIds)
{
if (entityStateIds == null)
{
return(Maybe.Nothing);
}
if (!entityStateIds.Any())
{
return(Maybe.Nothing);
}
var queryBuf = new List <string>();
IDocumentIndex entityStateIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, DocumentIndexTypeToken.EntityState);
queryBuf.AddRange(entityStateIds.Select(entityStateId => _indexDataFactory.CreateEntityStateData(entityStateId)));
string query = string.Join(" ", queryBuf.ToArray());
return(entityStateIndex.BuildExecutionPlan(new ParsedQuery(words: query), _profile.Initialized));
}
private QueryResult <EntityDocument> FindEntities(Maybe <QueryPlanResult> plan, Page page, out int entitiesTotalCount)
{
if (!plan.HasValue)
{
entitiesTotalCount = 0;
return(new QueryResult <EntityDocument>
{
Documents = Enumerable.Empty <EntityDocument>(),
LastIndexedId = 0
});
}
IDocumentIndex entityIndexes = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, DocumentIndexTypeToken.Entity);
IEnumerable <EntityDocument> found = entityIndexes.Find <EntityDocument>(plan.Value, page.Number, page.Size, 0, out entitiesTotalCount);
var lastDocument = entityIndexes.GetLastDocument <EntityDocument>();
return(new QueryResult <EntityDocument>
{
Documents = found,
LastIndexedId = lastDocument != null?int.Parse(lastDocument.FileName) : 0
});
}
private Maybe <Lazy <WAHBitArray> > BuildSquadPlan(IEnumerable <int> squadIds, bool includeNoTeam, DocumentIndexTypeToken squadContextType)
{
var queryBuf = new List <string>();
if (squadIds != null)
{
queryBuf.AddRange(squadIds.Select(squadId => _documentIdFactory.EncodeSquadId(squadId)));
}
if (includeNoTeam)
{
queryBuf.Add(_documentIdFactory.EncodeSquadId(null));
}
if (!queryBuf.Any())
{
return(Maybe.Nothing);
}
string query = String.Join(" ", queryBuf.ToArray());
IDocumentIndex squadIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_accountName, squadContextType);
return(squadIndex.BuildExecutionPlan(query, _profile.Initialized));
}
private Maybe <Lazy <WAHBitArray> > BuildProjectContextPlan(IEnumerable <int> projectIds, bool includeNoProject, DocumentIndexTypeToken documentIndexTypeToken)
{
var queryBuf = new List <string>();
if (projectIds != null)
{
queryBuf.AddRange(projectIds.Select(projectId => _documentIdFactory.EncodeProjectId(projectId)));
}
if (includeNoProject)
{
queryBuf.Add(_documentIdFactory.EncodeProjectId(null));
}
if (!queryBuf.Any())
{
return(Maybe.Nothing);
}
string query = String.Join(" ", queryBuf.ToArray());
IDocumentIndex projectContextIndices = _documentIndexProvider.GetOrCreateDocumentIndex(_accountName, documentIndexTypeToken);
return(projectContextIndices.BuildExecutionPlan(query, _profile.Initialized));
}
Maybe <QueryPlan> IProjectContextQueryPlanBuilder.BuildProjectContextPlan(IEnumerable <int> projectIds, bool includeNoProject, DocumentIndexTypeToken projectIndexTypeToken)
{
var queryBuf = new List <string>();
if (projectIds != null)
{
queryBuf.AddRange(projectIds.Select(projectId => _indexDataFactory.CreateProjectData(projectId)));
}
if (includeNoProject)
{
queryBuf.Add(_indexDataFactory.CreateProjectData(null));
}
if (!queryBuf.Any())
{
return(Maybe.Nothing);
}
string query = String.Join(" ", queryBuf.ToArray());
IDocumentIndex projectContextIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, projectIndexTypeToken);
return(projectContextIndex.BuildExecutionPlan(new ParsedQuery(words: query), _profile.Initialized));
}
private Maybe <QueryPlan> BuildSquadPlan(IEnumerable <int> squadIds, bool includeNoTeam, DocumentIndexTypeToken squadContextType)
{
var queryBuf = new List <string>();
if (squadIds != null)
{
queryBuf.AddRange(squadIds.Select(squadId => _indexDataFactory.CreateSquadData(squadId)));
}
if (includeNoTeam)
{
queryBuf.Add(_indexDataFactory.CreateSquadData(null));
}
if (!queryBuf.Any())
{
return(Maybe.Nothing);
}
string query = String.Join(" ", queryBuf.ToArray());
IDocumentIndex squadIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, squadContextType);
return(squadIndex.BuildExecutionPlan(new ParsedQuery(words: query), _profile.Initialized));
}
public void IsDirtyShouldBeTrueWhenIndexIsModified()
{
IDocumentIndex doc = _provider.New(1);
doc.Add("foo", "value");
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", false);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", (float)1.0);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", 1);
Assert.That(doc.IsDirty, Is.True);
doc = _provider.New(1);
doc.Add("foo", DateTime.Now);
Assert.That(doc.IsDirty, Is.True);
}
private Task SaveIndex(IDocumentIndex index, string host)
{
var state = new
{
Index = index,
Host = host
};
{
var indexFile = GetIndexFile(state.Host);
lock (state.Index)
{
using (var indexStream = indexFile.OpenWrite())
{
state.Index.Save(indexStream);
}
}
Console.WriteLine("Index updated - " + host);
}
return(Task.FromResult(true));
}
public IndexContentContext(ContentItem contentItem, IDocumentIndex documentIndex)
: base(contentItem) {
DocumentIndex = documentIndex;
}
public void Store(string indexName, IDocumentIndex indexDocument)
{
Store(indexName, new[] { (ElasticSearchDocumentIndex)indexDocument });
}
public void Index(ContentItem contentItem, IDocumentIndex documentIndex)
{
throw new NotImplementedException();
}
IndexerMemory()
{
this.lexicon = FactoryLexicon.GetLexicon();
this.documentIndex = FactoryDocumentIndex.GetDocumentIndex();
this.repDoc = FactoryRepositoryDocument.GetRepositoryDocument(EnumRepositoryType.Folder);
}
/// <summary>
/// Indexes a batch of content items
/// </summary>
/// <returns>
/// <c>true</c> if there are more items to process; otherwise, <c>false</c>.
/// </returns>
private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings)
{
var addToIndex = new List <IDocumentIndex>();
var deleteFromIndex = new List <int>();
bool loop = false;
// Rebuilding the index ?
if (indexSettings.Mode == IndexingMode.Rebuild)
{
Logger.Information("Rebuilding index");
_indexingStatus = IndexingStatus.Rebuilding;
do
{
loop = true;
// load all content items
var contentItems = _contentRepository
.Table.Where(versionRecord => versionRecord.Latest && versionRecord.Id > indexSettings.LastContentId)
.OrderBy(versionRecord => versionRecord.Id)
.Take(ContentItemsPerLoop)
.ToList()
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
.Distinct()
.ToList();
// if no more elements to index, switch to update mode
if (contentItems.Count == 0)
{
indexSettings.Mode = IndexingMode.Update;
}
foreach (var item in contentItems)
{
try {
var settings = GetTypeIndexingSettings(item);
// skip items from types which are not indexed
if (settings.List.Contains(indexName))
{
if (item.HasPublished())
{
var published = _contentManager.Get(item.Id, VersionOptions.Published);
IDocumentIndex documentIndex = ExtractDocumentIndex(published);
if (documentIndex != null && documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
}
}
else if (settings.List.Contains(indexName + ":latest"))
{
IDocumentIndex documentIndex = ExtractDocumentIndex(item);
if (documentIndex != null && documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
}
indexSettings.LastContentId = item.VersionRecord.Id;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id);
}
}
if (contentItems.Count < ContentItemsPerLoop)
{
loop = false;
}
else
{
_transactionManager.RequireNew();
}
} while (loop);
}
if (indexSettings.Mode == IndexingMode.Update)
{
Logger.Information("Updating index");
_indexingStatus = IndexingStatus.Updating;
do
{
var indexingTasks = _taskRepository
.Table.Where(x => x.Id > indexSettings.LastIndexedId)
.OrderBy(x => x.Id)
.Take(ContentItemsPerLoop)
.ToList()
.GroupBy(x => x.ContentItemRecord.Id)
.Select(group => new { TaskId = group.Max(task => task.Id), Delete = group.Last().Action == IndexingTaskRecord.Delete, Id = group.Key, ContentItem = _contentManager.Get(group.Key, VersionOptions.Latest) })
.OrderBy(x => x.TaskId)
.ToArray();
foreach (var item in indexingTasks)
{
try {
IDocumentIndex documentIndex = null;
// item.ContentItem can be null if the content item has been deleted
if (item.ContentItem != null)
{
// skip items from types which are not indexed
var settings = GetTypeIndexingSettings(item.ContentItem);
if (settings.List.Contains(indexName))
{
if (item.ContentItem.HasPublished())
{
var published = _contentManager.Get(item.Id, VersionOptions.Published);
documentIndex = ExtractDocumentIndex(published);
}
}
else if (settings.List.Contains(indexName + ":latest"))
{
var latest = _contentManager.Get(item.Id, VersionOptions.Latest);
documentIndex = ExtractDocumentIndex(latest);
}
}
if (documentIndex == null || item.Delete)
{
deleteFromIndex.Add(item.Id);
}
else if (documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
indexSettings.LastIndexedId = item.TaskId;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during update", item.Id);
}
}
if (indexingTasks.Length < ContentItemsPerLoop)
{
loop = false;
}
else
{
_transactionManager.RequireNew();
}
} while (loop);
}
// save current state of the index
indexSettings.LastIndexedUtc = _clock.UtcNow;
_appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
if (deleteFromIndex.Count == 0 && addToIndex.Count == 0)
{
// nothing more to do
_indexingStatus = IndexingStatus.Idle;
return(false);
}
// save new and updated documents to the index
try {
if (addToIndex.Count > 0)
{
_indexProvider.Store(indexName, addToIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
// removing documents from the index
try {
if (deleteFromIndex.Count > 0)
{
_indexProvider.Delete(indexName, deleteFromIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while removing a document from the index");
}
return(true);
}
public IndexContentContext(ContentItem contentItem, IDocumentIndex documentIndex)
: base(contentItem)
{
DocumentIndex = documentIndex;
}
public void IsDirtyShouldBeFalseForNewDocuments()
{
IDocumentIndex doc = _provider.New(1);
Assert.That(doc.IsDirty, Is.False);
}
private QueryPlan CreateEntityQueryPlan(ParsedQuery parsedQuery)
{
IDocumentIndex entityIndex = _documentIndexProvider.GetOrCreateDocumentIndex(_pluginContext, DocumentIndexTypeToken.Entity);
return(entityIndex.BuildExecutionPlan(parsedQuery, _profile.Initialized));
}
private InvertedFileManager()
{
this.invertedfileName = string.Empty;
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
}
public void Store(string indexName, IDocumentIndex indexDocument) {
Store(indexName, new [] { (LuceneDocumentIndex)indexDocument });
}
RankFunctionBM25()
{
this.engConf = EngineConfiguration.Instance;
this.docIndex = FactoryDocumentIndex.GetDocumentIndex();
this.totalDocQuantity = docIndex.GetQuantity();
}
public DocumentIndex(DocumentIndexType charactersIndexType, DocumentIndexType digitsIndexType, IPluginContext context, Action shuttedDown, DocumentIndexSetup documentIndexSetup, IActivityLoggerFactory loggerFactory, DocumentIndexOptimizeHintFactory optimizeHintFactory)
{
_charactersIndex = new DocumentIndexTyped(charactersIndexType, context, shuttedDown, documentIndexSetup, loggerFactory, optimizeHintFactory);
_numberIndex = new DocumentIndexTyped(digitsIndexType, context, shuttedDown, documentIndexSetup, loggerFactory, optimizeHintFactory);
}
