// caller: IndexPopulator.Populator public void RepopulateTree(string path) { using (var traceOperation = Logger.TraceOperation("IndexPopulator RepopulateTree")) { var writer = IndexManager.GetIndexWriter(false); writer.DeleteDocuments(new Term(LucObject.FieldName.InTree, path.ToLower())); try { foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(path)) { var doc = IndexDocumentInfo.GetDocument(docData); if (doc == null) // indexing disabled { continue; } writer.AddDocument(doc); OnNodeIndexed(docData.Path); } writer.Optimize(); } finally { writer.Close(); } traceOperation.IsSuccessful = true; } }
// caller: IndexPopulator.Populator public void RepopulateTree(string path) { using (var op = SnTrace.Index.StartOperation("IndexPopulator RepopulateTree")) { var writer = IndexManager.GetIndexWriter(false); writer.DeleteDocuments(new Term(LucObject.FieldName.InTree, path.ToLowerInvariant())); try { var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes(); foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(path, excludedNodeTypes)) { var doc = IndexDocumentInfo.GetDocument(docData); if (doc == null) // indexing disabled { continue; } writer.AddDocument(doc); OnNodeIndexed(docData.Path); } writer.Optimize(); } finally { writer.Close(); } op.Successful = true; } }
public object GetIndexDocumentInfo(Node node, bool skipBinaries, bool isNew, out bool hasBinary) { var x = IndexDocumentInfo.Create(node, skipBinaries, isNew); hasBinary = x.HasBinaryField; return(x); }
private static void ValidateField(IndexDocumentInfo doc, string fieldName) { var field = doc.Fields.FirstOrDefault(f => f.Name == fieldName); if (field == null || string.IsNullOrEmpty(field.Value)) { throw new InvalidOperationException("Invalid empty field value for field: " + fieldName); } }
public static IndexDocumentInfo Create(Node node) { if (node == null) { throw new ArgumentNullException("node"); } var textEtract = new StringBuilder(); var doc = new IndexDocumentInfo(); doc._hasCustomField = node is IHasCustomIndexField; var ixnode = node as IIndexableDocument; if (ixnode == null) { doc.AddField(LucObject.FieldName.NodeId, node.Id, Field.Store.YES, true); doc.AddField(LucObject.FieldName.VersionId, node.VersionId, Field.Store.YES, true); doc.AddField(LucObject.FieldName.Version, node.Version.ToString().ToLower(), Field.Store.YES, Field.Index.ANALYZED); doc.AddField(LucObject.FieldName.CreatedById, node.CreatedById, Field.Store.YES, true); doc.AddField(LucObject.FieldName.ModifiedById, node.ModifiedById, Field.Store.YES, true); } else { var fieldNames = new List <string>(); foreach (var field in ixnode.GetIndexableFields()) { if (PostponedFields.Contains(field.Name)) { continue; } string extract; var lucFields = field.GetIndexFieldInfos(out extract); textEtract.AppendLine(extract); if (lucFields != null) { foreach (var lucField in lucFields) { fieldNames.Add(lucField.Name); doc.AddField(lucField); } } } } //doc.AddField(LucObject.FieldName.NodeTimestamp, node.NodeTimestamp, Field.Store.YES, true); //doc.AddField(LucObject.FieldName.VersionTimestamp, node.VersionTimestamp, Field.Store.YES, true); doc.AddField(LucObject.FieldName.IsInherited, node.IsInherited ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsMajor, node.Version.IsMajor ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsPublic, node.Version.Status == VersionStatus.Approved ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.AllText, textEtract.ToString(), Field.Store.NO, Field.Index.ANALYZED); return(doc); }
private IEnumerable <Fieldable> GetFieldsPrivate(IndexDocumentInfo info, IndexDocumentData docData) { var fields = new List <Fieldable>(); foreach (var provider in _providers) { var f = provider.GetFields(docData); if (f != null) { fields.AddRange(f); } } return(fields.Count == 0 ? null : fields); }
private static void UpdateDirtyDocuments(List <VersionInfo> infoList) { //-- select dirty documents var dirtyVersions = infoList.Where(i => !i.IsActualDocument && (i.OriginalIsPublic != i.ExpectedIsPublic || i.OriginalIsMajor != i.ExpectedIsMajor || i.OriginalIsLastDraft != i.ExpectedIsLastDraft || i.OriginalIsLastPublic != i.ExpectedIsLastPublic)).ToArray(); //-- play dirty documents var docs = IndexDocumentInfo.GetDocuments(dirtyVersions.Select(d => d.VersionId)); foreach (var doc in docs) { var versionId = Int32.Parse(doc.Get(LucObject.FieldName.VersionId)); foreach (var dirtyVersion in dirtyVersions) { if (dirtyVersion.VersionId == versionId) { dirtyVersion.Document = doc; SetDocumentFlags(dirtyVersion); var delTerm = new Term(KeyFieldName, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(dirtyVersion.VersionId)); //_writerRestartLock.EnterReadLock(); //try //{ // //in case the system was shut down in the meantime // if (!LuceneManager.Running) // return; // _writer.UpdateDocument(delTerm, dirtyVersion.Document); //} //finally //{ // _writerRestartLock.ExitReadLock(); //} using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDirtyDocuments { //in case the system was shut down in the meantime if (!LuceneManager.Running) { return; } _writer.UpdateDocument(delTerm, dirtyVersion.Document); } break; } } } }
internal static bool AddTree(string treeRoot, bool moveOrRename, int activityId, bool executingUnprocessedActivities) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: AddTree skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot); return(false); } SnTrace.Index.Write("LM: AddTree. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot); using (var wrFrame = IndexWriterFrame.Get(false)) { if (executingUnprocessedActivities) // pessimistic compensation { wrFrame.IndexWriter.DeleteDocuments(new Term("InTree", treeRoot), new Term("Path", treeRoot)); } var excludedNodeTypes = GetNotIndexedNodeTypes(); foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(treeRoot, excludedNodeTypes)) { Document document; int versionId; try { document = IndexDocumentInfo.GetDocument(docData); if (document == null) // indexing disabled { continue; } versionId = GetVersionIdFromDocument(document); } catch (Exception e) { var path = docData == null ? string.Empty : docData.Path ?? string.Empty; SnLog.WriteException(e, "Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path); SnTrace.Index.WriteError("LM: Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path); SnTrace.Index.WriteError("LM: Error during indexing: " + e); throw; } // pessimistic approach: delete document before adding it to avoid duplicate index documents wrFrame.IndexWriter.DeleteDocuments(GetVersionIdTerm(versionId)); wrFrame.IndexWriter.AddDocument(document); } } return(true); }
private static IEnumerable <DocumentUpdate> GetUpdates(VersioningInfo versioning) { var result = new List <DocumentUpdate>(versioning.Reindex.Length); var updates = IndexDocumentInfo.GetDocuments(versioning.Reindex); foreach (var doc in updates) { var verId = GetVersionIdFromDocument(doc); SetDocumentFlags(doc, verId, versioning); result.Add(new DocumentUpdate { UpdateTerm = GetVersionIdTerm(verId), Document = doc }); } return(result); }
/*======================================================================================================= IIndexPopulator Members */ // caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce public void ClearAndPopulateAll(bool backup = true) { var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId(); var commitData = CompletionState.GetCommitUserData(lastActivityId); using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll")) { // recreate var writer = IndexManager.GetIndexWriter(true); try { var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes(); foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes)) { var doc = IndexDocumentInfo.GetDocument(docData); if (doc == null) // indexing disabled { continue; } writer.AddDocument(doc); OnNodeIndexed(docData.Path); } RepositoryInstance.Instance.ConsoleWrite(" Commiting ... "); writer.Commit(commitData); RepositoryInstance.Instance.ConsoleWriteLine("ok"); RepositoryInstance.Instance.ConsoleWrite(" Optimizing ... "); writer.Optimize(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); } finally { writer.Close(); } RepositoryInstance.Instance.ConsoleWrite(" Deleting indexing activities ... "); LuceneManager.DeleteAllIndexingActivities(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); if (backup) { RepositoryInstance.Instance.ConsoleWrite(" Making backup ... "); BackupTools.BackupIndexImmediatelly(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); } op.Successful = true; } }
/*======================================================================================================= IIndexPopulator Members */ // caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce public void ClearAndPopulateAll() { var lastActivityId = IndexingActivityManager.GetLastActivityId(); var commitData = IndexManager.CreateCommitUserData(lastActivityId); using (var traceOperation = Logger.TraceOperation("IndexPopulator ClearAndPopulateAll")) { //-- recreate var writer = IndexManager.GetIndexWriter(true); try { foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root")) { var doc = IndexDocumentInfo.GetDocument(docData); writer.AddDocument(doc); OnNodeIndexed(docData.Path); } RepositoryInstance.Instance.ConsoleWrite(" Commiting ... "); writer.Commit(commitData); RepositoryInstance.Instance.ConsoleWriteLine("ok"); RepositoryInstance.Instance.ConsoleWrite(" Optimizing ... "); writer.Optimize(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); } finally { writer.Close(); } RepositoryInstance.Instance.ConsoleWrite(" Deleting indexing activities ... "); IndexingActivityManager.DeleteAllActivities(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); RepositoryInstance.Instance.ConsoleWrite(" Making backup ... "); BackupTools.BackupIndexImmediatelly(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); traceOperation.IsSuccessful = true; } }
internal static IEnumerable <Fieldable> GetFields(IndexDocumentInfo info, SenseNet.ContentRepository.Storage.Data.IndexDocumentData docData) { Debug.WriteLine("%> adding custom fields for " + docData.Path); return(Instance.GetFieldsPrivate(info, docData)); }
internal static Document CreateDocument(IndexDocumentInfo info, IndexDocumentData docData) { var doc = new Document(); foreach (var fieldInfo in info.fields) { doc.Add(CreateField(fieldInfo)); } var path = docData.Path.ToLower(); //doc.Add(new Field(LucObject.FieldName.Name, RepositoryPath.GetFileName(path), Field.Store.YES, Field.Index.ANALYZED)); //doc.Add(new Field(LucObject.FieldName.Path, path, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.Add(CreateStringField(LucObject.FieldName.Name, RepositoryPath.GetFileName(path), NameFieldIndexingInfo)); doc.Add(CreateStringField(LucObject.FieldName.Path, path, NameFieldIndexingInfo)); //LucObject.FieldName.Depth var nf = new NumericField(LucObject.FieldName.Depth, Field.Store.YES, true); nf.SetIntValue(DepthIndexHandler.GetDepth(docData.Path)); doc.Add(nf); //LucObject.FieldName.InTree //var fields = InTreeIndexHandlerInstance.GetIndexFields(LucObject.FieldName.InTree, docData.Path); var fields = CreateInTreeFields(LucObject.FieldName.InTree, docData.Path); foreach (var field in fields) { doc.Add(field); } //LucObject.FieldName.InFolder doc.Add(CreateInFolderField(LucObject.FieldName.InFolder, path)); //LucObject.FieldName.ParentId nf = new NumericField(LucObject.FieldName.ParentId, Field.Store.NO, true); nf.SetIntValue(docData.ParentId); doc.Add(nf); // flags doc.Add(new Field(LucObject.FieldName.IsLastPublic, docData.IsLastPublic ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); doc.Add(new Field(LucObject.FieldName.IsLastDraft, docData.IsLastDraft ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); // timestamps nf = new NumericField(LucObject.FieldName.NodeTimestamp, Field.Store.YES, true); nf.SetLongValue(docData.NodeTimestamp); doc.Add(nf); nf = new NumericField(LucObject.FieldName.VersionTimestamp, Field.Store.YES, true); nf.SetLongValue(docData.VersionTimestamp); doc.Add(nf); // custom fields if (info.HasCustomField) { var customFields = CustomIndexFieldManager.GetFields(info, docData); if (customFields != null) { foreach (var field in customFields) { doc.Add(field); } } } return(doc); }
public object GetIndexDocumentInfo(Node node) { return(IndexDocumentInfo.Create(node)); }
internal static Document CreateDocument(IndexDocumentInfo info, IndexDocumentData docData) { var doc = new Document(); foreach (var fieldInfo in info.fields) doc.Add(CreateField(fieldInfo)); var path = docData.Path.ToLower(); //doc.Add(new Field(LucObject.FieldName.Name, RepositoryPath.GetFileName(path), Field.Store.YES, Field.Index.ANALYZED)); //doc.Add(new Field(LucObject.FieldName.Path, path, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); doc.Add(CreateStringField(LucObject.FieldName.Name, RepositoryPath.GetFileName(path), NameFieldIndexingInfo)); doc.Add(CreateStringField(LucObject.FieldName.Path, path, NameFieldIndexingInfo)); //LucObject.FieldName.Depth var nf = new NumericField(LucObject.FieldName.Depth, Field.Store.YES, true); nf.SetIntValue(DepthIndexHandler.GetDepth(docData.Path)); doc.Add(nf); //LucObject.FieldName.InTree //var fields = InTreeIndexHandlerInstance.GetIndexFields(LucObject.FieldName.InTree, docData.Path); var fields = CreateInTreeFields(LucObject.FieldName.InTree, docData.Path); foreach (var field in fields) doc.Add(field); //LucObject.FieldName.InFolder doc.Add(CreateInFolderField(LucObject.FieldName.InFolder, path)); //LucObject.FieldName.ParentId nf = new NumericField(LucObject.FieldName.ParentId, Field.Store.NO, true); nf.SetIntValue(docData.ParentId); doc.Add(nf); // flags doc.Add(new Field(LucObject.FieldName.IsLastPublic, docData.IsLastPublic ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); doc.Add(new Field(LucObject.FieldName.IsLastDraft, docData.IsLastDraft ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); // timestamps nf = new NumericField(LucObject.FieldName.NodeTimestamp, Field.Store.YES, true); nf.SetLongValue(docData.NodeTimestamp); doc.Add(nf); nf = new NumericField(LucObject.FieldName.VersionTimestamp, Field.Store.YES, true); nf.SetLongValue(docData.VersionTimestamp); doc.Add(nf); // custom fields if (info.HasCustomField) { var customFields = CustomIndexFieldManager.GetFields(info, docData); if (customFields != null) foreach (var field in customFields) doc.Add(field); } return doc; }
internal static IEnumerable<Fieldable> GetFields(IndexDocumentInfo info, SenseNet.ContentRepository.Storage.Data.IndexDocumentData docData) { Debug.WriteLine("%> adding custom fields for " + docData.Path); return Instance.GetFieldsPrivate(info, docData); }
private IEnumerable<Fieldable> GetFieldsPrivate(IndexDocumentInfo info, IndexDocumentData docData) { var fields = new List<Fieldable>(); foreach (var provider in _providers) { var f = provider.GetFields(docData); if (f != null) fields.AddRange(f); } return fields.Count == 0 ? null : fields; }
public static IndexDocumentInfo Create(Node node) { if (node == null) throw new ArgumentNullException("node"); var textEtract = new StringBuilder(); var doc = new IndexDocumentInfo(); doc._hasCustomField = node is IHasCustomIndexField; var ixnode = node as IIndexableDocument; if (ixnode == null) { doc.AddField(LucObject.FieldName.NodeId, node.Id, Field.Store.YES, true); doc.AddField(LucObject.FieldName.VersionId, node.VersionId, Field.Store.YES, true); doc.AddField(LucObject.FieldName.Version, node.Version.ToString().ToLower(), Field.Store.YES, Field.Index.ANALYZED); doc.AddField(LucObject.FieldName.CreatedById, node.CreatedById, Field.Store.YES, true); doc.AddField(LucObject.FieldName.ModifiedById, node.ModifiedById, Field.Store.YES, true); } else { var fieldNames = new List<string>(); foreach (var field in ixnode.GetIndexableFields()) { if (PostponedFields.Contains(field.Name)) continue; string extract; var lucFields = field.GetIndexFieldInfos(out extract); textEtract.AppendLine(extract); if (lucFields != null) { foreach (var lucField in lucFields) { fieldNames.Add(lucField.Name); doc.AddField(lucField); } } } } //doc.AddField(LucObject.FieldName.NodeTimestamp, node.NodeTimestamp, Field.Store.YES, true); //doc.AddField(LucObject.FieldName.VersionTimestamp, node.VersionTimestamp, Field.Store.YES, true); doc.AddField(LucObject.FieldName.IsInherited, node.IsInherited ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsMajor, node.Version.IsMajor ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsPublic, node.Version.Status == VersionStatus.Approved ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.AllText, textEtract.ToString(), Field.Store.NO, Field.Index.ANALYZED); return doc; }
private static void ValidateDocumentInfo(IndexDocumentInfo doc) { ValidateField(doc, LucObject.FieldName.NodeId); ValidateField(doc, LucObject.FieldName.VersionId); }
public static IndexDocumentInfo Create(Node node, bool skipBinaries, bool isNew) { if (node == null) { throw new ArgumentNullException("node"); } if (!Indexable(node)) { return(NotIndexedDocument); } var textEtract = new StringBuilder(); var doc = new IndexDocumentInfo(); doc._hasCustomField = node is IHasCustomIndexField; var ixnode = node as IIndexableDocument; var faultedFieldNames = new List <string>(); if (ixnode == null) { doc.AddField(LucObject.FieldName.NodeId, node.Id, Field.Store.YES, true); doc.AddField(LucObject.FieldName.VersionId, node.VersionId, Field.Store.YES, true); doc.AddField(LucObject.FieldName.Version, node.Version.ToString().ToLowerInvariant(), Field.Store.YES, Field.Index.ANALYZED); doc.AddField(LucObject.FieldName.OwnerId, node.OwnerId, Field.Store.YES, true); doc.AddField(LucObject.FieldName.CreatedById, node.CreatedById, Field.Store.YES, true); doc.AddField(LucObject.FieldName.ModifiedById, node.ModifiedById, Field.Store.YES, true); } else { var fieldNames = new List <string>(); foreach (var field in ixnode.GetIndexableFields()) { if (ForbiddenFields.Contains(field.Name)) { continue; } if (PostponedFields.Contains(field.Name)) { continue; } if (node.SavingState != ContentSavingState.Finalized && (field is BinaryField || SkippedMultistepFields.Contains(field.Name))) { continue; } if (skipBinaries && (field is BinaryField)) { if (TextExtractor.TextExtractingWillBePotentiallySlow((BinaryData)((BinaryField)field).GetData())) { doc.HasBinaryField = true; continue; } } IEnumerable <IndexFieldInfo> lucFields = null; string extract = null; try { lucFields = field.GetIndexFieldInfos(out extract); } catch (Exception) { faultedFieldNames.Add(field.Name); } if (!String.IsNullOrEmpty(extract)) // do not add extra line if extract is empty { try { textEtract.AppendLine(extract); } catch (OutOfMemoryException) { SnLog.WriteWarning("Out of memory error during indexing.", EventId.Indexing, properties: new Dictionary <string, object> { { "Path", node.Path }, { "Field", field.Name } }); } } if (lucFields != null) { foreach (var lucField in lucFields) { fieldNames.Add(lucField.Name); doc.AddField(lucField); } } } } var isInherited = true; if (!isNew) { isInherited = node.IsInherited; } doc.AddField(LucObject.FieldName.IsInherited, isInherited ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsMajor, node.Version.IsMajor ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.IsPublic, node.Version.Status == VersionStatus.Approved ? BooleanIndexHandler.YES : BooleanIndexHandler.NO, Field.Store.YES, Field.Index.NOT_ANALYZED); doc.AddField(LucObject.FieldName.AllText, textEtract.ToString(), Field.Store.NO, Field.Index.ANALYZED); if (faultedFieldNames.Any()) { doc.AddField(LucObject.FieldName.IsFaulted, BooleanIndexHandler.YES, Field.Store.YES, Field.Index.NOT_ANALYZED); foreach (var faultedFieldName in faultedFieldNames) { doc.AddField(LucObject.FieldName.FaultedFieldName, faultedFieldName, Field.Store.YES, Field.Index.NOT_ANALYZED); } } ValidateDocumentInfo(doc); return(doc); }