internal static bool DeleteDocument(int nodeId, int versionId, bool moveOrRename, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: DeleteDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, MoveOrRename:{2}", activityId, executingUnprocessedActivities, moveOrRename); return(false); } SnTrace.Index.Write("LM: DeleteDocument. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, NodeId:{2}, VersionId:{3}", activityId, executingUnprocessedActivities, nodeId, versionId); var deleteTerm = GetVersionIdTerm(versionId); var updates = GetUpdates(versioning); using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments { wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray()); foreach (var update in updates) { wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document); } wrFrame.IndexWriter.DeleteDocuments(deleteTerm); } return(true); }
internal static bool DeleteDocuments(Term[] deleteTerms, bool moveOrRename, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: DeleteDocuments skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, MoveOrRename:{2}", activityId, executingUnprocessedActivities, moveOrRename); return(false); } if (SnTrace.Index.Enabled) { SnTrace.Index.Write("LM: DeleteDocuments. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, terms:{2}", activityId, executingUnprocessedActivities, string.Join(", ", deleteTerms.Select(t => { var name = t.Field(); var value = t.Text(); if (name == "VersionId") { value = GetIntFromPrefixCode(value).ToString(); } return(name + ":" + value); })) ); } using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments { wrFrame.IndexWriter.DeleteDocuments(deleteTerms); } // don't need to check if indexing interfered here. If it did, change is detected in overlapped adddocument/updatedocument, and refresh (re-delete) is called there. // deletedocuments will never detect change in index, since it sets timestamp in indexhistory to maxvalue. return(true); }
internal static bool UpdateDocument(Document document, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: UpdateDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities); return(false); } var nodeId = GetNodeIdFromDocument(document); var versionId = GetVersionIdFromDocument(document); SnTrace.Index.Write("LM: UpdateDocument. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, NodeId:{2}, VersionId:{3}", activityId, executingUnprocessedActivities, nodeId, versionId); var updates = GetUpdates(versioning); SetDocumentFlags(document, versionId, versioning); using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDocument { wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray()); foreach (var update in updates) { wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document); } wrFrame.IndexWriter.UpdateDocument(GetVersionIdTerm(versionId), document); } return(true); }
internal static void Commit(bool reopenReader = true) { var gapData = MissingActivityHandler.GetGapString(); var gapString = gapData.Item1; var maxActivityId = gapData.Item2; //_writerRestartLock.EnterReadLock(); //try //{ // _writer.Commit(IndexManager.CreateCommitUserData(maxActivityId, gapString)); // if (reopenReader) // ReopenReader(); //} //finally //{ // _writerRestartLock.ExitReadLock(); //} using (var wrFrame = IndexWriterFrame.Get(false)) // // Commit { _writer.Commit(IndexManager.CreateCommitUserData(maxActivityId, gapString)); if (reopenReader) { ReopenReader(); } } //in case of shutdown, reopen is not needed WriteCommitLog("Committed", null, null, null, null, gapString, maxActivityId); Interlocked.Exchange(ref _activities, 0); _delayCycle = 0; }
internal static void Commit(bool reopenReader = true) { CompletionState commitState; using (var op = SnTrace.Index.StartOperation("LM: Commit. reopenReader:{0}", reopenReader)) { using (var wrFrame = IndexWriterFrame.Get(!reopenReader)) // // Commit { commitState = CompletionState.GetCurrent(); var commitStateMessage = commitState.ToString(); SnTrace.Index.Write("LM: Committing_writer. commitState: " + commitStateMessage); // Write a fake document to make sure that the index changes are written to the file system. wrFrame.IndexWriter.UpdateDocument(new Term(COMMITFIELDNAME, COMMITFIELDNAME), GetFakeDocument()); wrFrame.IndexWriter.Commit(CompletionState.GetCommitUserData(commitState)); if (reopenReader) { ReopenReader(); } } #pragma warning disable 420 Interlocked.Exchange(ref _activities, 0); #pragma warning restore 420 _delayCycle = 0; op.Successful = true; } }
internal static void ForceRestore() { try { PauseIndexing(); ApplyChanges(); BackupTools.RestoreIndex(true, null); var directory = FSDirectory.Open(new System.IO.DirectoryInfo(IndexDirectory.CurrentDirectory)); //_writerRestartLock.EnterWriteLock(); //try //{ // _writer = new IndexWriter(directory, IndexManager.GetAnalyzer(), false); // _reader = _writer.GetReader(); //} //finally //{ // _writerRestartLock.ExitWriteLock(); //} using (var wrFrame = IndexWriterFrame.Get(true)) // // ForceRestore { _writer = new IndexWriter(directory, IndexManager.GetAnalyzer(), false); _reader = _writer.GetReader(); } ExecuteUnprocessedIndexingActivities(null); } finally { ContinueIndexing(); } }
private static void UpdateDirtyDocuments(List <VersionInfo> infoList) { //-- select dirty documents var dirtyVersions = infoList.Where(i => !i.IsActualDocument && (i.OriginalIsPublic != i.ExpectedIsPublic || i.OriginalIsMajor != i.ExpectedIsMajor || i.OriginalIsLastDraft != i.ExpectedIsLastDraft || i.OriginalIsLastPublic != i.ExpectedIsLastPublic)).ToArray(); //-- play dirty documents var docs = IndexDocumentInfo.GetDocuments(dirtyVersions.Select(d => d.VersionId)); foreach (var doc in docs) { var versionId = Int32.Parse(doc.Get(LucObject.FieldName.VersionId)); foreach (var dirtyVersion in dirtyVersions) { if (dirtyVersion.VersionId == versionId) { dirtyVersion.Document = doc; SetDocumentFlags(dirtyVersion); var delTerm = new Term(KeyFieldName, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(dirtyVersion.VersionId)); //_writerRestartLock.EnterReadLock(); //try //{ // //in case the system was shut down in the meantime // if (!LuceneManager.Running) // return; // _writer.UpdateDocument(delTerm, dirtyVersion.Document); //} //finally //{ // _writerRestartLock.ExitReadLock(); //} using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDirtyDocuments { //in case the system was shut down in the meantime if (!LuceneManager.Running) { return; } _writer.UpdateDocument(delTerm, dirtyVersion.Document); } break; } } } }
internal static bool AddTree(string treeRoot, bool moveOrRename, int activityId, bool executingUnprocessedActivities) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: AddTree skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot); return(false); } SnTrace.Index.Write("LM: AddTree. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot); using (var wrFrame = IndexWriterFrame.Get(false)) { if (executingUnprocessedActivities) // pessimistic compensation { wrFrame.IndexWriter.DeleteDocuments(new Term("InTree", treeRoot), new Term("Path", treeRoot)); } var excludedNodeTypes = GetNotIndexedNodeTypes(); foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(treeRoot, excludedNodeTypes)) { Document document; int versionId; try { document = IndexDocumentInfo.GetDocument(docData); if (document == null) // indexing disabled { continue; } versionId = GetVersionIdFromDocument(document); } catch (Exception e) { var path = docData == null ? string.Empty : docData.Path ?? string.Empty; SnLog.WriteException(e, "Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path); SnTrace.Index.WriteError("LM: Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path); SnTrace.Index.WriteError("LM: Error during indexing: " + e); throw; } // pessimistic approach: delete document before adding it to avoid duplicate index documents wrFrame.IndexWriter.DeleteDocuments(GetVersionIdTerm(versionId)); wrFrame.IndexWriter.AddDocument(document); } } return(true); }
// ========================================================================================== Start, Restart, Shutdown, Warmup internal static void Restart() { if (Paused) { SnTrace.Index.Write("LM: LUCENEMANAGER RESTART called but it is not executed because indexing is paused."); return; } SnTrace.Index.Write("LM: LUCENEMANAGER RESTART"); using (var wrFrame = IndexWriterFrame.Get(true)) // // Restart { wrFrame.IndexWriter.Close(); CreateWriterAndReader(); } }
public static IndexReaderFrame GetIndexReaderFrame(bool dirty = false) { var needReopen = (DateTime.UtcNow - IndexReopenedAt) > ForceReopenFrequency; if (!dirty || needReopen) { if (!_reader.IsCurrent()) { using (var wrFrame = IndexWriterFrame.Get(false)) // // IndexReader getter ReopenReader(); } } Interlocked.Increment(ref _recentlyUsedReaderFrames); return(new IndexReaderFrame(_reader)); }
private static void ShutDown(bool log) { if (!_running) { return; } Debug.WriteLine("##> LUCENEMANAGER SHUTDOWN"); if (_writer != null) { _stopCommitWorker = true; lock (_commitLock) Commit(false); //_writerRestartLock.EnterWriteLock(); //try //{ // if(_reader != null) // _reader.Close(); // if (_writer != null) // _writer.Close(); //} //finally //{ // _running = false; // _writerRestartLock.ExitWriteLock(); //} using (var wrFrame = IndexWriterFrame.Get(true)) // // ShutDown { if (_reader != null) { _reader.Close(); } if (_writer != null) { _writer.Close(); } _running = false; } } if (log) { Logger.WriteInformation("LuceneManager has stopped. Max task id: " + MissingActivityHandler.MaxActivityId); } }
private static void ShutDown(bool log) { if (!_running) { return; } if (Paused) { throw GetPausedException(); } using (var op = SnTrace.Index.StartOperation("LUCENEMANAGER SHUTDOWN")) { if (_writer != null) { _stopCommitWorker = true; lock (_commitLock) Commit(false); using (var op2 = SnTrace.Index.StartOperation("LM.CloseReaders")) { using (var wrFrame = IndexWriterFrame.Get(true)) // // ShutDown { if (_reader != null) { _reader.Close(); } if (_writer != null) { _writer.Close(); } _running = false; } op2.Successful = true; } op.Successful = true; } } if (log) { SnLog.WriteInformation("LuceneManager has stopped. Max task id and exceptions: " + IndexingActivityQueue.GetCurrentCompletionState()); } }
internal static void AddDocument(Document document) { // the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed. var versionId = _history.GetVersionId(document); var timestamp = _history.GetTimestamp(document); if (!_history.CheckForAdd(versionId, timestamp)) { return; } SetFlagsForAdd(document); //_writerRestartLock.EnterReadLock(); //try //{ // //in case the system was shut down in the meantime // if (!LuceneManager.Running) // return; // if (_executingUnprocessedIndexingActivities) // _writer.DeleteDocuments(GetVersionIdTerm(versionId)); // _writer.AddDocument(document); //} //finally //{ // _writerRestartLock.ExitReadLock(); //} using (var wrFrame = IndexWriterFrame.Get(false)) // // AddDocument { if (!LuceneManager.Running) { return; } if (_executingUnprocessedIndexingActivities) { _writer.DeleteDocuments(GetVersionIdTerm(versionId)); } _writer.AddDocument(document); } // check if indexing has interfered with other indexing activity for the same versionid if (_history.CheckHistoryChange(versionId, timestamp)) { RefreshDocument(versionId); } }
//========================================================================================== Start, Restart, Shutdown, Warmup internal static void Restart() { Debug.WriteLine("##> LUCENEMANAGER RESTART"); //_writerRestartLock.EnterWriteLock(); //try //{ // _writer.Close(); // CreateWriterAndReader(); //} //finally //{ // _writerRestartLock.ExitWriteLock(); //} using (var wrFrame = IndexWriterFrame.Get(true)) // // Restart { _writer.Close(); CreateWriterAndReader(); } }
internal static void DeleteDocuments(Term[] deleteTerms, bool moveOrRename) { // the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed. if (moveOrRename) { _history.Remove(deleteTerms); } else { _history.ProcessDelete(deleteTerms); } SetFlagsForDelete(deleteTerms); //_writerRestartLock.EnterReadLock(); //try //{ // //in case the system was shut down in the meantime // if (!LuceneManager.Running) // return; // _writer.DeleteDocuments(deleteTerms); //} //finally //{ // _writerRestartLock.ExitReadLock(); //} using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments { //in case the system was shut down in the meantime if (!LuceneManager.Running) { return; } _writer.DeleteDocuments(deleteTerms); } // don't need to check if indexing interfered here. If it did, change is detected in overlapped adddocument/updatedocument, and refresh (re-delete) is called there. // deletedocuments will never detect change in index, since it sets timestamp in indexhistory to maxvalue. }
internal static bool UpdateDocument(int activityId, bool executingUnprocessedActivities, VersioningInfo versioning) { if (!IsActivityExecutable(executingUnprocessedActivities)) { SnTrace.Index.Write("LM: UpdateDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities); return(false); } SnTrace.Index.Write("LM: AddDocument without document. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities); var updates = GetUpdates(versioning); using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDocument { wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray()); foreach (var update in updates) { wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document); } } return(true); }