internal static bool DeleteDocument(int nodeId, int versionId, bool moveOrRename, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning)
{
if (!IsActivityExecutable(executingUnprocessedActivities))
{
SnTrace.Index.Write("LM: DeleteDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, MoveOrRename:{2}", activityId, executingUnprocessedActivities, moveOrRename);
return(false);
}
SnTrace.Index.Write("LM: DeleteDocument. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, NodeId:{2}, VersionId:{3}", activityId, executingUnprocessedActivities, nodeId, versionId);
var deleteTerm = GetVersionIdTerm(versionId);
var updates = GetUpdates(versioning);
using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments
{
wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray());
foreach (var update in updates)
{
wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document);
}
wrFrame.IndexWriter.DeleteDocuments(deleteTerm);
}
return(true);
}
internal static bool DeleteDocuments(Term[] deleteTerms, bool moveOrRename, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning)
{
if (!IsActivityExecutable(executingUnprocessedActivities))
{
SnTrace.Index.Write("LM: DeleteDocuments skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, MoveOrRename:{2}", activityId, executingUnprocessedActivities, moveOrRename);
return(false);
}
if (SnTrace.Index.Enabled)
{
SnTrace.Index.Write("LM: DeleteDocuments. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, terms:{2}", activityId,
executingUnprocessedActivities,
string.Join(", ", deleteTerms.Select(t =>
{
var name = t.Field();
var value = t.Text();
if (name == "VersionId")
{
value = GetIntFromPrefixCode(value).ToString();
}
return(name + ":" + value);
}))
);
}
using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments
{
wrFrame.IndexWriter.DeleteDocuments(deleteTerms);
}
// don't need to check if indexing interfered here. If it did, change is detected in overlapped adddocument/updatedocument, and refresh (re-delete) is called there.
// deletedocuments will never detect change in index, since it sets timestamp in indexhistory to maxvalue.
return(true);
}
internal static bool UpdateDocument(Document document, int activityId, bool executingUnprocessedActivities, VersioningInfo versioning)
{
if (!IsActivityExecutable(executingUnprocessedActivities))
{
SnTrace.Index.Write("LM: UpdateDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities);
return(false);
}
var nodeId = GetNodeIdFromDocument(document);
var versionId = GetVersionIdFromDocument(document);
SnTrace.Index.Write("LM: UpdateDocument. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, NodeId:{2}, VersionId:{3}", activityId, executingUnprocessedActivities, nodeId, versionId);
var updates = GetUpdates(versioning);
SetDocumentFlags(document, versionId, versioning);
using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDocument
{
wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray());
foreach (var update in updates)
{
wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document);
}
wrFrame.IndexWriter.UpdateDocument(GetVersionIdTerm(versionId), document);
}
return(true);
}
internal static void Commit(bool reopenReader = true)
{
var gapData = MissingActivityHandler.GetGapString();
var gapString = gapData.Item1;
var maxActivityId = gapData.Item2;
//_writerRestartLock.EnterReadLock();
//try
//{
// _writer.Commit(IndexManager.CreateCommitUserData(maxActivityId, gapString));
// if (reopenReader)
// ReopenReader();
//}
//finally
//{
// _writerRestartLock.ExitReadLock();
//}
using (var wrFrame = IndexWriterFrame.Get(false)) // // Commit
{
_writer.Commit(IndexManager.CreateCommitUserData(maxActivityId, gapString));
if (reopenReader)
{
ReopenReader();
}
}
//in case of shutdown, reopen is not needed
WriteCommitLog("Committed", null, null, null, null, gapString, maxActivityId);
Interlocked.Exchange(ref _activities, 0);
_delayCycle = 0;
}
internal static void Commit(bool reopenReader = true)
{
CompletionState commitState;
using (var op = SnTrace.Index.StartOperation("LM: Commit. reopenReader:{0}", reopenReader))
{
using (var wrFrame = IndexWriterFrame.Get(!reopenReader)) // // Commit
{
commitState = CompletionState.GetCurrent();
var commitStateMessage = commitState.ToString();
SnTrace.Index.Write("LM: Committing_writer. commitState: " + commitStateMessage);
// Write a fake document to make sure that the index changes are written to the file system.
wrFrame.IndexWriter.UpdateDocument(new Term(COMMITFIELDNAME, COMMITFIELDNAME), GetFakeDocument());
wrFrame.IndexWriter.Commit(CompletionState.GetCommitUserData(commitState));
if (reopenReader)
{
ReopenReader();
}
}
#pragma warning disable 420
Interlocked.Exchange(ref _activities, 0);
#pragma warning restore 420
_delayCycle = 0;
op.Successful = true;
}
}
internal static void ForceRestore()
{
try
{
PauseIndexing();
ApplyChanges();
BackupTools.RestoreIndex(true, null);
var directory = FSDirectory.Open(new System.IO.DirectoryInfo(IndexDirectory.CurrentDirectory));
//_writerRestartLock.EnterWriteLock();
//try
//{
// _writer = new IndexWriter(directory, IndexManager.GetAnalyzer(), false);
// _reader = _writer.GetReader();
//}
//finally
//{
// _writerRestartLock.ExitWriteLock();
//}
using (var wrFrame = IndexWriterFrame.Get(true)) // // ForceRestore
{
_writer = new IndexWriter(directory, IndexManager.GetAnalyzer(), false);
_reader = _writer.GetReader();
}
ExecuteUnprocessedIndexingActivities(null);
}
finally
{
ContinueIndexing();
}
}
private static void UpdateDirtyDocuments(List <VersionInfo> infoList)
{
//-- select dirty documents
var dirtyVersions = infoList.Where(i => !i.IsActualDocument &&
(i.OriginalIsPublic != i.ExpectedIsPublic || i.OriginalIsMajor != i.ExpectedIsMajor ||
i.OriginalIsLastDraft != i.ExpectedIsLastDraft || i.OriginalIsLastPublic != i.ExpectedIsLastPublic)).ToArray();
//-- play dirty documents
var docs = IndexDocumentInfo.GetDocuments(dirtyVersions.Select(d => d.VersionId));
foreach (var doc in docs)
{
var versionId = Int32.Parse(doc.Get(LucObject.FieldName.VersionId));
foreach (var dirtyVersion in dirtyVersions)
{
if (dirtyVersion.VersionId == versionId)
{
dirtyVersion.Document = doc;
SetDocumentFlags(dirtyVersion);
var delTerm = new Term(KeyFieldName, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(dirtyVersion.VersionId));
//_writerRestartLock.EnterReadLock();
//try
//{
// //in case the system was shut down in the meantime
// if (!LuceneManager.Running)
// return;
// _writer.UpdateDocument(delTerm, dirtyVersion.Document);
//}
//finally
//{
// _writerRestartLock.ExitReadLock();
//}
using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDirtyDocuments
{
//in case the system was shut down in the meantime
if (!LuceneManager.Running)
{
return;
}
_writer.UpdateDocument(delTerm, dirtyVersion.Document);
}
break;
}
}
}
}
internal static bool AddTree(string treeRoot, bool moveOrRename, int activityId, bool executingUnprocessedActivities)
{
if (!IsActivityExecutable(executingUnprocessedActivities))
{
SnTrace.Index.Write("LM: AddTree skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot);
return(false);
}
SnTrace.Index.Write("LM: AddTree. ActivityId:{0}, ExecutingUnprocessedActivities:{1}, TreeRoot:{2}", activityId, executingUnprocessedActivities, treeRoot);
using (var wrFrame = IndexWriterFrame.Get(false))
{
if (executingUnprocessedActivities) // pessimistic compensation
{
wrFrame.IndexWriter.DeleteDocuments(new Term("InTree", treeRoot), new Term("Path", treeRoot));
}
var excludedNodeTypes = GetNotIndexedNodeTypes();
foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(treeRoot, excludedNodeTypes))
{
Document document;
int versionId;
try
{
document = IndexDocumentInfo.GetDocument(docData);
if (document == null) // indexing disabled
{
continue;
}
versionId = GetVersionIdFromDocument(document);
}
catch (Exception e)
{
var path = docData == null ? string.Empty : docData.Path ?? string.Empty;
SnLog.WriteException(e, "Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path);
SnTrace.Index.WriteError("LM: Error during indexing: the document data loaded from the database or the generated Lucene Document is invalid. Please save the content to regenerate the index for it. Path: " + path);
SnTrace.Index.WriteError("LM: Error during indexing: " + e);
throw;
}
// pessimistic approach: delete document before adding it to avoid duplicate index documents
wrFrame.IndexWriter.DeleteDocuments(GetVersionIdTerm(versionId));
wrFrame.IndexWriter.AddDocument(document);
}
}
return(true);
}
// ========================================================================================== Start, Restart, Shutdown, Warmup
internal static void Restart()
{
if (Paused)
{
SnTrace.Index.Write("LM: LUCENEMANAGER RESTART called but it is not executed because indexing is paused.");
return;
}
SnTrace.Index.Write("LM: LUCENEMANAGER RESTART");
using (var wrFrame = IndexWriterFrame.Get(true)) // // Restart
{
wrFrame.IndexWriter.Close();
CreateWriterAndReader();
}
}
public static IndexReaderFrame GetIndexReaderFrame(bool dirty = false)
{
var needReopen = (DateTime.UtcNow - IndexReopenedAt) > ForceReopenFrequency;
if (!dirty || needReopen)
{
if (!_reader.IsCurrent())
{
using (var wrFrame = IndexWriterFrame.Get(false)) // // IndexReader getter
ReopenReader();
}
}
Interlocked.Increment(ref _recentlyUsedReaderFrames);
return(new IndexReaderFrame(_reader));
}
private static void ShutDown(bool log)
{
if (!_running)
{
return;
}
Debug.WriteLine("##> LUCENEMANAGER SHUTDOWN");
if (_writer != null)
{
_stopCommitWorker = true;
lock (_commitLock)
Commit(false);
//_writerRestartLock.EnterWriteLock();
//try
//{
// if(_reader != null)
// _reader.Close();
// if (_writer != null)
// _writer.Close();
//}
//finally
//{
// _running = false;
// _writerRestartLock.ExitWriteLock();
//}
using (var wrFrame = IndexWriterFrame.Get(true)) // // ShutDown
{
if (_reader != null)
{
_reader.Close();
}
if (_writer != null)
{
_writer.Close();
}
_running = false;
}
}
if (log)
{
Logger.WriteInformation("LuceneManager has stopped. Max task id: " + MissingActivityHandler.MaxActivityId);
}
}
private static void ShutDown(bool log)
{
if (!_running)
{
return;
}
if (Paused)
{
throw GetPausedException();
}
using (var op = SnTrace.Index.StartOperation("LUCENEMANAGER SHUTDOWN"))
{
if (_writer != null)
{
_stopCommitWorker = true;
lock (_commitLock)
Commit(false);
using (var op2 = SnTrace.Index.StartOperation("LM.CloseReaders"))
{
using (var wrFrame = IndexWriterFrame.Get(true)) // // ShutDown
{
if (_reader != null)
{
_reader.Close();
}
if (_writer != null)
{
_writer.Close();
}
_running = false;
}
op2.Successful = true;
}
op.Successful = true;
}
}
if (log)
{
SnLog.WriteInformation("LuceneManager has stopped. Max task id and exceptions: " + IndexingActivityQueue.GetCurrentCompletionState());
}
}
internal static void AddDocument(Document document)
{
// the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed.
var versionId = _history.GetVersionId(document);
var timestamp = _history.GetTimestamp(document);
if (!_history.CheckForAdd(versionId, timestamp))
{
return;
}
SetFlagsForAdd(document);
//_writerRestartLock.EnterReadLock();
//try
//{
// //in case the system was shut down in the meantime
// if (!LuceneManager.Running)
// return;
// if (_executingUnprocessedIndexingActivities)
// _writer.DeleteDocuments(GetVersionIdTerm(versionId));
// _writer.AddDocument(document);
//}
//finally
//{
// _writerRestartLock.ExitReadLock();
//}
using (var wrFrame = IndexWriterFrame.Get(false)) // // AddDocument
{
if (!LuceneManager.Running)
{
return;
}
if (_executingUnprocessedIndexingActivities)
{
_writer.DeleteDocuments(GetVersionIdTerm(versionId));
}
_writer.AddDocument(document);
}
// check if indexing has interfered with other indexing activity for the same versionid
if (_history.CheckHistoryChange(versionId, timestamp))
{
RefreshDocument(versionId);
}
}
//========================================================================================== Start, Restart, Shutdown, Warmup
internal static void Restart()
{
Debug.WriteLine("##> LUCENEMANAGER RESTART");
//_writerRestartLock.EnterWriteLock();
//try
//{
// _writer.Close();
// CreateWriterAndReader();
//}
//finally
//{
// _writerRestartLock.ExitWriteLock();
//}
using (var wrFrame = IndexWriterFrame.Get(true)) // // Restart
{
_writer.Close();
CreateWriterAndReader();
}
}
internal static void DeleteDocuments(Term[] deleteTerms, bool moveOrRename)
{
// the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed.
if (moveOrRename)
{
_history.Remove(deleteTerms);
}
else
{
_history.ProcessDelete(deleteTerms);
}
SetFlagsForDelete(deleteTerms);
//_writerRestartLock.EnterReadLock();
//try
//{
// //in case the system was shut down in the meantime
// if (!LuceneManager.Running)
// return;
// _writer.DeleteDocuments(deleteTerms);
//}
//finally
//{
// _writerRestartLock.ExitReadLock();
//}
using (var wrFrame = IndexWriterFrame.Get(false)) // // DeleteDocuments
{
//in case the system was shut down in the meantime
if (!LuceneManager.Running)
{
return;
}
_writer.DeleteDocuments(deleteTerms);
}
// don't need to check if indexing interfered here. If it did, change is detected in overlapped adddocument/updatedocument, and refresh (re-delete) is called there.
// deletedocuments will never detect change in index, since it sets timestamp in indexhistory to maxvalue.
}
internal static bool UpdateDocument(int activityId, bool executingUnprocessedActivities, VersioningInfo versioning)
{
if (!IsActivityExecutable(executingUnprocessedActivities))
{
SnTrace.Index.Write("LM: UpdateDocument skipped #1. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities);
return(false);
}
SnTrace.Index.Write("LM: AddDocument without document. ActivityId:{0}, ExecutingUnprocessedActivities:{1}", activityId, executingUnprocessedActivities);
var updates = GetUpdates(versioning);
using (var wrFrame = IndexWriterFrame.Get(false)) // // UpdateDocument
{
wrFrame.IndexWriter.DeleteDocuments(versioning.Delete.Select(i => GetVersionIdTerm(i)).ToArray());
foreach (var update in updates)
{
wrFrame.IndexWriter.UpdateDocument(update.UpdateTerm, update.Document);
}
}
return(true);
}
