internal static void Commit(bool reopenReader = true) { CompletionState commitState; using (var op = SnTrace.Index.StartOperation("LM: Commit. reopenReader:{0}", reopenReader)) { using (var wrFrame = IndexWriterFrame.Get(!reopenReader)) // // Commit { commitState = CompletionState.GetCurrent(); var commitStateMessage = commitState.ToString(); SnTrace.Index.Write("LM: Committing_writer. commitState: " + commitStateMessage); // Write a fake document to make sure that the index changes are written to the file system. wrFrame.IndexWriter.UpdateDocument(new Term(COMMITFIELDNAME, COMMITFIELDNAME), GetFakeDocument()); wrFrame.IndexWriter.Commit(CompletionState.GetCommitUserData(commitState)); if (reopenReader) { ReopenReader(); } } #pragma warning disable 420 Interlocked.Exchange(ref _activities, 0); #pragma warning restore 420 _delayCycle = 0; op.Successful = true; } }
public static void Startup(System.IO.TextWriter consoleOut) { // initalize from index CompletionState cud; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) cud = CompletionState.ParseFromReader(readerFrame.IndexReader); var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId(); using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}" , cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength)) { Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut); op.Successful = true; } }
/*======================================================================================================= IIndexPopulator Members */ // caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce public void ClearAndPopulateAll(bool backup = true) { var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId(); var commitData = CompletionState.GetCommitUserData(lastActivityId); using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll")) { // recreate var writer = IndexManager.GetIndexWriter(true); try { var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes(); foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes)) { var doc = IndexDocumentInfo.GetDocument(docData); if (doc == null) // indexing disabled { continue; } writer.AddDocument(doc); OnNodeIndexed(docData.Path); } RepositoryInstance.Instance.ConsoleWrite(" Commiting ... "); writer.Commit(commitData); RepositoryInstance.Instance.ConsoleWriteLine("ok"); RepositoryInstance.Instance.ConsoleWrite(" Optimizing ... "); writer.Optimize(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); } finally { writer.Close(); } RepositoryInstance.Instance.ConsoleWrite(" Deleting indexing activities ... "); LuceneManager.DeleteAllIndexingActivities(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); if (backup) { RepositoryInstance.Instance.ConsoleWrite(" Making backup ... "); BackupTools.BackupIndexImmediatelly(); RepositoryInstance.Instance.ConsoleWriteLine("ok"); } op.Successful = true; } }
internal static CompletionState ParseFromReader(IDictionary <string, string> commitUserData) { var result = new CompletionState(); if (commitUserData != null) { string value; int lastActivityId; if (commitUserData.TryGetValue(CompletionState.LastActivityIdKey, out value)) { if (!string.IsNullOrEmpty(value)) { if (int.TryParse(value, out lastActivityId)) { result.LastActivityId = lastActivityId; } } } if (commitUserData.TryGetValue(CompletionState.GapsKey, out value)) { if (!string.IsNullOrEmpty(value)) { result.Gaps = value .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) .Select(s => { int i; if (Int32.TryParse(s, out i)) { return(i); } return(0); }) .Where(i => i > 0) .ToArray(); } } } return(result); }
public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse) { var path = content == null ? null : content.Path; CompletionState completionState; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) completionState = CompletionState.ParseFromReader(readerFrame.IndexReader); var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId(); var channel = SenseNet.ContentRepository.DistributedApplication.ClusterChannel; var appDomainName = channel == null ? null : channel.ReceiverName; return(new { AppDomainName = appDomainName, LastStoredActivity = lastDatabaseId, LastProcessedActivity = completionState.LastActivityId, GapsLength = completionState.GapsLength, Gaps = completionState.Gaps, Differences = Check(path, recurse) }); }
internal static Dictionary <string, string> GetCommitUserData(CompletionState data) { return(GetCommitUserData(data.LastActivityId, data.Gaps)); }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { if (consoleOut != null) { consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3)); } SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", String.Join(", ", gaps) } }); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. LuceneManager.Commit(); } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }
/// <summary>Only for tests</summary> internal static void _setCurrentExecutionState(CompletionState state) { Serializer.Reset(state.LastActivityId); DependencyManager.Reset(); TerminationHistory.Reset(state.LastActivityId, state.Gaps); }