internal static void HealthCheck() { if (IsWorking()) { SnTrace.Index.Write("IAQ: Health check triggered but ignored."); return; } SnTrace.Index.Write("IAQ: Health check triggered."); var state = TerminationHistory.GetCurrentState(); var gapsLength = state.Gaps.Length; if (gapsLength > 0) { SnTrace.IndexQueue.Write("IAQ: Health checker is processing {0} gap{1}.", gapsLength, gapsLength > 1 ? "s" : ""); var notLoadedIds = state.Gaps.ToList(); foreach (var indexingActivity in new IndexingActivityLoader(state.Gaps, false)) { var activity = (IndexingActivityBase)indexingActivity; WaitIfOverloaded(); ExecuteActivity(activity); notLoadedIds.Remove(activity.Id); } if (notLoadedIds.Count > 0) { TerminationHistory.RemoveGaps(notLoadedIds); SnTrace.IndexQueue.Write("IAQ: Health checker ignores the following activity ids after processing the gaps: {0}", notLoadedIds); } } var lastId = TerminationHistory.GetLastTerminatedId(); var lastDbId = IndexManager.GetLastStoredIndexingActivityId(); if (lastId < lastDbId) { SnTrace.IndexQueue.Write("IAQ: Health checker is processing activities from {0} to {1}", (lastId + 1), lastDbId); foreach (var indexingActivity in new IndexingActivityLoader(lastId + 1, lastDbId, false)) { var activity = (IndexingActivityBase)indexingActivity; WaitIfOverloaded(); ExecuteActivity(activity); } } }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3)); SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) } }); IndexingActivityBase.NotIndexableContentCollection.Clear(); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } if (IndexingActivityBase.NotIndexableContentCollection.Any()) { const int maxWrittenCount = 100; // Collect the first n ContentId, VersionId pairs into one message // (key: version id, value: content id). var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount) .OrderBy(x => x.Value) .Select(x => $"{x.Value},{x.Key}")); var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount ? $" first {maxWrittenCount}" : string.Empty; // Write one aggregated message SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" + $" content. The{firstN} related ContentId, VersionId pairs: {data}"); // Do not keep memory for unnecessary data IndexingActivityBase.NotIndexableContentCollection.Clear(); } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult(); // explicit commit } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { if (consoleOut != null) { consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3)); } SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", String.Join(", ", gaps) } }); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. LuceneManager.Commit(); } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }