public static void EnqueueActivity(SecurityActivity activity) { SnTrace.SecurityQueue.Write("SAQ: SA{0} arrived{1}. {2}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.TypeName); SecurityActivityHistory.Arrive(activity); lock (_arrivalQueueLock) { if (activity.Id <= _lastQueued) { var sameActivity = _arrivalQueue.FirstOrDefault(a => a.Id == activity.Id); if (sameActivity != null) { sameActivity.Attach(activity); SnTrace.SecurityQueue.Write("SAQ: SA{0} attached to another one in the queue", activity.Id); return; } DependencyManager.AttachOrFinish(activity); return; } if (activity.Id > _lastQueued + 1) { //var loadedActivities = LoadActivities(_lastQueued + 1, activity.Id - 1); var from = _lastQueued + 1; var to = activity.Id - 1; var expectedCount = to - from + 1; var loadedActivities = Retrier.Retry( 3, 100, () => LoadActivities(from, to), (r, i, e) => { if (i < 3) { SnTrace.SecurityQueue.Write("SAQ: Loading attempt {0}", 4 - i); } if (e != null) { return(false); } return(r.Count() == expectedCount); }); foreach (var loadedActivity in loadedActivities) { SecurityActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; SnTrace.SecurityQueue.Write("SAQ: SA{0} enqueued from db.", loadedActivity.Id); DependencyManager.ActivityEnqueued(); } } _arrivalQueue.Enqueue(activity); _lastQueued = activity.Id; SnTrace.SecurityQueue.Write("SAQ: SA{0} enqueued.", activity.Id); DependencyManager.ActivityEnqueued(); } }
private static void WaitIfOverloaded(bool startProcessing = false) { // We prevent memory overflow by limiting the number of activities that we // keep in memory. This method waits for the queue to be able to process // new activities. var logCount = 1; while (IsOverloaded()) { // In case of startup, we have to start processing activities that are // already in the queue so that new ones can be added later. if (startProcessing) { DependencyManager.ActivityEnqueued(); } if (logCount++ % 10 == 1) { SnTrace.Index.Write("IAQ OVERLOAD waiting {0} milliseconds.", IndexingOverloadWaitingTime * 10); } Thread.Sleep(IndexingOverloadWaitingTime); } }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps) { var hasUnprocessed = gaps.Length > 0 || lastDatabaseId != lastExecutedId; SnLog.WriteInformation(EventMessage.Information.StartTheSystem, EventId.RepositoryLifecycle, // ReSharper disable once ArgumentsStyleOther properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", string.Join(", ", gaps) } }); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new SecurityActivityLoader(gaps, true); foreach (var loadedActivity in loadedActivities) { SnTrace.SecurityQueue.Write("SAQ: Startup: SA{0} enqueued from db.", loadedActivity.Id); SecurityActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new SecurityActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (var loadedActivity in loadedActivities) { SnTrace.SecurityQueue.Write("SAQ: Startup: SA{0} enqueued from db.", loadedActivity.Id); SecurityActivityHistory.Arrive(loadedActivity); SnTrace.SecurityQueue.Write("SecurityActivityArrived SA{0}", loadedActivity.Id); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } if (hasUnprocessed) { SnLog.WriteInformation(string.Format(EventMessage.Information.ExecutingUnprocessedActivitiesFinished, count), EventId.RepositoryLifecycle); } }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3)); SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) } }); IndexingActivityBase.NotIndexableContentCollection.Clear(); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } if (IndexingActivityBase.NotIndexableContentCollection.Any()) { const int maxWrittenCount = 100; // Collect the first n ContentId, VersionId pairs into one message // (key: version id, value: content id). var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount) .OrderBy(x => x.Value) .Select(x => $"{x.Value},{x.Key}")); var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount ? $" first {maxWrittenCount}" : string.Empty; // Write one aggregated message SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" + $" content. The{firstN} related ContentId, VersionId pairs: {data}"); // Do not keep memory for unnecessary data IndexingActivityBase.NotIndexableContentCollection.Clear(); } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult(); // explicit commit } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { if (consoleOut != null) { consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3)); } SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", String.Join(", ", gaps) } }); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. LuceneManager.Commit(); } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }