/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3)); SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) } }); IndexingActivityBase.NotIndexableContentCollection.Clear(); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } if (IndexingActivityBase.NotIndexableContentCollection.Any()) { const int maxWrittenCount = 100; // Collect the first n ContentId, VersionId pairs into one message // (key: version id, value: content id). var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount) .OrderBy(x => x.Value) .Select(x => $"{x.Value},{x.Key}")); var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount ? $" first {maxWrittenCount}" : string.Empty; // Write one aggregated message SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" + $" content. The{firstN} related ContentId, VersionId pairs: {data}"); // Do not keep memory for unnecessary data IndexingActivityBase.NotIndexableContentCollection.Clear(); } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult(); // explicit commit } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }
public static void EnqueueActivity(IndexingActivityBase activity) { SnTrace.IndexQueue.Write("IAQ: A{0} arrived{1}. {2}, {3}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.GetType().Name, activity.Path); IndexingActivityHistory.Arrive(activity); lock (ArrivalQueueLock) { if (activity.Id <= _lastQueued) { var sameActivity = ArrivalQueue.FirstOrDefault(a => a.Id == activity.Id); if (sameActivity != null) { sameActivity.Attach(activity); SnTrace.IndexQueue.Write("IAQ: A{0} attached to another one in the queue", activity.Id); return; } DependencyManager.AttachOrFinish(activity); return; } if (activity.Id > _lastQueued + 1) { var from = _lastQueued + 1; var to = activity.Id - 1; var expectedCount = to - from + 1; var loadedActivities = Retrier.Retry( 3, 100, () => LoadActivities(from, to), (r, i, e) => { if (i < 3) { SnTrace.IndexQueue.Write("IAQ: Loading attempt {0}", 4 - i); } if (e != null) { return(false); } return(r.Count() == expectedCount); }); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; SnTrace.IndexQueue.Write("IAQ: A{0} enqueued from db.", loadedActivity.Id); DependencyManager.ActivityEnqueued(); } } ArrivalQueue.Enqueue(activity); _lastQueued = activity.Id; SnTrace.IndexQueue.Write("IAQ: A{0} enqueued.", activity.Id); DependencyManager.ActivityEnqueued(); } }
/// <summary> /// MUST BE SYNCHRON /// GAPS MUST BE ORDERED /// </summary> internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut) { consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3)); SnLog.WriteInformation("Executing unprocessed activities.", EventId.RepositoryRuntime, properties: new Dictionary <string, object> { { "LastDatabaseId", lastDatabaseId }, { "LastExecutedId", lastExecutedId }, { "CountOfGaps", gaps.Length }, { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) } }); DependencyManager.Start(); var count = 0; if (gaps.Any()) { var loadedActivities = new IndexingActivityLoader(gaps, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (lastExecutedId < lastDatabaseId) { var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true); foreach (var indexingActivity in loadedActivities) { var loadedActivity = (IndexingActivityBase)indexingActivity; // wait and start processing loaded activities in the meantime WaitIfOverloaded(true); SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id); IndexingActivityHistory.Arrive(loadedActivity); ArrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; count++; } } if (_lastQueued < lastExecutedId) { _lastQueued = lastExecutedId; } // ensure that the arrival activity queue is not empty at this pont. DependencyManager.ActivityEnqueued(); if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any()) { while (IsWorking()) { Thread.Sleep(200); } } // At this point we know for sure that the original gap is not there anymore. // In case there is a false gap (e.g. because there are missing activity ids // in the db) we have to remove these ids manually from the in-memory gap. if (gaps.Any()) { TerminationHistory.RemoveGaps(gaps); // Commit is necessary because otherwise the gap is removed only in memory, but // the index is not updated in the file system. IndexManager.Commit(); // explicit commit } SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle); }