public static void ExecuteActivity(LuceneIndexingActivity activity, bool waitForComplete, bool distribute) { if (distribute) { activity.Distribute(); } // If there are too many activities in the queue, we have to drop at least the inner // data of the activity to prevent memory overflow. We still have to wait for the // activity to finish, but the inner data can (and will) be loaded from the db when // the time comes for this activity to be executed. if (IndexingActivityQueue.IsOverloaded()) { SnTrace.Index.Write("IAQ OVERLOAD drop activity FromPopulator A:" + activity.Id); activity.IndexDocumentData = null; } // all activities must be executed through the activity queue's API IndexingActivityQueue.ExecuteActivity(activity); if (waitForComplete) { activity.WaitForComplete(); } }
private static bool MustWait(LuceneIndexingActivity newerActivity, LuceneIndexingActivity olderActivity) { Debug.Assert(olderActivity.Id != newerActivity.Id); if (olderActivity.NodeId == newerActivity.NodeId) { return(true); } if (olderActivity.Path == newerActivity.Path) { return(true); } if (olderActivity is LuceneTreeActivity) { if (newerActivity is LuceneTreeActivity) { return(olderActivity.IsInTree(newerActivity) || newerActivity.IsInTree(olderActivity)); } else { return(newerActivity.IsInTree(olderActivity)); } } return(newerActivity is LuceneTreeActivity && olderActivity.IsInTree(newerActivity)); }
internal static void Finish(LuceneIndexingActivity activity) { lock (_waitingSetLock) { // activity is done in the ActivityQueue _waitingSet.Remove(activity); // terminate and release waiting threads if there is any. activity.Finish(); // register activity termination in the log. IndexingActivityHistory.Finish(activity.Id); // register activity termination if the activity was not skipped. if (activity.Executed) { TerminationHistory.FinishActivity(activity); } // execute all activities that are completely freed. foreach (var dependentItem in activity.WaitingForMe.ToArray()) { dependentItem.FinishWaiting(activity); if (dependentItem.WaitingFor.Count == 0) { Task.Run(() => Executor.Execute(dependentItem)); } } } }
internal static void Wait(LuceneIndexingActivity activity) { lock (_lock) { foreach (var item in _history) { if (item != null && item.Id == activity.Id) { item.WaitedFor = activity.WaitingFor.Select(a => a.Id).ToArray(); break; } } } }
internal static void AttachOrFinish(LuceneIndexingActivity activity) { lock (_waitingSetLock) { var sameActivity = _waitingSet.FirstOrDefault(a => a.Id == activity.Id); if (sameActivity != null) { sameActivity.Attach(activity); SnTrace.IndexQueue.Write("IAQ: A{0} attached to another in the waiting set.", activity.Id); return; } } activity.Finish(); // release blocked thread IndexingActivityHistory.Finish(activity.Id); SnTrace.IndexQueue.Write("IAQ: A{0} ignored: finished but not executed.", activity.Id); }
internal static void Arrive(LuceneIndexingActivity activity) { lock (_lock) { // avoiding duplication foreach (var item in _history) { if (item != null && item.Id == activity.Id) { return; } } var retired = _history[_position]; _history[_position] = new IndexingActivityHistoryItem { Id = activity.Id, TypeName = activity.ActivityType.ToString(), FromReceiver = activity.FromReceiver, FromDb = activity.FromDatabase, IsStartup = activity.IsUnprocessedActivity, ArrivedAt = DateTime.UtcNow, StartedAt = DateTime.MinValue, FinishedAt = DateTime.MinValue }; if (retired != null) { if (retired.FinishedAt == DateTime.MinValue) { _unfinished++; } } _position++; if (_position >= HistoryLength) { _position = 0; } } }
internal static LuceneIndexingActivity CreateLucActivity(IndexingActivity activity) { switch (activity.ActivityType) { case IndexingActivityType.AddDocument: return(LuceneIndexingActivity.CreateFromIndexingActivity <AddDocumentActivity>(activity)); case IndexingActivityType.AddTree: return(LuceneIndexingActivity.CreateFromIndexingActivity <AddTreeActivity>(activity)); case IndexingActivityType.UpdateDocument: return(LuceneIndexingActivity.CreateFromIndexingActivity <UpdateDocumentActivity>(activity)); case IndexingActivityType.RemoveTree: return(LuceneIndexingActivity.CreateFromIndexingActivity <RemoveTreeActivity>(activity)); case IndexingActivityType.RemoveDocument: return(LuceneIndexingActivity.CreateFromIndexingActivity <RemoveDocumentActivity>(activity)); } throw new ArgumentException("Invalid ActivityType value", activity.ActivityType.ToString()); }
internal static void FinishActivity(LuceneIndexingActivity activity) { var id = activity.Id; lock (_gapsLock) { if (id > _lastId) { if (id > _lastId + 1) { _gaps.AddRange(Enumerable.Range(_lastId + 1, id - _lastId - 1)); } _lastId = id; } else { _gaps.Remove(id); } SnTrace.IndexQueue.Write("IAQ: State after finishing A{0}: {1}", id, GetCurrentState()); } }
public static void Execute(LuceneIndexingActivity activity) { using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id)) { IndexingActivityHistory.Start(activity.Id); try { using (new SenseNet.ContentRepository.Storage.Security.SystemAccount()) activity.ExecuteIndexingActivity(); } catch (Exception e) { SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e); IndexingActivityHistory.Error(activity.Id, e); } finally { DependencyManager.Finish(activity); } op.Successful = true; } }
private static void MakeDependencies(LuceneIndexingActivity newerActivity) { lock (_waitingSetLock) { foreach (var olderActivity in _waitingSet) { if (MustWait(newerActivity, olderActivity)) { newerActivity.WaitFor(olderActivity); SnTrace.IndexQueue.Write("IAQ: A{0} depends from A{1}", newerActivity.Id, olderActivity.Id); IndexingActivityHistory.Wait(newerActivity); } } _waitingSet.Add(newerActivity); if (newerActivity.WaitingFor.Count == 0) { Task.Run(() => Executor.Execute(newerActivity)); } } }
private static void ExecuteActivity(LuceneIndexingActivity activity) { LuceneManager.RegisterActivity(activity); LuceneManager.ExecuteActivity(activity, true, true); }
// ========================================================================================== Register Activity public static void RegisterActivity(LuceneIndexingActivity activity) { DataProvider.Current.RegisterIndexingActivity(activity); }
public static void EnqueueActivity(LuceneIndexingActivity activity) { SnTrace.IndexQueue.Write("IAQ: A{0} arrived{1}. {2}, {3}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.GetType().Name, activity.Path); IndexingActivityHistory.Arrive(activity); lock (_arrivalQueueLock) { if (activity.Id <= _lastQueued) { var sameActivity = _arrivalQueue.FirstOrDefault(a => a.Id == activity.Id); if (sameActivity != null) { sameActivity.Attach(activity); SnTrace.IndexQueue.Write("IAQ: A{0} attached to another one in the queue", activity.Id); return; } DependencyManager.AttachOrFinish(activity); return; } if (activity.Id > _lastQueued + 1) { var from = _lastQueued + 1; var to = activity.Id - 1; var expectedCount = to - from + 1; var loadedActivities = Retrier.Retry <IEnumerable <IIndexingActivity> >( 3, 100, () => LoadActivities(from, to), (r, i, e) => { if (i < 3) { SnTrace.IndexQueue.Write("IAQ: Loading attempt {0}", 4 - i); } if (e != null) { return(false); } return(r.Count() == expectedCount); }); foreach (LuceneIndexingActivity loadedActivity in loadedActivities) { IndexingActivityHistory.Arrive(loadedActivity); _arrivalQueue.Enqueue(loadedActivity); _lastQueued = loadedActivity.Id; SnTrace.IndexQueue.Write("IAQ: A{0} enqueued from db.", loadedActivity.Id); DependencyManager.ActivityEnqueued(); } } _arrivalQueue.Enqueue(activity); _lastQueued = activity.Id; SnTrace.IndexQueue.Write("IAQ: A{0} enqueued.", activity.Id); DependencyManager.ActivityEnqueued(); } }
public static void ExecuteActivity(LuceneIndexingActivity activity) { Serializer.EnqueueActivity(activity); }