public static void ExecuteActivity(LuceneIndexingActivity activity, bool waitForComplete, bool distribute)
{
if (distribute)
{
activity.Distribute();
}
// If there are too many activities in the queue, we have to drop at least the inner
// data of the activity to prevent memory overflow. We still have to wait for the
// activity to finish, but the inner data can (and will) be loaded from the db when
// the time comes for this activity to be executed.
if (IndexingActivityQueue.IsOverloaded())
{
SnTrace.Index.Write("IAQ OVERLOAD drop activity FromPopulator A:" + activity.Id);
activity.IndexDocumentData = null;
}
// all activities must be executed through the activity queue's API
IndexingActivityQueue.ExecuteActivity(activity);
if (waitForComplete)
{
activity.WaitForComplete();
}
}
private static bool MustWait(LuceneIndexingActivity newerActivity, LuceneIndexingActivity olderActivity)
{
Debug.Assert(olderActivity.Id != newerActivity.Id);
if (olderActivity.NodeId == newerActivity.NodeId)
{
return(true);
}
if (olderActivity.Path == newerActivity.Path)
{
return(true);
}
if (olderActivity is LuceneTreeActivity)
{
if (newerActivity is LuceneTreeActivity)
{
return(olderActivity.IsInTree(newerActivity) || newerActivity.IsInTree(olderActivity));
}
else
{
return(newerActivity.IsInTree(olderActivity));
}
}
return(newerActivity is LuceneTreeActivity && olderActivity.IsInTree(newerActivity));
}
internal static void Finish(LuceneIndexingActivity activity)
{
lock (_waitingSetLock)
{
// activity is done in the ActivityQueue
_waitingSet.Remove(activity);
// terminate and release waiting threads if there is any.
activity.Finish();
// register activity termination in the log.
IndexingActivityHistory.Finish(activity.Id);
// register activity termination if the activity was not skipped.
if (activity.Executed)
{
TerminationHistory.FinishActivity(activity);
}
// execute all activities that are completely freed.
foreach (var dependentItem in activity.WaitingForMe.ToArray())
{
dependentItem.FinishWaiting(activity);
if (dependentItem.WaitingFor.Count == 0)
{
Task.Run(() => Executor.Execute(dependentItem));
}
}
}
}
internal static void Wait(LuceneIndexingActivity activity)
{
lock (_lock)
{
foreach (var item in _history)
{
if (item != null && item.Id == activity.Id)
{
item.WaitedFor = activity.WaitingFor.Select(a => a.Id).ToArray();
break;
}
}
}
}
internal static void AttachOrFinish(LuceneIndexingActivity activity)
{
lock (_waitingSetLock)
{
var sameActivity = _waitingSet.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another in the waiting set.", activity.Id);
return;
}
}
activity.Finish(); // release blocked thread
IndexingActivityHistory.Finish(activity.Id);
SnTrace.IndexQueue.Write("IAQ: A{0} ignored: finished but not executed.", activity.Id);
}
internal static void Arrive(LuceneIndexingActivity activity)
{
lock (_lock)
{
// avoiding duplication
foreach (var item in _history)
{
if (item != null && item.Id == activity.Id)
{
return;
}
}
var retired = _history[_position];
_history[_position] = new IndexingActivityHistoryItem
{
Id = activity.Id,
TypeName = activity.ActivityType.ToString(),
FromReceiver = activity.FromReceiver,
FromDb = activity.FromDatabase,
IsStartup = activity.IsUnprocessedActivity,
ArrivedAt = DateTime.UtcNow,
StartedAt = DateTime.MinValue,
FinishedAt = DateTime.MinValue
};
if (retired != null)
{
if (retired.FinishedAt == DateTime.MinValue)
{
_unfinished++;
}
}
_position++;
if (_position >= HistoryLength)
{
_position = 0;
}
}
}
internal static LuceneIndexingActivity CreateLucActivity(IndexingActivity activity)
{
switch (activity.ActivityType)
{
case IndexingActivityType.AddDocument:
return(LuceneIndexingActivity.CreateFromIndexingActivity <AddDocumentActivity>(activity));
case IndexingActivityType.AddTree:
return(LuceneIndexingActivity.CreateFromIndexingActivity <AddTreeActivity>(activity));
case IndexingActivityType.UpdateDocument:
return(LuceneIndexingActivity.CreateFromIndexingActivity <UpdateDocumentActivity>(activity));
case IndexingActivityType.RemoveTree:
return(LuceneIndexingActivity.CreateFromIndexingActivity <RemoveTreeActivity>(activity));
case IndexingActivityType.RemoveDocument:
return(LuceneIndexingActivity.CreateFromIndexingActivity <RemoveDocumentActivity>(activity));
}
throw new ArgumentException("Invalid ActivityType value", activity.ActivityType.ToString());
}
internal static void FinishActivity(LuceneIndexingActivity activity)
{
var id = activity.Id;
lock (_gapsLock)
{
if (id > _lastId)
{
if (id > _lastId + 1)
{
_gaps.AddRange(Enumerable.Range(_lastId + 1, id - _lastId - 1));
}
_lastId = id;
}
else
{
_gaps.Remove(id);
}
SnTrace.IndexQueue.Write("IAQ: State after finishing A{0}: {1}", id, GetCurrentState());
}
}
public static void Execute(LuceneIndexingActivity activity)
{
using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id))
{
IndexingActivityHistory.Start(activity.Id);
try
{
using (new SenseNet.ContentRepository.Storage.Security.SystemAccount())
activity.ExecuteIndexingActivity();
}
catch (Exception e)
{
SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e);
IndexingActivityHistory.Error(activity.Id, e);
}
finally
{
DependencyManager.Finish(activity);
}
op.Successful = true;
}
}
private static void MakeDependencies(LuceneIndexingActivity newerActivity)
{
lock (_waitingSetLock)
{
foreach (var olderActivity in _waitingSet)
{
if (MustWait(newerActivity, olderActivity))
{
newerActivity.WaitFor(olderActivity);
SnTrace.IndexQueue.Write("IAQ: A{0} depends from A{1}", newerActivity.Id, olderActivity.Id);
IndexingActivityHistory.Wait(newerActivity);
}
}
_waitingSet.Add(newerActivity);
if (newerActivity.WaitingFor.Count == 0)
{
Task.Run(() => Executor.Execute(newerActivity));
}
}
}
private static void ExecuteActivity(LuceneIndexingActivity activity)
{
LuceneManager.RegisterActivity(activity);
LuceneManager.ExecuteActivity(activity, true, true);
}
// ========================================================================================== Register Activity
public static void RegisterActivity(LuceneIndexingActivity activity)
{
DataProvider.Current.RegisterIndexingActivity(activity);
}
public static void EnqueueActivity(LuceneIndexingActivity activity)
{
SnTrace.IndexQueue.Write("IAQ: A{0} arrived{1}. {2}, {3}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.GetType().Name, activity.Path);
IndexingActivityHistory.Arrive(activity);
lock (_arrivalQueueLock)
{
if (activity.Id <= _lastQueued)
{
var sameActivity = _arrivalQueue.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another one in the queue", activity.Id);
return;
}
DependencyManager.AttachOrFinish(activity);
return;
}
if (activity.Id > _lastQueued + 1)
{
var from = _lastQueued + 1;
var to = activity.Id - 1;
var expectedCount = to - from + 1;
var loadedActivities = Retrier.Retry <IEnumerable <IIndexingActivity> >(
3,
100,
() => LoadActivities(from, to),
(r, i, e) =>
{
if (i < 3)
{
SnTrace.IndexQueue.Write("IAQ: Loading attempt {0}", 4 - i);
}
if (e != null)
{
return(false);
}
return(r.Count() == expectedCount);
});
foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
{
IndexingActivityHistory.Arrive(loadedActivity);
_arrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued from db.", loadedActivity.Id);
DependencyManager.ActivityEnqueued();
}
}
_arrivalQueue.Enqueue(activity);
_lastQueued = activity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued.", activity.Id);
DependencyManager.ActivityEnqueued();
}
}
public static void ExecuteActivity(LuceneIndexingActivity activity)
{
Serializer.EnqueueActivity(activity);
}
