private static bool MustWait(IndexingActivityBase newerActivity, IndexingActivityBase olderActivity)
{
Debug.Assert(olderActivity.Id != newerActivity.Id);
if (olderActivity.NodeId == newerActivity.NodeId)
{
return(true);
}
if (olderActivity.Path == newerActivity.Path)
{
return(true);
}
if (olderActivity is TreeIndexingActivity)
{
if (newerActivity is TreeIndexingActivity)
{
return(olderActivity.IsInTree(newerActivity) || newerActivity.IsInTree(olderActivity));
}
else
{
return(newerActivity.IsInTree(olderActivity));
}
}
return(newerActivity is TreeIndexingActivity && olderActivity.IsInTree(newerActivity));
}
public static void Execute(IndexingActivityBase activity)
{
using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id))
{
IndexingActivityHistory.Start(activity.Id);
try
{
using (new Storage.Security.SystemAccount())
activity.ExecuteIndexingActivity();
}
catch (Exception e)
{
//TODO: WARNING Do not fill the event log with repetitive messages.
SnLog.WriteException(e, $"Indexing activity execution error. Activity: #{activity.Id} ({activity.ActivityType})");
SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e);
IndexingActivityHistory.Error(activity.Id, e);
}
finally
{
DependencyManager.Finish(activity);
IndexManager.ActivityFinished(activity.Id);
}
op.Successful = true;
}
}
internal static void Finish(IndexingActivityBase activity)
{
lock (WaitingSetLock)
{
// activity is done in the ActivityQueue
WaitingSet.Remove(activity);
// terminate and release waiting threads if there is any.
activity.Finish();
// register activity termination in the log.
IndexingActivityHistory.Finish(activity.Id);
// register activity termination if the activity was not skipped.
if (activity.Executed)
{
TerminationHistory.FinishActivity(activity);
}
// execute all activities that are completely freed.
foreach (var dependentItem in activity.WaitingForMe.ToArray())
{
dependentItem.FinishWaiting(activity);
if (dependentItem.WaitingFor.Count == 0)
{
System.Threading.Tasks.Task.Run(() => Executor.Execute(dependentItem));
}
}
}
}
private static void ExecuteCentralizedActivity(IndexingActivityBase activity)
{
SnTrace.Index.Write("ExecuteCentralizedActivity: #{0}", activity.Id);
CentralizedIndexingActivityQueue.ExecuteActivity(activity);
activity.WaitForComplete();
}
private static STT.Task ExecuteCentralizedActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
{
SnTrace.Index.Write("ExecuteCentralizedActivity: #{0}", activity.Id);
CentralizedIndexingActivityQueue.ExecuteActivity(activity);
return(activity.WaitForCompleteAsync(cancellationToken));
}
public static void Execute(IndexingActivityBase activity)
{
using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id))
{
IndexingActivityHistory.Start(activity.Id);
try
{
using (new Storage.Security.SystemAccount())
activity.ExecuteIndexingActivityAsync(CancellationToken.None).GetAwaiter().GetResult();
}
catch (Exception e)
{
SnLog.WriteException(e, $"Indexing activity execution error. Activity: #{activity.Id} ({activity.ActivityType})");
SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e);
IndexingActivityHistory.Error(activity.Id, e);
}
finally
{
DependencyManager.Finish(activity);
IndexManager.ActivityFinished(activity.Id);
}
op.Successful = true;
}
}
/// <summary>
/// Executes an indexing activity taking into account the dependencies.
/// The execution is immediately (ie parallelized) when possible but the
/// dependent activities are executed in the order of registration.
/// Dependent activity execution starts after the blocker activity is completed.
/// </summary>
public static void ExecuteActivity(IndexingActivityBase activity)
{
if (SearchManager.SearchEngine.IndexingEngine.IndexIsCentralized)
{
ExecuteCentralizedActivity(activity);
}
else
{
ExecuteDistributedActivity(activity);
}
}
/// <summary>
/// Entry point of the centralized indexing activity queue.
/// Memorizes the activity in a waiting list and executes some available activities in asynchron way.
/// </summary>
public static void ExecuteActivity(IndexingActivityBase activity)
{
try
{
DisablePolling();
ExecuteActivities(activity, false);
}
finally
{
EnablePolling();
}
}
internal static void Wait(IndexingActivityBase activity)
{
lock (Lock)
{
foreach (var item in History)
{
if (item != null && item.Id == activity.Id)
{
item.WaitedFor = activity.WaitingFor.Select(a => a.Id).ToArray();
break;
}
}
}
}
internal static void AttachOrFinish(IndexingActivityBase activity)
{
lock (WaitingSetLock)
{
var sameActivity = WaitingSet.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another in the waiting set.", activity.Id);
return;
}
}
activity.Finish(); // release blocked thread
IndexingActivityHistory.Finish(activity.Id);
SnTrace.IndexQueue.Write("IAQ: A{0} ignored: finished but not executed.", activity.Id);
}
private static async STT.Task ExecuteDistributedActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
{
SnTrace.Index.Write("ExecuteDistributedActivity: #{0}", activity.Id);
await activity.DistributeAsync(cancellationToken).ConfigureAwait(false);
// If there are too many activities in the queue, we have to drop at least the inner
// data of the activity to prevent memory overflow. We still have to wait for the
// activity to finish, but the inner data can (and will) be loaded from the db when
// the time comes for this activity to be executed.
if (DistributedIndexingActivityQueue.IsOverloaded())
{
SnTrace.Index.Write("IAQ OVERLOAD drop activity FromPopulator A:" + activity.Id);
activity.IndexDocumentData = null;
}
// all activities must be executed through the activity queue's API
DistributedIndexingActivityQueue.ExecuteActivity(activity);
await activity.WaitForCompleteAsync(cancellationToken).ConfigureAwait(false);
}
internal static void Arrive(IndexingActivityBase activity)
{
lock (Lock)
{
// avoiding duplication
foreach (var item in History)
{
if (item != null && item.Id == activity.Id)
{
return;
}
}
var retired = History[_position];
History[_position] = new IndexingActivityHistoryItem
{
Id = activity.Id,
TypeName = activity.ActivityType.ToString(),
FromReceiver = activity.FromReceiver,
FromDb = activity.FromDatabase,
IsStartup = activity.IsUnprocessedActivity,
ArrivedAt = DateTime.UtcNow,
StartedAt = DateTime.MinValue,
FinishedAt = DateTime.MinValue
};
if (retired != null)
{
if (retired.FinishedAt == DateTime.MinValue)
{
_unfinished++;
}
}
_position++;
if (_position >= HistoryLength)
{
_position = 0;
}
}
}
internal static void FinishActivity(IndexingActivityBase activity)
{
var id = activity.Id;
lock (GapsLock)
{
if (id > _lastId)
{
if (id > _lastId + 1)
{
Gaps.AddRange(Enumerable.Range(_lastId + 1, id - _lastId - 1));
}
_lastId = id;
}
else
{
Gaps.Remove(id);
}
SnTrace.IndexQueue.Write("IAQ: State after finishing A{0}: {1}", id, GetCurrentState());
}
}
private static void MakeDependencies(IndexingActivityBase newerActivity)
{
lock (WaitingSetLock)
{
foreach (var olderActivity in WaitingSet)
{
if (MustWait(newerActivity, olderActivity))
{
newerActivity.WaitFor(olderActivity);
SnTrace.IndexQueue.Write("IAQ: A{0} depends from A{1}", newerActivity.Id, olderActivity.Id);
IndexingActivityHistory.Wait(newerActivity);
}
}
WaitingSet.Add(newerActivity);
if (newerActivity.WaitingFor.Count == 0)
{
System.Threading.Tasks.Task.Run(() => Executor.Execute(newerActivity));
}
}
}
private static void ExecuteActivity(IndexingActivityBase activity)
{
IndexManager.RegisterActivity(activity);
IndexManager.ExecuteActivity(activity);
}
private static async STT.Task ExecuteActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
{
await IndexManager.RegisterActivityAsync(activity, cancellationToken).ConfigureAwait(false);
await IndexManager.ExecuteActivityAsync(activity, cancellationToken).ConfigureAwait(false);
}
public static void EnqueueActivity(IndexingActivityBase activity)
{
SnTrace.IndexQueue.Write("IAQ: A{0} arrived{1}. {2}, {3}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.GetType().Name, activity.Path);
IndexingActivityHistory.Arrive(activity);
lock (ArrivalQueueLock)
{
if (activity.Id <= _lastQueued)
{
var sameActivity = ArrivalQueue.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another one in the queue", activity.Id);
return;
}
DependencyManager.AttachOrFinish(activity);
return;
}
if (activity.Id > _lastQueued + 1)
{
var from = _lastQueued + 1;
var to = activity.Id - 1;
var expectedCount = to - from + 1;
var loadedActivities = Retrier.Retry(
3,
100,
() => LoadActivities(from, to),
(r, i, e) =>
{
if (i < 3)
{
SnTrace.IndexQueue.Write("IAQ: Loading attempt {0}", 4 - i);
}
if (e != null)
{
return(false);
}
return(r.Count() == expectedCount);
});
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued from db.", loadedActivity.Id);
DependencyManager.ActivityEnqueued();
}
}
ArrivalQueue.Enqueue(activity);
_lastQueued = activity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued.", activity.Id);
DependencyManager.ActivityEnqueued();
}
}
public static void ExecuteActivity(IndexingActivityBase activity)
{
Serializer.EnqueueActivity(activity);
}
/* ========================================================================================== Activity */
/// <summary>
/// Registers an indexing activity in the database.
/// </summary>
/// <param name="activity">The activity to register.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests. The default value is None.</param>
/// <returns>A Task that represents the asynchronous operation.</returns>
public static STT.Task RegisterActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
{
return(DataStore.RegisterIndexingActivityAsync(activity, cancellationToken));
}
/// <summary>
/// Executes an indexing activity taking dependencies into account and waits for its completion asynchronously.
/// Dependent activities are executed in the order of registration.
/// Dependent activity execution starts after the previously blocker activity is completed.
/// </summary>
/// <param name="activity">The activity to execute.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests. The default value is None.</param>
/// <returns>A Task that represents the asynchronous operation.</returns>
public static STT.Task ExecuteActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
{
return(SearchManager.SearchEngine.IndexingEngine.IndexIsCentralized
? ExecuteCentralizedActivityAsync(activity, cancellationToken)
: ExecuteDistributedActivityAsync(activity, cancellationToken));
}
/// <summary>
/// Loads some executable activities and queries the state of the waiting activities in one database request.
/// Releases the finished activities and executes the loaded ones in asynchron way.
/// </summary>
private static int ExecuteActivities(IndexingActivityBase waitingActivity, bool systemStart)
{
int[] waitingActivityIds;
lock (WaitingActivitiesSync)
{
if (waitingActivity != null)
{
if (WaitingActivities.TryGetValue(waitingActivity.Id, out IndexingActivityBase olderWaitingActivity))
{
SnTrace.IndexQueue.Write($"CIAQ: Attaching new waiting A{waitingActivity.Id} to an older instance.");
// if exists, attach the current to existing.
olderWaitingActivity.Attach(waitingActivity);
// do not load executables because wait-polling cycle is active.
return(0);
}
SnTrace.IndexQueue.Write($"CIAQ: Adding arrived A{waitingActivity.Id} to waiting list.");
// add to waiting list
WaitingActivities.Add(waitingActivity.Id, waitingActivity);
}
// get id array of waiting activities
waitingActivityIds = WaitingActivities.Keys.ToArray();
if (SnTrace.IndexQueue.Enabled)
{
SnTrace.IndexQueue.Write($"Waiting set v1: {string.Join(", ", waitingActivityIds)}");
}
}
// load some executable activities and currently finished ones
var result = DataStore.LoadExecutableIndexingActivitiesAsync(
IndexingActivityFactory.Instance,
MaxCount,
RunningTimeoutInSeconds,
waitingActivityIds, CancellationToken.None).GetAwaiter().GetResult();
var loadedActivities = result.Activities;
var finishedActivitiyIds = result.FinishedActivitiyIds;
if (SnTrace.IndexQueue.Enabled)
{
SnTrace.IndexQueue.Write("CIAQ: loaded: {0} ({1}), waiting: {2}, finished: {3}, tasks: {4}",
loadedActivities.Length,
string.Join(", ", loadedActivities.Select(la => la.Id)),
waitingActivityIds.Length, finishedActivitiyIds.Length, _activeTasks);
}
// release finished activities
if (finishedActivitiyIds.Length > 0)
{
lock (WaitingActivitiesSync)
{
foreach (var finishedActivitiyId in finishedActivitiyIds)
{
if (WaitingActivities.TryGetValue(finishedActivitiyId, out IndexingActivityBase finishedActivity))
{
WaitingActivities.Remove(finishedActivitiyId);
finishedActivity.Finish();
}
}
}
}
if (loadedActivities.Any())
{
lock (WaitingActivitiesSync)
{
if (SnTrace.IndexQueue.Enabled)
{
SnTrace.IndexQueue.Write($"Waiting set v2: {string.Join(", ", WaitingActivities.Keys)}");
}
// execute loaded activities
foreach (var loadedActivity in loadedActivities)
{
if (systemStart)
{
loadedActivity.IsUnprocessedActivity = true;
}
// If a loaded activity is the same as the current activity or
// the same as any of the already waiting activities, we have to
// drop the loaded instance and execute the waiting instance, otherwise
// the algorithm would not notice when the activity is finished and
// the finish signal is released.
IIndexingActivity executableActivity;
if (loadedActivity.Id == waitingActivity?.Id)
{
executableActivity = waitingActivity;
}
else
{
if (WaitingActivities.TryGetValue(loadedActivity.Id, out var otherWaitingActivity))
{
// Found in the waiting list: drop the loaded one and execute the waiting.
executableActivity = otherWaitingActivity;
SnTrace.IndexQueue.Write($"CIAQ: Loaded A{loadedActivity.Id} found in the waiting list.");
}
else
{
// If a loaded activity is not in the waiting list, we have to add it here
// so that other threads may find it and be able to attach to it.
SnTrace.IndexQueue.Write($"CIAQ: Adding loaded A{loadedActivity.Id} to waiting list.");
WaitingActivities.Add(loadedActivity.Id, loadedActivity as IndexingActivityBase);
executableActivity = loadedActivity;
}
}
System.Threading.Tasks.Task.Run(() => Execute(executableActivity));
}
}
}
// memorize last running time
_lastExecutionTime = DateTime.UtcNow;
return(loadedActivities.Length);
}
/* ========================================================================================== Activity */
/// <summary>
/// Registers an indexing aztivity in the database.
/// </summary>
public static void RegisterActivity(IndexingActivityBase activity)
{
DataProvider.Current.RegisterIndexingActivity(activity);
}
