/// <summary>
/// Generates a history from the recent items.
/// </summary>
/// <returns></returns>
public static IndexingActivityHistory GetHistory()
{
IndexingActivityHistory result;
var list = new List <IndexingActivityHistoryItem>(History.Length);
lock (Lock)
{
for (int i = _position; i < History.Length; i++)
{
if (History[i] != null)
{
list.Add(History[i]);
}
}
for (int i = 0; i < _position; i++)
{
if (History[i] != null)
{
list.Add(History[i]);
}
}
result = new IndexingActivityHistory()
{
State = DistributedIndexingActivityQueue.GetCurrentState(),
Recent = list.ToArray()
};
}
return(result);
}
public static void Execute(IndexingActivityBase activity)
{
using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id))
{
IndexingActivityHistory.Start(activity.Id);
try
{
using (new Storage.Security.SystemAccount())
activity.ExecuteIndexingActivityAsync(CancellationToken.None).GetAwaiter().GetResult();
}
catch (Exception e)
{
SnLog.WriteException(e, $"Indexing activity execution error. Activity: #{activity.Id} ({activity.ActivityType})");
SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e);
IndexingActivityHistory.Error(activity.Id, e);
}
finally
{
DependencyManager.Finish(activity);
IndexManager.ActivityFinished(activity.Id);
}
op.Successful = true;
}
}
internal static void Finish(IndexingActivityBase activity)
{
lock (WaitingSetLock)
{
// activity is done in the ActivityQueue
WaitingSet.Remove(activity);
// terminate and release waiting threads if there is any.
activity.Finish();
// register activity termination in the log.
IndexingActivityHistory.Finish(activity.Id);
// register activity termination if the activity was not skipped.
if (activity.Executed)
{
TerminationHistory.FinishActivity(activity);
}
// execute all activities that are completely freed.
foreach (var dependentItem in activity.WaitingForMe.ToArray())
{
dependentItem.FinishWaiting(activity);
if (dependentItem.WaitingFor.Count == 0)
{
System.Threading.Tasks.Task.Run(() => Executor.Execute(dependentItem));
}
}
}
}
public static void Execute(IndexingActivityBase activity)
{
using (var op = SnTrace.Index.StartOperation("IAQ: A{0} EXECUTION.", activity.Id))
{
IndexingActivityHistory.Start(activity.Id);
try
{
using (new Storage.Security.SystemAccount())
activity.ExecuteIndexingActivity();
}
catch (Exception e)
{
//TODO: WARNING Do not fill the event log with repetitive messages.
SnLog.WriteException(e, $"Indexing activity execution error. Activity: #{activity.Id} ({activity.ActivityType})");
SnTrace.Index.WriteError("IAQ: A{0} EXECUTION ERROR: {1}", activity.Id, e);
IndexingActivityHistory.Error(activity.Id, e);
}
finally
{
DependencyManager.Finish(activity);
IndexManager.ActivityFinished(activity.Id);
}
op.Successful = true;
}
}
private static void Startup(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
Serializer.Reset();
DependencyManager.Reset();
TerminationHistory.Reset(lastExecutedId, gaps);
Serializer.Start(lastDatabaseId, lastExecutedId, gaps, consoleOut);
IndexingActivityHistory.Reset();
}
internal static void AttachOrFinish(IndexingActivityBase activity)
{
lock (WaitingSetLock)
{
var sameActivity = WaitingSet.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another in the waiting set.", activity.Id);
return;
}
}
activity.Finish(); // release blocked thread
IndexingActivityHistory.Finish(activity.Id);
SnTrace.IndexQueue.Write("IAQ: A{0} ignored: finished but not executed.", activity.Id);
}
/// <summary>
/// Resets the history and returns with the starting state.
/// </summary>
public static IndexingActivityHistory Reset()
{
IndexingActivityHistory result;
lock (Lock)
{
for (int i = 0; i < History.Length; i++)
{
History[i] = null;
}
_position = 0;
_unfinished = 0;
result = new IndexingActivityHistory()
{
State = DistributedIndexingActivityQueue.GetCurrentState(),
Recent = new IndexingActivityHistoryItem[0]
};
}
return(result);
}
private static void MakeDependencies(IndexingActivityBase newerActivity)
{
lock (WaitingSetLock)
{
foreach (var olderActivity in WaitingSet)
{
if (MustWait(newerActivity, olderActivity))
{
newerActivity.WaitFor(olderActivity);
SnTrace.IndexQueue.Write("IAQ: A{0} depends from A{1}", newerActivity.Id, olderActivity.Id);
IndexingActivityHistory.Wait(newerActivity);
}
}
WaitingSet.Add(newerActivity);
if (newerActivity.WaitingFor.Count == 0)
{
System.Threading.Tasks.Task.Run(() => Executor.Execute(newerActivity));
}
}
}
public static void EnqueueActivity(IndexingActivityBase activity)
{
SnTrace.IndexQueue.Write("IAQ: A{0} arrived{1}. {2}, {3}", activity.Id, activity.FromReceiver ? " from another computer" : "", activity.GetType().Name, activity.Path);
IndexingActivityHistory.Arrive(activity);
lock (ArrivalQueueLock)
{
if (activity.Id <= _lastQueued)
{
var sameActivity = ArrivalQueue.FirstOrDefault(a => a.Id == activity.Id);
if (sameActivity != null)
{
sameActivity.Attach(activity);
SnTrace.IndexQueue.Write("IAQ: A{0} attached to another one in the queue", activity.Id);
return;
}
DependencyManager.AttachOrFinish(activity);
return;
}
if (activity.Id > _lastQueued + 1)
{
var from = _lastQueued + 1;
var to = activity.Id - 1;
var expectedCount = to - from + 1;
var loadedActivities = Retrier.Retry(
3,
100,
() => LoadActivities(from, to),
(r, i, e) =>
{
if (i < 3)
{
SnTrace.IndexQueue.Write("IAQ: Loading attempt {0}", 4 - i);
}
if (e != null)
{
return(false);
}
return(r.Count() == expectedCount);
});
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued from db.", loadedActivity.Id);
DependencyManager.ActivityEnqueued();
}
}
ArrivalQueue.Enqueue(activity);
_lastQueued = activity.Id;
SnTrace.IndexQueue.Write("IAQ: A{0} enqueued.", activity.Id);
DependencyManager.ActivityEnqueued();
}
}
/// <summary>
/// MUST BE SYNCHRON
/// GAPS MUST BE ORDERED
/// </summary>
internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3));
SnLog.WriteInformation("Executing unprocessed activities.",
EventId.RepositoryRuntime,
properties: new Dictionary <string, object> {
{ "LastDatabaseId", lastDatabaseId },
{ "LastExecutedId", lastExecutedId },
{ "CountOfGaps", gaps.Length },
{ "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) }
});
IndexingActivityBase.NotIndexableContentCollection.Clear();
DependencyManager.Start();
var count = 0;
if (gaps.Any())
{
var loadedActivities = new IndexingActivityLoader(gaps, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (lastExecutedId < lastDatabaseId)
{
var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (_lastQueued < lastExecutedId)
{
_lastQueued = lastExecutedId;
}
// ensure that the arrival activity queue is not empty at this pont.
DependencyManager.ActivityEnqueued();
if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
{
while (IsWorking())
{
Thread.Sleep(200);
}
}
if (IndexingActivityBase.NotIndexableContentCollection.Any())
{
const int maxWrittenCount = 100;
// Collect the first n ContentId, VersionId pairs into one message
// (key: version id, value: content id).
var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount)
.OrderBy(x => x.Value)
.Select(x => $"{x.Value},{x.Key}"));
var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount
? $" first {maxWrittenCount}"
: string.Empty;
// Write one aggregated message
SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" +
$" content. The{firstN} related ContentId, VersionId pairs: {data}");
// Do not keep memory for unnecessary data
IndexingActivityBase.NotIndexableContentCollection.Clear();
}
// At this point we know for sure that the original gap is not there anymore.
// In case there is a false gap (e.g. because there are missing activity ids
// in the db) we have to remove these ids manually from the in-memory gap.
if (gaps.Any())
{
TerminationHistory.RemoveGaps(gaps);
// Commit is necessary because otherwise the gap is removed only in memory, but
// the index is not updated in the file system.
IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult(); // explicit commit
}
SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
}
/// <summary>
/// MUST BE SYNCHRON
/// GAPS MUST BE ORDERED
/// </summary>
internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3));
SnLog.WriteInformation("Executing unprocessed activities.",
EventId.RepositoryRuntime,
properties: new Dictionary <string, object> {
{ "LastDatabaseId", lastDatabaseId },
{ "LastExecutedId", lastExecutedId },
{ "CountOfGaps", gaps.Length },
{ "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) }
});
DependencyManager.Start();
var count = 0;
if (gaps.Any())
{
var loadedActivities = new IndexingActivityLoader(gaps, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (lastExecutedId < lastDatabaseId)
{
var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (_lastQueued < lastExecutedId)
{
_lastQueued = lastExecutedId;
}
// ensure that the arrival activity queue is not empty at this pont.
DependencyManager.ActivityEnqueued();
if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
{
while (IsWorking())
{
Thread.Sleep(200);
}
}
// At this point we know for sure that the original gap is not there anymore.
// In case there is a false gap (e.g. because there are missing activity ids
// in the db) we have to remove these ids manually from the in-memory gap.
if (gaps.Any())
{
TerminationHistory.RemoveGaps(gaps);
// Commit is necessary because otherwise the gap is removed only in memory, but
// the index is not updated in the file system.
IndexManager.Commit(); // explicit commit
}
SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
}
