/*======================================================================================================= IIndexPopulator Members */
// caller: IndexPopulator.Populator, Import.Importer, Tests
public async STT.Task ClearAndPopulateAllAsync(CancellationToken cancellationToken, TextWriter consoleWriter = null)
{
using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll"))
{
// recreate
consoleWriter?.Write(" Cleanup index ... ");
await IndexManager.ClearIndexAsync(cancellationToken).ConfigureAwait(false);
consoleWriter?.WriteLine("ok");
await IndexManager.AddDocumentsAsync(LoadIndexDocumentsByPath("/Root"), cancellationToken).ConfigureAwait(false);
// delete progress characters
consoleWriter?.Write(" \n");
consoleWriter?.Write(" Commiting ... ");
await IndexManager.CommitAsync(cancellationToken).ConfigureAwait(false); // explicit commit
consoleWriter?.WriteLine("ok");
consoleWriter?.Write(" Deleting indexing activities ... ");
await IndexManager.DeleteAllIndexingActivitiesAsync(cancellationToken).ConfigureAwait(false);
op.Successful = true;
}
}
private void Commit()
{
//TODO: [async] make the whole mechanism async
IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult();
Interlocked.Exchange(ref _uncommittedActivityCount, 0);
_lastCommitTime = DateTime.UtcNow;
}
private async STT.Task CommitAsync()
{
await _indexManager.CommitAsync(CancellationToken.None).ConfigureAwait(false);
Interlocked.Exchange(ref _uncommittedActivityCount, 0);
_lastCommitTime = DateTime.UtcNow;
}
/*======================================================================================================= IIndexPopulator Members */
// caller: IndexPopulator.Populator, Import.Importer, Tests
public async STT.Task ClearAndPopulateAllAsync(CancellationToken cancellationToken, TextWriter consoleWriter = null)
{
using var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll");
using var sa = new SystemAccount();
var stopIndexing = false;
// We must not assume that the indexing engine works correctly without running.
// Local engines may work, but centralized engines do not. This is why we have to
// start it if it is not running already. In that case we have to stop it
// after the operation.
if (!IndexManager.Running)
{
await IndexManager.StartAsync(consoleWriter, cancellationToken).ConfigureAwait(false);
stopIndexing = true;
}
// recreate
consoleWriter?.Write(" Cleanup index ... ");
await IndexManager.ClearIndexAsync(cancellationToken).ConfigureAwait(false);
consoleWriter?.WriteLine("ok");
await IndexManager.AddDocumentsAsync(LoadIndexDocumentsByPath("/Root"), cancellationToken).ConfigureAwait(false);
// delete progress characters
consoleWriter?.Write(" \n");
consoleWriter?.Write(" Commiting ... ");
await IndexManager.CommitAsync(cancellationToken).ConfigureAwait(false); // explicit commit
consoleWriter?.WriteLine("ok");
consoleWriter?.Write(" Deleting indexing activities ... ");
await IndexManager.DeleteAllIndexingActivitiesAsync(cancellationToken).ConfigureAwait(false);
if (stopIndexing)
{
IndexManager.ShutDown();
}
op.Successful = true;
}
/// <summary>
/// MUST BE SYNCHRON
/// GAPS MUST BE ORDERED
/// </summary>
internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3));
SnLog.WriteInformation("Executing unprocessed activities.",
EventId.RepositoryRuntime,
properties: new Dictionary <string, object> {
{ "LastDatabaseId", lastDatabaseId },
{ "LastExecutedId", lastExecutedId },
{ "CountOfGaps", gaps.Length },
{ "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) }
});
IndexingActivityBase.NotIndexableContentCollection.Clear();
DependencyManager.Start();
var count = 0;
if (gaps.Any())
{
var loadedActivities = new IndexingActivityLoader(gaps, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (lastExecutedId < lastDatabaseId)
{
var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
foreach (var indexingActivity in loadedActivities)
{
var loadedActivity = (IndexingActivityBase)indexingActivity;
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
ArrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (_lastQueued < lastExecutedId)
{
_lastQueued = lastExecutedId;
}
// ensure that the arrival activity queue is not empty at this pont.
DependencyManager.ActivityEnqueued();
if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
{
while (IsWorking())
{
Thread.Sleep(200);
}
}
if (IndexingActivityBase.NotIndexableContentCollection.Any())
{
const int maxWrittenCount = 100;
// Collect the first n ContentId, VersionId pairs into one message
// (key: version id, value: content id).
var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount)
.OrderBy(x => x.Value)
.Select(x => $"{x.Value},{x.Key}"));
var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount
? $" first {maxWrittenCount}"
: string.Empty;
// Write one aggregated message
SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" +
$" content. The{firstN} related ContentId, VersionId pairs: {data}");
// Do not keep memory for unnecessary data
IndexingActivityBase.NotIndexableContentCollection.Clear();
}
// At this point we know for sure that the original gap is not there anymore.
// In case there is a false gap (e.g. because there are missing activity ids
// in the db) we have to remove these ids manually from the in-memory gap.
if (gaps.Any())
{
TerminationHistory.RemoveGaps(gaps);
// Commit is necessary because otherwise the gap is removed only in memory, but
// the index is not updated in the file system.
IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult(); // explicit commit
}
SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
}
public void ActivityFinished()
{
IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult();
}
public STT.Task ActivityFinishedAsync()
{
return(_indexManager.CommitAsync(CancellationToken.None));
}