public void Indexing_Centralized_Sql_Allocate07_MaxRecords()
{
var connectionStringBackup = Configuration.ConnectionStrings.ConnectionString;
Configuration.ConnectionStrings.ConnectionString = _connectionString;
try
{
CleanupIndexingActivitiesTable();
var start = new IIndexingActivity[15];
for (int i = 1; i <= start.Length; i++)
{
start[i - 1] = RegisterActivity(IndexingActivityType.AddDocument, IndexingActivityRunningState.Waiting, i, i, $"/Root/Path0{i}");
}
var allocated = DataProvider.Current.LoadExecutableIndexingActivities(new IndexingActivityFactory(), 10, 60);
Assert.AreEqual(10, allocated.Length);
for (var i = 0; i < 10; i++)
{
Assert.AreEqual(start[i].Id, allocated[i].Id);
}
}
finally
{
Configuration.ConnectionStrings.ConnectionString = connectionStringBackup;
}
}
private STT.Task ExecuteCentralizedActivityAsync(IIndexingActivity activity, CancellationToken cancellationToken)
{
var activityBase = (IndexingActivityBase)activity;
SnTrace.Index.Write("ExecuteCentralizedActivity: #{0}", activity.Id);
CentralizedIndexingActivityQueue.ExecuteActivity(activityBase);
return(activityBase.WaitForCompleteAsync(cancellationToken));
}
/// <summary>
/// Executes an activity synchronously.
/// Updates the activity's runningState to "Done" to indicate the end of execution.
/// Calls the activity's Finish to release the waiting thread.
/// Removes the activity from the waiting list.
/// </summary>
private static void Execute(IIndexingActivity activity)
{
var act = (IndexingActivityBase)activity;
using (var op = SnTrace.Index.StartOperation("CIAQ: A{0} EXECUTION.", act.Id))
{
try
{
#pragma warning disable 420
Interlocked.Increment(ref _activeTasks);
#pragma warning restore 420
//TODO: [async] refactor this method to be async
// execute synchronously
using (new Storage.Security.SystemAccount())
act.ExecuteIndexingActivityAsync(CancellationToken.None).GetAwaiter().GetResult();
// publish the finishing state
DataStore.UpdateIndexingActivityRunningStateAsync(act.Id, IndexingActivityRunningState.Done, CancellationToken.None)
.GetAwaiter().GetResult();
}
catch (Exception e)
{
//TODO: WARNING Do not fill the event log with repetitive messages.
SnLog.WriteException(e, $"Indexing activity execution error. Activity: #{act.Id} ({act.ActivityType})");
SnTrace.Index.WriteError("CIAQ: A{0} EXECUTION ERROR: {1}", act.Id, e);
}
finally
{
// release the waiting thread and remove from the waiting list
lock (WaitingActivitiesSync)
{
act.Finish();
WaitingActivities.Remove(act.Id);
}
#pragma warning disable 420
Interlocked.Decrement(ref _activeTasks);
#pragma warning restore 420
}
op.Successful = true;
}
}
public void Indexing_Centralized_InMemory_Allocate07_MaxRecords()
{
using (new Tests.Tools.DataProviderSwindler(new InMemoryDataProvider()))
{
var start = new IIndexingActivity[15];
for (int i = 1; i <= start.Length; i++)
{
start[i - 1] = RegisterActivity(IndexingActivityType.AddDocument, IndexingActivityRunningState.Waiting, i, i, $"/Root/Path0{i}");
}
var allocated = DataProvider.Current.LoadExecutableIndexingActivities(new IndexingActivityFactory(), 10, 60);
Assert.AreEqual(10, allocated.Length);
for (var i = 0; i < 10; i++)
{
Assert.AreEqual(start[i].Id, allocated[i].Id);
}
}
}
public async Task Indexing_Centralized_Allocate07_MaxRecords()
{
await NoRepoIntegrationTestAsync(async() =>
{
await DP.DeleteAllIndexingActivitiesAsync(CancellationToken.None);
var start = new IIndexingActivity[15];
for (int i = 1; i <= start.Length; i++)
{
start[i - 1] = await RegisterActivityAsync(IndexingActivityType.AddDocument, IndexingActivityRunningState.Waiting, i, i, $"/Root/Path0{i}");
}
var loaded = await DP.LoadExecutableIndexingActivitiesAsync(new IndexingActivityFactory(), 10, 60, new int[0], CancellationToken.None);
var activities = loaded.Activities;
Assert.AreEqual(10, activities.Length);
for (var i = 0; i < 10; i++)
{
Assert.AreEqual(start[i].Id, activities[i].Id);
}
});
}
private async STT.Task ExecuteDistributedActivityAsync(IIndexingActivity activity, CancellationToken cancellationToken)
{
var activityBase = (IndexingActivityBase)activity;
SnTrace.Index.Write("ExecuteDistributedActivity: #{0}", activity.Id);
await activityBase.DistributeAsync(cancellationToken).ConfigureAwait(false);
// If there are too many activities in the queue, we have to drop at least the inner
// data of the activity to prevent memory overflow. We still have to wait for the
// activity to finish, but the inner data can (and will) be loaded from the db when
// the time comes for this activity to be executed.
if (DistributedIndexingActivityQueue.IsOverloaded())
{
SnTrace.Index.Write("IAQ OVERLOAD drop activity FromPopulator A:" + activity.Id);
activity.IndexDocumentData = null;
}
// all activities must be executed through the activity queue's API
DistributedIndexingActivityQueue.ExecuteActivity(activityBase);
await activityBase.WaitForCompleteAsync(cancellationToken).ConfigureAwait(false);
}
public IIndexingActivity CreateActivity(IndexingActivityType activityType)
{
IIndexingActivity activity = null;
switch (activityType)
{
case IndexingActivityType.AddDocument: activity = new AddDocumentActivity(); break;
case IndexingActivityType.AddTree: activity = new AddTreeActivity(); break;
case IndexingActivityType.UpdateDocument: activity = new UpdateDocumentActivity(); break;
case IndexingActivityType.RemoveTree: activity = new RemoveTreeActivity(); break;
case IndexingActivityType.RemoveDocument: activity = new RemoveDocumentActivity(); break;
case IndexingActivityType.Rebuild: activity = new RebuildActivity(); break;
default: throw new NotSupportedException("Unknown IndexingActivityType: " + activityType);
}
activity.ActivityType = activityType;
return(activity);
}
public abstract void RegisterIndexingActivity(IIndexingActivity activity);
public STT.Task ExecuteActivityAsync(IIndexingActivity activity, CancellationToken cancellationToken)
{
return(_searchManager.SearchEngine.IndexingEngine.IndexIsCentralized
? ExecuteCentralizedActivityAsync(activity, cancellationToken)
: ExecuteDistributedActivityAsync(activity, cancellationToken));
}
/* ------------------------------------------------------------------------------------------ Activity */
public STT.Task RegisterActivityAsync(IIndexingActivity activity, CancellationToken cancellationToken)
{
return(_dataStore.RegisterIndexingActivityAsync(activity, cancellationToken));
}
public Task RegisterIndexingActivityAsync(IIndexingActivity activity, CancellationToken cancellationToken)
{
return(DataProvider.RegisterIndexingActivityAsync(activity, cancellationToken));
}
