Ejemplo n.º 1
0
 private void Commit()
 {
     IndexManager.Commit();
     Interlocked.Exchange(ref _uncommittedActivityCount, 0);
     _lastCommitTime = DateTime.UtcNow;
 }
Ejemplo n.º 2
0
            /// <summary>
            /// MUST BE SYNCHRON
            /// GAPS MUST BE ORDERED
            /// </summary>
            internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
            {
                consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3));

                SnLog.WriteInformation("Executing unprocessed activities.",
                                       EventId.RepositoryRuntime,
                                       properties: new Dictionary <string, object> {
                    { "LastDatabaseId", lastDatabaseId },
                    { "LastExecutedId", lastExecutedId },
                    { "CountOfGaps", gaps.Length },
                    { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) }
                });

                IndexingActivityBase.NotIndexableContentCollection.Clear();
                DependencyManager.Start();

                var count = 0;

                if (gaps.Any())
                {
                    var loadedActivities = new IndexingActivityLoader(gaps, true);
                    foreach (var indexingActivity in loadedActivities)
                    {
                        var loadedActivity = (IndexingActivityBase)indexingActivity;
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        ArrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }
                if (lastExecutedId < lastDatabaseId)
                {
                    var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
                    foreach (var indexingActivity in loadedActivities)
                    {
                        var loadedActivity = (IndexingActivityBase)indexingActivity;
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        ArrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }

                if (_lastQueued < lastExecutedId)
                {
                    _lastQueued = lastExecutedId;
                }

                // ensure that the arrival activity queue is not empty at this pont.
                DependencyManager.ActivityEnqueued();

                if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
                {
                    while (IsWorking())
                    {
                        Thread.Sleep(200);
                    }
                }

                if (IndexingActivityBase.NotIndexableContentCollection.Any())
                {
                    const int maxWrittenCount = 100;

                    // Collect the first n ContentId, VersionId pairs into one message
                    // (key: version id, value: content id).
                    var data = string.Join("; ", IndexingActivityBase.NotIndexableContentCollection.Take(maxWrittenCount)
                                           .OrderBy(x => x.Value)
                                           .Select(x => $"{x.Value},{x.Key}"));
                    var firstN = IndexingActivityBase.NotIndexableContentCollection.Count > maxWrittenCount
                        ? $" first {maxWrittenCount}"
                        : string.Empty;

                    // Write one aggregated message
                    SnLog.WriteWarning($"Cannot index {IndexingActivityBase.NotIndexableContentCollection.Count}" +
                                       $" content. The{firstN} related ContentId, VersionId pairs: {data}");

                    // Do not keep memory for unnecessary data
                    IndexingActivityBase.NotIndexableContentCollection.Clear();
                }

                // At this point we know for sure that the original gap is not there anymore.
                // In case there is a false gap (e.g. because there are missing activity ids
                // in the db) we have to remove these ids manually from the in-memory gap.
                if (gaps.Any())
                {
                    TerminationHistory.RemoveGaps(gaps);

                    // Commit is necessary because otherwise the gap is removed only in memory, but
                    // the index is not updated in the file system.
                    IndexManager.Commit(); // explicit commit
                }

                SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
            }
Ejemplo n.º 3
0
        private static async STT.Task ExecuteActivityAsync(IndexingActivityBase activity, CancellationToken cancellationToken)
        {
            await IndexManager.RegisterActivityAsync(activity, cancellationToken).ConfigureAwait(false);

            await IndexManager.ExecuteActivityAsync(activity, cancellationToken).ConfigureAwait(false);
        }
Ejemplo n.º 4
0
 public void ActivityFinished()
 {
     IndexManager.Commit();
 }
            /// <summary>
            /// MUST BE SYNCHRON
            /// GAPS MUST BE ORDERED
            /// </summary>
            internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
            {
                consoleOut?.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, IndexingActivityStatus.GapsToString(gaps, 5, 3));

                SnLog.WriteInformation("Executing unprocessed activities.",
                                       EventId.RepositoryRuntime,
                                       properties: new Dictionary <string, object> {
                    { "LastDatabaseId", lastDatabaseId },
                    { "LastExecutedId", lastExecutedId },
                    { "CountOfGaps", gaps.Length },
                    { "Gaps", IndexingActivityStatus.GapsToString(gaps, 100, 3) }
                });

                DependencyManager.Start();

                var count = 0;

                if (gaps.Any())
                {
                    var loadedActivities = new IndexingActivityLoader(gaps, true);
                    foreach (var indexingActivity in loadedActivities)
                    {
                        var loadedActivity = (IndexingActivityBase)indexingActivity;
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        ArrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }
                if (lastExecutedId < lastDatabaseId)
                {
                    var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
                    foreach (var indexingActivity in loadedActivities)
                    {
                        var loadedActivity = (IndexingActivityBase)indexingActivity;
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        ArrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }

                if (_lastQueued < lastExecutedId)
                {
                    _lastQueued = lastExecutedId;
                }

                // ensure that the arrival activity queue is not empty at this pont.
                DependencyManager.ActivityEnqueued();

                if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
                {
                    while (IsWorking())
                    {
                        Thread.Sleep(200);
                    }
                }

                // At this point we know for sure that the original gap is not there anymore.
                // In case there is a false gap (e.g. because there are missing activity ids
                // in the db) we have to remove these ids manually from the in-memory gap.
                if (gaps.Any())
                {
                    TerminationHistory.RemoveGaps(gaps);

                    // Commit is necessary because otherwise the gap is removed only in memory, but
                    // the index is not updated in the file system.
                    IndexManager.Commit(); // explicit commit
                }

                SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
            }
Ejemplo n.º 6
0
 public void ActivityFinished()
 {
     IndexManager.CommitAsync(CancellationToken.None).GetAwaiter().GetResult();
 }
Ejemplo n.º 7
0
 public NearRealTimeCommitManager(IndexManager indexManager)
 {
     _indexManager = indexManager;
 }
Ejemplo n.º 8
0
 public NoDelayCommitManager(IndexManager indexManager)
 {
     _indexManager = indexManager;
 }
Ejemplo n.º 9
0
 private static void ExecuteActivity(IndexingActivityBase activity)
 {
     IndexManager.RegisterActivity(activity);
     IndexManager.ExecuteActivity(activity);
 }