Exemple #1
0
        internal static void Commit(bool reopenReader = true)
        {
            CompletionState commitState;

            using (var op = SnTrace.Index.StartOperation("LM: Commit. reopenReader:{0}", reopenReader))
            {
                using (var wrFrame = IndexWriterFrame.Get(!reopenReader)) // // Commit
                {
                    commitState = CompletionState.GetCurrent();
                    var commitStateMessage = commitState.ToString();

                    SnTrace.Index.Write("LM: Committing_writer. commitState: " + commitStateMessage);

                    // Write a fake document to make sure that the index changes are written to the file system.
                    wrFrame.IndexWriter.UpdateDocument(new Term(COMMITFIELDNAME, COMMITFIELDNAME), GetFakeDocument());

                    wrFrame.IndexWriter.Commit(CompletionState.GetCommitUserData(commitState));

                    if (reopenReader)
                    {
                        ReopenReader();
                    }
                }

#pragma warning disable 420
                Interlocked.Exchange(ref _activities, 0);
#pragma warning restore 420
                _delayCycle = 0;

                op.Successful = true;
            }
        }
Exemple #2
0
        public static void Startup(System.IO.TextWriter consoleOut)
        {
            // initalize from index
            CompletionState cud;

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                cud = CompletionState.ParseFromReader(readerFrame.IndexReader);
            var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();

            using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}"
                                                         , cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength))
            {
                Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut);

                op.Successful = true;
            }
        }
Exemple #3
0
        /*======================================================================================================= IIndexPopulator Members */

        // caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce
        public void ClearAndPopulateAll(bool backup = true)
        {
            var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId();
            var commitData     = CompletionState.GetCommitUserData(lastActivityId);

            using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll"))
            {
                // recreate
                var writer = IndexManager.GetIndexWriter(true);
                try
                {
                    var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
                    foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes))
                    {
                        var doc = IndexDocumentInfo.GetDocument(docData);
                        if (doc == null) // indexing disabled
                        {
                            continue;
                        }
                        writer.AddDocument(doc);
                        OnNodeIndexed(docData.Path);
                    }
                    RepositoryInstance.Instance.ConsoleWrite("  Commiting ... ");
                    writer.Commit(commitData);
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                    RepositoryInstance.Instance.ConsoleWrite("  Optimizing ... ");
                    writer.Optimize();
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                }
                finally
                {
                    writer.Close();
                }
                RepositoryInstance.Instance.ConsoleWrite("  Deleting indexing activities ... ");
                LuceneManager.DeleteAllIndexingActivities();
                RepositoryInstance.Instance.ConsoleWriteLine("ok");
                if (backup)
                {
                    RepositoryInstance.Instance.ConsoleWrite("  Making backup ... ");
                    BackupTools.BackupIndexImmediatelly();
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                }
                op.Successful = true;
            }
        }
Exemple #4
0
        internal static CompletionState ParseFromReader(IDictionary <string, string> commitUserData)
        {
            var result = new CompletionState();

            if (commitUserData != null)
            {
                string value;

                int lastActivityId;
                if (commitUserData.TryGetValue(CompletionState.LastActivityIdKey, out value))
                {
                    if (!string.IsNullOrEmpty(value))
                    {
                        if (int.TryParse(value, out lastActivityId))
                        {
                            result.LastActivityId = lastActivityId;
                        }
                    }
                }

                if (commitUserData.TryGetValue(CompletionState.GapsKey, out value))
                {
                    if (!string.IsNullOrEmpty(value))
                    {
                        result.Gaps = value
                                      .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
                                      .Select(s => { int i; if (Int32.TryParse(s, out i))
                                                     {
                                                         return(i);
                                                     }
                                                     return(0); })
                                      .Where(i => i > 0)
                                      .ToArray();
                    }
                }
            }
            return(result);
        }
Exemple #5
0
        public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse)
        {
            var path = content == null ? null : content.Path;

            CompletionState completionState;

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                completionState = CompletionState.ParseFromReader(readerFrame.IndexReader);
            var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();

            var channel       = SenseNet.ContentRepository.DistributedApplication.ClusterChannel;
            var appDomainName = channel == null ? null : channel.ReceiverName;

            return(new
            {
                AppDomainName = appDomainName,
                LastStoredActivity = lastDatabaseId,
                LastProcessedActivity = completionState.LastActivityId,
                GapsLength = completionState.GapsLength,
                Gaps = completionState.Gaps,
                Differences = Check(path, recurse)
            });
        }
Exemple #6
0
 internal static Dictionary <string, string> GetCommitUserData(CompletionState data)
 {
     return(GetCommitUserData(data.LastActivityId, data.Gaps));
 }
Exemple #7
0
            /// <summary>
            /// MUST BE SYNCHRON
            /// GAPS MUST BE ORDERED
            /// </summary>
            internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
            {
                if (consoleOut != null)
                {
                    consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3));
                }

                SnLog.WriteInformation("Executing unprocessed activities.",
                                       EventId.RepositoryRuntime,
                                       properties: new Dictionary <string, object> {
                    { "LastDatabaseId", lastDatabaseId },
                    { "LastExecutedId", lastExecutedId },
                    { "CountOfGaps", gaps.Length },
                    { "Gaps", String.Join(", ", gaps) }
                });

                DependencyManager.Start();

                var count = 0;

                if (gaps.Any())
                {
                    var loadedActivities = new IndexingActivityLoader(gaps, true);
                    foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
                    {
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        _arrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }
                if (lastExecutedId < lastDatabaseId)
                {
                    var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
                    foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
                    {
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        _arrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }

                if (_lastQueued < lastExecutedId)
                {
                    _lastQueued = lastExecutedId;
                }

                // ensure that the arrival activity queue is not empty at this pont.
                DependencyManager.ActivityEnqueued();

                if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
                {
                    while (IsWorking())
                    {
                        Thread.Sleep(200);
                    }
                }

                // At this point we know for sure that the original gap is not there anymore.
                // In case there is a false gap (e.g. because there are missing activity ids
                // in the db) we have to remove these ids manually from the in-memory gap.
                if (gaps.Any())
                {
                    TerminationHistory.RemoveGaps(gaps);

                    // Commit is necessary because otherwise the gap is removed only in memory, but
                    // the index is not updated in the file system.
                    LuceneManager.Commit();
                }

                SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
            }
Exemple #8
0
 /// <summary>Only for tests</summary>
 internal static void _setCurrentExecutionState(CompletionState state)
 {
     Serializer.Reset(state.LastActivityId);
     DependencyManager.Reset();
     TerminationHistory.Reset(state.LastActivityId, state.Gaps);
 }