예제 #1
0
        /*==================================================================== Document operations */

        internal static void RefreshDocument(int versionId)
        {
            // the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed.

            var node = Node.LoadNodeByVersionId(versionId);

            // if node exists, refresh index from db
            // if node does not exist, remove from index
            if (node != null)
            {
                // delete from indexhistory first, because we are trying to refresh to the last timestamp
                // also check if there is a more fresh indexing pending which would surely correct the index, and we don't need this here
                if (!_history.RemoveIfLast(versionId, node.NodeTimestamp))
                {
                    return;
                }

                StorageContext.Search.SearchEngine.GetPopulator().RefreshIndex(node, false);
            }
            else
            {
                var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(versionId));
                LuceneManager.DeleteDocuments(new[] { delTerm }, false);
            }
        }
예제 #2
0
        /// <summary>
        /// Synchronous backup method for console applications.
        /// </summary>
        public static void SynchronousBackupIndex()
        {
            using (var op = SnTrace.Repository.StartOperation("Backup index immediatelly."))
            {
                EnsureEmptyDirctory(_backupDirectoryPath);
                LuceneManager.PauseIndexing();

                while (!LuceneManager.Paused)
                {
                    Thread.Sleep(100);
                }
                Thread.Sleep(1000);
                try
                {
                    CopyIndexToBackupDirectory();
                }
                finally
                {
                    LuceneManager.ContinueIndexing();
                }

                OptimizeCompressAndStore();

                op.Successful = true;
            }
        }
예제 #3
0
        private ScoreDoc[] GetDocsUnderTree(string path, bool recurse)
        {
            var field = recurse ? "InTree" : "Path";
            var lq    = LucQuery.Parse(String.Format("{0}:'{1}'", field, path.ToLower()));

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
            {
                var idxReader = readerFrame.IndexReader;
                var searcher  = new IndexSearcher(idxReader);
                var numDocs   = idxReader.NumDocs();
                try
                {
                    var collector = TopScoreDocCollector.Create(numDocs, false);
                    searcher.Search(lq.Query, collector);
                    var topDocs = collector.TopDocs(0, numDocs);
                    return(topDocs.ScoreDocs);
                }
                finally
                {
                    if (searcher != null)
                    {
                        searcher.Close();
                    }
                    searcher = null;
                }
            }
        }
예제 #4
0
 // caller: IndexPopulator.Populator
 public void RepopulateTree(string path)
 {
     using (var op = SnTrace.Index.StartOperation("IndexPopulator RepopulateTree"))
     {
         var writer = IndexManager.GetIndexWriter(false);
         writer.DeleteDocuments(new Term(LucObject.FieldName.InTree, path.ToLowerInvariant()));
         try
         {
             var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
             foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(path, excludedNodeTypes))
             {
                 var doc = IndexDocumentInfo.GetDocument(docData);
                 if (doc == null) // indexing disabled
                 {
                     continue;
                 }
                 writer.AddDocument(doc);
                 OnNodeIndexed(docData.Path);
             }
             writer.Optimize();
         }
         finally
         {
             writer.Close();
         }
         op.Successful = true;
     }
 }
예제 #5
0
        public override void DoAction(bool onRemote, bool isFromMe)
        {
            if (onRemote && isFromMe)
            {
                return;
            }
            if (!SenseNet.ContentRepository.RepositoryInstance.LuceneManagerIsRunning)
            {
                return;
            }

            Debug.WriteLine(String.Format("@> {0} =========== Executing Restore action.", AppDomain.CurrentDomain.FriendlyName));
            Exception ex = null;

            try
            {
                new IndexRestoringStartedMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName).Send();
                LuceneManager.ForceRestore();
                new IndexRestoringFinishedMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName).Send();
            }
            catch (Exception e)
            {
                ex = e;
                Logger.WriteException(ex);//TODO: xx " RestoreIndex" szerepeljen a message-ben
            }
            if (ex != null)
            {
                new IndexRestoringErrorMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName, ex).Send();
            }
            Debug.WriteLine(String.Format("@> {0} =========== Executing Restore action is finished.", AppDomain.CurrentDomain.FriendlyName));
        }
예제 #6
0
 public static List <Document> GetDocumentsByNodeId(int nodeId)
 {
     using (var readerFrame = LuceneManager.GetIndexReaderFrame())
     {
         var termDocs = readerFrame.IndexReader.TermDocs(new Term(LucObject.FieldName.NodeId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(nodeId)));
         return(GetDocumentsFromTermDocs(termDocs, readerFrame));
     }
 }
예제 #7
0
 private static void ContinueIndexingAndWriteResult(HttpContext context)
 {
     try
     {
         LuceneManager.ContinueIndexing();
         context.Response.Write("OK");
     }
     catch (Exception ex)
     {
         Logger.WriteException(ex);
         context.Response.Write(ex.Message);
     }
 }
예제 #8
0
        public static void Startup(System.IO.TextWriter consoleOut)
        {
            // initalize from index
            CompletionState cud;

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                cud = CompletionState.ParseFromReader(readerFrame.IndexReader);
            var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();

            using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}"
                                                         , cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength))
            {
                Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut);

                op.Successful = true;
            }
        }
예제 #9
0
        /*==================================================================================== Instance part */

        private IEnumerable <Difference> CheckNode(string path)
        {
            var result = new List <Difference>();

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
            {
                var ixreader = readerFrame.IndexReader;
                //var sql = String.Format(checkNodeSql, path);
                //var proc = SenseNet.ContentRepository.Storage.Data.DataProvider.CreateDataProcedure(sql);
                //proc.CommandType = System.Data.CommandType.Text;
                var docids = new List <int>();
                var proc   = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path);
                using (var dbreader = proc.ExecuteReader())
                {
                    while (dbreader.Read())
                    {
                        var docid = CheckDbAndIndex(dbreader, ixreader, result);
                        if (docid >= 0)
                        {
                            docids.Add(docid);
                        }
                    }
                }
                var scoredocs = GetDocsUnderTree(path, false);
                foreach (var scoredoc in scoredocs)
                {
                    var docid = scoredoc.Doc;
                    var doc   = ixreader.Document(docid);
                    if (!docids.Contains(docid))
                    {
                        result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
                        {
                            DocId              = scoredoc.Doc,
                            VersionId          = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
                            NodeId             = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
                            Path               = path,
                            Version            = doc.Get(LucObject.FieldName.Version),
                            IxNodeTimestamp    = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
                            IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
                        });
                    }
                }
            }
            return(result);
        }
예제 #10
0
        /*======================================================================================================= IIndexPopulator Members */

        // caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce
        public void ClearAndPopulateAll(bool backup = true)
        {
            var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId();
            var commitData     = CompletionState.GetCommitUserData(lastActivityId);

            using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll"))
            {
                // recreate
                var writer = IndexManager.GetIndexWriter(true);
                try
                {
                    var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
                    foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes))
                    {
                        var doc = IndexDocumentInfo.GetDocument(docData);
                        if (doc == null) // indexing disabled
                        {
                            continue;
                        }
                        writer.AddDocument(doc);
                        OnNodeIndexed(docData.Path);
                    }
                    RepositoryInstance.Instance.ConsoleWrite("  Commiting ... ");
                    writer.Commit(commitData);
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                    RepositoryInstance.Instance.ConsoleWrite("  Optimizing ... ");
                    writer.Optimize();
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                }
                finally
                {
                    writer.Close();
                }
                RepositoryInstance.Instance.ConsoleWrite("  Deleting indexing activities ... ");
                LuceneManager.DeleteAllIndexingActivities();
                RepositoryInstance.Instance.ConsoleWriteLine("ok");
                if (backup)
                {
                    RepositoryInstance.Instance.ConsoleWrite("  Making backup ... ");
                    BackupTools.BackupIndexImmediatelly();
                    RepositoryInstance.Instance.ConsoleWriteLine("ok");
                }
                op.Successful = true;
            }
        }
예제 #11
0
        /*==================================================================================== Instance part */

        private IEnumerable <Difference> CheckNode(string path)
        {
            var result = new List <Difference>();

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
            {
                var ixreader = readerFrame.IndexReader;
                var docids   = new List <int>();
                var proc     = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path, GetExcludedNodeTypeIds());
                using (var dbreader = proc.ExecuteReader())
                {
                    while (dbreader.Read())
                    {
                        var docid = CheckDbAndIndex(dbreader, ixreader, result);
                        if (docid >= 0)
                        {
                            docids.Add(docid);
                        }
                    }
                }
                var scoredocs = GetDocsUnderTree(path, false);
                foreach (var scoredoc in scoredocs)
                {
                    var docid = scoredoc.Doc;
                    var doc   = ixreader.Document(docid);
                    if (!docids.Contains(docid))
                    {
                        result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
                        {
                            DocId              = scoredoc.Doc,
                            VersionId          = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
                            NodeId             = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
                            Path               = path,
                            Version            = doc.Get(LucObject.FieldName.Version),
                            IxNodeTimestamp    = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
                            IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
                        });
                    }
                }
            }
            return(result);
        }
예제 #12
0
 static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
 {
     if (StorageContext.Search.IsOuterEngineEnabled && StorageContext.Search.SearchEngine != InternalSearchEngine.Instance)
     {
         var timerEnabled = _timer.Enabled;
         _timer.Enabled = false;
         try
         {
             LuceneManager.ExecuteLostIndexingActivities();
         }
         catch (Exception ex) //logged
         {
             SenseNet.Diagnostics.Logger.WriteException(ex);
         }
         finally
         {
             _timer.Enabled = timerEnabled;
         }
     }
 }
예제 #13
0
        internal static void HealthCheck()
        {
            if (IsWorking())
            {
                SnTrace.Index.Write("IAQ: Health check triggered but ignored.");

                return;
            }

            SnTrace.Index.Write("IAQ: Health check triggered.");


            var state      = TerminationHistory.GetCurrentState();
            var gapsLength = state.Gaps.Length;

            if (gapsLength > 0)
            {
                SnTrace.IndexQueue.Write("IAQ: Health checker is processing {0} gap{1}.", gapsLength, gapsLength > 1 ? "s" : "");

                foreach (LuceneIndexingActivity activity in new IndexingActivityLoader(state.Gaps, false))
                {
                    WaitIfOverloaded();
                    IndexingActivityQueue.ExecuteActivity(activity);
                }
            }

            var lastId     = TerminationHistory.GetLastTerminatedId();
            var lastDbId   = LuceneManager.GetLastStoredIndexingActivityId();
            var newerCount = lastDbId - lastId;

            if (lastId < lastDbId)
            {
                SnTrace.IndexQueue.Write("IAQ: Health checker is processing activities from {0} to {1}", (lastId + 1), lastDbId);

                foreach (LuceneIndexingActivity activity in new IndexingActivityLoader(lastId + 1, lastDbId, false))
                {
                    WaitIfOverloaded();
                    IndexingActivityQueue.ExecuteActivity(activity);
                }
            }
        }
예제 #14
0
        public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse)
        {
            var path = content == null ? null : content.Path;

            CompletionState completionState;

            using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                completionState = CompletionState.ParseFromReader(readerFrame.IndexReader);
            var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();

            var channel       = SenseNet.ContentRepository.DistributedApplication.ClusterChannel;
            var appDomainName = channel == null ? null : channel.ReceiverName;

            return(new
            {
                AppDomainName = appDomainName,
                LastStoredActivity = lastDatabaseId,
                LastProcessedActivity = completionState.LastActivityId,
                GapsLength = completionState.GapsLength,
                Gaps = completionState.Gaps,
                Differences = Check(path, recurse)
            });
        }
예제 #15
0
 private static void ExecuteActivity(LuceneIndexingActivity activity)
 {
     LuceneManager.RegisterActivity(activity);
     LuceneManager.ExecuteActivity(activity, true, true);
 }
예제 #16
0
        //---- Caller: Startup, ForceRestore
        internal static void ExecuteUnprocessedIndexingActivities(System.IO.TextWriter consoleOut)
        {
            lock (_executingUnprocessedIndexingActivitiesLock)
            {
                try
                {
                    _executingUnprocessedIndexingActivities = true;

                    CommitUserData cud;
                    using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                    {
                        cud = IndexManager.ReadCommitUserData(readerFrame.IndexReader);
                    }
                    MissingActivityHandler.MaxActivityId = cud.LastActivityId;
                    MissingActivityHandler.SetGap(cud.Gap);

                    var logProps = new Dictionary <string, object> {
                        { "LastActivityID", cud.LastActivityId }, { "Size of gap", cud.Gap.Count }
                    };
                    Logger.WriteInformation("Executing unprocessed indexing activities from the stored commit point.",
                                            Logger.EmptyCategoryList, logProps);

                    var i        = 0;
                    var sumCount = 0;

                    // This loop was created to avoid loading too many activities at once that are present in the gap.
                    while (i * ACTIVITIESFRAGMENTSIZE <= cud.Gap.Count)
                    {
                        // get activities from the DB that are in the current gap fragment
                        var gapSegment = cud.Gap.Skip(i * ACTIVITIESFRAGMENTSIZE).Take(ACTIVITIESFRAGMENTSIZE).ToArray();
                        var activities = IndexingActivityManager.GetUnprocessedActivities(gapSegment);
                        ProcessTasks(activities, consoleOut);
                        sumCount += activities.Length;
                        i++;
                    }

                    // Execute activities where activity id is bigger than than our last (activity) task id
                    var maxIdInDb = 0;
                    var newtasks  = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out maxIdInDb, ACTIVITIESFRAGMENTSIZE);
                    while (newtasks.Length > 0)
                    {
                        ProcessTasks(newtasks, consoleOut);
                        sumCount += newtasks.Length;

                        //load the remaining activities, but only if they were created before this operation started
                        var tempMax = 0;
                        newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out tempMax, ACTIVITIESFRAGMENTSIZE, maxIdInDb);
                    }

                    if (consoleOut != null)
                    {
                        consoleOut.WriteLine("ok.");
                    }

                    logProps.Add("Processed tasks", sumCount);

                    //write the latest max activity id and gap size to log
                    logProps["LastActivityID"] = MissingActivityHandler.MaxActivityId;
                    logProps["Size of gap"]    = MissingActivityHandler.GetGap().Count;

                    Logger.WriteInformation("Executing unprocessed tasks is finished.", Logger.EmptyCategoryList, logProps);
                }
                finally
                {
                    _executingUnprocessedIndexingActivities = false;
                }
            }
        }
예제 #17
0
        private IEnumerable <Difference> CheckRecurse(string path)
        {
            var result = new List <Difference>();

            using (var op = SnTrace.Index.StartOperation("Index Integrity Checker: CheckRecurse {0}", path))
            {
                using (var readerFrame = LuceneManager.GetIndexReaderFrame())
                {
                    var ixreader = readerFrame.IndexReader;
                    numdocs = ixreader.NumDocs() + ixreader.NumDeletedDocs();
                    var x = numdocs / intsize;
                    var y = numdocs % intsize;
                    docbits = new int[x + (y > 0 ? 1 : 0)];
                    if (path == null)
                    {
                        if (y > 0)
                        {
                            var q = 0;
                            for (int i = 0; i < y; i++)
                            {
                                q += 1 << i;
                            }
                            docbits[docbits.Length - 1] = q ^ (-1);
                        }
                    }
                    else
                    {
                        for (int i = 0; i < docbits.Length; i++)
                        {
                            docbits[i] = -1;
                        }
                        var scoredocs = GetDocsUnderTree(path, true);
                        for (int i = 0; i < scoredocs.Length; i++)
                        {
                            var docid = scoredocs[i].Doc;
                            docbits[docid / intsize] ^= 1 << docid % intsize;
                        }
                    }
                    var proc     = DataProvider.Current.GetTimestampDataForRecursiveIntegrityCheck(path, GetExcludedNodeTypeIds());
                    var progress = 0;
                    using (var dbreader = proc.ExecuteReader())
                    {
                        while (dbreader.Read())
                        {
                            if ((++progress % 10000) == 0)
                            {
                                SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex: progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
                            }

                            var docid = CheckDbAndIndex(dbreader, ixreader, result);
                            if (docid > -1)
                            {
                                docbits[docid / intsize] |= 1 << docid % intsize;
                            }
                        }
                    }
                    SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex finished. Progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
                    for (int i = 0; i < docbits.Length; i++)
                    {
                        if (docbits[i] != -1)
                        {
                            var bits = docbits[i];
                            for (int j = 0; j < intsize; j++)
                            {
                                if ((bits & (1 << j)) == 0)
                                {
                                    var docid = i * intsize + j;
                                    if (docid >= numdocs)
                                    {
                                        break;
                                    }
                                    if (!ixreader.IsDeleted(docid))
                                    {
                                        var doc = ixreader.Document(docid);
                                        result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
                                        {
                                            DocId              = docid,
                                            VersionId          = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
                                            NodeId             = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
                                            Path               = doc.Get(LucObject.FieldName.Path),
                                            Version            = doc.Get(LucObject.FieldName.Version),
                                            IxNodeTimestamp    = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
                                            IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
                                        });
                                    }
                                }
                            }
                        }
                    }
                }
                op.Successful = true;
            }
            return(result.ToArray());
        }
예제 #18
0
            /// <summary>
            /// MUST BE SYNCHRON
            /// GAPS MUST BE ORDERED
            /// </summary>
            internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
            {
                if (consoleOut != null)
                {
                    consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3));
                }

                SnLog.WriteInformation("Executing unprocessed activities.",
                                       EventId.RepositoryRuntime,
                                       properties: new Dictionary <string, object> {
                    { "LastDatabaseId", lastDatabaseId },
                    { "LastExecutedId", lastExecutedId },
                    { "CountOfGaps", gaps.Length },
                    { "Gaps", String.Join(", ", gaps) }
                });

                DependencyManager.Start();

                var count = 0;

                if (gaps.Any())
                {
                    var loadedActivities = new IndexingActivityLoader(gaps, true);
                    foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
                    {
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        _arrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }
                if (lastExecutedId < lastDatabaseId)
                {
                    var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
                    foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
                    {
                        // wait and start processing loaded activities in the meantime
                        WaitIfOverloaded(true);

                        SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);

                        IndexingActivityHistory.Arrive(loadedActivity);
                        _arrivalQueue.Enqueue(loadedActivity);
                        _lastQueued = loadedActivity.Id;
                        count++;
                    }
                }

                if (_lastQueued < lastExecutedId)
                {
                    _lastQueued = lastExecutedId;
                }

                // ensure that the arrival activity queue is not empty at this pont.
                DependencyManager.ActivityEnqueued();

                if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
                {
                    while (IsWorking())
                    {
                        Thread.Sleep(200);
                    }
                }

                // At this point we know for sure that the original gap is not there anymore.
                // In case there is a false gap (e.g. because there are missing activity ids
                // in the db) we have to remove these ids manually from the in-memory gap.
                if (gaps.Any())
                {
                    TerminationHistory.RemoveGaps(gaps);

                    // Commit is necessary because otherwise the gap is removed only in memory, but
                    // the index is not updated in the file system.
                    LuceneManager.Commit();
                }

                SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
            }
예제 #19
0
        //=========================================================================== Helper methods

        private static void WriteFilePaths(HttpContext context)
        {
            context.Response.Write(string.Join(";", LuceneManager.PauseIndexingAndGetIndexFilePaths()));
        }