private ScoreDoc[] GetDocsUnderTree(string path, bool recurse) { var field = recurse ? "InTree" : "Path"; var lq = LucQuery.Parse(String.Format("{0}:'{1}'", field, path.ToLower())); using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var idxReader = readerFrame.IndexReader; var searcher = new IndexSearcher(idxReader); var numDocs = idxReader.NumDocs(); try { var collector = TopScoreDocCollector.Create(numDocs, false); searcher.Search(lq.Query, collector); var topDocs = collector.TopDocs(0, numDocs); return(topDocs.ScoreDocs); } finally { if (searcher != null) { searcher.Close(); } searcher = null; } } }
public static List <Document> GetDocumentsByNodeId(int nodeId) { using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var termDocs = readerFrame.IndexReader.TermDocs(new Term(LucObject.FieldName.NodeId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(nodeId))); return(GetDocumentsFromTermDocs(termDocs, readerFrame)); } }
public static void Startup(System.IO.TextWriter consoleOut) { // initalize from index CompletionState cud; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) cud = CompletionState.ParseFromReader(readerFrame.IndexReader); var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId(); using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}" , cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength)) { Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut); op.Successful = true; } }
/*==================================================================================== Instance part */ private IEnumerable <Difference> CheckNode(string path) { var result = new List <Difference>(); using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var ixreader = readerFrame.IndexReader; //var sql = String.Format(checkNodeSql, path); //var proc = SenseNet.ContentRepository.Storage.Data.DataProvider.CreateDataProcedure(sql); //proc.CommandType = System.Data.CommandType.Text; var docids = new List <int>(); var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path); using (var dbreader = proc.ExecuteReader()) { while (dbreader.Read()) { var docid = CheckDbAndIndex(dbreader, ixreader, result); if (docid >= 0) { docids.Add(docid); } } } var scoredocs = GetDocsUnderTree(path, false); foreach (var scoredoc in scoredocs) { var docid = scoredoc.Doc; var doc = ixreader.Document(docid); if (!docids.Contains(docid)) { result.Add(new Difference(IndexDifferenceKind.NotInDatabase) { DocId = scoredoc.Doc, VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)), NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)), Path = path, Version = doc.Get(LucObject.FieldName.Version), IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)), IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp)) }); } } } return(result); }
/*==================================================================================== Instance part */ private IEnumerable <Difference> CheckNode(string path) { var result = new List <Difference>(); using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var ixreader = readerFrame.IndexReader; var docids = new List <int>(); var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path, GetExcludedNodeTypeIds()); using (var dbreader = proc.ExecuteReader()) { while (dbreader.Read()) { var docid = CheckDbAndIndex(dbreader, ixreader, result); if (docid >= 0) { docids.Add(docid); } } } var scoredocs = GetDocsUnderTree(path, false); foreach (var scoredoc in scoredocs) { var docid = scoredoc.Doc; var doc = ixreader.Document(docid); if (!docids.Contains(docid)) { result.Add(new Difference(IndexDifferenceKind.NotInDatabase) { DocId = scoredoc.Doc, VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)), NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)), Path = path, Version = doc.Get(LucObject.FieldName.Version), IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)), IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp)) }); } } } return(result); }
public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse) { var path = content == null ? null : content.Path; CompletionState completionState; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) completionState = CompletionState.ParseFromReader(readerFrame.IndexReader); var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId(); var channel = SenseNet.ContentRepository.DistributedApplication.ClusterChannel; var appDomainName = channel == null ? null : channel.ReceiverName; return(new { AppDomainName = appDomainName, LastStoredActivity = lastDatabaseId, LastProcessedActivity = completionState.LastActivityId, GapsLength = completionState.GapsLength, Gaps = completionState.Gaps, Differences = Check(path, recurse) }); }
private IEnumerable <Difference> CheckRecurse(string path) { var result = new List <Difference>(); using (var op = SnTrace.Index.StartOperation("Index Integrity Checker: CheckRecurse {0}", path)) { using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { var ixreader = readerFrame.IndexReader; numdocs = ixreader.NumDocs() + ixreader.NumDeletedDocs(); var x = numdocs / intsize; var y = numdocs % intsize; docbits = new int[x + (y > 0 ? 1 : 0)]; if (path == null) { if (y > 0) { var q = 0; for (int i = 0; i < y; i++) { q += 1 << i; } docbits[docbits.Length - 1] = q ^ (-1); } } else { for (int i = 0; i < docbits.Length; i++) { docbits[i] = -1; } var scoredocs = GetDocsUnderTree(path, true); for (int i = 0; i < scoredocs.Length; i++) { var docid = scoredocs[i].Doc; docbits[docid / intsize] ^= 1 << docid % intsize; } } var proc = DataProvider.Current.GetTimestampDataForRecursiveIntegrityCheck(path, GetExcludedNodeTypeIds()); var progress = 0; using (var dbreader = proc.ExecuteReader()) { while (dbreader.Read()) { if ((++progress % 10000) == 0) { SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex: progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count); } var docid = CheckDbAndIndex(dbreader, ixreader, result); if (docid > -1) { docbits[docid / intsize] |= 1 << docid % intsize; } } } SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex finished. Progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count); for (int i = 0; i < docbits.Length; i++) { if (docbits[i] != -1) { var bits = docbits[i]; for (int j = 0; j < intsize; j++) { if ((bits & (1 << j)) == 0) { var docid = i * intsize + j; if (docid >= numdocs) { break; } if (!ixreader.IsDeleted(docid)) { var doc = ixreader.Document(docid); result.Add(new Difference(IndexDifferenceKind.NotInDatabase) { DocId = docid, VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)), NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)), Path = doc.Get(LucObject.FieldName.Path), Version = doc.Get(LucObject.FieldName.Version), IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)), IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp)) }); } } } } } } op.Successful = true; } return(result.ToArray()); }
//---- Caller: Startup, ForceRestore internal static void ExecuteUnprocessedIndexingActivities(System.IO.TextWriter consoleOut) { lock (_executingUnprocessedIndexingActivitiesLock) { try { _executingUnprocessedIndexingActivities = true; CommitUserData cud; using (var readerFrame = LuceneManager.GetIndexReaderFrame()) { cud = IndexManager.ReadCommitUserData(readerFrame.IndexReader); } MissingActivityHandler.MaxActivityId = cud.LastActivityId; MissingActivityHandler.SetGap(cud.Gap); var logProps = new Dictionary <string, object> { { "LastActivityID", cud.LastActivityId }, { "Size of gap", cud.Gap.Count } }; Logger.WriteInformation("Executing unprocessed indexing activities from the stored commit point.", Logger.EmptyCategoryList, logProps); var i = 0; var sumCount = 0; // This loop was created to avoid loading too many activities at once that are present in the gap. while (i * ACTIVITIESFRAGMENTSIZE <= cud.Gap.Count) { // get activities from the DB that are in the current gap fragment var gapSegment = cud.Gap.Skip(i * ACTIVITIESFRAGMENTSIZE).Take(ACTIVITIESFRAGMENTSIZE).ToArray(); var activities = IndexingActivityManager.GetUnprocessedActivities(gapSegment); ProcessTasks(activities, consoleOut); sumCount += activities.Length; i++; } // Execute activities where activity id is bigger than than our last (activity) task id var maxIdInDb = 0; var newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out maxIdInDb, ACTIVITIESFRAGMENTSIZE); while (newtasks.Length > 0) { ProcessTasks(newtasks, consoleOut); sumCount += newtasks.Length; //load the remaining activities, but only if they were created before this operation started var tempMax = 0; newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out tempMax, ACTIVITIESFRAGMENTSIZE, maxIdInDb); } if (consoleOut != null) { consoleOut.WriteLine("ok."); } logProps.Add("Processed tasks", sumCount); //write the latest max activity id and gap size to log logProps["LastActivityID"] = MissingActivityHandler.MaxActivityId; logProps["Size of gap"] = MissingActivityHandler.GetGap().Count; Logger.WriteInformation("Executing unprocessed tasks is finished.", Logger.EmptyCategoryList, logProps); } finally { _executingUnprocessedIndexingActivities = false; } } }