private ScoreDoc[] GetDocsUnderTree(string path, bool recurse)
{
var field = recurse ? "InTree" : "Path";
var lq = LucQuery.Parse(String.Format("{0}:'{1}'", field, path.ToLower()));
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var idxReader = readerFrame.IndexReader;
var searcher = new IndexSearcher(idxReader);
var numDocs = idxReader.NumDocs();
try
{
var collector = TopScoreDocCollector.Create(numDocs, false);
searcher.Search(lq.Query, collector);
var topDocs = collector.TopDocs(0, numDocs);
return(topDocs.ScoreDocs);
}
finally
{
if (searcher != null)
{
searcher.Close();
}
searcher = null;
}
}
}
public static List <Document> GetDocumentsByNodeId(int nodeId)
{
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var termDocs = readerFrame.IndexReader.TermDocs(new Term(LucObject.FieldName.NodeId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(nodeId)));
return(GetDocumentsFromTermDocs(termDocs, readerFrame));
}
}
public static void Startup(System.IO.TextWriter consoleOut)
{
// initalize from index
CompletionState cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
cud = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}"
, cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength))
{
Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut);
op.Successful = true;
}
}
/*==================================================================================== Instance part */
private IEnumerable <Difference> CheckNode(string path)
{
var result = new List <Difference>();
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
//var sql = String.Format(checkNodeSql, path);
//var proc = SenseNet.ContentRepository.Storage.Data.DataProvider.CreateDataProcedure(sql);
//proc.CommandType = System.Data.CommandType.Text;
var docids = new List <int>();
var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path);
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid >= 0)
{
docids.Add(docid);
}
}
}
var scoredocs = GetDocsUnderTree(path, false);
foreach (var scoredoc in scoredocs)
{
var docid = scoredoc.Doc;
var doc = ixreader.Document(docid);
if (!docids.Contains(docid))
{
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = scoredoc.Doc,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = path,
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
return(result);
}
/*==================================================================================== Instance part */
private IEnumerable <Difference> CheckNode(string path)
{
var result = new List <Difference>();
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
var docids = new List <int>();
var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path, GetExcludedNodeTypeIds());
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid >= 0)
{
docids.Add(docid);
}
}
}
var scoredocs = GetDocsUnderTree(path, false);
foreach (var scoredoc in scoredocs)
{
var docid = scoredoc.Doc;
var doc = ixreader.Document(docid);
if (!docids.Contains(docid))
{
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = scoredoc.Doc,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = path,
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
return(result);
}
public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse)
{
var path = content == null ? null : content.Path;
CompletionState completionState;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
completionState = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
var channel = SenseNet.ContentRepository.DistributedApplication.ClusterChannel;
var appDomainName = channel == null ? null : channel.ReceiverName;
return(new
{
AppDomainName = appDomainName,
LastStoredActivity = lastDatabaseId,
LastProcessedActivity = completionState.LastActivityId,
GapsLength = completionState.GapsLength,
Gaps = completionState.Gaps,
Differences = Check(path, recurse)
});
}
private IEnumerable <Difference> CheckRecurse(string path)
{
var result = new List <Difference>();
using (var op = SnTrace.Index.StartOperation("Index Integrity Checker: CheckRecurse {0}", path))
{
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
numdocs = ixreader.NumDocs() + ixreader.NumDeletedDocs();
var x = numdocs / intsize;
var y = numdocs % intsize;
docbits = new int[x + (y > 0 ? 1 : 0)];
if (path == null)
{
if (y > 0)
{
var q = 0;
for (int i = 0; i < y; i++)
{
q += 1 << i;
}
docbits[docbits.Length - 1] = q ^ (-1);
}
}
else
{
for (int i = 0; i < docbits.Length; i++)
{
docbits[i] = -1;
}
var scoredocs = GetDocsUnderTree(path, true);
for (int i = 0; i < scoredocs.Length; i++)
{
var docid = scoredocs[i].Doc;
docbits[docid / intsize] ^= 1 << docid % intsize;
}
}
var proc = DataProvider.Current.GetTimestampDataForRecursiveIntegrityCheck(path, GetExcludedNodeTypeIds());
var progress = 0;
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
if ((++progress % 10000) == 0)
{
SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex: progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
}
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid > -1)
{
docbits[docid / intsize] |= 1 << docid % intsize;
}
}
}
SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex finished. Progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
for (int i = 0; i < docbits.Length; i++)
{
if (docbits[i] != -1)
{
var bits = docbits[i];
for (int j = 0; j < intsize; j++)
{
if ((bits & (1 << j)) == 0)
{
var docid = i * intsize + j;
if (docid >= numdocs)
{
break;
}
if (!ixreader.IsDeleted(docid))
{
var doc = ixreader.Document(docid);
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = docid,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = doc.Get(LucObject.FieldName.Path),
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
}
}
}
op.Successful = true;
}
return(result.ToArray());
}
//---- Caller: Startup, ForceRestore
internal static void ExecuteUnprocessedIndexingActivities(System.IO.TextWriter consoleOut)
{
lock (_executingUnprocessedIndexingActivitiesLock)
{
try
{
_executingUnprocessedIndexingActivities = true;
CommitUserData cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
cud = IndexManager.ReadCommitUserData(readerFrame.IndexReader);
}
MissingActivityHandler.MaxActivityId = cud.LastActivityId;
MissingActivityHandler.SetGap(cud.Gap);
var logProps = new Dictionary <string, object> {
{ "LastActivityID", cud.LastActivityId }, { "Size of gap", cud.Gap.Count }
};
Logger.WriteInformation("Executing unprocessed indexing activities from the stored commit point.",
Logger.EmptyCategoryList, logProps);
var i = 0;
var sumCount = 0;
// This loop was created to avoid loading too many activities at once that are present in the gap.
while (i * ACTIVITIESFRAGMENTSIZE <= cud.Gap.Count)
{
// get activities from the DB that are in the current gap fragment
var gapSegment = cud.Gap.Skip(i * ACTIVITIESFRAGMENTSIZE).Take(ACTIVITIESFRAGMENTSIZE).ToArray();
var activities = IndexingActivityManager.GetUnprocessedActivities(gapSegment);
ProcessTasks(activities, consoleOut);
sumCount += activities.Length;
i++;
}
// Execute activities where activity id is bigger than than our last (activity) task id
var maxIdInDb = 0;
var newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out maxIdInDb, ACTIVITIESFRAGMENTSIZE);
while (newtasks.Length > 0)
{
ProcessTasks(newtasks, consoleOut);
sumCount += newtasks.Length;
//load the remaining activities, but only if they were created before this operation started
var tempMax = 0;
newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out tempMax, ACTIVITIESFRAGMENTSIZE, maxIdInDb);
}
if (consoleOut != null)
{
consoleOut.WriteLine("ok.");
}
logProps.Add("Processed tasks", sumCount);
//write the latest max activity id and gap size to log
logProps["LastActivityID"] = MissingActivityHandler.MaxActivityId;
logProps["Size of gap"] = MissingActivityHandler.GetGap().Count;
Logger.WriteInformation("Executing unprocessed tasks is finished.", Logger.EmptyCategoryList, logProps);
}
finally
{
_executingUnprocessedIndexingActivities = false;
}
}
}
