/*==================================================================== Document operations */
internal static void RefreshDocument(int versionId)
{
// the optimistic overlapping detection algorithm here is tested in IndexingHistory_Fix**Overlap tests. change tests if this algorithm is changed.
var node = Node.LoadNodeByVersionId(versionId);
// if node exists, refresh index from db
// if node does not exist, remove from index
if (node != null)
{
// delete from indexhistory first, because we are trying to refresh to the last timestamp
// also check if there is a more fresh indexing pending which would surely correct the index, and we don't need this here
if (!_history.RemoveIfLast(versionId, node.NodeTimestamp))
{
return;
}
StorageContext.Search.SearchEngine.GetPopulator().RefreshIndex(node, false);
}
else
{
var delTerm = new Term(LuceneManager.KeyFieldName, NumericUtils.IntToPrefixCoded(versionId));
LuceneManager.DeleteDocuments(new[] { delTerm }, false);
}
}
/// <summary>
/// Synchronous backup method for console applications.
/// </summary>
public static void SynchronousBackupIndex()
{
using (var op = SnTrace.Repository.StartOperation("Backup index immediatelly."))
{
EnsureEmptyDirctory(_backupDirectoryPath);
LuceneManager.PauseIndexing();
while (!LuceneManager.Paused)
{
Thread.Sleep(100);
}
Thread.Sleep(1000);
try
{
CopyIndexToBackupDirectory();
}
finally
{
LuceneManager.ContinueIndexing();
}
OptimizeCompressAndStore();
op.Successful = true;
}
}
private ScoreDoc[] GetDocsUnderTree(string path, bool recurse)
{
var field = recurse ? "InTree" : "Path";
var lq = LucQuery.Parse(String.Format("{0}:'{1}'", field, path.ToLower()));
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var idxReader = readerFrame.IndexReader;
var searcher = new IndexSearcher(idxReader);
var numDocs = idxReader.NumDocs();
try
{
var collector = TopScoreDocCollector.Create(numDocs, false);
searcher.Search(lq.Query, collector);
var topDocs = collector.TopDocs(0, numDocs);
return(topDocs.ScoreDocs);
}
finally
{
if (searcher != null)
{
searcher.Close();
}
searcher = null;
}
}
}
// caller: IndexPopulator.Populator
public void RepopulateTree(string path)
{
using (var op = SnTrace.Index.StartOperation("IndexPopulator RepopulateTree"))
{
var writer = IndexManager.GetIndexWriter(false);
writer.DeleteDocuments(new Term(LucObject.FieldName.InTree, path.ToLowerInvariant()));
try
{
var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath(path, excludedNodeTypes))
{
var doc = IndexDocumentInfo.GetDocument(docData);
if (doc == null) // indexing disabled
{
continue;
}
writer.AddDocument(doc);
OnNodeIndexed(docData.Path);
}
writer.Optimize();
}
finally
{
writer.Close();
}
op.Successful = true;
}
}
public override void DoAction(bool onRemote, bool isFromMe)
{
if (onRemote && isFromMe)
{
return;
}
if (!SenseNet.ContentRepository.RepositoryInstance.LuceneManagerIsRunning)
{
return;
}
Debug.WriteLine(String.Format("@> {0} =========== Executing Restore action.", AppDomain.CurrentDomain.FriendlyName));
Exception ex = null;
try
{
new IndexRestoringStartedMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName).Send();
LuceneManager.ForceRestore();
new IndexRestoringFinishedMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName).Send();
}
catch (Exception e)
{
ex = e;
Logger.WriteException(ex);//TODO: xx " RestoreIndex" szerepeljen a message-ben
}
if (ex != null)
{
new IndexRestoringErrorMessage(Environment.MachineName, AppDomain.CurrentDomain.FriendlyName, ex).Send();
}
Debug.WriteLine(String.Format("@> {0} =========== Executing Restore action is finished.", AppDomain.CurrentDomain.FriendlyName));
}
public static List <Document> GetDocumentsByNodeId(int nodeId)
{
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var termDocs = readerFrame.IndexReader.TermDocs(new Term(LucObject.FieldName.NodeId, Lucene.Net.Util.NumericUtils.IntToPrefixCoded(nodeId)));
return(GetDocumentsFromTermDocs(termDocs, readerFrame));
}
}
private static void ContinueIndexingAndWriteResult(HttpContext context)
{
try
{
LuceneManager.ContinueIndexing();
context.Response.Write("OK");
}
catch (Exception ex)
{
Logger.WriteException(ex);
context.Response.Write(ex.Message);
}
}
public static void Startup(System.IO.TextWriter consoleOut)
{
// initalize from index
CompletionState cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
cud = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}"
, cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength))
{
Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut);
op.Successful = true;
}
}
/*==================================================================================== Instance part */
private IEnumerable <Difference> CheckNode(string path)
{
var result = new List <Difference>();
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
//var sql = String.Format(checkNodeSql, path);
//var proc = SenseNet.ContentRepository.Storage.Data.DataProvider.CreateDataProcedure(sql);
//proc.CommandType = System.Data.CommandType.Text;
var docids = new List <int>();
var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path);
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid >= 0)
{
docids.Add(docid);
}
}
}
var scoredocs = GetDocsUnderTree(path, false);
foreach (var scoredoc in scoredocs)
{
var docid = scoredoc.Doc;
var doc = ixreader.Document(docid);
if (!docids.Contains(docid))
{
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = scoredoc.Doc,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = path,
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
return(result);
}
/*======================================================================================================= IIndexPopulator Members */
// caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce
public void ClearAndPopulateAll(bool backup = true)
{
var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId();
var commitData = CompletionState.GetCommitUserData(lastActivityId);
using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll"))
{
// recreate
var writer = IndexManager.GetIndexWriter(true);
try
{
var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes))
{
var doc = IndexDocumentInfo.GetDocument(docData);
if (doc == null) // indexing disabled
{
continue;
}
writer.AddDocument(doc);
OnNodeIndexed(docData.Path);
}
RepositoryInstance.Instance.ConsoleWrite(" Commiting ... ");
writer.Commit(commitData);
RepositoryInstance.Instance.ConsoleWriteLine("ok");
RepositoryInstance.Instance.ConsoleWrite(" Optimizing ... ");
writer.Optimize();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
}
finally
{
writer.Close();
}
RepositoryInstance.Instance.ConsoleWrite(" Deleting indexing activities ... ");
LuceneManager.DeleteAllIndexingActivities();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
if (backup)
{
RepositoryInstance.Instance.ConsoleWrite(" Making backup ... ");
BackupTools.BackupIndexImmediatelly();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
}
op.Successful = true;
}
}
/*==================================================================================== Instance part */
private IEnumerable <Difference> CheckNode(string path)
{
var result = new List <Difference>();
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
var docids = new List <int>();
var proc = DataProvider.Current.GetTimestampDataForOneNodeIntegrityCheck(path, GetExcludedNodeTypeIds());
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid >= 0)
{
docids.Add(docid);
}
}
}
var scoredocs = GetDocsUnderTree(path, false);
foreach (var scoredoc in scoredocs)
{
var docid = scoredoc.Doc;
var doc = ixreader.Document(docid);
if (!docids.Contains(docid))
{
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = scoredoc.Doc,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = path,
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
return(result);
}
static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
if (StorageContext.Search.IsOuterEngineEnabled && StorageContext.Search.SearchEngine != InternalSearchEngine.Instance)
{
var timerEnabled = _timer.Enabled;
_timer.Enabled = false;
try
{
LuceneManager.ExecuteLostIndexingActivities();
}
catch (Exception ex) //logged
{
SenseNet.Diagnostics.Logger.WriteException(ex);
}
finally
{
_timer.Enabled = timerEnabled;
}
}
}
internal static void HealthCheck()
{
if (IsWorking())
{
SnTrace.Index.Write("IAQ: Health check triggered but ignored.");
return;
}
SnTrace.Index.Write("IAQ: Health check triggered.");
var state = TerminationHistory.GetCurrentState();
var gapsLength = state.Gaps.Length;
if (gapsLength > 0)
{
SnTrace.IndexQueue.Write("IAQ: Health checker is processing {0} gap{1}.", gapsLength, gapsLength > 1 ? "s" : "");
foreach (LuceneIndexingActivity activity in new IndexingActivityLoader(state.Gaps, false))
{
WaitIfOverloaded();
IndexingActivityQueue.ExecuteActivity(activity);
}
}
var lastId = TerminationHistory.GetLastTerminatedId();
var lastDbId = LuceneManager.GetLastStoredIndexingActivityId();
var newerCount = lastDbId - lastId;
if (lastId < lastDbId)
{
SnTrace.IndexQueue.Write("IAQ: Health checker is processing activities from {0} to {1}", (lastId + 1), lastDbId);
foreach (LuceneIndexingActivity activity in new IndexingActivityLoader(lastId + 1, lastDbId, false))
{
WaitIfOverloaded();
IndexingActivityQueue.ExecuteActivity(activity);
}
}
}
public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse)
{
var path = content == null ? null : content.Path;
CompletionState completionState;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
completionState = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
var channel = SenseNet.ContentRepository.DistributedApplication.ClusterChannel;
var appDomainName = channel == null ? null : channel.ReceiverName;
return(new
{
AppDomainName = appDomainName,
LastStoredActivity = lastDatabaseId,
LastProcessedActivity = completionState.LastActivityId,
GapsLength = completionState.GapsLength,
Gaps = completionState.Gaps,
Differences = Check(path, recurse)
});
}
private static void ExecuteActivity(LuceneIndexingActivity activity)
{
LuceneManager.RegisterActivity(activity);
LuceneManager.ExecuteActivity(activity, true, true);
}
//---- Caller: Startup, ForceRestore
internal static void ExecuteUnprocessedIndexingActivities(System.IO.TextWriter consoleOut)
{
lock (_executingUnprocessedIndexingActivitiesLock)
{
try
{
_executingUnprocessedIndexingActivities = true;
CommitUserData cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
cud = IndexManager.ReadCommitUserData(readerFrame.IndexReader);
}
MissingActivityHandler.MaxActivityId = cud.LastActivityId;
MissingActivityHandler.SetGap(cud.Gap);
var logProps = new Dictionary <string, object> {
{ "LastActivityID", cud.LastActivityId }, { "Size of gap", cud.Gap.Count }
};
Logger.WriteInformation("Executing unprocessed indexing activities from the stored commit point.",
Logger.EmptyCategoryList, logProps);
var i = 0;
var sumCount = 0;
// This loop was created to avoid loading too many activities at once that are present in the gap.
while (i * ACTIVITIESFRAGMENTSIZE <= cud.Gap.Count)
{
// get activities from the DB that are in the current gap fragment
var gapSegment = cud.Gap.Skip(i * ACTIVITIESFRAGMENTSIZE).Take(ACTIVITIESFRAGMENTSIZE).ToArray();
var activities = IndexingActivityManager.GetUnprocessedActivities(gapSegment);
ProcessTasks(activities, consoleOut);
sumCount += activities.Length;
i++;
}
// Execute activities where activity id is bigger than than our last (activity) task id
var maxIdInDb = 0;
var newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out maxIdInDb, ACTIVITIESFRAGMENTSIZE);
while (newtasks.Length > 0)
{
ProcessTasks(newtasks, consoleOut);
sumCount += newtasks.Length;
//load the remaining activities, but only if they were created before this operation started
var tempMax = 0;
newtasks = IndexingActivityManager.GetUnprocessedActivities(MissingActivityHandler.MaxActivityId, out tempMax, ACTIVITIESFRAGMENTSIZE, maxIdInDb);
}
if (consoleOut != null)
{
consoleOut.WriteLine("ok.");
}
logProps.Add("Processed tasks", sumCount);
//write the latest max activity id and gap size to log
logProps["LastActivityID"] = MissingActivityHandler.MaxActivityId;
logProps["Size of gap"] = MissingActivityHandler.GetGap().Count;
Logger.WriteInformation("Executing unprocessed tasks is finished.", Logger.EmptyCategoryList, logProps);
}
finally
{
_executingUnprocessedIndexingActivities = false;
}
}
}
private IEnumerable <Difference> CheckRecurse(string path)
{
var result = new List <Difference>();
using (var op = SnTrace.Index.StartOperation("Index Integrity Checker: CheckRecurse {0}", path))
{
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
{
var ixreader = readerFrame.IndexReader;
numdocs = ixreader.NumDocs() + ixreader.NumDeletedDocs();
var x = numdocs / intsize;
var y = numdocs % intsize;
docbits = new int[x + (y > 0 ? 1 : 0)];
if (path == null)
{
if (y > 0)
{
var q = 0;
for (int i = 0; i < y; i++)
{
q += 1 << i;
}
docbits[docbits.Length - 1] = q ^ (-1);
}
}
else
{
for (int i = 0; i < docbits.Length; i++)
{
docbits[i] = -1;
}
var scoredocs = GetDocsUnderTree(path, true);
for (int i = 0; i < scoredocs.Length; i++)
{
var docid = scoredocs[i].Doc;
docbits[docid / intsize] ^= 1 << docid % intsize;
}
}
var proc = DataProvider.Current.GetTimestampDataForRecursiveIntegrityCheck(path, GetExcludedNodeTypeIds());
var progress = 0;
using (var dbreader = proc.ExecuteReader())
{
while (dbreader.Read())
{
if ((++progress % 10000) == 0)
{
SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex: progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
}
var docid = CheckDbAndIndex(dbreader, ixreader, result);
if (docid > -1)
{
docbits[docid / intsize] |= 1 << docid % intsize;
}
}
}
SnTrace.Index.Write("Index Integrity Checker: CheckDbAndIndex finished. Progress={0}/{1}, diffs:{2}", progress, numdocs, result.Count);
for (int i = 0; i < docbits.Length; i++)
{
if (docbits[i] != -1)
{
var bits = docbits[i];
for (int j = 0; j < intsize; j++)
{
if ((bits & (1 << j)) == 0)
{
var docid = i * intsize + j;
if (docid >= numdocs)
{
break;
}
if (!ixreader.IsDeleted(docid))
{
var doc = ixreader.Document(docid);
result.Add(new Difference(IndexDifferenceKind.NotInDatabase)
{
DocId = docid,
VersionId = ParseInt(doc.Get(LucObject.FieldName.VersionId)),
NodeId = ParseInt(doc.Get(LucObject.FieldName.NodeId)),
Path = doc.Get(LucObject.FieldName.Path),
Version = doc.Get(LucObject.FieldName.Version),
IxNodeTimestamp = ParseLong(doc.Get(LucObject.FieldName.NodeTimestamp)),
IxVersionTimestamp = ParseLong(doc.Get(LucObject.FieldName.VersionTimestamp))
});
}
}
}
}
}
}
op.Successful = true;
}
return(result.ToArray());
}
/// <summary>
/// MUST BE SYNCHRON
/// GAPS MUST BE ORDERED
/// </summary>
internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
if (consoleOut != null)
{
consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3));
}
SnLog.WriteInformation("Executing unprocessed activities.",
EventId.RepositoryRuntime,
properties: new Dictionary <string, object> {
{ "LastDatabaseId", lastDatabaseId },
{ "LastExecutedId", lastExecutedId },
{ "CountOfGaps", gaps.Length },
{ "Gaps", String.Join(", ", gaps) }
});
DependencyManager.Start();
var count = 0;
if (gaps.Any())
{
var loadedActivities = new IndexingActivityLoader(gaps, true);
foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
{
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
_arrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (lastExecutedId < lastDatabaseId)
{
var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
{
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
_arrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (_lastQueued < lastExecutedId)
{
_lastQueued = lastExecutedId;
}
// ensure that the arrival activity queue is not empty at this pont.
DependencyManager.ActivityEnqueued();
if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
{
while (IsWorking())
{
Thread.Sleep(200);
}
}
// At this point we know for sure that the original gap is not there anymore.
// In case there is a false gap (e.g. because there are missing activity ids
// in the db) we have to remove these ids manually from the in-memory gap.
if (gaps.Any())
{
TerminationHistory.RemoveGaps(gaps);
// Commit is necessary because otherwise the gap is removed only in memory, but
// the index is not updated in the file system.
LuceneManager.Commit();
}
SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
}
//=========================================================================== Helper methods
private static void WriteFilePaths(HttpContext context)
{
context.Response.Write(string.Join(";", LuceneManager.PauseIndexingAndGetIndexFilePaths()));
}