internal static void Commit(bool reopenReader = true)
{
CompletionState commitState;
using (var op = SnTrace.Index.StartOperation("LM: Commit. reopenReader:{0}", reopenReader))
{
using (var wrFrame = IndexWriterFrame.Get(!reopenReader)) // // Commit
{
commitState = CompletionState.GetCurrent();
var commitStateMessage = commitState.ToString();
SnTrace.Index.Write("LM: Committing_writer. commitState: " + commitStateMessage);
// Write a fake document to make sure that the index changes are written to the file system.
wrFrame.IndexWriter.UpdateDocument(new Term(COMMITFIELDNAME, COMMITFIELDNAME), GetFakeDocument());
wrFrame.IndexWriter.Commit(CompletionState.GetCommitUserData(commitState));
if (reopenReader)
{
ReopenReader();
}
}
#pragma warning disable 420
Interlocked.Exchange(ref _activities, 0);
#pragma warning restore 420
_delayCycle = 0;
op.Successful = true;
}
}
public static void Startup(System.IO.TextWriter consoleOut)
{
// initalize from index
CompletionState cud;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
cud = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
using (var op = SnTrace.Index.StartOperation("IAQ: InitializeFromIndex. LastIndexedActivityId: {0}, LastDatabaseId: {1}, TotalUnprocessed: {2}"
, cud.LastActivityId, lastDatabaseId, lastDatabaseId - cud.LastActivityId + cud.GapsLength))
{
Startup(lastDatabaseId, cud.LastActivityId, cud.Gaps, consoleOut);
op.Successful = true;
}
}
/*======================================================================================================= IIndexPopulator Members */
// caller: IndexPopulator.Populator, Import.Importer, Tests.Initializer, RunOnce
public void ClearAndPopulateAll(bool backup = true)
{
var lastActivityId = LuceneManager.GetLastStoredIndexingActivityId();
var commitData = CompletionState.GetCommitUserData(lastActivityId);
using (var op = SnTrace.Index.StartOperation("IndexPopulator ClearAndPopulateAll"))
{
// recreate
var writer = IndexManager.GetIndexWriter(true);
try
{
var excludedNodeTypes = LuceneManager.GetNotIndexedNodeTypes();
foreach (var docData in StorageContext.Search.LoadIndexDocumentsByPath("/Root", excludedNodeTypes))
{
var doc = IndexDocumentInfo.GetDocument(docData);
if (doc == null) // indexing disabled
{
continue;
}
writer.AddDocument(doc);
OnNodeIndexed(docData.Path);
}
RepositoryInstance.Instance.ConsoleWrite(" Commiting ... ");
writer.Commit(commitData);
RepositoryInstance.Instance.ConsoleWriteLine("ok");
RepositoryInstance.Instance.ConsoleWrite(" Optimizing ... ");
writer.Optimize();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
}
finally
{
writer.Close();
}
RepositoryInstance.Instance.ConsoleWrite(" Deleting indexing activities ... ");
LuceneManager.DeleteAllIndexingActivities();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
if (backup)
{
RepositoryInstance.Instance.ConsoleWrite(" Making backup ... ");
BackupTools.BackupIndexImmediatelly();
RepositoryInstance.Instance.ConsoleWriteLine("ok");
}
op.Successful = true;
}
}
internal static CompletionState ParseFromReader(IDictionary <string, string> commitUserData)
{
var result = new CompletionState();
if (commitUserData != null)
{
string value;
int lastActivityId;
if (commitUserData.TryGetValue(CompletionState.LastActivityIdKey, out value))
{
if (!string.IsNullOrEmpty(value))
{
if (int.TryParse(value, out lastActivityId))
{
result.LastActivityId = lastActivityId;
}
}
}
if (commitUserData.TryGetValue(CompletionState.GapsKey, out value))
{
if (!string.IsNullOrEmpty(value))
{
result.Gaps = value
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => { int i; if (Int32.TryParse(s, out i))
{
return(i);
}
return(0); })
.Where(i => i > 0)
.ToArray();
}
}
}
return(result);
}
public static object CheckIndexIntegrity(SenseNet.ContentRepository.Content content, bool recurse)
{
var path = content == null ? null : content.Path;
CompletionState completionState;
using (var readerFrame = LuceneManager.GetIndexReaderFrame())
completionState = CompletionState.ParseFromReader(readerFrame.IndexReader);
var lastDatabaseId = LuceneManager.GetLastStoredIndexingActivityId();
var channel = SenseNet.ContentRepository.DistributedApplication.ClusterChannel;
var appDomainName = channel == null ? null : channel.ReceiverName;
return(new
{
AppDomainName = appDomainName,
LastStoredActivity = lastDatabaseId,
LastProcessedActivity = completionState.LastActivityId,
GapsLength = completionState.GapsLength,
Gaps = completionState.Gaps,
Differences = Check(path, recurse)
});
}
internal static Dictionary <string, string> GetCommitUserData(CompletionState data)
{
return(GetCommitUserData(data.LastActivityId, data.Gaps));
}
/// <summary>
/// MUST BE SYNCHRON
/// GAPS MUST BE ORDERED
/// </summary>
internal static void Start(int lastDatabaseId, int lastExecutedId, int[] gaps, System.IO.TextWriter consoleOut)
{
if (consoleOut != null)
{
consoleOut.WriteLine("Executing unprocessed activities. {0}-{1} {2}", lastExecutedId, lastDatabaseId, CompletionState.GapsToString(gaps, 5, 3));
}
SnLog.WriteInformation("Executing unprocessed activities.",
EventId.RepositoryRuntime,
properties: new Dictionary <string, object> {
{ "LastDatabaseId", lastDatabaseId },
{ "LastExecutedId", lastExecutedId },
{ "CountOfGaps", gaps.Length },
{ "Gaps", String.Join(", ", gaps) }
});
DependencyManager.Start();
var count = 0;
if (gaps.Any())
{
var loadedActivities = new IndexingActivityLoader(gaps, true);
foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
{
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
_arrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (lastExecutedId < lastDatabaseId)
{
var loadedActivities = new IndexingActivityLoader(lastExecutedId + 1, lastDatabaseId, true);
foreach (LuceneIndexingActivity loadedActivity in loadedActivities)
{
// wait and start processing loaded activities in the meantime
WaitIfOverloaded(true);
SnTrace.IndexQueue.Write("IAQ: Startup: A{0} enqueued from db.", loadedActivity.Id);
IndexingActivityHistory.Arrive(loadedActivity);
_arrivalQueue.Enqueue(loadedActivity);
_lastQueued = loadedActivity.Id;
count++;
}
}
if (_lastQueued < lastExecutedId)
{
_lastQueued = lastExecutedId;
}
// ensure that the arrival activity queue is not empty at this pont.
DependencyManager.ActivityEnqueued();
if (lastDatabaseId != 0 || lastExecutedId != 0 || gaps.Any())
{
while (IsWorking())
{
Thread.Sleep(200);
}
}
// At this point we know for sure that the original gap is not there anymore.
// In case there is a false gap (e.g. because there are missing activity ids
// in the db) we have to remove these ids manually from the in-memory gap.
if (gaps.Any())
{
TerminationHistory.RemoveGaps(gaps);
// Commit is necessary because otherwise the gap is removed only in memory, but
// the index is not updated in the file system.
LuceneManager.Commit();
}
SnLog.WriteInformation($"Executing unprocessed activities ({count}) finished.", EventId.RepositoryLifecycle);
}
/// <summary>Only for tests</summary>
internal static void _setCurrentExecutionState(CompletionState state)
{
Serializer.Reset(state.LastActivityId);
DependencyManager.Reset();
TerminationHistory.Reset(state.LastActivityId, state.Gaps);
}