public static void RemoveIndex(IndexItem indexItem, string indexPath)
{
if (indexItem == null)
{
return;
}
if (indexPath == null)
{
return;
}
if (indexPath.Length == 0)
{
return;
}
IndexingQueue queueItem = new IndexingQueue();
queueItem.IndexPath = indexPath;
queueItem.ItemKey = indexItem.Key;
queueItem.RemoveOnly = true;
queueItem.SerializedItem = SerializationHelper.SerializeToString(indexItem);
queueItem.Save();
// the above queues the items to be indexed. Edit page must also call SiteUtils.QueueIndexing(); after the content is deleted.
}
public void OnItemProcessed(object sender, EventArgs args)
{
var arguments = args as ItemProcessedEventArgs;
var context = arguments?.Context;
Item item = context?.VersionToPublish;
if (item == null)
{
return;
}
if (!this.indexingService.ItemShouldBeIndexed(item))
{
return;
}
if (context.Action == PublishAction.DeleteTargetItem)
{
IndexingQueue.Delete(item);
}
else if (context.Action != PublishAction.Skip)
{
IndexingQueue.Add(item);
}
}
public void Run()
{
if (IndexingQueue.Items.IsEmpty)
{
return;
}
TimeSpan timeSinceLastAdd = DateTime.UtcNow - IndexingQueue.LastChange;
if (timeSinceLastAdd.TotalMinutes < 1 && IndexingQueue.Items.Count < 1000)
{
return; // wait, probably publish is running and not too many items in queue
}
List <IndexingQueueItem> items = new List <IndexingQueueItem>();
while (items.Count < 1000)
{
var queueItem = IndexingQueue.Dequeue();
if (queueItem.Equals(IndexingQueueItem.Empty))
{
break;
}
items.Add(queueItem);
}
Log.Info("Start indexing: " + items.Count + " items", this);
this.indexingService.IndexPageItems(items);
Log.Info("Indexing finished", this);
}
private void ProcessIndexingQueue()
{
bool markAsComplete = false;
// this gets executed while content is still being queued.
// if we start right away we may finish and miss some content
// so pause a bit but log progress so it knows this task is running
for (int i = 0; i < 10; i++)
{
rowsToProcess = 1;
Thread.Sleep(5000); // 5 seconds
ReportStatus(markAsComplete);
}
rowsToProcess = 0;
DataTable indexPaths = IndexingQueue.GetIndexPaths();
foreach (DataRow row in indexPaths.Rows)
{
string indexPath = row["IndexPath"].ToString();
try
{
if (indexPath.Length > 0)
{
if (!Directory.Exists(indexPath))
{
Directory.CreateDirectory(indexPath);
}
DataTable q = IndexingQueue.GetByPath(indexPath);
ProcessQueue(q, indexPath);
}
}
catch (IOException ex)
{
errorCount += 1;
log.Error(ex);
}
}
int countOfRowsNotProcessed = IndexingQueue.GetCount();
if ((countOfRowsNotProcessed > 0) && (errorCount < 3))
{
// recurse
ProcessIndexingQueue();
}
else
{
this.rowsToProcess = 0;
markAsComplete = true;
ReportStatus(markAsComplete);
}
}
protected void btnRebuildSearchIndex_Click(object sender, EventArgs e)
{
IndexingQueue.DeleteAll();
IndexHelper.DeleteSearchIndex(siteSettings);
IndexHelper.VerifySearchIndex(siteSettings);
lblMessage.Text = Resource.SearchResultsBuildingIndexMessage;
Thread.Sleep(5000); //wait 1 seconds
SiteUtils.QueueIndexing();
}
void btnRebuildSearchIndex_Click(object sender, EventArgs e)
{
IndexingQueue.DeleteAll();
CanhCam.SearchIndex.IndexHelper.DeleteSearchIndex(siteSettings);
CanhCam.SearchIndex.IndexHelper.VerifySearchIndex(siteSettings);
this.lblMessage.Text = ResourceHelper.GetResourceString("Resource", "SearchResultsBuildingIndexMessage");
Thread.Sleep(5000); //wait 5 seconds
SiteUtils.QueueIndexing();
}
public void OnItemDeleted(object sender, EventArgs args)
{
Item item = Event.ExtractParameter(args, 0) as Item;
if (item == null)
{
return;
}
if (!this.indexingService.ItemShouldBeIndexed(item))
{
return;
}
IndexingQueue.Delete(item);
}
public static bool VerifySearchIndex(SiteSettings siteSettings)
{
string indexPath = GetIndexPath(siteSettings.SiteId);
if (indexPath.Length == 0)
{
return(false);
}
if (!Directory.Exists(indexPath))
{
Directory.CreateDirectory(indexPath);
}
if (Directory.Exists(indexPath))
{
DirectoryInfo directoryInfo = new DirectoryInfo(indexPath);
int fileCount = directoryInfo.GetFiles().Length;
int configFileCount = directoryInfo.GetFiles(".config").Length;
if (
(fileCount == 0) ||
(
(fileCount == 1) &&
(configFileCount == 1)
)
)
{
int rowsToIndex = IndexingQueue.GetCount();
if (rowsToIndex > 0)
{
// already started the indexing process
return(true);
}
// no search index exists so build it
IndexHelper.RebuildSiteIndexAsync(indexPath, CacheHelper.GetMenuPages());
return(false);
}
}
return(true);
}
private void ProcessIndexingQueue()
{
try
{
bool markAsComplete = false;
// this gets executed while content is still being queued.
// if we start right away we may finish and miss some content
// so pause a bit but log progress so it knows this task is running
for (int i = 0; i < 10; i++)
{
rowsToProcess = 1;
Thread.Sleep(5000); // 5 seconds
ReportStatus(markAsComplete);
}
rowsToProcess = 0;
DataTable siteIds = IndexingQueue.GetSiteIDs();
foreach (DataRow row in siteIds.Rows)
{
int siteId = Convert.ToInt32(row["SiteID"]);
string indexPath = IndexHelper.GetSearchIndexPath(siteId);
try
{
if (indexPath.Length > 0)
{
if (!System.IO.Directory.Exists(indexPath))
{
System.IO.Directory.CreateDirectory(indexPath);
}
DataTable q = IndexingQueue.GetByPath(indexPath);
ProcessQueue(q, siteId, indexPath);
}
}
catch (System.IO.IOException ex)
{
errorCount += 1;
log.Error(ex);
}
}
int countOfRowsNotProcessed = IndexingQueue.GetCount();
if ((countOfRowsNotProcessed > 0) && (errorCount < 3))
{
// recurse
ProcessIndexingQueue();
}
else
{
this.rowsToProcess = 0;
markAsComplete = true;
ReportStatus(markAsComplete);
}
}
catch (Exception ex)
{
log.Error(ex);
}
}
private void ProcessQueue(DataTable q, int siteId, string indexPath)
{
rowsProcessed = 0;
rowsToProcess = q.Rows.Count;
// first process deletes with reader
try
{
using (Lucene.Net.Store.Directory searchDirectory = IndexHelper.GetDirectory(siteId))
{
using (IndexReader reader = IndexReader.Open(searchDirectory, false))
{
foreach (DataRow row in q.Rows)
{
Term term = new Term("Key", row["ItemKey"].ToString());
try
{
reader.DeleteDocuments(term);
log.Debug("reader.DeleteDocuments(term) for Key " + row["ItemKey"].ToString());
}
catch (Exception ge)
{
// TODO: monitor what real exceptions if any occur and then
// change this catch to catch only the expected ones
// instead of non specific exception
log.Error(ge);
}
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (removeOnly)
{
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
}
}
}
catch (System.IO.IOException ex)
{
log.Info("IndexWriter swallowed exception this is not unexpected if building or rebuilding the search index ", ex);
errorCount += 1;
}
catch (TypeInitializationException ex)
{
log.Info("IndexWriter swallowed exception ", ex);
errorCount += 1;
}
// next add items with writer
using (Lucene.Net.Store.Directory searchDirectory = IndexHelper.GetDirectory(siteId))
{
using (IndexWriter indexWriter = GetWriter(siteId, searchDirectory))
{
if (indexWriter == null)
{
log.Error("failed to get IndexWriter for path: " + indexPath);
errorCount += 1;
return;
}
foreach (DataRow row in q.Rows)
{
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (!removeOnly)
{
try
{
IndexItem indexItem
= (IndexItem)SerializationHelper.DeserializeFromString(typeof(IndexItem), row["SerializedItem"].ToString());
// if the content is locked down to only admins it is a special case
// we just won't add it to the search index
// because at search time we avoid the role check for all admins, content admins and siteeditors
// we don't have a good way to prevent content admins and site editors from seeing the content
// in search
if ((indexItem.ViewRoles != "Admins;") && (indexItem.ModuleViewRoles != "Admins;"))
{
if (indexItem.ViewPage.Length > 0)
{
Document doc = GetDocument(indexItem);
WriteToIndex(doc, indexWriter);
log.Debug("called WriteToIndex(doc, indexWriter) for key " + indexItem.Key);
}
}
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
catch (Exception ex)
{
log.Error(ex);
}
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
try
{
indexWriter.Optimize();
}
catch (System.IO.IOException ex)
{
log.Error(ex);
}
}
}
}
private void ProcessQueue(DataTable q, string indexPath)
{
rowsProcessed = 0;
rowsToProcess = q.Rows.Count;
// first process deletes with reader
try
{
IndexReader reader = IndexReader.Open(indexPath);
foreach (DataRow row in q.Rows)
{
Term term = new Term("Key", row["ItemKey"].ToString());
try
{
reader.DeleteDocuments(term);
log.Debug("reader.DeleteDocuments(term) for Key " + row["ItemKey"].ToString());
}
catch (Exception ge)
{
// TODO: monitor what real exceptions if any occur and then
// change this catch to catch only the expected ones
// instead of non specific exception
log.Error(ge);
}
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (removeOnly)
{
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
reader.Close();
}
catch (IOException ex)
{
log.Info("IndexWriter swallowed exception this is not unexpected if building or rebuilding the search index ", ex);
errorCount += 1;
}
catch (TypeInitializationException ex)
{
log.Info("IndexWriter swallowed exception ", ex);
errorCount += 1;
}
// next add items with writer
IndexWriter indexWriter = GetWriter(indexPath);
if (indexWriter == null)
{
log.Error("failed to get IndexWriter for path: " + indexPath);
errorCount += 1;
return;
}
foreach (DataRow row in q.Rows)
{
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (!removeOnly)
{
try
{
IndexItem indexItem
= (IndexItem)SerializationHelper.DeserializeFromString(typeof(IndexItem), row["SerializedItem"].ToString());
Document doc = GetDocument(indexItem);
WriteToIndex(doc, indexWriter);
log.Debug("called WriteToIndex(doc, indexWriter) for key " + indexItem.Key);
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
catch (Exception ex)
{
log.Error(ex);
}
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
try
{
indexWriter.Optimize();
}
catch (IOException ex)
{
log.Error(ex);
}
try
{
indexWriter.Close();
}
catch (IOException ex)
{
log.Error(ex);
}
}
