private bool UpdateIndex(string indexName, string settingsFilename, IndexSettings indexSettings)
{
var addToIndex = new List <IDocumentIndex>();
// Rebuilding the inde
logger.Value.Info("Rebuilding index");
indexingStatus = IndexingStatus.Rebuilding;
foreach (var contentProvider in contentProviders)
{
addToIndex.AddRange(contentProvider.GetDocuments(id => indexProvider.New(id)));
}
// save current state of the index
indexSettings.LastIndexedUtc = DateTime.UtcNow;
appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
if (addToIndex.Count == 0)
{
// nothing more to do
indexingStatus = IndexingStatus.Idle;
return(false);
}
// save new and updated documents to the index
indexProvider.Store(indexName, addToIndex);
logger.Value.InfoFormat("Added documents to index: {0}", addToIndex.Count);
return(true);
}
/// <summary>
/// Indexes a batch of content items
/// </summary>
/// <returns>
/// <c>true</c> if there are more items to process; otherwise, <c>false</c>.
/// </returns>
private bool BatchIndex(string indexName, string settingsFilename, IndexSettings indexSettings)
{
var addToIndex = new List <IDocumentIndex>();
var deleteFromIndex = new List <int>();
bool loop = false;
// Rebuilding the index ?
if (indexSettings.Mode == IndexingMode.Rebuild)
{
Logger.Information("Rebuilding index");
_indexingStatus = IndexingStatus.Rebuilding;
do
{
loop = true;
// load all content items
var contentItems = _contentRepository
.Table.Where(versionRecord => versionRecord.Latest && versionRecord.Id > indexSettings.LastContentId)
.OrderBy(versionRecord => versionRecord.Id)
.Take(ContentItemsPerLoop)
.ToList()
.Select(versionRecord => _contentManager.Get(versionRecord.ContentItemRecord.Id, VersionOptions.VersionRecord(versionRecord.Id)))
.Distinct()
.ToList();
// if no more elements to index, switch to update mode
if (contentItems.Count == 0)
{
indexSettings.Mode = IndexingMode.Update;
}
foreach (var item in contentItems)
{
try {
var settings = GetTypeIndexingSettings(item);
// skip items from types which are not indexed
if (settings.List.Contains(indexName))
{
if (item.HasPublished())
{
var published = _contentManager.Get(item.Id, VersionOptions.Published);
IDocumentIndex documentIndex = ExtractDocumentIndex(published);
if (documentIndex != null && documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
}
}
else if (settings.List.Contains(indexName + ":latest"))
{
IDocumentIndex documentIndex = ExtractDocumentIndex(item);
if (documentIndex != null && documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
}
indexSettings.LastContentId = item.VersionRecord.Id;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during rebuild", item.Id);
}
}
if (contentItems.Count < ContentItemsPerLoop)
{
loop = false;
}
else
{
_transactionManager.RequireNew();
}
} while (loop);
}
if (indexSettings.Mode == IndexingMode.Update)
{
Logger.Information("Updating index");
_indexingStatus = IndexingStatus.Updating;
do
{
var indexingTasks = _taskRepository
.Table.Where(x => x.Id > indexSettings.LastIndexedId)
.OrderBy(x => x.Id)
.Take(ContentItemsPerLoop)
.ToList()
.GroupBy(x => x.ContentItemRecord.Id)
.Select(group => new { TaskId = group.Max(task => task.Id), Delete = group.Last().Action == IndexingTaskRecord.Delete, Id = group.Key, ContentItem = _contentManager.Get(group.Key, VersionOptions.Latest) })
.OrderBy(x => x.TaskId)
.ToArray();
foreach (var item in indexingTasks)
{
try {
IDocumentIndex documentIndex = null;
// item.ContentItem can be null if the content item has been deleted
if (item.ContentItem != null)
{
// skip items from types which are not indexed
var settings = GetTypeIndexingSettings(item.ContentItem);
if (settings.List.Contains(indexName))
{
if (item.ContentItem.HasPublished())
{
var published = _contentManager.Get(item.Id, VersionOptions.Published);
documentIndex = ExtractDocumentIndex(published);
}
}
else if (settings.List.Contains(indexName + ":latest"))
{
var latest = _contentManager.Get(item.Id, VersionOptions.Latest);
documentIndex = ExtractDocumentIndex(latest);
}
}
if (documentIndex == null || item.Delete)
{
deleteFromIndex.Add(item.Id);
}
else if (documentIndex.IsDirty)
{
addToIndex.Add(documentIndex);
}
indexSettings.LastIndexedId = item.TaskId;
}
catch (Exception ex) {
Logger.Warning(ex, "Unable to index content item #{0} during update", item.Id);
}
}
if (indexingTasks.Length < ContentItemsPerLoop)
{
loop = false;
}
else
{
_transactionManager.RequireNew();
}
} while (loop);
}
// save current state of the index
indexSettings.LastIndexedUtc = _clock.UtcNow;
_appDataFolder.CreateFile(settingsFilename, indexSettings.ToXml());
if (deleteFromIndex.Count == 0 && addToIndex.Count == 0)
{
// nothing more to do
_indexingStatus = IndexingStatus.Idle;
return(false);
}
// save new and updated documents to the index
try {
if (addToIndex.Count > 0)
{
_indexProvider.Store(indexName, addToIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while adding a document to the index");
}
// removing documents from the index
try {
if (deleteFromIndex.Count > 0)
{
_indexProvider.Delete(indexName, deleteFromIndex);
Logger.Information("Added content items to index: {0}", addToIndex.Count);
}
}
catch (Exception ex) {
Logger.Warning(ex, "An error occured while removing a document from the index");
}
return(true);
}
public void IndexProviderShouldOverwriteAlreadyExistingIndex()
{
_provider.CreateIndex("default");
_provider.Store("default", _provider.New(1).Add("body", null));
Assert.That(_provider.IsEmpty("default"), Is.False);
_provider.CreateIndex("default");
Assert.That(_provider.IsEmpty("default"), Is.True);
}
public void SearchTermsShouldBeFoundInMultipleFields()
{
_provider.CreateIndex("default");
_provider.Store("default",
_provider.New(42)
.Add("title", "title1 title2 title3").Analyze()
.Add("date", new DateTime(2010, 05, 28, 14, 13, 56, 123))
);
Assert.IsNotNull(_provider.CreateSearchBuilder("default").Get(42));
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title1").Search().FirstOrDefault());
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title2").Search().FirstOrDefault());
Assert.IsNotNull(_provider.CreateSearchBuilder("default").WithField("title", "title3").Search().FirstOrDefault());
Assert.IsNull(_provider.CreateSearchBuilder("default").WithField("title", "title4").Search().FirstOrDefault());
}
public async Task InitializeAsync()
{
_settingsPath = Path.GetFullPath(Path.Combine($"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}", @"..\..\..\settings"));
_indexPath = $"{_settingsPath}\\PersonalBlog\\index";
_mockLocalIndexPathFactory = new Mock <ILocalIndexPathFactory>();
_mockLocalIndexPathFactory.Setup(x => x.GetLocalIndexPath())
.Returns(_indexPath);
var services = new ServiceCollection()
.AddLuceneDocumentMapper()
.AddLuceneProvider()
.AddLogging(x => x.AddConsole());
services.Add(new ServiceDescriptor(typeof(ILocalIndexPathFactory), _mockLocalIndexPathFactory.Object));
_serviceProvider = services.BuildServiceProvider();
var dateTime = DateTime.Now;
_indexProvider = _serviceProvider.GetService <IIndexProvider>();
await _indexProvider.CreateIndexIfNotExists(typeof(BlogPost));
await _indexProvider.Store(new List <BlogPost>
{
new BlogPost
{
Name = "My Test Blog Post",
Id = "1",
PublishedDate = DateTime.Now.AddDays(-1)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "2",
PublishedDate = DateTime.Now.AddDays(-100)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "3",
PublishedDate = DateTime.Now.AddDays(-120)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "4",
PublishedDate = DateTime.Now.AddDays(-11)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "5",
PublishedDate = DateTime.Now.AddDays(-19)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "6",
PublishedDate = DateTime.Now.AddDays(-13)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "7",
PublishedDate = DateTime.Now.AddDays(-7)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "8",
PublishedDate = DateTime.Now.AddDays(-8)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "9",
Body = "My test body",
Tags = new List <Tag>()
{
new Tag()
{
Id = "1",
Name = "my-test-tag"
}
},
PublishedDate = DateTime.Now.AddDays(-9)
},
new BlogPost
{
Name = "My Test Blog Post",
Id = "10",
TagIds = new List <string>()
{
"11",
"2"
},
PublishedDate = DateTime.Now.AddDays(-10)
}
});
}
