public IEnumerable <LearningCourseDTO> IndexCourse(LearningCourseDTO course)
{
var indexer = new LuceneIndexer();
indexer.AddUpdateLuceneIndex(course);
return(indexer.GetAllIndexRecords());
}
/// <summary>
/// Return the total number of fields in the index
/// </summary>
/// <param name="indexer"></param>
/// <returns></returns>
public static int GetIndexFieldCount(this LuceneIndexer indexer)
{
using (var reader = indexer.GetIndexWriter().GetReader())
{
return(reader.GetFieldNames(IndexReader.FieldOption.ALL).Count);
}
}
protected override void Process()
{
var config = GetConfiguration();
indexer = new LuceneIndexer();
IndexContent();
}
/// <summary>
/// The number of documents deleted in the index
/// </summary>
/// <param name="indexer"></param>
/// <returns></returns>
public static int GetDeletedDocumentsCount(this LuceneIndexer indexer)
{
using (var reader = indexer.GetIndexWriter().GetReader())
{
return(reader.NumDeletedDocs());
}
}
/// <summary>
/// Returns true if the index is optimized or not
/// </summary>
/// <param name="indexer"></param>
/// <returns></returns>
public static bool IsIndexOptimized(this LuceneIndexer indexer)
{
using (var reader = indexer.GetIndexWriter().GetReader())
{
return(reader.IsOptimized());
}
}
public void Add(Faq faq)
{
if (_ModuleID != Guid.Empty)
{
Guid FaqID = Guid.NewGuid();
using (SqlConnection cn = new SqlConnection(ConfigurationManager.ConnectionStrings["ApplicationServices"].ConnectionString)) {
using (SqlCommand cmd = new SqlCommand("INSERT INTO cms_FAQ (FaqID, ModuleID, Locale, FAQCategorySerial, Question, Answer, SortOrder) VALUES (@FaqID, @ModuleID, @Locale, @FAQCategorySerial, @Question, @Answer, @SortOrder)", cn)) {
cmd.CommandType = CommandType.Text;
cmd.Parameters.Add("FaqID", SqlDbType.UniqueIdentifier).Value = FaqID;
cmd.Parameters.Add("ModuleID", SqlDbType.UniqueIdentifier).Value = _ModuleID;
cmd.Parameters.Add("Locale", SqlDbType.VarChar, 10).Value = _Locale;
cmd.Parameters.Add("FAQCategorySerial", SqlDbType.Int).Value = faq.FAQCategorySerial > 0 ? faq.FAQCategorySerial : SqlInt32.Null;
cmd.Parameters.Add("Question", SqlDbType.VarChar).Value = faq.Question;
cmd.Parameters.Add("Answer", SqlDbType.VarChar).Value = faq.Answer;
cmd.Parameters.Add("SortOrder", SqlDbType.Int).Value = faq.SortOrder;
cmd.Connection.Open();
cmd.ExecuteNonQuery();
cmd.Connection.Close();
}
}
// Update Lucene search index
Faq newFaq = new Faq(FaqID);
LuceneIndexer li = new LuceneIndexer();
li.CreateIndexWriter();
li.UpdateWebPage(FaqID.ToString(), newFaq.URL, newFaq.Question, newFaq.Question + " " + newFaq.Answer, "FAQ");
li.Close();
li.IndexWords();
}
}
public static void ReIndexAllPages()
{
// to extract urls: http://www.dotnetcoders.com/web/Learning/Regex/exHrefExtractor.aspx
// or http://regexlib.com/REDetails.aspx?regexp_id=1525
List <IndexableFileInfo> fileInfos = new List <IndexableFileInfo>();
Dictionary <int, CmsPage> AllPages = CmsContext.HomePage.getLinearizedPages();
// -- translate from [path] => {CmsPage} to IndexableFileInfo objects
foreach (int pageId in AllPages.Keys)
{
CmsPage page = AllPages[pageId];
if (page.isVisibleForCurrentUser)
{
if (!page.hasPlaceholder("PageRedirect") && page.ShowInMenu) // do not index redirect pages.
{
// index the content of placeholders
fileInfos.AddRange(indexStandardPage(page));
}
}
} // foreach page
IndexableFileInfo[] fInfos = fileInfos.ToArray();
LuceneIndexer.doIndex(IndexStorageDirectory, SpellCheckIndexStorageDirectory, LuceneIndexer.IndexCreationMode.CreateNewIndex, fInfos, new Object());
} // ReIndexAllPages
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var codeGen = GetLocalStorageDirectory(indexFolder);
var master = new DirectoryInfo(luceneIndexFolder);
return new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(codeGen));
}
/// <summary>
/// Gets the lastindexed date.
/// </summary>
/// <returns></returns>
public List <IndexInformation> SearchIndex(ProductType productType, string searchText)
{
LuceneIndexer li = new LuceneIndexer();
string indexFolder = ConfigurationManager.AppSettings.Get(IndexFolder) + productType.ToString();
return(li.SearchIndex(indexFolder, searchText));
}
public virtual Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var tempFolder = GetLocalStorageDirectory(indexFolder);
return(new SimpleFSDirectory(tempFolder));
}
public override Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var tempFolder = GetLocalStorageDirectory(indexFolder);
var master = new DirectoryInfo(luceneIndexFolder);
return(new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(tempFolder)));
}
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var codeGen = GetLocalStorageDirectory(indexFolder);
var master = new DirectoryInfo(luceneIndexFolder);
return(new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(codeGen)));
}
public void AnItemCanBeUpdatedInTheIndex()
{
TestDataGenerator tdg = new TestDataGenerator();
Directory directory = new RAMDirectory();
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
indexer = new LuceneIndexer(directory, analyzer);
indexer.CreateIndex(tdg.AllData, true);
// we need a searcher for this test
LuceneIndexSearcher searcher = new LuceneIndexSearcher(directory, analyzer);
// get the 1st item
SearchOptions options = new SearchOptions("*****@*****.**", "Email");
var initialResults = searcher.ScoredSearch(options);
foreach (var item in initialResults.Results)
{
output.WriteLine($"{item.Score}\t{item.Document.Get("Id")}\t{item.Document.Get("FirstName")}\t{item.Document.Get("Email")}");
}
//Assert.Equal(1, initialResults.TotalHits);
Document rambler = initialResults.Results.First().Document;
// convert to ILuceneIndexable
User user = new User()
{
Id = int.Parse(rambler.Get("Id")),
IndexId = new Guid(rambler.Get("IndexId")),
FirstName = rambler.Get("FirstName"),
Surname = rambler.Get("Surname"),
Email = rambler.Get("Email"),
JobTitle = rambler.Get("JobTitle")
};
// make an edit
user.FirstName = "Duke";
user.Surname = "Nukem";
// add the update to the indexer
indexer.Update(user);
// search again
var endResults = searcher.ScoredSearch(options);
foreach (var item in endResults.Results)
{
output.WriteLine($"{item.Score}\t{item.Document.Get("Id")}\t{item.Document.Get("FirstName")}\t{item.Document.Get("Email")}");
}
// Assert.Equal(1, endResults.TotalHits);
Assert.Equal(user.IndexId.ToString(), endResults.Results.First().Document.Get("IndexId"));
Assert.Equal(user.Id.ToString(), endResults.Results.First().Document.Get("Id"));
Assert.Equal(user.FirstName, endResults.Results.First().Document.Get("FirstName"));
Assert.Equal(user.Surname, endResults.Results.First().Document.Get("Surname"));
directory.ClearLock("write.lock");
}
public override void SetUp()
{
base.SetUp();
accessor = new LuceneAccesor(new ThreadContext(), new DatabaseSection());
indexer = new LuceneIndexer(accessor, new TextExtractor(new IndexableDefinitionExtractor(definitions)));
root = CreateOneItem <PersistableItem1>(1, "The Root Page", null);
indexer.Clear();
}
public void UpdateIndeces(IPublishingStrategy sender, PublishEventArgs <IContent> args)
{
foreach (var node in args.PublishedEntities.Where(x => x.ContentType.Alias == Blogpost.ModelTypeAlias))
{
MongoIndexer.Index(node.Id);
LuceneIndexer.Index(node.Id);
PetaPocoIndexer.Index(node.Id);
}
}
private void UpdateIndex()
{
// Update Lucene search index
LuceneIndexer li = new LuceneIndexer();
li.CreateIndexWriter();
li.UpdateWebPage(FaqID.ToString(), URL, Question, Question + " " + Answer, "FAQ");
li.Close();
li.IndexWords();
}
public virtual Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var tempFolder = GetLocalStorageDirectory(indexFolder);
var simpleFsDirectory = new SimpleFSDirectory(tempFolder);
simpleFsDirectory.SetLockFactory(DirectoryTracker.DefaultLockFactory(tempFolder));
return(simpleFsDirectory);
}
/// <summary>
/// Returns a DateTimeOffset with an offset of zero (UTC) for the value passed in, if it cannot be parsed a null is returned.
/// </summary>
/// <param name="val"></param>
/// <returns></returns>
internal static DateTimeOffset?FromExamineDateTime(string val)
{
long output;
if (long.TryParse(val, out output))
{
return(new DateTimeOffset(LuceneIndexer.DateTimeFromTicks(output), new TimeSpan(0, 0, 0)));
}
return(null);
}
/// <summary>
/// Return the AzureDirectory.
/// It stores the master index in Blob storage.
/// Only a master server can write to it.
/// For each slave server, the blob storage index files are synced to the local machine.
/// </summary>
/// <param name="indexer">
/// The indexer.
/// </param>
/// <param name="luceneIndexFolder">
/// The lucene index folder.
/// </param>
/// <returns>
/// The <see cref="Lucene.Net.Store.Directory"/>.
/// </returns>
public override Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var tempFolder = GetLocalStorageDirectory(indexFolder);
return(new AzureDirectory(
CloudStorageAccount.Parse(ConfigurationManager.AppSettings[ConfigStorageKey]),
ConfigurationManager.AppSettings[ConfigContainerKey],
new SimpleFSDirectory(tempFolder),
rootFolder: indexer.IndexSetName));
}
private HttpResponseMessage HandleException(Exception ex, LuceneIndexer indexer)
{
//ensure it's not listening
indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
LogHelper.Error <ExamineManagementApiController>("An error occurred rebuilding index", ex);
var response = Request.CreateResponse(HttpStatusCode.Conflict);
response.Content = new StringContent(string.Format("The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {0}", ex));
response.ReasonPhrase = "Could Not Rebuild";
return(response);
}
private InstanceServices CreateServices(string instanceName)
{
var path = Path.Combine(indexPath, instanceName);
var accessor = new LuceneAccesor(path);
var indexer = new LuceneIndexer(accessor);
var searcher = new LuceneLightweightSearcher(accessor);
return(new InstanceServices {
indexer = indexer, searcher = searcher
});
}
public static IndexWriter GetAzureIndexWriter(this LuceneIndexer indexer)
{
indexer.EnsureIndex(false);
var writer = new IndexWriter(indexer.GetLuceneDirectory(), indexer.IndexingAnalyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetRAMBufferSizeMB(10.0);
writer.SetUseCompoundFile(false);
writer.SetMaxMergeDocs(10000);
writer.SetMergeFactor(100);
return(writer);
}
public void AnItemCanBeRemovedFromTheIndex()
{
TestDataGenerator tdg = new TestDataGenerator();
Directory directory = new RAMDirectory();
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
_indexer = new LuceneIndexer(directory, analyzer);
_indexer.CreateIndex(tdg.AllData);
_indexer.Delete(tdg.AllData.First());
Assert.True(tdg.AllData.Count > _indexer.Count());
directory.ClearLock("write.lock");
}
public static LuceneSearcher GetSearcherForIndexer(this LuceneIndexer indexer)
{
var indexSet = indexer.IndexSetName;
var searcher = ExamineManager.Instance.SearchProviderCollection.OfType <LuceneSearcher>()
.FirstOrDefault(x => x.IndexSetName == indexSet);
if (searcher == null)
{
throw new InvalidOperationException("No searcher assigned to the index set " + indexer.IndexSetName);
}
return(searcher);
}
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder)
{
var indexFolder = new DirectoryInfo(luceneIndexFolder);
var codeGen = GetLocalStorageDirectory(indexFolder);
var master = new DirectoryInfo(luceneIndexFolder);
var masterDir = new SimpleFSDirectory(master);
var cacheDir = new SimpleFSDirectory(codeGen);
masterDir.SetLockFactory(DirectoryTracker.DefaultLockFactory(master));
cacheDir.SetLockFactory(DirectoryTracker.DefaultLockFactory(codeGen));
return(new SyncDirectory(masterDir, cacheDir));
}
public void Execute(IJobExecutionContext context)
{
LuceneIndexer.ClearLuceneIndex();
Console.WriteLine("Search index was cleared successfully!");
var items = FeedParser.GetModels();
LuceneIndexer.AddUpdateLuceneIndex(items);
Console.WriteLine("Search index was created successfully!");
LuceneIndexer.Optimize();
Console.WriteLine("Search index was optimized successfully!");
}
/// <summary>
/// Indexes the files.
/// </summary>
/// <param name="servers">The servers.</param>
/// <returns></returns>
public bool CreateIndexFiles(ProductType productType)
{
string ImpersonatedUser = ConfigurationManager.AppSettings.Get("User");
string ImpersonatedUserPassword = ConfigurationManager.AppSettings.Get("Password");
string ImpersonatedUserDomain = ConfigurationManager.AppSettings.Get("Domain");
using (UserImpersonation user = new UserImpersonation(ImpersonatedUser, ImpersonatedUserDomain, ImpersonatedUserPassword))
{
if (user.ImpersonateValidUser())
{
XmlProcessor processor = new XmlProcessor();
Product product = processor.ReadProduct(productType);
if (product.LastIndexedDate.HasValue && DateTime.Compare(product.LastIndexedDate.Value, DateTime.Today) == 0)
{
return(true);
}
DateTime startDate = product.LastIndexedDate ?? product.IndexStartDate;
DateTime endDate = DateTime.Today.AddDays(-1);
string targetDirectory = ConfigurationManager.AppSettings.Get("DecompressedFolder");
string indexLocation = ConfigurationManager.AppSettings.Get("IndexFolder") + productType.ToString();
ClearLogDecompress(targetDirectory);
DeleteEarlierIndexes(product, processor);
while (startDate <= endDate)
{
foreach (string file in processor.CopyFiles(startDate.ToShortDateString(), productType))
{
string fileName;
FileInfo fi = new DirectoryInfo(targetDirectory).GetFiles("*.zip").FirstOrDefault();
fileName = fi != null?processor.DecompressFile(fi) : string.Empty;
if (!string.IsNullOrEmpty(fileName))
{
LuceneIndexer li = new LuceneIndexer();
HashSet <string> set = processor.ReadFile(product, fileName);
li.IndexFile(indexLocation, fileName, startDate.ToShortDateString(), set);
}
}
startDate = startDate.AddDays(1);
this.UpdateProductDate(product, startDate, "LastIndexedDate");
}
}
}
return(true);
}
public void AnIndexCanBeCreated()
{
TestDataGenerator tdg = new TestDataGenerator();
Directory directory = new RAMDirectory();
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
_indexer = new LuceneIndexer(directory, analyzer);
_indexer.CreateIndex(tdg.AllData);
Assert.Equal(2000, _indexer.Count());
analyzer.Dispose();
directory.ClearLock("write.lock");
directory.Dispose();
}
public ActionResult Index(FormCollection data)
{
LuceneIndexer li = new LuceneIndexer();
li.DeleteIndex(false);
li.CreateIndexWriter();
//Index blog posts
BlogPosts blogPosts = new BlogPosts("en-US");
foreach (BlogPostViewModel blogPost in blogPosts.EveryPost())
{
li.AddWebPage(blogPost.PostID.ToString(), blogPost.URL, blogPost.Title, blogPost.PostContent, "Blog");
}
////Index Documents
//DocumentSet docs = new DocumentSet("en-US");
//foreach (TKS.Document doc in docs.Documents()) {
// if (!string.IsNullOrEmpty(doc.DocumentTitle)) {
// li.AddWebPage(doc.DocumentID.ToString(), doc.URL, doc.DocumentTitle, doc.Description, "Document");
// } else {
// li.AddWebPage(doc.DocumentID.ToString(), doc.URL, doc.LinkText, doc.Description, "Document");
// }
//}
////Index FAQs
//FaqSet faqs = new FaqSet("en-US");
//foreach (Faq faq in faqs.FAQs()) {
// li.AddWebPage(faq.FaqID.ToString(), faq.URL, faq.Question, faq.Question + " " + faq.Answer, "FAQ");
//}
////Index News
//NewsSet newsSet = new NewsSet();
//foreach (News news in newsSet.News()) {
// li.AddWebPage(news.NewsID.ToString(), news.URL, news.Headline, news.Content, "News");
//}
//Index Content Blocks
ContentTextSet contentSet = new ContentTextSet("en-US");
foreach (ContentText content in contentSet.AllContentTextBlocks())
{
li.AddWebPage(content.ContentID.ToString(), content.URL, content.PageTitle, content.Contents, "Page");
}
li.Close();
li.IndexWords();
return(View());
}
public void AnItemCanBeAddedToTheIndex()
{
TestDataGenerator tdg = new TestDataGenerator();
Directory directory = new RAMDirectory();
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
indexer = new LuceneIndexer(directory, analyzer);
indexer.CreateIndex(tdg.AllData, true);
Assert.Equal(2000, indexer.Count());
indexer.Add(tdg.ANewUser());
Assert.Equal(2001, indexer.Count());
directory.ClearLock("write.lock");
}
/// <summary>
/// Returns true if the index is optimized or not
/// </summary>
/// <param name="indexer"></param>
/// <returns></returns>
public static bool IsIndexOptimized(this LuceneIndexer indexer)
{
try
{
using (var reader = indexer.GetIndexWriter().GetReader())
{
return(reader.IsOptimized());
}
}
catch (AlreadyClosedException)
{
LogHelper.Warn(typeof(ExamineExtensions), "Cannot get IsIndexOptimized, the writer is already closed");
return(false);
}
}
private HttpResponseMessage ValidateLuceneIndexer(string indexerName, out LuceneIndexer indexer)
{
if (ExamineManager.Instance.IndexProviderCollection.Any(x => x.Name == indexerName))
{
indexer = ExamineManager.Instance.IndexProviderCollection[indexerName] as LuceneIndexer;
if (indexer == null)
{
var response1 = Request.CreateResponse(HttpStatusCode.BadRequest);
response1.Content = new StringContent(string.Format("The indexer {0} is not of type {1}", indexerName, typeof(LuceneIndexer)));
response1.ReasonPhrase = "Wrong Indexer Type";
return response1;
}
//return Ok!
return Request.CreateResponse(HttpStatusCode.OK);
}
indexer = null;
var response = Request.CreateResponse(HttpStatusCode.BadRequest);
response.Content = new StringContent(string.Format("No indexer found with name = {0}", indexerName));
response.ReasonPhrase = "Indexer Not Found";
return response;
}
