public IEnumerable <LearningCourseDTO> IndexCourse(LearningCourseDTO course) { var indexer = new LuceneIndexer(); indexer.AddUpdateLuceneIndex(course); return(indexer.GetAllIndexRecords()); }
/// <summary> /// Return the total number of fields in the index /// </summary> /// <param name="indexer"></param> /// <returns></returns> public static int GetIndexFieldCount(this LuceneIndexer indexer) { using (var reader = indexer.GetIndexWriter().GetReader()) { return(reader.GetFieldNames(IndexReader.FieldOption.ALL).Count); } }
protected override void Process() { var config = GetConfiguration(); indexer = new LuceneIndexer(); IndexContent(); }
/// <summary> /// The number of documents deleted in the index /// </summary> /// <param name="indexer"></param> /// <returns></returns> public static int GetDeletedDocumentsCount(this LuceneIndexer indexer) { using (var reader = indexer.GetIndexWriter().GetReader()) { return(reader.NumDeletedDocs()); } }
/// <summary> /// Returns true if the index is optimized or not /// </summary> /// <param name="indexer"></param> /// <returns></returns> public static bool IsIndexOptimized(this LuceneIndexer indexer) { using (var reader = indexer.GetIndexWriter().GetReader()) { return(reader.IsOptimized()); } }
public void Add(Faq faq) { if (_ModuleID != Guid.Empty) { Guid FaqID = Guid.NewGuid(); using (SqlConnection cn = new SqlConnection(ConfigurationManager.ConnectionStrings["ApplicationServices"].ConnectionString)) { using (SqlCommand cmd = new SqlCommand("INSERT INTO cms_FAQ (FaqID, ModuleID, Locale, FAQCategorySerial, Question, Answer, SortOrder) VALUES (@FaqID, @ModuleID, @Locale, @FAQCategorySerial, @Question, @Answer, @SortOrder)", cn)) { cmd.CommandType = CommandType.Text; cmd.Parameters.Add("FaqID", SqlDbType.UniqueIdentifier).Value = FaqID; cmd.Parameters.Add("ModuleID", SqlDbType.UniqueIdentifier).Value = _ModuleID; cmd.Parameters.Add("Locale", SqlDbType.VarChar, 10).Value = _Locale; cmd.Parameters.Add("FAQCategorySerial", SqlDbType.Int).Value = faq.FAQCategorySerial > 0 ? faq.FAQCategorySerial : SqlInt32.Null; cmd.Parameters.Add("Question", SqlDbType.VarChar).Value = faq.Question; cmd.Parameters.Add("Answer", SqlDbType.VarChar).Value = faq.Answer; cmd.Parameters.Add("SortOrder", SqlDbType.Int).Value = faq.SortOrder; cmd.Connection.Open(); cmd.ExecuteNonQuery(); cmd.Connection.Close(); } } // Update Lucene search index Faq newFaq = new Faq(FaqID); LuceneIndexer li = new LuceneIndexer(); li.CreateIndexWriter(); li.UpdateWebPage(FaqID.ToString(), newFaq.URL, newFaq.Question, newFaq.Question + " " + newFaq.Answer, "FAQ"); li.Close(); li.IndexWords(); } }
public static void ReIndexAllPages() { // to extract urls: http://www.dotnetcoders.com/web/Learning/Regex/exHrefExtractor.aspx // or http://regexlib.com/REDetails.aspx?regexp_id=1525 List <IndexableFileInfo> fileInfos = new List <IndexableFileInfo>(); Dictionary <int, CmsPage> AllPages = CmsContext.HomePage.getLinearizedPages(); // -- translate from [path] => {CmsPage} to IndexableFileInfo objects foreach (int pageId in AllPages.Keys) { CmsPage page = AllPages[pageId]; if (page.isVisibleForCurrentUser) { if (!page.hasPlaceholder("PageRedirect") && page.ShowInMenu) // do not index redirect pages. { // index the content of placeholders fileInfos.AddRange(indexStandardPage(page)); } } } // foreach page IndexableFileInfo[] fInfos = fileInfos.ToArray(); LuceneIndexer.doIndex(IndexStorageDirectory, SpellCheckIndexStorageDirectory, LuceneIndexer.IndexCreationMode.CreateNewIndex, fInfos, new Object()); } // ReIndexAllPages
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var codeGen = GetLocalStorageDirectory(indexFolder); var master = new DirectoryInfo(luceneIndexFolder); return new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(codeGen)); }
/// <summary> /// Gets the lastindexed date. /// </summary> /// <returns></returns> public List <IndexInformation> SearchIndex(ProductType productType, string searchText) { LuceneIndexer li = new LuceneIndexer(); string indexFolder = ConfigurationManager.AppSettings.Get(IndexFolder) + productType.ToString(); return(li.SearchIndex(indexFolder, searchText)); }
public virtual Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var tempFolder = GetLocalStorageDirectory(indexFolder); return(new SimpleFSDirectory(tempFolder)); }
public override Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var tempFolder = GetLocalStorageDirectory(indexFolder); var master = new DirectoryInfo(luceneIndexFolder); return(new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(tempFolder))); }
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var codeGen = GetLocalStorageDirectory(indexFolder); var master = new DirectoryInfo(luceneIndexFolder); return(new SyncDirectory(new SimpleFSDirectory(master), new SimpleFSDirectory(codeGen))); }
public void AnItemCanBeUpdatedInTheIndex() { TestDataGenerator tdg = new TestDataGenerator(); Directory directory = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48); indexer = new LuceneIndexer(directory, analyzer); indexer.CreateIndex(tdg.AllData, true); // we need a searcher for this test LuceneIndexSearcher searcher = new LuceneIndexSearcher(directory, analyzer); // get the 1st item SearchOptions options = new SearchOptions("*****@*****.**", "Email"); var initialResults = searcher.ScoredSearch(options); foreach (var item in initialResults.Results) { output.WriteLine($"{item.Score}\t{item.Document.Get("Id")}\t{item.Document.Get("FirstName")}\t{item.Document.Get("Email")}"); } //Assert.Equal(1, initialResults.TotalHits); Document rambler = initialResults.Results.First().Document; // convert to ILuceneIndexable User user = new User() { Id = int.Parse(rambler.Get("Id")), IndexId = new Guid(rambler.Get("IndexId")), FirstName = rambler.Get("FirstName"), Surname = rambler.Get("Surname"), Email = rambler.Get("Email"), JobTitle = rambler.Get("JobTitle") }; // make an edit user.FirstName = "Duke"; user.Surname = "Nukem"; // add the update to the indexer indexer.Update(user); // search again var endResults = searcher.ScoredSearch(options); foreach (var item in endResults.Results) { output.WriteLine($"{item.Score}\t{item.Document.Get("Id")}\t{item.Document.Get("FirstName")}\t{item.Document.Get("Email")}"); } // Assert.Equal(1, endResults.TotalHits); Assert.Equal(user.IndexId.ToString(), endResults.Results.First().Document.Get("IndexId")); Assert.Equal(user.Id.ToString(), endResults.Results.First().Document.Get("Id")); Assert.Equal(user.FirstName, endResults.Results.First().Document.Get("FirstName")); Assert.Equal(user.Surname, endResults.Results.First().Document.Get("Surname")); directory.ClearLock("write.lock"); }
public override void SetUp() { base.SetUp(); accessor = new LuceneAccesor(new ThreadContext(), new DatabaseSection()); indexer = new LuceneIndexer(accessor, new TextExtractor(new IndexableDefinitionExtractor(definitions))); root = CreateOneItem <PersistableItem1>(1, "The Root Page", null); indexer.Clear(); }
public void UpdateIndeces(IPublishingStrategy sender, PublishEventArgs <IContent> args) { foreach (var node in args.PublishedEntities.Where(x => x.ContentType.Alias == Blogpost.ModelTypeAlias)) { MongoIndexer.Index(node.Id); LuceneIndexer.Index(node.Id); PetaPocoIndexer.Index(node.Id); } }
private void UpdateIndex() { // Update Lucene search index LuceneIndexer li = new LuceneIndexer(); li.CreateIndexWriter(); li.UpdateWebPage(FaqID.ToString(), URL, Question, Question + " " + Answer, "FAQ"); li.Close(); li.IndexWords(); }
public virtual Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var tempFolder = GetLocalStorageDirectory(indexFolder); var simpleFsDirectory = new SimpleFSDirectory(tempFolder); simpleFsDirectory.SetLockFactory(DirectoryTracker.DefaultLockFactory(tempFolder)); return(simpleFsDirectory); }
/// <summary> /// Returns a DateTimeOffset with an offset of zero (UTC) for the value passed in, if it cannot be parsed a null is returned. /// </summary> /// <param name="val"></param> /// <returns></returns> internal static DateTimeOffset?FromExamineDateTime(string val) { long output; if (long.TryParse(val, out output)) { return(new DateTimeOffset(LuceneIndexer.DateTimeFromTicks(output), new TimeSpan(0, 0, 0))); } return(null); }
/// <summary> /// Return the AzureDirectory. /// It stores the master index in Blob storage. /// Only a master server can write to it. /// For each slave server, the blob storage index files are synced to the local machine. /// </summary> /// <param name="indexer"> /// The indexer. /// </param> /// <param name="luceneIndexFolder"> /// The lucene index folder. /// </param> /// <returns> /// The <see cref="Lucene.Net.Store.Directory"/>. /// </returns> public override Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var tempFolder = GetLocalStorageDirectory(indexFolder); return(new AzureDirectory( CloudStorageAccount.Parse(ConfigurationManager.AppSettings[ConfigStorageKey]), ConfigurationManager.AppSettings[ConfigContainerKey], new SimpleFSDirectory(tempFolder), rootFolder: indexer.IndexSetName)); }
private HttpResponseMessage HandleException(Exception ex, LuceneIndexer indexer) { //ensure it's not listening indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; LogHelper.Error <ExamineManagementApiController>("An error occurred rebuilding index", ex); var response = Request.CreateResponse(HttpStatusCode.Conflict); response.Content = new StringContent(string.Format("The index could not be rebuilt at this time, most likely there is another thread currently writing to the index. Error: {0}", ex)); response.ReasonPhrase = "Could Not Rebuild"; return(response); }
private InstanceServices CreateServices(string instanceName) { var path = Path.Combine(indexPath, instanceName); var accessor = new LuceneAccesor(path); var indexer = new LuceneIndexer(accessor); var searcher = new LuceneLightweightSearcher(accessor); return(new InstanceServices { indexer = indexer, searcher = searcher }); }
public static IndexWriter GetAzureIndexWriter(this LuceneIndexer indexer) { indexer.EnsureIndex(false); var writer = new IndexWriter(indexer.GetLuceneDirectory(), indexer.IndexingAnalyzer, false, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetRAMBufferSizeMB(10.0); writer.SetUseCompoundFile(false); writer.SetMaxMergeDocs(10000); writer.SetMergeFactor(100); return(writer); }
public void AnItemCanBeRemovedFromTheIndex() { TestDataGenerator tdg = new TestDataGenerator(); Directory directory = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48); _indexer = new LuceneIndexer(directory, analyzer); _indexer.CreateIndex(tdg.AllData); _indexer.Delete(tdg.AllData.First()); Assert.True(tdg.AllData.Count > _indexer.Count()); directory.ClearLock("write.lock"); }
public static LuceneSearcher GetSearcherForIndexer(this LuceneIndexer indexer) { var indexSet = indexer.IndexSetName; var searcher = ExamineManager.Instance.SearchProviderCollection.OfType <LuceneSearcher>() .FirstOrDefault(x => x.IndexSetName == indexSet); if (searcher == null) { throw new InvalidOperationException("No searcher assigned to the index set " + indexer.IndexSetName); } return(searcher); }
public Lucene.Net.Store.Directory CreateDirectory(LuceneIndexer indexer, string luceneIndexFolder) { var indexFolder = new DirectoryInfo(luceneIndexFolder); var codeGen = GetLocalStorageDirectory(indexFolder); var master = new DirectoryInfo(luceneIndexFolder); var masterDir = new SimpleFSDirectory(master); var cacheDir = new SimpleFSDirectory(codeGen); masterDir.SetLockFactory(DirectoryTracker.DefaultLockFactory(master)); cacheDir.SetLockFactory(DirectoryTracker.DefaultLockFactory(codeGen)); return(new SyncDirectory(masterDir, cacheDir)); }
public void Execute(IJobExecutionContext context) { LuceneIndexer.ClearLuceneIndex(); Console.WriteLine("Search index was cleared successfully!"); var items = FeedParser.GetModels(); LuceneIndexer.AddUpdateLuceneIndex(items); Console.WriteLine("Search index was created successfully!"); LuceneIndexer.Optimize(); Console.WriteLine("Search index was optimized successfully!"); }
/// <summary> /// Indexes the files. /// </summary> /// <param name="servers">The servers.</param> /// <returns></returns> public bool CreateIndexFiles(ProductType productType) { string ImpersonatedUser = ConfigurationManager.AppSettings.Get("User"); string ImpersonatedUserPassword = ConfigurationManager.AppSettings.Get("Password"); string ImpersonatedUserDomain = ConfigurationManager.AppSettings.Get("Domain"); using (UserImpersonation user = new UserImpersonation(ImpersonatedUser, ImpersonatedUserDomain, ImpersonatedUserPassword)) { if (user.ImpersonateValidUser()) { XmlProcessor processor = new XmlProcessor(); Product product = processor.ReadProduct(productType); if (product.LastIndexedDate.HasValue && DateTime.Compare(product.LastIndexedDate.Value, DateTime.Today) == 0) { return(true); } DateTime startDate = product.LastIndexedDate ?? product.IndexStartDate; DateTime endDate = DateTime.Today.AddDays(-1); string targetDirectory = ConfigurationManager.AppSettings.Get("DecompressedFolder"); string indexLocation = ConfigurationManager.AppSettings.Get("IndexFolder") + productType.ToString(); ClearLogDecompress(targetDirectory); DeleteEarlierIndexes(product, processor); while (startDate <= endDate) { foreach (string file in processor.CopyFiles(startDate.ToShortDateString(), productType)) { string fileName; FileInfo fi = new DirectoryInfo(targetDirectory).GetFiles("*.zip").FirstOrDefault(); fileName = fi != null?processor.DecompressFile(fi) : string.Empty; if (!string.IsNullOrEmpty(fileName)) { LuceneIndexer li = new LuceneIndexer(); HashSet <string> set = processor.ReadFile(product, fileName); li.IndexFile(indexLocation, fileName, startDate.ToShortDateString(), set); } } startDate = startDate.AddDays(1); this.UpdateProductDate(product, startDate, "LastIndexedDate"); } } } return(true); }
public void AnIndexCanBeCreated() { TestDataGenerator tdg = new TestDataGenerator(); Directory directory = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48); _indexer = new LuceneIndexer(directory, analyzer); _indexer.CreateIndex(tdg.AllData); Assert.Equal(2000, _indexer.Count()); analyzer.Dispose(); directory.ClearLock("write.lock"); directory.Dispose(); }
public ActionResult Index(FormCollection data) { LuceneIndexer li = new LuceneIndexer(); li.DeleteIndex(false); li.CreateIndexWriter(); //Index blog posts BlogPosts blogPosts = new BlogPosts("en-US"); foreach (BlogPostViewModel blogPost in blogPosts.EveryPost()) { li.AddWebPage(blogPost.PostID.ToString(), blogPost.URL, blogPost.Title, blogPost.PostContent, "Blog"); } ////Index Documents //DocumentSet docs = new DocumentSet("en-US"); //foreach (TKS.Document doc in docs.Documents()) { // if (!string.IsNullOrEmpty(doc.DocumentTitle)) { // li.AddWebPage(doc.DocumentID.ToString(), doc.URL, doc.DocumentTitle, doc.Description, "Document"); // } else { // li.AddWebPage(doc.DocumentID.ToString(), doc.URL, doc.LinkText, doc.Description, "Document"); // } //} ////Index FAQs //FaqSet faqs = new FaqSet("en-US"); //foreach (Faq faq in faqs.FAQs()) { // li.AddWebPage(faq.FaqID.ToString(), faq.URL, faq.Question, faq.Question + " " + faq.Answer, "FAQ"); //} ////Index News //NewsSet newsSet = new NewsSet(); //foreach (News news in newsSet.News()) { // li.AddWebPage(news.NewsID.ToString(), news.URL, news.Headline, news.Content, "News"); //} //Index Content Blocks ContentTextSet contentSet = new ContentTextSet("en-US"); foreach (ContentText content in contentSet.AllContentTextBlocks()) { li.AddWebPage(content.ContentID.ToString(), content.URL, content.PageTitle, content.Contents, "Page"); } li.Close(); li.IndexWords(); return(View()); }
public void AnItemCanBeAddedToTheIndex() { TestDataGenerator tdg = new TestDataGenerator(); Directory directory = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48); indexer = new LuceneIndexer(directory, analyzer); indexer.CreateIndex(tdg.AllData, true); Assert.Equal(2000, indexer.Count()); indexer.Add(tdg.ANewUser()); Assert.Equal(2001, indexer.Count()); directory.ClearLock("write.lock"); }
/// <summary> /// Returns true if the index is optimized or not /// </summary> /// <param name="indexer"></param> /// <returns></returns> public static bool IsIndexOptimized(this LuceneIndexer indexer) { try { using (var reader = indexer.GetIndexWriter().GetReader()) { return(reader.IsOptimized()); } } catch (AlreadyClosedException) { LogHelper.Warn(typeof(ExamineExtensions), "Cannot get IsIndexOptimized, the writer is already closed"); return(false); } }
private HttpResponseMessage ValidateLuceneIndexer(string indexerName, out LuceneIndexer indexer) { if (ExamineManager.Instance.IndexProviderCollection.Any(x => x.Name == indexerName)) { indexer = ExamineManager.Instance.IndexProviderCollection[indexerName] as LuceneIndexer; if (indexer == null) { var response1 = Request.CreateResponse(HttpStatusCode.BadRequest); response1.Content = new StringContent(string.Format("The indexer {0} is not of type {1}", indexerName, typeof(LuceneIndexer))); response1.ReasonPhrase = "Wrong Indexer Type"; return response1; } //return Ok! return Request.CreateResponse(HttpStatusCode.OK); } indexer = null; var response = Request.CreateResponse(HttpStatusCode.BadRequest); response.Content = new StringContent(string.Format("No indexer found with name = {0}", indexerName)); response.ReasonPhrase = "Indexer Not Found"; return response; }