private static void _addToLuceneIndex(LuceneSearchData sampleData, IndexWriter writer)
        {
            // remove older index entry
            var searchQuery = new TermQuery(new Term("Id", sampleData.Id.ToString()));

            writer.DeleteDocuments(searchQuery);

            // add new index entry
            var doc = new Document();

            // add lucene fields mapped to db fields
            doc.Add(new Field("Id", sampleData.Id.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("ArticleId", sampleData.ArticleId.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("BaseArticleId", sampleData.BaseArticleId.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("CategoryId", sampleData.CategoryId.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("Type", sampleData.Type.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("Name", sampleData.Name.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("Description", sampleData.Description.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("name_en", sampleData.name_en.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("name_zh", sampleData.name_zh.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("name_cn", sampleData.name_cn.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("desc_en", sampleData.desc_en.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("desc_zh", sampleData.desc_zh.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("desc_cn", sampleData.desc_cn.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("Url", sampleData.Url.ToString(), Field.Store.YES, Field.Index.ANALYZED));
            doc.Add(new Field("is_page", sampleData.is_page.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_pdf", sampleData.is_pdf.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_doc", sampleData.is_doc.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_docx", sampleData.is_docx.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_trading", sampleData.is_trading.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_member", sampleData.is_member.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.Add(new Field("is_visitor", sampleData.is_visitor.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));

            // add entry to index
            writer.AddDocument(doc);
        }
 public static void AddUpdateLuceneIndex(LuceneSearchData sampleData)
 {
     AddUpdateLuceneIndex(new List <LuceneSearchData> {
         sampleData
     });
 }
        public static List <LuceneSearchData> GetAll()
        {
            List <LuceneSearchData> searchData = new List <LuceneSearchData>();
            List <ArticlePublished> items      = ArticlePublishedDbContext.getInstance().findPublishedArticlesGroupByBaseVersion("en", "trading");

            List <string> innerHTMLLinks = new List <string>();

            for (int i = 0; i < items.Count; i++)
            {
                try
                {
                    ArticlePublished article = items[i];
                    LuceneSearchData data    = new LuceneSearchData();

                    data.is_page = 0;
                    data.is_pdf  = 0;
                    data.is_doc  = 0;
                    data.is_docx = 0;

                    data.is_visitor = 0;
                    data.is_member  = 0;
                    data.is_trading = 0;

                    if (article != null &&
                        article.categoryID != null &&
                        article.categoryID.HasValue)
                    {
                        data.Id            = i;
                        data.ArticleId     = article.ArticleID;
                        data.BaseArticleId = article.BaseArticleID;
                        data.CategoryId    = article.categoryID.Value;
                        data.Url           = article.category.url;
                        data.Type          = "page";
                        data.Name          = article.Name;
                        data.Description   = article.Desc;

                        if (data.Name == null)
                        {
                            continue;
                        }

                        if (data.Description == null)
                        {
                            data.Description = "";
                        }

                        data.name_en = article.Name;
                        data.desc_en = article.Desc;

                        ArticlePublished a_zh = ArticlePublishedDbContext.getInstance().getArticlePublishedByBaseArticleID(article.BaseArticleID, "zh");
                        if (a_zh != null)
                        {
                            data.name_zh = a_zh.Name;
                            data.desc_zh = a_zh.Desc;
                        }

                        ArticlePublished a_cn = ArticlePublishedDbContext.getInstance().getArticlePublishedByBaseArticleID(article.BaseArticleID, "cn");
                        if (a_cn != null)
                        {
                            data.name_cn = a_cn.Name;
                            data.desc_cn = a_cn.Desc;
                        }

                        if (data.name_en == null)
                        {
                            data.name_en = "";
                        }

                        if (data.name_zh == null)
                        {
                            data.name_zh = "";
                        }

                        if (data.name_cn == null)
                        {
                            data.name_cn = "";
                        }

                        if (data.desc_en == null)
                        {
                            data.desc_en = "";
                        }

                        if (data.desc_zh == null)
                        {
                            data.desc_zh = "";
                        }

                        if (data.desc_cn == null)
                        {
                            data.desc_cn = "";
                        }

                        data.Description = MyRazorExtensions.Render(null, data.Description, null, true);
                        data.desc_en     = MyRazorExtensions.Render(null, data.desc_en, null, true);
                        data.desc_zh     = MyRazorExtensions.Render(null, data.desc_zh, null, true);
                        data.desc_cn     = MyRazorExtensions.Render(null, data.desc_cn, null, true);


                        data.Description = HtmlToPlainText(data.Description);
                        data.desc_en     = HtmlToPlainText(data.desc_en);
                        data.desc_zh     = HtmlToPlainText(data.desc_zh);
                        data.desc_cn     = HtmlToPlainText(data.desc_cn);


                        data.is_page = 1;

                        data.is_trading = article.category.isVisibleToTradingOnly ? 1 : 0;
                        data.is_member  = article.category.isVisibleToMembersOnly ? 1 : 0;
                        data.is_visitor = article.category.isVisibleToVisitorOnly ? 1 : 0;

                        searchData.Add(data);

                        List <string> innerLinks_en = LinkScanner.getInnerUrlsFromHTML(data.desc_en);
                        List <string> innerLinks_zh = LinkScanner.getInnerUrlsFromHTML(data.desc_zh);
                        List <string> innerLinks_cn = LinkScanner.getInnerUrlsFromHTML(data.desc_cn);

                        innerLinks_en.AddRange(innerLinks_zh.Except(innerLinks_en));
                        innerLinks_zh.AddRange(innerLinks_cn.Except(innerLinks_zh));

                        for (var m = innerLinks_en.Count - 1; m >= 0; m--)
                        {
                            string link = innerLinks_en[m];
                            if (innerHTMLLinks.Contains(link))
                            {
                                innerLinks_en.RemoveAt(m);
                            }
                        }

                        for (var m = innerLinks_en.Count - 1; m >= 0; m--)
                        {
                            string link = innerLinks_en[m];
                            string name = LinkScannerParser.getFilenameFromInnerUrl(link);
                            string ext  = LinkScannerParser.getFiletypeFromInnerUrl(link);

                            LuceneSearchData _data = new LuceneSearchData();

                            _data.ArticleId     = 0;
                            _data.BaseArticleId = 0;
                            _data.CategoryId    = 0;

                            _data.is_page = 0;
                            _data.is_pdf  = 0;
                            _data.is_doc  = 0;
                            _data.is_docx = 0;

                            if (ext == ".pdf")
                            {
                                _data.is_pdf = 1;
                            }
                            else if (ext == ".doc")
                            {
                                _data.is_doc = 1;
                            }
                            else if (ext == ".docx")
                            {
                                _data.is_docx = 1;
                            }
                            else
                            {
                                continue;
                            }

                            string desc = LinkScannerParser.getDocumentFromInnerUrl(link);

                            _data.is_visitor = data.is_visitor;
                            _data.is_member  = data.is_member;
                            _data.is_trading = data.is_trading;
                            _data.Name       = name;

                            if (name == "")
                            {
                                continue;
                            }

                            _data.name_en = "";
                            _data.name_zh = "";
                            _data.name_cn = "";

                            _data.Url         = "/ckfinder/userfiles/files/" + _data.Name;
                            _data.Type        = ext.ToUpper();
                            _data.Description = desc;

                            _data.desc_en = "";
                            _data.desc_zh = "";
                            _data.desc_cn = "";

                            searchData.Add(_data);
                        }
                    }
                }
                catch (Exception e)
                {
                    AuditLogDbContext.getInstance().createAuditLog(new AuditLog
                    {
                        action  = "Search Indexing",
                        remarks = "Exception GetAll: " + e.Message,
                    });
                }
            }

            return(searchData);
        }