public IndexPage() { var settings = UserSettingsSingleton.CurrentSettings(); SiteUrl = settings.SiteUrl; SiteName = settings.SiteName; SiteKeywords = settings.SiteKeywords; SiteSummary = settings.SiteSummary; SiteAuthors = settings.SiteAuthors; PageUrl = settings.IndexPageUrl(); IndexContent = Db.MainFeedRecentDynamicContent(20).Result.OrderByDescending(x => x.CreatedOn).ToList(); var mainImageGuid = IndexContent .FirstOrDefault(x => x.GetType() == typeof(PostContent) && x.MainPicture != null)?.MainPicture; if (mainImageGuid != null) { MainImage = new PictureSiteInformation(mainImageGuid); } if (!IndexContent.Any()) { PreviousPosts = new List <IContentCommon>(); } else { DateTime previousDate = IndexContent.Skip(_numberOfContentItemsToDisplay - 1).Max(x => x.CreatedOn); var previousLater = Tags.MainFeedPreviousAndLaterContent(6, previousDate); PreviousPosts = previousLater.previousContent; } }
public void WriteLocalHtml() { WriteRss(); foreach (var loopPosts in IndexContent.Take(_numberOfContentItemsToDisplay)) { if (BracketCodeCommon.ContainsSpatialBracketCodes(loopPosts) || loopPosts.GetType() == typeof(PointContentDto)) { IncludeSpatialScripts = true; } } var parser = new HtmlParser(); var htmlDoc = parser.ParseDocument(TransformText()); var stringWriter = new StringWriter(); htmlDoc.ToHtml(stringWriter, new PrettyMarkupFormatter()); var htmlString = stringWriter.ToString(); var htmlFileInfo = new FileInfo($@"{UserSettingsSingleton.CurrentSettings().LocalSiteRootDirectory}\index.html"); if (htmlFileInfo.Exists) { htmlFileInfo.Delete(); htmlFileInfo.Refresh(); } FileManagement.WriteAllTextToFileAndLog(htmlFileInfo.FullName, htmlString); }
public async Task <IEnumerable <ScheduleDiff> > ProcessAsync(CancellationToken ct = default) { var indexPage = await _flexKidsClient.GetAvailableSchedulesPage(ct); IndexContent indexContent = _parser.GetIndexContent(indexPage); var weekSchedulesToImport = new List <WeekAndImportedSchedules>(indexContent.Weeks.Count); foreach (KeyValuePair <int, WeekItem> item in indexContent.Weeks) { var htmlComboboxIndex = item.Key; var year = item.Value.Year; var weekNumber = item.Value.WeekNr; var htmlSchedule = await _flexKidsClient.GetSchedulePage(htmlComboboxIndex, ct); var parsedSchedules = _parser.GetScheduleFromContent(htmlSchedule, year).ToList(); WeekSchedule weekSchedule = await _repo.Get(year, weekNumber); weekSchedulesToImport.Add(new WeekAndImportedSchedules { WeekSchedule = weekSchedule, ScheduleItems = parsedSchedules, }); } return(await ProcessRawData(weekSchedulesToImport)); }
private void IndexEntry(ZipArchiveEntry zipEntry, int curr) { string file = Site.FTRobot_PATH + zipEntry.FullName; IndexContent ic = new IndexContent(); ic.AliasName = Path.GetFileName(file).Replace(".txt", ""); ic.ContentText = null; //read file if (zipEntry != null) { using (StreamReader sr = new StreamReader(zipEntry.Open(), Archive.Encoding)) { ic.ContentText = sr.ReadToEnd(); } } //index file if (ic.ContentText != null) { while (_cq.Count > 10000) { Thread.Sleep(100); } _cq.Enqueue(ic); } }
public HtmlTag IndexPosts() { if (!IndexContent.Any()) { return(HtmlTag.Empty()); } var indexBodyContainer = new DivTag().AddClass("index-posts-container"); foreach (var loopPosts in IndexContent.Take(_numberOfContentItemsToDisplay)) { if (loopPosts.GetType() == typeof(PostContent)) { var post = new SinglePostDiv(loopPosts); var indexPostContentDiv = new DivTag().AddClass("index-posts-content"); indexPostContentDiv.Encoded(false).Text(post.TransformText()); indexBodyContainer.Children.Add(indexPostContentDiv); indexBodyContainer.Children.Add(HorizontalRule.StandardRule()); } if (loopPosts.GetType() == typeof(NoteContent)) { var post = new SingleNoteDiv(loopPosts); var indexPostContentDiv = new DivTag().AddClass("index-posts-content"); indexPostContentDiv.Encoded(false).Text(post.TransformText()); indexBodyContainer.Children.Add(indexPostContentDiv); indexBodyContainer.Children.Add(HorizontalRule.StandardRule()); } if (loopPosts.GetType() == typeof(PhotoContent)) { var post = new SinglePhotoDiv(loopPosts); var indexPostContentDiv = new DivTag().AddClass("index-posts-content"); indexPostContentDiv.Encoded(false).Text(post.TransformText()); indexBodyContainer.Children.Add(indexPostContentDiv); indexBodyContainer.Children.Add(HorizontalRule.StandardRule()); } if (loopPosts.GetType() == typeof(ImageContent)) { var post = new SingleImageDiv(loopPosts); var indexPostContentDiv = new DivTag().AddClass("index-posts-content"); indexPostContentDiv.Encoded(false).Text(post.TransformText()); indexBodyContainer.Children.Add(indexPostContentDiv); indexBodyContainer.Children.Add(HorizontalRule.StandardRule()); } if (loopPosts.GetType() == typeof(FileContent)) { var post = new SingleFileDiv(loopPosts); var indexPostContentDiv = new DivTag().AddClass("index-posts-content"); indexPostContentDiv.Encoded(false).Text(post.TransformText()); indexBodyContainer.Children.Add(indexPostContentDiv); indexBodyContainer.Children.Add(HorizontalRule.StandardRule()); } } return(indexBodyContainer); }
public IndexModel(IBlogService blogService, ILogger <IndexModel> logger) { _blogService = blogService; _logger = logger; Content = new IndexContent( "Hi, I'm Phil Broderick", "Full stack developer specialising in Azure and DevOps practices"); }
/// <summary> /// 向队列中添加要删除数据 /// </summary> /// <param name="Id"></param> public void DeleteQueue(string Id) { IndexContent indexContent = new IndexContent(); indexContent.Id = Id; indexContent.LuceneEnum = LuceneEnum.DeleType;//删除 queue.Enqueue(indexContent); }
public IndexContent Parse() { var result = new IndexContent(); result.Email = ExtractEmailFromContent(); result.IsLoggedin = (result.Email != ""); result.Weeks = ExtractWeeksFromContent(); return(result); }
public async Task IndexPageTest() { // arrange var htmlContent = GetFileContent("index.txt"); var indexParser = new IndexParser(NullLogger.Instance); // act IndexContent indexContent = indexParser.Parse(htmlContent); // assert await Verifier.Verify(indexContent); }
/// <summary> /// 向队列中添加数据 /// </summary> /// <param name="Id"></param> /// <param name="title"></param> /// <param name="content"></param> public void AddQueue(Article article) { IndexContent indexContent = new IndexContent() { Id = article.ArticleId.ToString(), Title = article.Title, Content = article.Content, CreateTime = article.CreateTime.ToString(), LuceneEnum = LuceneEnum.AddType,// 添加 }; queue.Enqueue(indexContent); }
/// <summary> /// 添加数据到队列 /// </summary> /// <param name="id"></param> /// <param name="title"></param> /// <param name="content"></param> /// <param name="author"></param> /// <param name="creation_time"></param> public void AddQueue(string id, string title, string content, string author_nick, string author_name, DateTime?creation_time) { IndexContent indexContent = new IndexContent { Id = id, Title = title, Content = content, Author_Nick = author_nick, Author_Name = author_name, Creation_Time = creation_time }; queue.Enqueue(indexContent); }
private void CreateIndexContent() { try { FSDirectory directory = FSDirectory.Open(new DirectoryInfo(IndexPath), new NativeFSLockFactory()); //指定索引文件(打开索引目录) FS指的是就是FileSystem bool isUpdate = IndexReader.IndexExists(directory); //IndexReader:对索引进行读取的类。该语句的作用:判断索引库文件夹是否存在以及索引特征文件是否存在。 if (isUpdate) { //同时只能有一段代码对索引库进行写操作。当使用IndexWriter打开directory时会自动对索引库文件上锁。 //如果索引目录被锁定(比如索引过程中程序异常退出),则首先解锁 if (IndexWriter.IsLocked(directory)) { IndexWriter.Unlock(directory); } } using (IndexWriter writer = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED))//向索引库中写索引。这时在这里加锁。 { while (queue.Count > 0) { IndexContent indexContent = queue.Dequeue() as IndexContent; //将队列中的数据出队 writer.DeleteDocuments(new Term("Id", indexContent.Id)); if (indexContent.LuceneEnum == LuceneEnum.DeleType) { continue; } Document document = new Document(); //表示一篇文档。 //Field.Store.YES:表示是否存储原值。只有当Field.Store.YES在后面才能用doc.Get("Id")取出值来.Field.Index. NOT_ANALYZED:不进行分词保存 document.Add(new Field("Id", indexContent.Id, Field.Store.YES, Field.Index.NOT_ANALYZED)); //Field.Index. ANALYZED:进行分词保存:也就是要进行全文的字段要设置分词 保存(因为要进行模糊查询) //Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS:不仅保存分词还保存分词的距离。 //Field.Store.YES不仅要对文章进行分词记录,也要保存原文,就不用去数据库里查一次了 //需要进行全文检索的字段加 Field.Index. ANALYZED document.Add(new Field("Title", indexContent.Title, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS)); document.Add(new Field("Content", indexContent.Content, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS)); document.Add(new Field("CreateTime", indexContent.CreateTime.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); //防止存在的数据//delete from t where id=i writer.DeleteDocuments(new Term("Id", indexContent.Id)); writer.AddDocument(document); } } directory.Close();//不要忘了Close,否则索引结果搜不到 } catch (Exception ex) { throw ex; } }
private void IndexFile(ZipArchive archive, int curr, SqlDataReader reader) { string file = Site.FTRobot_PATH + (string)reader["URL"]; IndexContent ic = new IndexContent(); ic.AliasName = Path.GetFileName(file).Replace(".txt", ""); ic.ContentText = null; //read file if (archive != null) { string path = file.ToLower().Replace(Site.FTRobot_PATH.ToLower(), string.Empty); ZipArchiveEntry zipEntry = archive.GetEntry(path); if (zipEntry != null) { using (StreamReader sr = new StreamReader(zipEntry.Open(), Archive.Encoding)) { ic.ContentText = sr.ReadToEnd(); } } } else { if (File.Exists(file)) { ic.ContentText = File.ReadAllText(file, Archive.Encoding); } else { ic.ContentText = null; } } //index file if (ic.ContentText != null) { while (_cq.Count > 10000) { Thread.Sleep(100); } _cq.Enqueue(ic); } }
bool CreateIndex(IndexContent indexContent) { var isExists = IndexReader.IndexExists(directory_luce); if (isExists) { if (IndexWriter.IsLocked(directory_luce)) { IndexWriter.Unlock(directory_luce); } } IndexWriter writer = null; try { //false表示追加(true表示删除之前的重新写入) writer = new IndexWriter(directory_luce, analyzer, !isExists, IndexWriter.MaxFieldLength.LIMITED); if (indexContent == null) { return(false); } Document document = new Document(); document.Add(new Field("Id", indexContent.Id, Field.Store.YES, Field.Index.NOT_ANALYZED)); document.Add(new Field("Title", indexContent.Title, Field.Store.YES, Field.Index.ANALYZED)); document.Add(new Field("Content", indexContent.Content, Field.Store.YES, Field.Index.ANALYZED)); document.Add(new Field("CreationTime", indexContent.Creation_Time?.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); document.Add(new Field("Author_Nick", indexContent.Author_Nick, Field.Store.YES, Field.Index.NOT_ANALYZED)); document.Add(new Field("Author_Name", indexContent.Author_Name, Field.Store.YES, Field.Index.NOT_ANALYZED)); writer.AddDocument(document); } catch (Exception e) { throw e; } finally { if (writer != null) { writer.Optimize(); writer.Dispose(); } } return(true); }
void WriteIndexContent() { while (true) { if (queue.Count > 0) { IndexContent indexContent = null; queue.TryDequeue(out indexContent); if (indexContent != null) { CreateIndex(indexContent); } } else { Thread.Sleep(3000); } } }