public IndexPage()
{
var settings = UserSettingsSingleton.CurrentSettings();
SiteUrl = settings.SiteUrl;
SiteName = settings.SiteName;
SiteKeywords = settings.SiteKeywords;
SiteSummary = settings.SiteSummary;
SiteAuthors = settings.SiteAuthors;
PageUrl = settings.IndexPageUrl();
IndexContent = Db.MainFeedRecentDynamicContent(20).Result.OrderByDescending(x => x.CreatedOn).ToList();
var mainImageGuid = IndexContent
.FirstOrDefault(x => x.GetType() == typeof(PostContent) && x.MainPicture != null)?.MainPicture;
if (mainImageGuid != null)
{
MainImage = new PictureSiteInformation(mainImageGuid);
}
if (!IndexContent.Any())
{
PreviousPosts = new List <IContentCommon>();
}
else
{
DateTime previousDate = IndexContent.Skip(_numberOfContentItemsToDisplay - 1).Max(x => x.CreatedOn);
var previousLater = Tags.MainFeedPreviousAndLaterContent(6, previousDate);
PreviousPosts = previousLater.previousContent;
}
}
public void WriteLocalHtml()
{
WriteRss();
foreach (var loopPosts in IndexContent.Take(_numberOfContentItemsToDisplay))
{
if (BracketCodeCommon.ContainsSpatialBracketCodes(loopPosts) ||
loopPosts.GetType() == typeof(PointContentDto))
{
IncludeSpatialScripts = true;
}
}
var parser = new HtmlParser();
var htmlDoc = parser.ParseDocument(TransformText());
var stringWriter = new StringWriter();
htmlDoc.ToHtml(stringWriter, new PrettyMarkupFormatter());
var htmlString = stringWriter.ToString();
var htmlFileInfo =
new FileInfo($@"{UserSettingsSingleton.CurrentSettings().LocalSiteRootDirectory}\index.html");
if (htmlFileInfo.Exists)
{
htmlFileInfo.Delete();
htmlFileInfo.Refresh();
}
FileManagement.WriteAllTextToFileAndLog(htmlFileInfo.FullName, htmlString);
}
public async Task <IEnumerable <ScheduleDiff> > ProcessAsync(CancellationToken ct = default)
{
var indexPage = await _flexKidsClient.GetAvailableSchedulesPage(ct);
IndexContent indexContent = _parser.GetIndexContent(indexPage);
var weekSchedulesToImport = new List <WeekAndImportedSchedules>(indexContent.Weeks.Count);
foreach (KeyValuePair <int, WeekItem> item in indexContent.Weeks)
{
var htmlComboboxIndex = item.Key;
var year = item.Value.Year;
var weekNumber = item.Value.WeekNr;
var htmlSchedule = await _flexKidsClient.GetSchedulePage(htmlComboboxIndex, ct);
var parsedSchedules = _parser.GetScheduleFromContent(htmlSchedule, year).ToList();
WeekSchedule weekSchedule = await _repo.Get(year, weekNumber);
weekSchedulesToImport.Add(new WeekAndImportedSchedules
{
WeekSchedule = weekSchedule,
ScheduleItems = parsedSchedules,
});
}
return(await ProcessRawData(weekSchedulesToImport));
}
private void IndexEntry(ZipArchiveEntry zipEntry, int curr)
{
string file = Site.FTRobot_PATH + zipEntry.FullName;
IndexContent ic = new IndexContent();
ic.AliasName = Path.GetFileName(file).Replace(".txt", "");
ic.ContentText = null;
//read file
if (zipEntry != null)
{
using (StreamReader sr = new StreamReader(zipEntry.Open(), Archive.Encoding))
{
ic.ContentText = sr.ReadToEnd();
}
}
//index file
if (ic.ContentText != null)
{
while (_cq.Count > 10000)
{
Thread.Sleep(100);
}
_cq.Enqueue(ic);
}
}
public HtmlTag IndexPosts()
{
if (!IndexContent.Any())
{
return(HtmlTag.Empty());
}
var indexBodyContainer = new DivTag().AddClass("index-posts-container");
foreach (var loopPosts in IndexContent.Take(_numberOfContentItemsToDisplay))
{
if (loopPosts.GetType() == typeof(PostContent))
{
var post = new SinglePostDiv(loopPosts);
var indexPostContentDiv = new DivTag().AddClass("index-posts-content");
indexPostContentDiv.Encoded(false).Text(post.TransformText());
indexBodyContainer.Children.Add(indexPostContentDiv);
indexBodyContainer.Children.Add(HorizontalRule.StandardRule());
}
if (loopPosts.GetType() == typeof(NoteContent))
{
var post = new SingleNoteDiv(loopPosts);
var indexPostContentDiv = new DivTag().AddClass("index-posts-content");
indexPostContentDiv.Encoded(false).Text(post.TransformText());
indexBodyContainer.Children.Add(indexPostContentDiv);
indexBodyContainer.Children.Add(HorizontalRule.StandardRule());
}
if (loopPosts.GetType() == typeof(PhotoContent))
{
var post = new SinglePhotoDiv(loopPosts);
var indexPostContentDiv = new DivTag().AddClass("index-posts-content");
indexPostContentDiv.Encoded(false).Text(post.TransformText());
indexBodyContainer.Children.Add(indexPostContentDiv);
indexBodyContainer.Children.Add(HorizontalRule.StandardRule());
}
if (loopPosts.GetType() == typeof(ImageContent))
{
var post = new SingleImageDiv(loopPosts);
var indexPostContentDiv = new DivTag().AddClass("index-posts-content");
indexPostContentDiv.Encoded(false).Text(post.TransformText());
indexBodyContainer.Children.Add(indexPostContentDiv);
indexBodyContainer.Children.Add(HorizontalRule.StandardRule());
}
if (loopPosts.GetType() == typeof(FileContent))
{
var post = new SingleFileDiv(loopPosts);
var indexPostContentDiv = new DivTag().AddClass("index-posts-content");
indexPostContentDiv.Encoded(false).Text(post.TransformText());
indexBodyContainer.Children.Add(indexPostContentDiv);
indexBodyContainer.Children.Add(HorizontalRule.StandardRule());
}
}
return(indexBodyContainer);
}
public IndexModel(IBlogService blogService, ILogger <IndexModel> logger)
{
_blogService = blogService;
_logger = logger;
Content = new IndexContent(
"Hi, I'm Phil Broderick",
"Full stack developer specialising in Azure and DevOps practices");
}
/// <summary>
/// 向队列中添加要删除数据
/// </summary>
/// <param name="Id"></param>
public void DeleteQueue(string Id)
{
IndexContent indexContent = new IndexContent();
indexContent.Id = Id;
indexContent.LuceneEnum = LuceneEnum.DeleType;//删除
queue.Enqueue(indexContent);
}
public IndexContent Parse()
{
var result = new IndexContent();
result.Email = ExtractEmailFromContent();
result.IsLoggedin = (result.Email != "");
result.Weeks = ExtractWeeksFromContent();
return(result);
}
public async Task IndexPageTest()
{
// arrange
var htmlContent = GetFileContent("index.txt");
var indexParser = new IndexParser(NullLogger.Instance);
// act
IndexContent indexContent = indexParser.Parse(htmlContent);
// assert
await Verifier.Verify(indexContent);
}
/// <summary>
/// 向队列中添加数据
/// </summary>
/// <param name="Id"></param>
/// <param name="title"></param>
/// <param name="content"></param>
public void AddQueue(Article article)
{
IndexContent indexContent = new IndexContent()
{
Id = article.ArticleId.ToString(),
Title = article.Title,
Content = article.Content,
CreateTime = article.CreateTime.ToString(),
LuceneEnum = LuceneEnum.AddType,// 添加
};
queue.Enqueue(indexContent);
}
/// <summary>
/// 添加数据到队列
/// </summary>
/// <param name="id"></param>
/// <param name="title"></param>
/// <param name="content"></param>
/// <param name="author"></param>
/// <param name="creation_time"></param>
public void AddQueue(string id, string title, string content, string author_nick, string author_name, DateTime?creation_time)
{
IndexContent indexContent = new IndexContent
{
Id = id,
Title = title,
Content = content,
Author_Nick = author_nick,
Author_Name = author_name,
Creation_Time = creation_time
};
queue.Enqueue(indexContent);
}
private void CreateIndexContent()
{
try
{
FSDirectory directory = FSDirectory.Open(new DirectoryInfo(IndexPath), new NativeFSLockFactory()); //指定索引文件(打开索引目录) FS指的是就是FileSystem
bool isUpdate = IndexReader.IndexExists(directory); //IndexReader:对索引进行读取的类。该语句的作用：判断索引库文件夹是否存在以及索引特征文件是否存在。
if (isUpdate)
{
//同时只能有一段代码对索引库进行写操作。当使用IndexWriter打开directory时会自动对索引库文件上锁。
//如果索引目录被锁定（比如索引过程中程序异常退出），则首先解锁
if (IndexWriter.IsLocked(directory))
{
IndexWriter.Unlock(directory);
}
}
using (IndexWriter writer = new IndexWriter(directory, new PanGuAnalyzer(), !isUpdate, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED))//向索引库中写索引。这时在这里加锁。
{
while (queue.Count > 0)
{
IndexContent indexContent = queue.Dequeue() as IndexContent; //将队列中的数据出队
writer.DeleteDocuments(new Term("Id", indexContent.Id));
if (indexContent.LuceneEnum == LuceneEnum.DeleType)
{
continue;
}
Document document = new Document(); //表示一篇文档。
//Field.Store.YES:表示是否存储原值。只有当Field.Store.YES在后面才能用doc.Get("Id")取出值来.Field.Index. NOT_ANALYZED:不进行分词保存
document.Add(new Field("Id", indexContent.Id, Field.Store.YES, Field.Index.NOT_ANALYZED));
//Field.Index. ANALYZED:进行分词保存:也就是要进行全文的字段要设置分词 保存（因为要进行模糊查询）
//Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS:不仅保存分词还保存分词的距离。
//Field.Store.YES不仅要对文章进行分词记录，也要保存原文，就不用去数据库里查一次了
//需要进行全文检索的字段加 Field.Index. ANALYZED
document.Add(new Field("Title", indexContent.Title, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
document.Add(new Field("Content", indexContent.Content, Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
document.Add(new Field("CreateTime", indexContent.CreateTime.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
//防止存在的数据//delete from t where id=i
writer.DeleteDocuments(new Term("Id", indexContent.Id));
writer.AddDocument(document);
}
}
directory.Close();//不要忘了Close，否则索引结果搜不到
}
catch (Exception ex)
{
throw ex;
}
}
private void IndexFile(ZipArchive archive, int curr, SqlDataReader reader)
{
string file = Site.FTRobot_PATH + (string)reader["URL"];
IndexContent ic = new IndexContent();
ic.AliasName = Path.GetFileName(file).Replace(".txt", "");
ic.ContentText = null;
//read file
if (archive != null)
{
string path = file.ToLower().Replace(Site.FTRobot_PATH.ToLower(), string.Empty);
ZipArchiveEntry zipEntry = archive.GetEntry(path);
if (zipEntry != null)
{
using (StreamReader sr = new StreamReader(zipEntry.Open(), Archive.Encoding))
{
ic.ContentText = sr.ReadToEnd();
}
}
}
else
{
if (File.Exists(file))
{
ic.ContentText = File.ReadAllText(file, Archive.Encoding);
}
else
{
ic.ContentText = null;
}
}
//index file
if (ic.ContentText != null)
{
while (_cq.Count > 10000)
{
Thread.Sleep(100);
}
_cq.Enqueue(ic);
}
}
bool CreateIndex(IndexContent indexContent)
{
var isExists = IndexReader.IndexExists(directory_luce);
if (isExists)
{
if (IndexWriter.IsLocked(directory_luce))
{
IndexWriter.Unlock(directory_luce);
}
}
IndexWriter writer = null;
try
{
//false表示追加（true表示删除之前的重新写入）
writer = new IndexWriter(directory_luce, analyzer, !isExists, IndexWriter.MaxFieldLength.LIMITED);
if (indexContent == null)
{
return(false);
}
Document document = new Document();
document.Add(new Field("Id", indexContent.Id, Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("Title", indexContent.Title, Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("Content", indexContent.Content, Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("CreationTime", indexContent.Creation_Time?.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("Author_Nick", indexContent.Author_Nick, Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("Author_Name", indexContent.Author_Name, Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.AddDocument(document);
}
catch (Exception e)
{
throw e;
}
finally
{
if (writer != null)
{
writer.Optimize();
writer.Dispose();
}
}
return(true);
}
void WriteIndexContent()
{
while (true)
{
if (queue.Count > 0)
{
IndexContent indexContent = null;
queue.TryDequeue(out indexContent);
if (indexContent != null)
{
CreateIndex(indexContent);
}
}
else
{
Thread.Sleep(3000);
}
}
}
