public SearchResultModel GetSearchResult(string keyword)
{
if (string.IsNullOrWhiteSpace(keyword))
{
return(null);
}
var requestId = Guid.NewGuid().ToString();
var result = new SearchResultModel
{
RequestId = requestId
};
cache.Set(requestId, result, new TimeSpan(0, 1, 0));
var english = !keyword.HasOtherLetter();
result.Results = new List <SearchResult>();
var logger = NiologManager.CreateLogger();
Task.Run(() =>
{
NiologManager.Logger = logger;
Parallel.ForEach(engines, engine =>
{
NiologManager.Logger = logger;
SearchPerEngine(engine, keyword, english, result);
});
result.Finished = true;
});
SpinWait.SpinUntil(() => result.Searched > 0 || result.Finished);
SearchResultModel newResult = null;
try
{
lock (result)
{
newResult = result.Copy();
}
}
catch (Exception e)
{
logger.Error()
.Message("catched an exception when copying result.")
.Exception(e, true)
.Write();
}
return(newResult);
}
public ComprehensiveSearcher(IConfiguration config)
{
this.config = config;
var logger = NiologManager.CreateLogger();
try
{
engines = this.config["engines"].Split(',').Select <string, ISearchEngine>(engine =>
{
var strs = engine.Split(':');
engine = strs[0];
if (strs.Length > 1 && float.TryParse(strs[1], out float factor))
{
factorDic.TryAdd(engine, factor);
logger.Info()
.Message($"Adding search engine {engine}, search factor: {factorDic[engine]}")
.Write();
}
else
{
factorDic.TryAdd(engine, 1f);
logger.Info()
.Message($"Adding search engine {engine}, search factor: 1.0")
.Write();
}
switch (engine)
{
case "360":
case "baidu":
case "bing":
default:
logger.Info()
.Message("Parsed as GenericSearch")
.Write();
return(new GenericSearch(this.config, engine));
}
});
badUrls = this.config["badurls"].Split(';');
}
catch (Exception e)
{
logger.Error()
.Message("Failed to init SearchManager")
.Exception(e)
.Write();
badUrls = new string[0];
}
}
private void Consumer()
{
while (true)
{
if (this.queue.Count <= 0)
{
Thread.Sleep(1000);
}
if (!this.queue.TryDequeue(out string feed))
{
continue;
}
try
{
var sf = SyndicationFeed.Load(XmlReader.Create(feed));
using (var connection = this.helper.GetDbConnection())
{
this.helper.ParseArticles(sf, feed.Md5(), connection);
}
NiologManager.CreateLogger().Info()
.Message($"refreshed {feed}")
.Write();
}
catch
{
try
{
var feedEntity = FeedReader.ReadAsync(feed).Result;
using (var connection = this.helper.GetDbConnection())
{
this.helper.ParseArticles(feedEntity, feed.Md5(), connection);
}
NiologManager.CreateLogger().Info()
.Message($"refreshed {feed}")
.Write();
}
catch (Exception e)
{
NiologManager.CreateLogger().Error()
.Message($"error occured when refreshing {feed}")
.Exception(e)
.Write();
}
}
}
}
private void SearchPerEngine(ISearchEngine engine, string keyword, bool english,
SearchResultModel result)
{
var logger = NiologManager.CreateLogger();
try
{
var searchResults = engine.Search(keyword, english);
if (searchResults == null || searchResults.Count() <= 0)
{
logger.Warn()
.Message($"The result is null or empty, when searching {keyword} by {engine.Name}")
.Write();
return;
}
logger.Info()
.Message($"count of {engine.Name} results: {searchResults.Count()}")
.Write();
try
{
lock (result)
{
MergeResult(keyword, searchResults, result.Results, factorDic[engine.Name]);
}
}
catch (Exception e)
{
logger.Error()
.Message("catched an exception when merging result.")
.Exception(e, true)
.Write();
}
logger.Info()
.Message($"{engine.Name} results merged.")
.Write();
var count = result.Searched;
result.Searched = Interlocked.Increment(ref count);
}
catch (Exception e)
{
logger.Error()
.Message($"An exception occurred while searching for {keyword}")
.Exception(e, true)
.Write();
}
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory,
IOptions <AppSettings> appSettings, ILogger <IScheduler> schedulerLogger)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
NiologManager.DefaultWriters = new ILogWriter[]
{
new FileLogWriter(appSettings?.Value?.Log?.Path, 10),
new ConsoleLogWriter()
};
loggerFactory.AddProvider(new LoggerProvider());
var provider = app.ApplicationServices;
provider.UseScheduler(scheduler =>
{
scheduler.Schedule <RssFetcher>()
.EveryFiveMinutes()
.PreventOverlapping("RssFetcher");
})
.LogScheduledTaskProgress(schedulerLogger)
.OnError(e =>
{
var logger = NiologManager.CreateLogger();
logger.Warn()
.Message("Something goes wrong...")
.Exception(e, true)
.Write();
});
app.UseRouting();
app.UseAuthorization();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
}
public IEnumerable <SearchResult> Search(string keyword, bool english = false)
{
if (string.IsNullOrWhiteSpace(this.baseUrl))
{
return(null);
}
var logger = NiologManager.CreateLogger();
var lang = english ? "en" : "cn";
var ensearch = english ? "1" : "0";
var url = this.baseUrl.Replace("{keyword}", System.Web.HttpUtility.UrlEncode(keyword)).Replace("{lang}", lang).Replace("{ensearch}", ensearch);
logger.Info()
.Message($"{this.Name} url: {url}")
.Write();
var html = HttpUtility.Get(url);
if (string.IsNullOrWhiteSpace(html))
{
logger.Warn()
.Message($"{this.Name} response is null or white space")
.Write();
return(null);
}
var doc = new HtmlDocument();
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectAllNodes(this.nodesSelection);
if (nodes == null || nodes.Count <= 0)
{
logger.Warn()
.Message($"cannot select nodes from {this.Name} response")
.Write();
return(null);
}
var searchResults = nodes.AsParallel().AsOrdered().Select(node =>
{
var result = new SearchResult
{
Source = this.Name
};
var link = node.SelectFirstNode(this.linkSelection);
if (link == null)
{
return(null);
}
result.Url = link.Attributes["href"]?.Value.Trim();
if (string.IsNullOrWhiteSpace(result.Url))
{
return(null);
}
result.Title = System.Web.HttpUtility.HtmlDecode(link.InnerText.Trim());
var desc = node.SelectFirstNode(this.descSelection);
result.Desc = System.Web.HttpUtility.HtmlDecode(desc?.InnerText.Trim());
return(result);
})
.Where(result => result != null).ToList();
var count = searchResults.Count();
for (int i = 0; i < count; i++)
{
searchResults[i].Score = i + 1;
searchResults[i].Base = count + 1;
}
return(searchResults);
}
public List <Article> ParseArticles(SyndicationFeed sf, string feedId, IDbConnection connection)
{
if (sf == null || sf.Items == null || !sf.Items.Any())
{
NiologManager.CreateLogger().Warn()
.Message("SyndicationFeed is empty, so no articles.")
.Write();
return(null);
}
var articleDao = new ArticleDao(connection);
var articles = new List <Article>();
foreach (var item in sf.Items)
{
var articleUrl = item?.Links?.FirstOrDefault()?.Uri?.AbsoluteUri;
var articleTitle = item?.Title?.Text;
if (string.IsNullOrWhiteSpace(articleUrl) || string.IsNullOrWhiteSpace(articleTitle))
{
continue;
}
var articleId = feedId + articleUrl.Md5();
var content = string.Empty;
if (item.Content is TextSyndicationContent textContent)
{
content = textContent.Text;
}
Article article = new Article
{
Id = articleId,
Url = articleUrl,
FeedId = feedId,
Title = articleTitle,
Summary = Simplify(item.Summary?.Text),
Published = item.PublishDate.LocalDateTime,
Updated = item.LastUpdatedTime.LocalDateTime,
Keyword = string.Join(',', item.Categories?.Select(c => c?.Name)),
Content = Simplify(content),
Contributors = string.Join(',', item.Contributors?.Select(c => c?.Name)),
Authors = string.Join(',', item.Authors?.Select(c => c?.Name)),
Copyright = item.Copyright?.Text
};
articles.Add(article);
try
{
if (articleDao.GetArticle(articleId) == null)
{
articleDao.InsertArticle(article);
}
else
{
articleDao.UpdateArticle(article);
}
}
catch (Exception e)
{
NiologManager.CreateLogger().Error()
.Message($"error occured when insert {articleUrl}")
.Exception(e)
.Write();
}
}
return(articles);
}
public List <Article> ParseArticles(CodeHollow.FeedReader.Feed feed, string feedId, IDbConnection connection)
{
if (feed == null || feed.Items == null || !feed.Items.Any())
{
NiologManager.CreateLogger().Warn()
.Message("Feed is empty, so no articles.")
.Write();
return(null);
}
var articleDao = new ArticleDao(connection);
var articles = new List <Article>();
foreach (var item in feed.Items)
{
var articleUrl = item?.Link;
var articleTitle = item?.Title;
if (string.IsNullOrWhiteSpace(articleUrl) || string.IsNullOrWhiteSpace(articleTitle))
{
continue;
}
var articleId = feedId + articleUrl.Md5();
var content = this.Simplify(item.Content);
Article article = new Article
{
Id = articleId,
Url = articleUrl,
FeedId = feedId,
Title = articleTitle,
Summary = content,
Published = item.PublishingDate ?? DateTime.Now,
Updated = item.PublishingDate ?? DateTime.Now,
Keyword = string.Join(',', item.Categories),
Content = content,
Contributors = item.Author,
Authors = item.Author
};
articles.Add(article);
try
{
if (articleDao.GetArticle(articleId) == null)
{
articleDao.InsertArticle(article);
}
else
{
articleDao.UpdateArticle(article);
}
}
catch (Exception e)
{
NiologManager.CreateLogger().Error()
.Message($"error occured when insert {articleUrl}")
.Exception(e)
.Write();
}
}
return(articles);
}