public const string DouBanTagsUrl = "https://book.douban.com/tag/?view=type";//https://book.douban.com/tag/?view=cloud
public BookDetail_middle NewDetailMiddle()
{
var result = new BookDetail_middle();
result.DouBanBookInfo.DataSource = DataSource.DouBan;
return(result);
}
public void VerifyBookData(BookDetail_middle middle)
{
string msg = "[DouBanBookRepository]:";
if (string.IsNullOrEmpty(middle.DouBanBookInfo.Code))
{
throw new Exception(msg + "没有BookCode");
}
if (string.IsNullOrEmpty(middle.DouBanBookInfo.AuthorCode))
{
throw new Exception(msg + "没有作者Code");
}
}
private void InitData(string entryUrl)
{
_entryUrl = entryUrl;
int sp = _entryUrl.IndexOf("subject") + "subject/".Length;
int ep = _entryUrl.IndexOf("/", sp + 1);
if (ep == -1)
{
ep = _entryUrl.Length;
}
_DouBanBookId = _entryUrl.Substring(sp, ep - sp);
_bookDetailData = NewDetailMiddle();
_bookDetailData.DouBanBookInfo.SourceBookId = _DouBanBookId;
_bookDetailData.DouBanBookInfo.Code = GenCodeHelper.Book_Code(DouBanAbbr, _DouBanBookId); //$"{DouBanAbbr}_{_DouBanBookId}";
}
public async void runAsync(string url)
{
NLogUtil.InfoTxt($"开始抓爬单本书:{url}");
BookDetail_middle task_midData = null;
try
{
task_midData = await _DetailCrawler.CrawlerAsync(url);
}
catch (ExceptionProxyConnect epc)
{
throw epc;
}
NLogUtil.InfoTxt($"抓爬结束");
// _DouBanBookRepository.Test(task_midData);
// var rBook = _BookDb.AddOrUpdate_MasterData<EBookInfo>(middle.DouBanBookInfo);
await _DouBanBookRepository.HandleBookMiddleAsync(task_midData);
}
/// <summary>
/// 处理抓爬的书本Middle数据
/// </summary>
/// <param name="middle"></param>
public async Task <bool> HandleBookMiddleAsync(BookDetail_middle middle)
{
try
{
NLogUtil.InfoTxt($"开始处理书本到数据库:{middle.DouBanBookInfo.Code}-{middle.DouBanBookInfo.Title}");
VerifyBookData(middle);
var rAll = await _Db.Ado.UseTranAsync(() => {
try
{
//书本作者
var ePerson = _PersonDb.AddOrUpdate_MasterData <EPerson>(middle.Author);
}
catch (Exception ex)
{
NLogUtil.ErrorTxt($"【错误】写入人物:{ex.Message}");
}
//Section
// var rSection = _SectionDb.AddOrUpdate_MasterData<ESection>(middle.SectionList);
try
{
//书本信息
var rBook = _BookDb.AddOrUpdate_MasterData <EBookInfo>(middle.DouBanBookInfo);
}
catch (Exception ex)
{
NLogUtil.ErrorTxt($"【错误】写入书本:{ex.Message}");
}
try
{
//书本Tag
var rTag = _TagDb.AddOrUpdate_MasterData <ETag>(middle.tagList);
}
catch (Exception ex)
{
NLogUtil.ErrorTxt($"【错误】写入更新Tags:{ex.Message}");
}
// HandleDataSection(middle.GetDataSections());
//书本和Tag关系
HandleBookTag(middle.GetBookTags());
});
if (!rAll.IsSuccess)
{
NLogUtil.ErrorTxt($"[数据库]录入书本失败:{middle.DouBanBookInfo.Code}-{middle.DouBanBookInfo.Title} -- {rAll.ErrorMessage}");
}
else
{
NLogUtil.InfoTxt($"【成功】处理书本到数据库:{middle.DouBanBookInfo.Code}-{middle.DouBanBookInfo.Title}");
}
return(rAll.IsSuccess);
}
catch (Exception ex)
{
// _Db.Ado.RollbackTran();
NLogUtil.ErrorTxt($"[数据库]录入书本失败:{middle.DouBanBookInfo.Code}-{middle.DouBanBookInfo.Title}--{ex.Message}");
return(false);
}
// return true;
}
public BookDetailCrawler(string douBanId)
{
_DouBanBookId = douBanId;
_entryUrl = $"{DouBanBookPrefix}/subject/{douBanId}";
_bookDetailData = NewDetailMiddle();
}
public BookDetailCrawler()
{
_bookDetailData = NewDetailMiddle();
}
