public BookContentDto GetBookContent(string chapterLink)
{
string url = chapterLink;
string jsonRes = Utils.HttpHelper.Get(url);
var jsonObj = JsonConvert.DeserializeObject <BookContentResModel>(jsonRes);
if (jsonObj.Code < 0)
{
throw new UserFriendlyException(jsonObj.Msg);
}
if (jsonObj.Data == null)
{
return(null);
}
var responseModel = new BookContentDto()
{
BookName = jsonObj.Data.Book_Name,
BookLink = $"{SpiderRemoteUrl}/Book/LiteratureForeign/GetChapters?id={jsonObj.Data.BookId}",
ChapterName = jsonObj.Data.Chapter_Name,
ChapterLink = chapterLink,
Content = jsonObj.Data.Content,
NextChapterLink = $"{SpiderRemoteUrl}/Book/LiteratureForeign/GetContent?id={jsonObj.Data.Next_Chapter_Id}",
PrevChapterLink = $"{SpiderRemoteUrl}/Book/LiteratureForeign/GetContent?id={jsonObj.Data.Prev_Chapter_Id}",
Number_Of_Words = jsonObj.Data.Words
};
return(responseModel);
}
public async Task Setup()
{
var book = BookBuilder.WithLibrary(LibraryId).WithContents(3).Build();
_expected = BookBuilder.Contents.PickRandom();
_response = await Client.DeleteAsync($"/libraries/{LibraryId}/books/{book.Id}/contents", _expected.Language, _expected.MimeType);
}
public async Task Setup()
{
_book = BookBuilder.WithLibrary(LibraryId).WithContents(5).Build();
_expected = BookBuilder.Contents.PickRandom();
_response = await Client.GetAsync($"/libraries/{LibraryId}/books/{_book.Id}/contents", _expected.Language, _expected.MimeType);
_assert = new BookContentAssert(_response, LibraryId);
}
public static void AddBookFile(this IDbConnection connection, int bookId, BookContentDto contentDto)
{
var sql = @"Insert Into BookContent (BookId, FileId, Language)
Output Inserted.Id
Values (@BookId, @FileId, @Language)";
var id = connection.ExecuteScalar <int>(sql, new { BookId = bookId, FileId = contentDto.FileId, Language = contentDto.Language });
contentDto.Id = id;
}
public async Task Setup()
{
_book = BookBuilder.WithLibrary(LibraryId).WithContents(2).Build();
_file = BookBuilder.Contents.PickRandom();
_expected = new Faker().Image.Random.Bytes(50);
_response = await Client.PutFile($"/libraries/{LibraryId}/books/{_book.Id}/contents", _expected, _file.Language, _file.MimeType);
_assert = new BookContentAssert(_response, LibraryId);
}
//internal BookContentAssert ShouldMatch( request, BookDto bookDto)
//{
// _bookContent.BookId.Should().Be(bookDto.Id);
// _bookContent.MimeType.Should().Be(request.MimeType());
// _bookContent.Language.Should().Be(request.Language());
// return this;
//}
internal BookContentAssert ShouldMatch(BookContentDto content, int bookId, IDbConnection dbConnection)
{
_bookContent.BookId.Should().Be(content.BookId);
_bookContent.BookId.Should().Be(bookId);
_bookContent.Language.Should().Be(content.Language);
var dbFile = dbConnection.GetFileById(content.FileId);
_bookContent.MimeType.Should().Be(dbFile.MimeType);
return(this);
}
public async Task Setup()
{
_newMimeType = "text/markdown";
_book = BookBuilder.WithLibrary(LibraryId).WithContents(2, "application/pdf").Build();
_file = BookBuilder.Contents.PickRandom();
_contents = RandomData.Bytes;
_response = await Client.PutFile($"/libraries/{LibraryId}/books/{_book.Id}/contents", _contents, _file.Language, _newMimeType);
_assert = new BookContentAssert(_response, LibraryId);
}
public async Task Setup()
{
_newLanguage = RandomData.Locale;
_book = BookBuilder.WithLibrary(LibraryId).WithContents(2).WithContentLanguage($"{_newLanguage}_old").Build();
_file = BookBuilder.Contents.PickRandom();
_contents = new Faker().Image.Random.Bytes(50);
_response = await Client.PutFile($"/libraries/{LibraryId}/books/{_book.Id}/contents", _contents, _newLanguage, _file.MimeType);
_assert = new BookContentAssert(_response, LibraryId);
}
/// <summary>
/// 获取小说内容
/// </summary>
/// <param name="chapterLink"></param>
/// <returns></returns>
public BookContentDto GetBookContent(string chapterLink)
{
HtmlWeb webClient = new HtmlWeb();
HtmlDocument doc;
//webClient.OverrideEncoding = Encoding.UTF8;
SetGZipHeader(webClient);
//这里两次请求是为了。。。 不解释了
try
{
try
{
doc = webClient.Load(chapterLink);
}
catch
{
Thread.Sleep(2000);
doc = webClient.Load(chapterLink);
}
}
catch (Exception ex)
{
throw new UserFriendlyException($"抓取网站请求失败，{ex.Message}。请退出后重试");
}
var nodes = doc.DocumentNode.SelectNodes("//div[@class='bookname']/div[@class='bottem1']/a[@href]");
if (nodes == null || nodes.Count == 0)
{
throw new UserFriendlyException("解析网页异常，请重试");
}
//var _domain = StringHelper.GetUrlDomain(link);
Uri uri = new Uri(chapterLink);
var _domain = $"{uri.Scheme}://{uri.Host}";
var _content = doc.DocumentNode.SelectSingleNode("//div[@id='content']").InnerHtml;
//
var bookContent = new BookContentDto()
{
BookName = doc.DocumentNode.SelectSingleNode("//div[@class='con_top']/a[2]").Attributes["title"].Value.Trim(),
BookLink = CompleteDomain(doc.DocumentNode.SelectSingleNode("//div[@class='bookname']/div[@class='bottem1']/a[@class='back']").Attributes["href"].Value.Trim()),
ChapterName = doc.DocumentNode.SelectSingleNode("//div[@class='bookname']/h1").InnerText.Trim(),
ChapterLink = chapterLink,
Content = ClearSensitiveCharacter(_content).TrimEnd(),
NextChapterLink = CompleteDomain(doc.DocumentNode.SelectSingleNode("//div[@class='bookname']/div[@class='bottem1']/a[@class='next']").Attributes["href"].Value.Trim()),
PrevChapterLink = CompleteDomain(doc.DocumentNode.SelectSingleNode("//div[@class='bookname']/div[@class='bottem1']/a[@class='pre']").Attributes["href"].Value.Trim())
};
bookContent.Number_Of_Words = ClearSensitiveCharacter(doc.DocumentNode.SelectSingleNode("//div[@id='content']").InnerText).TrimEnd().Length;
//有些网站喜欢将最后一章的“下一章的链接地址”设置为返回目录，所以有下面的处理
var a = chapterLink.Substring(0, chapterLink.LastIndexOf("/")) + "/";
if (a == bookContent.NextChapterLink)
{
bookContent.NextChapterLink = "";
}
return(bookContent);
}
