private void ParseRecordGroupPage(IDbContainer dbContainer, IAudioBookGroup recordGroup, HtmlDocument document,
CancellationToken cancellationToken)
{
var topics = new Dictionary <string, string>();
foreach (var htmlNode in GetTopics(document))
{
var topicIdNode = htmlNode.GetNodesByClass("div", "biography-title").First().Descendants("a").First();
var topicLink = topicIdNode.GetAttributeValue("href", string.Empty);
var topicTitle = topicIdNode.InnerText;
var topicId = topicLink.Split('/').Last();
if (dbContainer.RecordsCache.Contains(topicId))
{
continue;
}
dbContainer.RecordsCache.Add(topicId);
topics.Add(topicId, topicTitle);
}
var records =
dbContainer.AudioBookSet.GetRecordsByKeys(new HashSet <string>(topics.Keys))
.ToDictionary(record => record.Key, record => record);
foreach (var topic in topics)
{
try
{
if (!records.TryGetValue(topic.Key, out var audioBook))
{
audioBook = dbContainer.AudioBookSet.CreateRecord();
audioBook.Created = DateTime.Now;
audioBook.Key = topic.Key;
audioBook.Genre = string.Empty;
audioBook.Author = string.Empty;
records.Add(topic.Key, audioBook);
dbContainer.AudioBookSet.AddRecord(audioBook);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBook.GroupKey = recordGroup.Key;
audioBook.Title = topic.Value;
}
catch (Exception)
{
// ignored
}
}
}
private void ParseRecordGroupPage(IDbContainer dbContainer, IAudioBookGroup recordGroup, HtmlDocument document,
CancellationToken cancellationToken)
{
var topics = new Dictionary <string, HtmlNode>();
foreach (var htmlNode in GetTopics(document))
{
var topicId = htmlNode.GetNodesByClass("td", "vf-col-icon vf-topic-icon-cell").First()
.GetAttributeValue("id", "");
if (dbContainer.RecordsCache.Contains(topicId))
{
continue;
}
dbContainer.RecordsCache.Add(topicId);
topics.Add(topicId, htmlNode);
}
var records =
dbContainer.AudioBookSet.GetRecordsByKeys(new HashSet <string>(topics.Keys))
.ToDictionary(record => record.Key, record => record);
foreach (var topic in topics)
{
try
{
if (!records.TryGetValue(topic.Key, out var audioBook))
{
audioBook = dbContainer.AudioBookSet.CreateRecord();
audioBook.Created = DateTime.Now;
audioBook.Key = topic.Key;
audioBook.Genre = string.Empty;
audioBook.Author = string.Empty;
records.Add(topic.Key, audioBook);
dbContainer.AudioBookSet.AddRecord(audioBook);
}
var topicTitle = GetTopicTitle(topic.Value, topic.Key);
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBook.GroupKey = recordGroup.Key;
audioBook.Title = topicTitle;
}
catch (Exception)
{
// ignored
}
}
}
protected override void DownloadRecordGroup(IDbContainer dbContainer, IAudioBookGroup audioBookGroup,
CancellationToken cancellationToken)
{
dbContainer.SaveChanges();
var url = $"http://rutracker.org/forum/viewforum.php?f={audioBookGroup.Key}";
var pageIndex = 0;
var pageCount = 0;
do
{
var progressMessage = audioBookGroup.Title +
(pageCount > 0 ? $": {pageIndex} / {pageCount}" : string.Empty);
ProgressMessage.Report(pageIndex, pageCount, progressMessage);
string pageHtml;
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = $"{url}&start={pageIndex * RecordsOnPageCount}";
pageHtml = webClient.Target.DownloadString(pageUrl);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
var document = new HtmlDocument();
document.LoadHtml(pageHtml);
if (pageCount == 0)
{
UpdateRecordGroupInfo(audioBookGroup, document);
dbContainer.AudioBookGroupSet.AddChangedRecordGroups(audioBookGroup);
pageCount = audioBookGroup.LastPageCount;
}
ParseRecordGroupPage(dbContainer, audioBookGroup, document, cancellationToken);
Thread.Sleep(200);
pageIndex++;
} while (pageIndex < pageCount);
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBookGroup.LastUpdate = DateTime.Now;
}
public void AddRecordGroup(IAudioBookGroup group)
{
_addedAudioBookGroups.Enqueue((AudioBookGroup)group);
}
protected abstract void DownloadRecordGroup(IDbContainer db, IAudioBookGroup recordGroup,
CancellationToken cancellationToken);
private void UpdateRecordGroupInfo(IAudioBookGroup recordGroup, HtmlDocument document)
{
recordGroup.Title = GetRecordGroupTitle(document);
recordGroup.LastPageCount = GetRecordGroupPageCount(document);
}