public override string GetAbsoluteLibraryPath(IAudioBook record)
{
var config = Config.Load <RutrackerTorrentFilesDownloaderConfig>();
var bookPath = GetRecordTargetLibraryPath(record);
return(Path.Combine(config.AudioCatalogFolder, bookPath));
}
public override void DownloadRecordTarget(string loginCoockies, IAudioBook record, IDbContainer dbContainer,
CancellationToken cancellationToken)
{
var config = Config.Load <RutrackerTorrentFilesDownloaderConfig>();
Directory.CreateDirectory(config.TorrentFilesFolder);
var userData = dbContainer.UserDataSet.CreateUserData();
userData.RecordGroupKey = record.GroupKey;
userData.RecordKey = record.Key;
var targetLibraryPath = GetAbsoluteLibraryPath(record);
userData.LocalPath = targetLibraryPath;
var downloader = Context.I.ComponentFactory.GetCreators <ITorrentFileDownloaderPlugin>()
.Select(item => item.GetInstance <ITorrentFileDownloaderPlugin>())
.FirstOrDefault(item => item.DisplayName == config.TorrentClientName);
if (downloader != null && downloader.IsExists)
{
downloader.Download(record.MagnetLink, targetLibraryPath);
}
dbContainer.UserDataSet.AddUserData(userData);
dbContainer.SaveChanges();
}
public Uri GetRecordPageUrl(IAudioBook record)
{
var groupKey = record.GroupKey;
var plugin = _siteParserPlugins.FirstOrDefault(item => item.GetGroupKeys(false).Contains(groupKey));
return(plugin?.GetRecordPageUrl(record));
}
protected override void ParseRecord(IDbContainer dbContainer, IAudioBook record, string postHtml)
{
record.ClearMetaInfo();
var document = new HtmlDocument();
document.LoadHtml(postHtml);
var topicMain = document.GetElementbyId("topic_main");
var postBody = topicMain?.Descendants().FirstOrDefault(item => item.HasClass("post_body"));
if (postBody != null)
{
var elementsByRows = ParsePostBodyElementsByRows(postBody.InnerHtml);
var sizeElement = document.DocumentNode.Descendants()
.FirstOrDefault(item => item.HasClass("attach_link") && item.HasClass("guest"));
if (sizeElement != null && sizeElement.ChildNodes.Count >= 2)
{
var sizeNode = sizeElement.LastChild;
var size = GetSizeInBytes(sizeNode.InnerText.ReplaceAll(new[] { "·", " " }, " ")
.Trim(' ', '\t'));
record.Size = size;
}
foreach (var element in elementsByRows)
{
FillRecordElement(record, element.Key.TrimEnd(':'), element.Value);
}
}
}
protected override void ParseRecord(IDbContainer dbContainer, IAudioBook record, string postHtml)
{
record.ClearMetaInfo();
var document = new HtmlDocument();
document.LoadHtml(postHtml);
var postBody = document.DocumentNode.Descendants("div")
.FirstOrDefault(item => item.GetAttributeValue("id", "").StartsWith("news-id-"));
if (postBody != null)
{
var elementsByRows = ParsePostBodyElementsByRows(postBody.InnerHtml);
if (elementsByRows.TryGetValue("Общий размер", out var sizeText))
{
var size = GetSizeInBytes(sizeText);
record.Size = size;
}
foreach (var element in elementsByRows)
{
FillRecordElement(record, element.Key.TrimEnd(':'), element.Value);
}
}
}
private string DownloadRecordMetaPageFromWeb(IAudioBook record, IDbContainer dbContainer,
CancellationToken cancellationToken)
{
while (_lastLoadFromInternet.IsRunning && _lastLoadFromInternet.ElapsedMilliseconds < 1000)
{
Thread.Sleep(200);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
}
string page;
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = GetRecordPageUrl(record);
page = webClient.Target.DownloadString(pageUrl);
_lastLoadFromInternet.Restart();
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
}
var document = new HtmlDocument();
document.LoadHtml(page);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
//ClearRecordPage(document);
page = document.DocumentNode.InnerHtml;
var pageKey = record.GetPageKey();
var binaryData = dbContainer.BinaryDataSet.CreateBinaryData();
binaryData.Key = pageKey;
binaryData.SetString(page, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(binaryData);
var magnetA = document.DocumentNode.SelectSingleNode("//*[@class=\"magnet-link\"]");
if (magnetA != null)
{
record.MagnetLink = magnetA.GetAttributeValue("href", null);
}
return(page);
}
public async Task <string> DownloadRecordSourcePage(IAudioBook record, CancellationToken cancellationToken)
{
var groupKey = record.GroupKey;
var plugin = _siteParserPlugins.FirstOrDefault(item => item.GetGroupKeys(false).Contains(groupKey));
if (plugin != null)
{
return(await plugin.DownloadRecordSourcePage(record, cancellationToken));
}
return(string.Empty);
}
public virtual string CalcAuthorName(IAudioBook record)
{
if (!record.AuthorNameForParse.IsNullOrEmpty() && !record.AuthorSurnameForParse.IsNullOrEmpty())
{
return($"{record.AuthorSurnameForParse}, {record.AuthorNameForParse}");
}
if (record.Author.IsNullOrEmpty())
{
return(record.AuthorSurnameForParse + record.AuthorNameForParse);
}
return(record.Author);
}
protected override void DownloadRecord(IDbContainer dbContainer, IAudioBook record, PageSources pageSource,
CancellationToken cancellationToken)
{
string pageHtml = null;
if (pageSource != PageSources.WebOnly)
{
//var pageMetaId = record.GetPageMetaKey();
//var metaPage = dbContainer.BinaryDataSet.GetByKey(pageMetaId);
//if (metaPage != null)
//{
// pageHtml = metaPage.GetString();
//}
var page = dbContainer.BinaryDataSet.GetByKey(record.GetPageKey());
if (page != null)
{
using (var ms = new MemoryStream(page.GetData()))
{
var doc = new HtmlDocument();
doc.Load(ms);
var savedHtml = doc.DocumentNode.OuterHtml;
CleanupRecordPage(doc);
pageHtml = doc.DocumentNode.InnerHtml;
if (savedHtml != pageHtml)
{
page.SetString(doc.DocumentNode.InnerHtml, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(page);
}
}
}
}
if (pageHtml.IsNullOrEmpty() && pageSource != PageSources.CacheOnly)
{
pageHtml = DownloadRecordMetaPageFromWeb(record, dbContainer, cancellationToken);
}
if (pageHtml.IsNullOrEmpty() || cancellationToken.IsCancellationRequested)
{
return;
}
ParseRecord(dbContainer, record, pageHtml);
}
protected abstract IReadOnlyCollection <string> GetPropertyValues(IAudioBook audioBook);
protected override IReadOnlyCollection <string> GetPropertyValues(IAudioBook audioBook)
{
return(audioBook.GetGenres());
}
public ItemDoubleClickRowEventArgs(IAudioBook target)
{
Target = target;
}
public abstract string GetRecordTargetLibraryPath(IAudioBook record);
public abstract string GetAbsoluteLibraryPath(IAudioBook record);
public abstract void DownloadRecordTarget(string loginCoockies, IAudioBook record, IDbContainer dbContainer,
CancellationToken cancellationToken);
public override string GetRecordTargetLibraryPath(IAudioBook record)
{
var result = GetFileName(record.Author ?? record.Title);
return(result);
}
protected abstract void ParseRecord(IDbContainer db, IAudioBook record, string postBodyHtml);
protected abstract string GetRecordSourcePageString(IAudioBook audioBook, PageSources pageSource,
CancellationToken cancellationToken);
protected abstract void DownloadRecord(IDbContainer db, IAudioBook record, PageSources pageSource,
CancellationToken cancellationToken);
public override Uri GetRecordPageUrl(IAudioBook record)
{
return(new Uri($"http://audioboo.ru/{record.GroupKey}/{record.Key}"));
}
public abstract Uri GetRecordPageUrl(IAudioBook record);
protected override string GetRecordSourcePageString([NotNull] IAudioBook audioBook, PageSources pageSource,
CancellationToken cancellationToken)
{
string result = null;
using (var autoSave = Context.I.DbContainerAutoSave)
{
var dbContainer = autoSave.DBContainer;
var recordPageKey = audioBook.GetPageKey();
var pageData = pageSource == PageSources.WebOnly
? null
: dbContainer.BinaryDataSet.GetByKey(recordPageKey);
if (pageData != null)
{
var data = pageData.GetData();
using (var ms = new MemoryStream(data))
{
var document = new HtmlDocument();
document.Load(ms);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
CleanupRecordPage(document);
result = document.DocumentNode.InnerHtml;
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
pageData = dbContainer.BinaryDataSet.CreateBinaryData();
pageData.Key = recordPageKey;
pageData.SetString(result, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
}
}
else if (pageSource != PageSources.CacheOnly)
{
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = GetRecordPageUrl(audioBook);
result = webClient.Target.DownloadString(pageUrl);
var document = new HtmlDocument();
document.LoadHtml(result);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
CleanupRecordPage(document);
result = document.DocumentNode.InnerHtml;
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
pageData = dbContainer.BinaryDataSet.CreateBinaryData();
pageData.Key = recordPageKey;
pageData.SetString(result, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(pageData);
}
}
}
return(result);
}
public override Uri GetRecordPageUrl(IAudioBook record)
{
return(new Uri($"http://rutracker.org/forum/viewtopic.php?t={record.Key}"));
}
public void AddRecord(IAudioBook record)
{
_addedAudioBooks.Enqueue((AudioBook)record);
}
public async Task <string> DownloadRecordSourcePage(IAudioBook audioBook, CancellationToken cancellationToken)
{
return(await Task.Factory.StartNew(
() => GetRecordSourcePageString(audioBook, PageSources.CacheOrWeb, cancellationToken),
cancellationToken));
}
//private static class KeysCollection
//{
// private static Stopwatch _sw = new Stopwatch();
// private static readonly Dictionary<string, HashSet<string>> _values = new Dictionary<string, HashSet<string>>(StringComparer.InvariantCultureIgnoreCase);
// public static void Add(string key, string value)
// {
// if (!_values.TryGetValue(key, out var values))
// {
// values = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
// _values.Add(key, values);
// }
// values.Add(value);
// if (!_sw.IsRunning)
// _sw.Start();
// if (_sw.Elapsed.TotalSeconds > 15)
// {
// Debug.WriteLine("===============================");
// var top50 = _values.Where(item => item.Value.Count > 10);
// foreach (var keyValuePair in top50.OrderByDescending(item=>item.Value.Count))
// {
// Debug.WriteLine($"{keyValuePair.Key}\t{keyValuePair.Value.Count}");
// }
// Debug.WriteLine("===============================");
// _sw.Restart();
// }
// }
//}
protected virtual void FillRecordElement(IAudioBook record, string key, string value)
{
key = key.Trim(' ', '[', ']', '<', '>', ';', '\n', '\t', '.', ',', '"', '(');
//#if DEBUG
// KeysCollection.Add(key, value);
//#endif
switch (key.ToLower())
{
case "автор":
case "авторы":
case "фамилия, имя автора":
case "фамилии авторов":
case "авторы произведений":
record.Author = CleanupHumanNameValue(value, 500);
break;
case "фамилия автора":
record.AuthorSurnameForParse = CleanupHumanNameValue(value, 250);
break;
case "имя автора":
record.AuthorNameForParse = CleanupHumanNameValue(value, 250);
break;
case "издательство":
record.Publisher =
CleanupRecordValue(value, false, 500).ChangeCase(Extensions.CaseTypes.FirstWord, true, true);
break;
case "исполнитель":
case "исполнители":
case "запись и обработка":
record.Reader = CleanupHumanNameValue(value, 500);
break;
case "жанр":
record.Genre =
CleanupRecordValue(value, false, 200)
.ChangeCase(Extensions.CaseTypes.AfterSplitter, false, true);
break;
case "битрейт":
case "битрейт аудио":
record.Bitrate = CleanupRecordValue(value, false, 100)
.ChangeCase(Extensions.CaseTypes.FirstWord, true, true);
break;
case "длительность":
case "прдолжительность":
case "продолжительность":
case "продолжительность звучания":
case "продолжительность книги":
case "общее время звучания":
case "общее время воспроизведения":
case "bремя звучания":
case "время звучания":
case "время воспроизведения":
case "время чтения":
case "общая продолжительность":
case "продолжительность аудиокниги":
case "продолжительность (время звучания)":
case "время":
case "длина записи":
record.Length = CleanupRecordValue(value, false, 500);
break;
case "описание":
case "аннотация":
record.Description = CleanupRecordValue(value, true, 1000);
break;
case "доп. информация":
if (ParseKeyValue(value, out var secondKey, out var secondValue))
{
FillRecordElement(record, secondKey, secondValue);
}
break;
}
}