protected override void ParseRecord(IDbContainer dbContainer, IAudioBook record, string postHtml)
{
record.ClearMetaInfo();
var document = new HtmlDocument();
document.LoadHtml(postHtml);
var topicMain = document.GetElementbyId("topic_main");
var postBody = topicMain?.Descendants().FirstOrDefault(item => item.HasClass("post_body"));
if (postBody != null)
{
var elementsByRows = ParsePostBodyElementsByRows(postBody.InnerHtml);
var sizeElement = document.DocumentNode.Descendants()
.FirstOrDefault(item => item.HasClass("attach_link") && item.HasClass("guest"));
if (sizeElement != null && sizeElement.ChildNodes.Count >= 2)
{
var sizeNode = sizeElement.LastChild;
var size = GetSizeInBytes(sizeNode.InnerText.ReplaceAll(new[] { "·", " " }, " ")
.Trim(' ', '\t'));
record.Size = size;
}
foreach (var element in elementsByRows)
{
FillRecordElement(record, element.Key.TrimEnd(':'), element.Value);
}
}
}
public void CreateStoredProcedures <T>(IDbContainer dbContainer, INestedSetsDbSetManager <T> manager)
where T : NestedSetsEntity
{
CreateStoredProcedurePreCreateNestedSetsNode(dbContainer, manager);
CreateStoredProcedurePostRemoveNestedSetsNode(dbContainer, manager);
CreateStoredProcedureMoveNestedSetsNode(dbContainer, manager);
}
public override IEnumerable <IAudioBookGroup> GetAllRecordGroups(IDbContainer dbContainer)
{
var result = GroupKeys.Select(groupKey => GetRecordGroup(groupKey, dbContainer));
dbContainer.SaveChanges();
return(result);
}
protected override void ParseRecord(IDbContainer dbContainer, IAudioBook record, string postHtml)
{
record.ClearMetaInfo();
var document = new HtmlDocument();
document.LoadHtml(postHtml);
var postBody = document.DocumentNode.Descendants("div")
.FirstOrDefault(item => item.GetAttributeValue("id", "").StartsWith("news-id-"));
if (postBody != null)
{
var elementsByRows = ParsePostBodyElementsByRows(postBody.InnerHtml);
if (elementsByRows.TryGetValue("Общий размер", out var sizeText))
{
var size = GetSizeInBytes(sizeText);
record.Size = size;
}
foreach (var element in elementsByRows)
{
FillRecordElement(record, element.Key.TrimEnd(':'), element.Value);
}
}
}
private void CreateStoredProcedureMoveNestedSetsNode <T>(IDbContainer dbContainer, INestedSetsDbSetManager <T> manager)
where T : NestedSetsEntity
{
string procedureName = GetMoveNestedSetsNodeSpName(manager);
dbContainer.Database.ExecuteSqlCommand(string.Format(@"IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[{0}]') AND type in (N'P', N'PC'))
DROP PROCEDURE [dbo].[{0}]", procedureName));
dbContainer.Database.ExecuteSqlCommand(string.Format(@"CREATE PROCEDURE [dbo].[{0}]
@NodeLeftKey int,
@NodeRightKey int,
@NodeLevel int,
@ParentLeftKey int,
@ParentRightKey int,
@ParentLevel int
AS
BEGIN
DECLARE @d int = @NodeRightKey - @NodeLeftKey + 1;
UPDATE dbo.[{1}] SET LeftKey = 0 - LeftKey, RightKey = 0 - RightKey WHERE LeftKey >= @NodeLeftKey AND RightKey <= @NodeRightKey;
UPDATE dbo.[{1}] SET LeftKey = LeftKey - @d WHERE LeftKey > @NodeRightKey;
UPDATE dbo.[{1}] SET RightKey = RightKey - @d WHERE RightKey > @NodeRightKey;
DECLARE @pr int = CASE WHEN @ParentRightKey > @NodeRightKey THEN @ParentRightKey - (@NodeRightKey - @NodeLeftKey + 1) ELSE @ParentRightKey END;
UPDATE dbo.[{1}] SET LeftKey = LeftKey + @d WHERE LeftKey >= @pr;
UPDATE dbo.[{1}] SET RightKey = RightKey + @d WHERE RightKey >= @pr;
DECLARE @pd int = CASE WHEN @ParentRightKey > @NodeRightKey THEN @ParentRightKey - @NodeRightKey - 1 ELSE @ParentRightKey - @NodeRightKey - 1 + @d END;
DECLARE @dl int = @ParentLevel + 1 - @NodeLevel;
UPDATE dbo.[{1}] SET LeftKey = @pd - LeftKey, RightKey = @pd - RightKey, Level = Level + @dl WHERE LeftKey <= 0-@NodeLeftKey AND RightKey >= 0-@NodeRightKey;
END", procedureName, manager.TableName));
}
private static object transactionLocker = new object(); //事故执行线程锁
/// <summary>
/// 创建事故执行器
/// </summary>
/// <param name="container">数据容器</param>
internal DbExecutor(IDbContainer container)
{
//赋值
this.container = container;
this.operators = new Dictionary <Type, dynamic>();
//开启事故
this.container.DbHelper.BeginTransaction(out this.connection, out this.transaction);
}
public TokenService(
IDbContainer container,
IOptions <JWT> config,
IRoleService roles)
{
Container = container;
JwtConfig = config.Value;
Roles = roles;
}
/// <summary>
/// 静态初始化
/// </summary>
static Program()
{
//获取连接字符串
string connectionString = "Data Source=39.106.212.169;User Id=root;Password=123456;Initial Catalog=cheij_management;";
//初始化数据容器
DbContainer.Init <MySqlInitializer>(connectionString);
//获取数据容器
_container = DbContainer.Get(connectionString);
}
private void ParseRecordGroupPage(IDbContainer dbContainer, IAudioBookGroup recordGroup, HtmlDocument document,
CancellationToken cancellationToken)
{
var topics = new Dictionary <string, string>();
foreach (var htmlNode in GetTopics(document))
{
var topicIdNode = htmlNode.GetNodesByClass("div", "biography-title").First().Descendants("a").First();
var topicLink = topicIdNode.GetAttributeValue("href", string.Empty);
var topicTitle = topicIdNode.InnerText;
var topicId = topicLink.Split('/').Last();
if (dbContainer.RecordsCache.Contains(topicId))
{
continue;
}
dbContainer.RecordsCache.Add(topicId);
topics.Add(topicId, topicTitle);
}
var records =
dbContainer.AudioBookSet.GetRecordsByKeys(new HashSet <string>(topics.Keys))
.ToDictionary(record => record.Key, record => record);
foreach (var topic in topics)
{
try
{
if (!records.TryGetValue(topic.Key, out var audioBook))
{
audioBook = dbContainer.AudioBookSet.CreateRecord();
audioBook.Created = DateTime.Now;
audioBook.Key = topic.Key;
audioBook.Genre = string.Empty;
audioBook.Author = string.Empty;
records.Add(topic.Key, audioBook);
dbContainer.AudioBookSet.AddRecord(audioBook);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBook.GroupKey = recordGroup.Key;
audioBook.Title = topic.Value;
}
catch (Exception)
{
// ignored
}
}
}
private string DownloadRecordMetaPageFromWeb(IAudioBook record, IDbContainer dbContainer,
CancellationToken cancellationToken)
{
while (_lastLoadFromInternet.IsRunning && _lastLoadFromInternet.ElapsedMilliseconds < 1000)
{
Thread.Sleep(200);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
}
string page;
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = GetRecordPageUrl(record);
page = webClient.Target.DownloadString(pageUrl);
_lastLoadFromInternet.Restart();
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
}
var document = new HtmlDocument();
document.LoadHtml(page);
if (cancellationToken.IsCancellationRequested)
{
return(null);
}
//ClearRecordPage(document);
page = document.DocumentNode.InnerHtml;
var pageKey = record.GetPageKey();
var binaryData = dbContainer.BinaryDataSet.CreateBinaryData();
binaryData.Key = pageKey;
binaryData.SetString(page, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(binaryData);
var magnetA = document.DocumentNode.SelectSingleNode("//*[@class=\"magnet-link\"]");
if (magnetA != null)
{
record.MagnetLink = magnetA.GetAttributeValue("href", null);
}
return(page);
}
private void ParseRecordGroupPage(IDbContainer dbContainer, IAudioBookGroup recordGroup, HtmlDocument document,
CancellationToken cancellationToken)
{
var topics = new Dictionary <string, HtmlNode>();
foreach (var htmlNode in GetTopics(document))
{
var topicId = htmlNode.GetNodesByClass("td", "vf-col-icon vf-topic-icon-cell").First()
.GetAttributeValue("id", "");
if (dbContainer.RecordsCache.Contains(topicId))
{
continue;
}
dbContainer.RecordsCache.Add(topicId);
topics.Add(topicId, htmlNode);
}
var records =
dbContainer.AudioBookSet.GetRecordsByKeys(new HashSet <string>(topics.Keys))
.ToDictionary(record => record.Key, record => record);
foreach (var topic in topics)
{
try
{
if (!records.TryGetValue(topic.Key, out var audioBook))
{
audioBook = dbContainer.AudioBookSet.CreateRecord();
audioBook.Created = DateTime.Now;
audioBook.Key = topic.Key;
audioBook.Genre = string.Empty;
audioBook.Author = string.Empty;
records.Add(topic.Key, audioBook);
dbContainer.AudioBookSet.AddRecord(audioBook);
}
var topicTitle = GetTopicTitle(topic.Value, topic.Key);
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBook.GroupKey = recordGroup.Key;
audioBook.Title = topicTitle;
}
catch (Exception)
{
// ignored
}
}
}
protected override void DownloadRecordGroup(IDbContainer dbContainer, IAudioBookGroup audioBookGroup,
CancellationToken cancellationToken)
{
dbContainer.SaveChanges();
var url = $"http://rutracker.org/forum/viewforum.php?f={audioBookGroup.Key}";
var pageIndex = 0;
var pageCount = 0;
do
{
var progressMessage = audioBookGroup.Title +
(pageCount > 0 ? $": {pageIndex} / {pageCount}" : string.Empty);
ProgressMessage.Report(pageIndex, pageCount, progressMessage);
string pageHtml;
using (var webClient = WebClientPool.GetPoolItem())
{
var pageUrl = $"{url}&start={pageIndex * RecordsOnPageCount}";
pageHtml = webClient.Target.DownloadString(pageUrl);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
var document = new HtmlDocument();
document.LoadHtml(pageHtml);
if (pageCount == 0)
{
UpdateRecordGroupInfo(audioBookGroup, document);
dbContainer.AudioBookGroupSet.AddChangedRecordGroups(audioBookGroup);
pageCount = audioBookGroup.LastPageCount;
}
ParseRecordGroupPage(dbContainer, audioBookGroup, document, cancellationToken);
Thread.Sleep(200);
pageIndex++;
} while (pageIndex < pageCount);
if (cancellationToken.IsCancellationRequested)
{
return;
}
audioBookGroup.LastUpdate = DateTime.Now;
}
private IAudioBookGroup GetRecordGroup(string recordGroupKey, IDbContainer dbContainer)
{
var result = dbContainer.AudioBookGroupSet.GetRecordGroupByKey(recordGroupKey);
if (result == null)
{
result = dbContainer.AudioBookGroupSet.CreateRecordGroup();
dbContainer.AudioBookGroupSet.AddRecordGroup(result);
result.Key = recordGroupKey;
result.Title = recordGroupKey;
}
return(result);
}
private void CreateStoredProcedurePreCreateNestedSetsNode <T>(IDbContainer dbContainer, INestedSetsDbSetManager <T> manager)
where T : NestedSetsEntity
{
string procedureName = GetPreCreateNestedSetsNodeSpName(manager);
dbContainer.Database.ExecuteSqlCommand(string.Format(@"IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[{0}]') AND type in (N'P', N'PC'))
DROP PROCEDURE [dbo].[{0}]", procedureName));
dbContainer.Database.ExecuteSqlCommand(string.Format(@"CREATE PROCEDURE [dbo].[{0}]
@RightKey int
AS
BEGIN
UPDATE dbo.[{1}] SET LeftKey = LeftKey + 2, RightKey = RightKey + 2 WHERE LeftKey > @RightKey;
UPDATE dbo.[{1}] SET RightKey = RightKey + 2 WHERE RightKey >= @RightKey AND LeftKey < @RightKey;
END", procedureName, manager.TableName));
}
protected override IEnumerable <IAudioBook> GetRecordsInner(IDbContainer dbContainer, Group group,
CancellationToken cancellationToken)
{
IEnumerable <IAudioBook> result;
if (group == null || group.Level == 0)
{
result = dbContainer.AudioBookSet.GetRecordsAllWithCache();
}
else
{
result = dbContainer.AudioBookSet.GetRecordsByKeys(group.LinkedRecords);
}
return(result);
}
protected override void DownloadRecord(IDbContainer dbContainer, IAudioBook record, PageSources pageSource,
CancellationToken cancellationToken)
{
string pageHtml = null;
if (pageSource != PageSources.WebOnly)
{
//var pageMetaId = record.GetPageMetaKey();
//var metaPage = dbContainer.BinaryDataSet.GetByKey(pageMetaId);
//if (metaPage != null)
//{
// pageHtml = metaPage.GetString();
//}
var page = dbContainer.BinaryDataSet.GetByKey(record.GetPageKey());
if (page != null)
{
using (var ms = new MemoryStream(page.GetData()))
{
var doc = new HtmlDocument();
doc.Load(ms);
var savedHtml = doc.DocumentNode.OuterHtml;
CleanupRecordPage(doc);
pageHtml = doc.DocumentNode.InnerHtml;
if (savedHtml != pageHtml)
{
page.SetString(doc.DocumentNode.InnerHtml, true);
dbContainer.BinaryDataSet.AddChangedBinaryData(page);
}
}
}
}
if (pageHtml.IsNullOrEmpty() && pageSource != PageSources.CacheOnly)
{
pageHtml = DownloadRecordMetaPageFromWeb(record, dbContainer, cancellationToken);
}
if (pageHtml.IsNullOrEmpty() || cancellationToken.IsCancellationRequested)
{
return;
}
ParseRecord(dbContainer, record, pageHtml);
}
protected override IEnumerable <IAudioBook> GetRecordsInner(IDbContainer dbContainer, Group group,
CancellationToken cancellationToken)
{
if (group == null || group.Level == 0)
{
return(dbContainer.AudioBookSet.GetRecordsAllWithCache());
}
if (group.Level == 1)
{
return(dbContainer.AudioBookSet.GetRecordsByWebSite(((WebSiteGroup)group).WebSite.Id).ToArray());
}
if (group.Level == 2)
{
return(dbContainer.AudioBookSet.GetRecordsByGroup(group.LinkedObjectString).ToArray());
}
return(dbContainer.AudioBookSet.GetRecordsByKeys(group.LinkedRecords).ToArray());
}
public void Normalize(IReadOnlyCollection <IAudioBook> records, IDbContainer dbContainer)
{
var recordArray = records as IAudioBook[] ?? records.ToArray();
var cache = dbContainer.ReplacementStringSet.GetReplacementStringsAll().ToArray();
foreach (var record in recordArray)
{
var bitrateReplacement =
cache.FirstOrDefault(
str =>
str.RecordPropertyName == "bitrate" &&
string.Compare(str.PossibleValue, record.Bitrate, StringComparison.OrdinalIgnoreCase) == 0);
if (bitrateReplacement != null)
{
record.Bitrate = bitrateReplacement.ReplaceValue;
}
var publisherReplacement =
cache.FirstOrDefault(
str =>
str.RecordPropertyName == "publisher" &&
string.Compare(str.PossibleValue, record.Publisher, StringComparison.OrdinalIgnoreCase) ==
0);
if (publisherReplacement != null)
{
record.Publisher = publisherReplacement.ReplaceValue;
}
var readerReplacement =
cache.FirstOrDefault(
str =>
str.RecordPropertyName == "reader" &&
string.Compare(str.PossibleValue, record.Reader, StringComparison.OrdinalIgnoreCase) == 0);
if (readerReplacement != null)
{
record.Reader = readerReplacement.ReplaceValue;
}
var authorName = CalcAuthorName(record);
var authorReplacement =
cache.FirstOrDefault(
str =>
str.RecordPropertyName == "author" &&
string.Compare(str.PossibleValue, authorName, StringComparison.OrdinalIgnoreCase) == 0);
record.Author = authorReplacement != null ? authorReplacement.ReplaceValue : authorName;
if (record.Author.IsNullOrEmpty())
{
if (!record.Title.IsNullOrEmpty())
{
var parts = record.Title.Split(" - ");
if (parts.Length > 1)
{
var possibleAuthor = parts[0].Trim().ToLower();
if (_authors.TryGetValue(possibleAuthor, out var author))
{
record.Author = author;
}
}
}
}
else
{
_authors.TryAdd(authorName, authorName);
}
var genreName = record.Genre ?? string.Empty;
var genreReplacement =
cache.FirstOrDefault(
str =>
str.RecordPropertyName == "genre" &&
string.Compare(str.PossibleValue, genreName, StringComparison.OrdinalIgnoreCase) == 0);
if (genreReplacement != null)
{
record.Genre = genreReplacement.ReplaceValue;
}
else
{
record.Genre = genreName;
}
}
}
public UserRepository(IDbContainer dbContainer) : base(dbContainer)
{
this.DbContainer = dbContainer ?? throw new ArgumentNullException(nameof(dbContainer));
}
public RoleService(IDbContainer container)
{
Container = container;
Preload();
}
/// <summary>
/// ExtendMethod: 删除实体类Mapping的表
/// </summary>
/// <typeparam name="TEntity">实体类型</typeparam>
/// <param name="container">数据容器</param>
public static void DropTable <TEntity>(this IDbContainer container)
where TEntity : class
{
container.DropTable(typeof(TEntity));
}
/// <summary>
/// ExtendMethod: 从旧表重命名获取所Mapping的Table
/// </summary>
/// <typeparam name="TEntity">实体类型</typeparam>
/// <param name="container">数据容器</param>
/// <param name="oldTableName">旧的表名</param>
public static void RenameTable <TEntity>(this IDbContainer container, string oldTableName)
where TEntity : class
{
container.RenameTable(typeof(TEntity), oldTableName);
}
public PhotoRepository(IDbContainer dbContainer) : base(dbContainer)
{
}
public FileTypeRepository(IDbContainer dbContainer) : base(dbContainer)
{
}
public CategoryRepository(IDbContainer dbContainer)
: base(dbContainer)
{
}
protected abstract void ParseRecord(IDbContainer db, IAudioBook record, string postBodyHtml);
public CharacteristicDbSetManager(DbSet <Characteristic> entities, IDbContainer dbContainer)
: base(entities, dbContainer, "Characteristics")
{
}
public abstract void DownloadRecordTarget(string loginCoockies, IAudioBook record, IDbContainer dbContainer,
CancellationToken cancellationToken);
public DbContainerAutoSave(IDbContainer dbContainer)
{
DBContainer = dbContainer;
}
protected abstract void DownloadRecordGroup(IDbContainer db, IAudioBookGroup recordGroup,
CancellationToken cancellationToken);
