public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
{
var type = eng.GetComicSourceProviderType(context.Url);
if (type is null)
{
return(null);
}
using (var scope = serviceScopeFactory.CreateScope())
{
var ser = (IComicSourceProvider)scope.ServiceProvider.GetRequiredService(type.ProviderType);
var entity = await ser.GetChaptersAsync(context.Url);
var now = DateTime.Now.Ticks;
return(new AnfComicEntityTruck
{
Chapters = entity.Chapters,
ComicUrl = entity.ComicUrl,
CreateTime = now,
Descript = entity.Descript,
ImageUrl = entity.ImageUrl,
UpdateTime = now,
Name = entity.Name,
});
}
}
public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
{
var type = eng.GetComicSourceProviderType(context.Url);
if (type is null)
{
return(null);
}
using (var scope = serviceScopeFactory.CreateScope())
{
var ser = (IComicSourceProvider)scope.ServiceProvider.GetRequiredService(type.ProviderType);
var entity = await ser.GetPagesAsync(context.Url);
var now = DateTime.Now.Ticks;
var ctx = context.Copy(context.EntityUrl);
var ent = await context.Root.FetchEntityAsync(ctx);
var chp = ent?.Chapters.FirstOrDefault(x => x.TargetUrl == context.Url);
if (chp is null)
{
return(null);
}
return(new WithPageChapter
{
Pages = entity,
CreateTime = now,
UpdateTime = now,
TargetUrl = context.Url,
Title = chp.Title
});
}
}
public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
{
var entity = await inRedisComicService.GetEntityAsync(context.Url);
if (entity != null)
{
context.SetIsCache();
}
return(entity);
}
public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
{
var chp = await inRedisComicService.GetChapterAsync(context.Url);
if (chp != null)
{
context.SetIsCache();
}
return(chp);
}
public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
{
for (int i = 0; i < Count; i++)
{
var c = this[i];
var res = await c.FetchEntityAsync(context);
if (res != null)
{
return(res);
}
}
return(null);
}
public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
{
for (int i = 0; i < Count; i++)
{
var c = this[i];
var res = await c.FetchChapterAsync(context);
if (res != null)
{
return(res);
}
}
return(null);
}
public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
{
var coll = mongoClient.GetComicEntityCollection();
var filter = Builders <AnfComicEntity> .Filter.ElemMatch(x => x.WithPageChapters, x => x.TargetUrl == context.Url);
var entity = await coll.Find(filter)
.Project(x => x.WithPageChapters.FirstOrDefault(y => y.TargetUrl == context.Url))
.FirstOrDefaultAsync();
var now = DateTime.Now.Ticks;
var isUpdate = entity != null;
var needUpdate = entity != null && (now - entity.UpdateTime) >= fetchOptions.Value.DataTimeout.Ticks;
if (entity is null)
{
return(null);
}
if (needUpdate)
{
using (var locker = await context.CreateChapterLockerAsync())
{
if (!locker.IsAcquired)
{
if (context.RequireReloopFetcher != this)
{
context.SetRequireReloop();
}
return(null);
}
entity = await remoteFetcher.FetchChapterAsync(context);
if (entity != null)
{
var updater = MakeUpdater(entity);
await coll.UpdateOneAsync(filter, updater);
}
}
}
return(entity);
}
public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
{
var coll = mongoClient.GetComicEntityCollection();
var filter = Builders <AnfComicEntity> .Filter.Eq(x => x.ComicUrl, context.Url);
var entity = await coll.Find(filter)
.FirstOrDefaultAsync();
var now = DateTime.Now.Ticks;
var isUpdate = entity != null;
var needUpdate = entity != null && (now - entity.UpdateTime) >= fetchOptions.Value.DataTimeout.Ticks;
AnfComicEntityTruck truck = null;
if (entity is null || needUpdate)
{
using (var locker = await context.CreateEntityLockerAsync())
{
if (!locker.IsAcquired)
{
if (context.RequireReloopFetcher != this)
{
context.SetRequireReloop();
}
return(null);
}
truck = await remoteFetcher.FetchEntityAsync(context);
if (truck != null)
{
if (needUpdate)
{
var updater = MakeUpdater(entity);
var chps = truck.Chapters.Select(x => new WithPageChapter
{
TargetUrl = x.TargetUrl,
CreateTime = now,
Title = x.Title
}).ToArray();
var originMap = entity.WithPageChapters.GroupBy(x => x.TargetUrl)
.ToDictionary(x => x.Key, x => x.First());
foreach (var item in chps)
{
if (originMap.TryGetValue(item.TargetUrl, out var origin))
{
item.Pages = origin.Pages;
item.UpdateTime = origin.UpdateTime;
item.RefCount = origin.RefCount;
item.CreateTime = origin.CreateTime;
}
}
updater = updater.Set(x => x.WithPageChapters, chps);
var r = await coll.UpdateOneAsync(filter, updater);
}
else
{
var val = new AnfComicEntity
{
ComicUrl = truck.ComicUrl,
CreateTime = truck.CreateTime,
Descript = truck.Descript,
ImageUrl = truck.ImageUrl,
Name = truck.Name,
RefCount = truck.RefCount,
UpdateTime = truck.UpdateTime,
WithPageChapters = truck.Chapters.Select(x => new WithPageChapter
{
CreateTime = now,
Title = x.Title,
TargetUrl = x.TargetUrl
}).ToArray()
};
await coll.InsertOneAsync(val);
}
}
}
}
if (truck is null && entity != null)
{
truck = new AnfComicEntityTruck
{
CreateTime = entity.CreateTime,
ComicUrl = entity.ComicUrl,
Descript = entity.Descript,
ImageUrl = entity.ImageUrl,
Name = entity.Name,
RefCount = entity.RefCount,
UpdateTime = entity.UpdateTime,
Chapters = entity.WithPageChapters.Select(x => new ComicChapter
{
TargetUrl = x.TargetUrl,
Title = x.Title
}).ToArray()
};
}
return(truck);
}
private async Task CoreDoneFetchEntityAsync(IResourceFetchContext ctx, AnfComicEntityTruck val)
{
var set = dbContextTransfer.GetComicEntitySet();
var now = DateTime.Now.Ticks;
var upRes = await set.AsNoTracking()
.Where(x => x.ComicUrl == ctx.Url)
.Take(1)
.UpdateFromQueryAsync(x => new KvComicEntity
{
ComicUrl = val.ComicUrl,
Name = val.Name,
Descript = val.Descript,
ImageUrl = val.ImageUrl,
UpdateTime = now
});
if (upRes != 0)
{
var chpSet = dbContextTransfer.GetComicChapterSet();
var includeUrls = val.Chapters.Select(x => x.TargetUrl).Distinct().ToArray();
var query = chpSet.AsNoTracking();
if (includeUrls.Length > 50)
{
var urlEntity = includeUrls.Select(x => new { TargetUrl = x }).ToArray();
query = query.WhereBulkContains(urlEntity, nameof(KvComicChapter.TargetUrl));
}
else
{
query = query.Where(x => includeUrls.Contains(x.TargetUrl));
}
var exists = await query.Select(x => x.TargetUrl).ToArrayAsync();
var existsHash = new HashSet <string>(exists);
var notExists = new List <KvComicChapter>();
long id = -1;
for (int i = 0; i < val.Chapters.Length; i++)
{
var chp = val.Chapters[i];
if (!existsHash.Contains(chp.TargetUrl))
{
if (id == -1)
{
id = await set.AsNoTracking()
.Where(x => x.ComicUrl == ctx.Url)
.Select(x => x.Id)
.FirstAsync();
}
var kvChp = new KvComicChapter
{
TargetUrl = chp.TargetUrl,
Title = chp.Title,
CreateTime = now,
EnitityId = id,
Order = existsHash.Count + notExists.Count + 1
};
notExists.Add(kvChp);
}
}
if (notExists.Count != 0)
{
await chpSet.BulkInsertAsync(notExists);
}
}
else
{
var index = 1;
var entity = new KvComicEntity
{
ComicUrl = val.ComicUrl,
Descript = val.Descript,
ImageUrl = val.ImageUrl,
Name = val.Name,
UpdateTime = now,
CreateTime = now,
Chapters = val.Chapters.Select(x => new KvComicChapter
{
Order = index++,
CreateTime = now,
Title = x.Title,
}).ToArray(),
};
await set.SingleInsertAsync(entity);
var chpSet = dbContextTransfer.GetComicChapterSet();
await chpSet.BulkInsertAsync(entity.Chapters);
}
}
public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
{
var datas = await FetchEntityAsync(new[] { context });
return(datas.FirstOrDefault());
}
public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
{
var datas = await FetchChapterAsync(new[] { context });
return(datas.FirstOrDefault());
}
public FetchResult(IResourceFetchContext resourceFetchContext, ISingleResourceFetcher triggerFetcher, T value)
{
ResourceFetchContext = resourceFetchContext;
TriggerFetcher = triggerFetcher;
Value = value;
}