예제 #1
0
        public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
        {
            var type = eng.GetComicSourceProviderType(context.Url);

            if (type is null)
            {
                return(null);
            }
            using (var scope = serviceScopeFactory.CreateScope())
            {
                var ser    = (IComicSourceProvider)scope.ServiceProvider.GetRequiredService(type.ProviderType);
                var entity = await ser.GetChaptersAsync(context.Url);

                var now = DateTime.Now.Ticks;
                return(new AnfComicEntityTruck
                {
                    Chapters = entity.Chapters,
                    ComicUrl = entity.ComicUrl,
                    CreateTime = now,
                    Descript = entity.Descript,
                    ImageUrl = entity.ImageUrl,
                    UpdateTime = now,
                    Name = entity.Name,
                });
            }
        }
예제 #2
0
        public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
        {
            var type = eng.GetComicSourceProviderType(context.Url);

            if (type is null)
            {
                return(null);
            }
            using (var scope = serviceScopeFactory.CreateScope())
            {
                var ser    = (IComicSourceProvider)scope.ServiceProvider.GetRequiredService(type.ProviderType);
                var entity = await ser.GetPagesAsync(context.Url);

                var now = DateTime.Now.Ticks;
                var ctx = context.Copy(context.EntityUrl);
                var ent = await context.Root.FetchEntityAsync(ctx);

                var chp = ent?.Chapters.FirstOrDefault(x => x.TargetUrl == context.Url);
                if (chp is null)
                {
                    return(null);
                }
                return(new WithPageChapter
                {
                    Pages = entity,
                    CreateTime = now,
                    UpdateTime = now,
                    TargetUrl = context.Url,
                    Title = chp.Title
                });
            }
        }
예제 #3
0
        public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
        {
            var entity = await inRedisComicService.GetEntityAsync(context.Url);

            if (entity != null)
            {
                context.SetIsCache();
            }
            return(entity);
        }
예제 #4
0
        public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
        {
            var chp = await inRedisComicService.GetChapterAsync(context.Url);

            if (chp != null)
            {
                context.SetIsCache();
            }
            return(chp);
        }
예제 #5
0
        public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
        {
            for (int i = 0; i < Count; i++)
            {
                var c   = this[i];
                var res = await c.FetchEntityAsync(context);

                if (res != null)
                {
                    return(res);
                }
            }
            return(null);
        }
예제 #6
0
        public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
        {
            for (int i = 0; i < Count; i++)
            {
                var c   = this[i];
                var res = await c.FetchChapterAsync(context);

                if (res != null)
                {
                    return(res);
                }
            }
            return(null);
        }
예제 #7
0
파일: MongoFetcher.cs 프로젝트: Cricle/Anf
        public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
        {
            var coll   = mongoClient.GetComicEntityCollection();
            var filter = Builders <AnfComicEntity> .Filter.ElemMatch(x => x.WithPageChapters, x => x.TargetUrl == context.Url);

            var entity = await coll.Find(filter)
                         .Project(x => x.WithPageChapters.FirstOrDefault(y => y.TargetUrl == context.Url))
                         .FirstOrDefaultAsync();

            var now        = DateTime.Now.Ticks;
            var isUpdate   = entity != null;
            var needUpdate = entity != null && (now - entity.UpdateTime) >= fetchOptions.Value.DataTimeout.Ticks;

            if (entity is null)
            {
                return(null);
            }
            if (needUpdate)
            {
                using (var locker = await context.CreateChapterLockerAsync())
                {
                    if (!locker.IsAcquired)
                    {
                        if (context.RequireReloopFetcher != this)
                        {
                            context.SetRequireReloop();
                        }
                        return(null);
                    }
                    entity = await remoteFetcher.FetchChapterAsync(context);

                    if (entity != null)
                    {
                        var updater = MakeUpdater(entity);
                        await coll.UpdateOneAsync(filter, updater);
                    }
                }
            }
            return(entity);
        }
예제 #8
0
파일: MongoFetcher.cs 프로젝트: Cricle/Anf
        public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
        {
            var coll   = mongoClient.GetComicEntityCollection();
            var filter = Builders <AnfComicEntity> .Filter.Eq(x => x.ComicUrl, context.Url);

            var entity = await coll.Find(filter)
                         .FirstOrDefaultAsync();

            var now                   = DateTime.Now.Ticks;
            var isUpdate              = entity != null;
            var needUpdate            = entity != null && (now - entity.UpdateTime) >= fetchOptions.Value.DataTimeout.Ticks;
            AnfComicEntityTruck truck = null;

            if (entity is null || needUpdate)
            {
                using (var locker = await context.CreateEntityLockerAsync())
                {
                    if (!locker.IsAcquired)
                    {
                        if (context.RequireReloopFetcher != this)
                        {
                            context.SetRequireReloop();
                        }
                        return(null);
                    }
                    truck = await remoteFetcher.FetchEntityAsync(context);

                    if (truck != null)
                    {
                        if (needUpdate)
                        {
                            var updater = MakeUpdater(entity);
                            var chps    = truck.Chapters.Select(x => new WithPageChapter
                            {
                                TargetUrl  = x.TargetUrl,
                                CreateTime = now,
                                Title      = x.Title
                            }).ToArray();
                            var originMap = entity.WithPageChapters.GroupBy(x => x.TargetUrl)
                                            .ToDictionary(x => x.Key, x => x.First());
                            foreach (var item in chps)
                            {
                                if (originMap.TryGetValue(item.TargetUrl, out var origin))
                                {
                                    item.Pages      = origin.Pages;
                                    item.UpdateTime = origin.UpdateTime;
                                    item.RefCount   = origin.RefCount;
                                    item.CreateTime = origin.CreateTime;
                                }
                            }
                            updater = updater.Set(x => x.WithPageChapters, chps);
                            var r = await coll.UpdateOneAsync(filter, updater);
                        }
                        else
                        {
                            var val = new AnfComicEntity
                            {
                                ComicUrl         = truck.ComicUrl,
                                CreateTime       = truck.CreateTime,
                                Descript         = truck.Descript,
                                ImageUrl         = truck.ImageUrl,
                                Name             = truck.Name,
                                RefCount         = truck.RefCount,
                                UpdateTime       = truck.UpdateTime,
                                WithPageChapters = truck.Chapters.Select(x => new WithPageChapter
                                {
                                    CreateTime = now,
                                    Title      = x.Title,
                                    TargetUrl  = x.TargetUrl
                                }).ToArray()
                            };
                            await coll.InsertOneAsync(val);
                        }
                    }
                }
            }
            if (truck is null && entity != null)
            {
                truck = new AnfComicEntityTruck
                {
                    CreateTime = entity.CreateTime,
                    ComicUrl   = entity.ComicUrl,
                    Descript   = entity.Descript,
                    ImageUrl   = entity.ImageUrl,
                    Name       = entity.Name,
                    RefCount   = entity.RefCount,
                    UpdateTime = entity.UpdateTime,
                    Chapters   = entity.WithPageChapters.Select(x => new ComicChapter
                    {
                        TargetUrl = x.TargetUrl,
                        Title     = x.Title
                    }).ToArray()
                };
            }
            return(truck);
        }
예제 #9
0
        private async Task CoreDoneFetchEntityAsync(IResourceFetchContext ctx, AnfComicEntityTruck val)
        {
            var set   = dbContextTransfer.GetComicEntitySet();
            var now   = DateTime.Now.Ticks;
            var upRes = await set.AsNoTracking()
                        .Where(x => x.ComicUrl == ctx.Url)
                        .Take(1)
                        .UpdateFromQueryAsync(x => new KvComicEntity
            {
                ComicUrl   = val.ComicUrl,
                Name       = val.Name,
                Descript   = val.Descript,
                ImageUrl   = val.ImageUrl,
                UpdateTime = now
            });

            if (upRes != 0)
            {
                var chpSet      = dbContextTransfer.GetComicChapterSet();
                var includeUrls = val.Chapters.Select(x => x.TargetUrl).Distinct().ToArray();
                var query       = chpSet.AsNoTracking();
                if (includeUrls.Length > 50)
                {
                    var urlEntity = includeUrls.Select(x => new { TargetUrl = x }).ToArray();
                    query = query.WhereBulkContains(urlEntity, nameof(KvComicChapter.TargetUrl));
                }
                else
                {
                    query = query.Where(x => includeUrls.Contains(x.TargetUrl));
                }
                var exists = await query.Select(x => x.TargetUrl).ToArrayAsync();

                var  existsHash = new HashSet <string>(exists);
                var  notExists  = new List <KvComicChapter>();
                long id         = -1;
                for (int i = 0; i < val.Chapters.Length; i++)
                {
                    var chp = val.Chapters[i];
                    if (!existsHash.Contains(chp.TargetUrl))
                    {
                        if (id == -1)
                        {
                            id = await set.AsNoTracking()
                                 .Where(x => x.ComicUrl == ctx.Url)
                                 .Select(x => x.Id)
                                 .FirstAsync();
                        }
                        var kvChp = new KvComicChapter
                        {
                            TargetUrl  = chp.TargetUrl,
                            Title      = chp.Title,
                            CreateTime = now,
                            EnitityId  = id,
                            Order      = existsHash.Count + notExists.Count + 1
                        };
                        notExists.Add(kvChp);
                    }
                }
                if (notExists.Count != 0)
                {
                    await chpSet.BulkInsertAsync(notExists);
                }
            }
            else
            {
                var index  = 1;
                var entity = new KvComicEntity
                {
                    ComicUrl   = val.ComicUrl,
                    Descript   = val.Descript,
                    ImageUrl   = val.ImageUrl,
                    Name       = val.Name,
                    UpdateTime = now,
                    CreateTime = now,
                    Chapters   = val.Chapters.Select(x => new KvComicChapter
                    {
                        Order      = index++,
                        CreateTime = now,
                        Title      = x.Title,
                    }).ToArray(),
                };
                await set.SingleInsertAsync(entity);

                var chpSet = dbContextTransfer.GetComicChapterSet();
                await chpSet.BulkInsertAsync(entity.Chapters);
            }
        }
예제 #10
0
        public async Task <AnfComicEntityTruck> FetchEntityAsync(IResourceFetchContext context)
        {
            var datas = await FetchEntityAsync(new[] { context });

            return(datas.FirstOrDefault());
        }
예제 #11
0
        public async Task <WithPageChapter> FetchChapterAsync(IResourceFetchContext context)
        {
            var datas = await FetchChapterAsync(new[] { context });

            return(datas.FirstOrDefault());
        }
예제 #12
0
 public FetchResult(IResourceFetchContext resourceFetchContext, ISingleResourceFetcher triggerFetcher, T value)
 {
     ResourceFetchContext = resourceFetchContext;
     TriggerFetcher       = triggerFetcher;
     Value = value;
 }