Exemplo n.º 1
0
 public BeatmapDownloader(FileStore store, Storage dataStorage, PisstaubeCacheDbContextFactory cache, RequestLimiter limiter)
 {
     _store      = store;
     _cache      = cache;
     _limiter    = limiter;
     _tmpStorage = dataStorage.GetStorageForDirectory("tmp");
 }
Exemplo n.º 2
0
 public DatabaseHouseKeeper(Storage storage, RequestLimiter requestLimiter, IAPIProvider apiProvider, IBeatmapSearchEngineProvider searchEngine, BeatmapDownloader beatmapDownloader, DbContextPool <PisstaubeDbContext> dbContextPool) : base(storage, requestLimiter, apiProvider, searchEngine, beatmapDownloader, dbContextPool)
 {
     _requestLimiter = requestLimiter;
     _apiProvider    = apiProvider;
     _dbContextPool  = dbContextPool;
     _searchEngine   = searchEngine;
 }
Exemplo n.º 3
0
 public OsuCrawler(Storage storage, RequestLimiter requestLimiter, IAPIProvider apiProvider,
                   IBeatmapSearchEngineProvider searchEngine, BeatmapDownloader beatmapDownloader, DbContextPool <PisstaubeDbContext> dbContextPool)
 {
     _storage           = storage;
     _beatmapDownloader = beatmapDownloader;
     _dbContextPool     = dbContextPool;
     SearchEngine       = searchEngine;
     ApiProvider        = apiProvider;
     RequestLimiter     = requestLimiter;
 }
Exemplo n.º 4
0
 public SetDownloader(Storage storage,
                      APIAccess apiAccess,
                      PisstaubeDbContextFactory factory,
                      PisstaubeCacheDbContextFactory cfactory,
                      Cleaner cleaner,
                      RequestLimiter limiter,
                      BeatmapSearchEngine search
                      )
 {
     _storage   = storage;
     _apiAccess = apiAccess;
     _factory   = factory;
     _cfactory  = cfactory;
     _cleaner   = cleaner;
     _limiter   = limiter;
     _search    = search;
 }
Exemplo n.º 5
0
        public BMUpdater(PisstaubeDbContextFactory factory,
                         APIAccess apiAccess,
                         BeatmapDownloader bmDl,
                         PisstaubeCacheDbContextFactory cFactory,
                         RulesetStore store,
                         Storage storage,
                         BeatmapSearchEngine search,

                         int limit = 100 /* 100 beatmaps can be updated at the same time per minute! */)
        {
            _factory   = factory;
            _apiAccess = apiAccess;
            _bmDl      = bmDl;
            _cFactory  = cFactory;
            _store     = store;
            _storage   = storage;
            _search    = search;
            _rl        = new RequestLimiter(limit, TimeSpan.FromMinutes(1));
        }
Exemplo n.º 6
0
 public SetDownloader(Storage storage,
                      IAPIProvider apiProvider,
                      PisstaubeDbContext dbContext,
                      PisstaubeCacheDbContextFactory cacheFactory,
                      SmartStorage smartStorage,
                      RequestLimiter limiter,
                      IBeatmapSearchEngineProvider search,
                      IpfsCache ipfsCache
                      )
 {
     _storage      = storage;
     _apiProvider  = apiProvider;
     _dbContext    = dbContext;
     _cacheFactory = cacheFactory;
     _smartStorage = smartStorage;
     _limiter      = limiter;
     _search       = search;
     _ipfsCache    = ipfsCache;
 }
Exemplo n.º 7
0
        protected virtual void ThreadWorker()
        {
            var dbContext = _dbContextPool.Rent();

            try {
                LatestId = dbContext.BeatmapSet
                           .OrderByDescending(bs => bs.SetId)
                           .Take(1)
                           .ToList()
                           .FirstOrDefault()?.SetId + 1 ?? 1;

                while (!CancellationToken.IsCancellationRequested)
                {
                    if (_errorCount > 1024)
                    {
                        LatestId = dbContext.BeatmapSet
                                   .OrderByDescending(bs => bs.SetId)
                                   .Take(1)
                                   .ToList()
                                   .FirstOrDefault()?.SetId + 1 ?? 1;

                        _dbContextPool.Return(dbContext);

                        Logger.LogPrint("Error count too high! will continue tomorrow...", LoggingTarget.Network, LogLevel.Important);
                        Thread.Sleep(TimeSpan.FromDays(1));

                        dbContext = _dbContextPool.Rent();
                    }
                    if (Tasks.Count > 32)
                    {
                        Task.WaitAll(Tasks.ToArray(), CancellationToken); // wait for all tasks
                        Tasks.Clear();                                    // Remove all previous tasks.
                    }

                    RequestLimiter.Limit();

                    Tasks.Add(Crawl(LatestId++));
                }
            } finally {
                _dbContextPool.Return(dbContext);
            }
        }