public BeatmapDownloader(FileStore store, Storage dataStorage, PisstaubeCacheDbContextFactory cache, RequestLimiter limiter)
{
_store = store;
_cache = cache;
_limiter = limiter;
_tmpStorage = dataStorage.GetStorageForDirectory("tmp");
}
public DatabaseHouseKeeper(Storage storage, RequestLimiter requestLimiter, IAPIProvider apiProvider, IBeatmapSearchEngineProvider searchEngine, BeatmapDownloader beatmapDownloader, DbContextPool <PisstaubeDbContext> dbContextPool) : base(storage, requestLimiter, apiProvider, searchEngine, beatmapDownloader, dbContextPool)
{
_requestLimiter = requestLimiter;
_apiProvider = apiProvider;
_dbContextPool = dbContextPool;
_searchEngine = searchEngine;
}
public OsuCrawler(Storage storage, RequestLimiter requestLimiter, IAPIProvider apiProvider,
IBeatmapSearchEngineProvider searchEngine, BeatmapDownloader beatmapDownloader, DbContextPool <PisstaubeDbContext> dbContextPool)
{
_storage = storage;
_beatmapDownloader = beatmapDownloader;
_dbContextPool = dbContextPool;
SearchEngine = searchEngine;
ApiProvider = apiProvider;
RequestLimiter = requestLimiter;
}
public SetDownloader(Storage storage,
APIAccess apiAccess,
PisstaubeDbContextFactory factory,
PisstaubeCacheDbContextFactory cfactory,
Cleaner cleaner,
RequestLimiter limiter,
BeatmapSearchEngine search
)
{
_storage = storage;
_apiAccess = apiAccess;
_factory = factory;
_cfactory = cfactory;
_cleaner = cleaner;
_limiter = limiter;
_search = search;
}
public BMUpdater(PisstaubeDbContextFactory factory,
APIAccess apiAccess,
BeatmapDownloader bmDl,
PisstaubeCacheDbContextFactory cFactory,
RulesetStore store,
Storage storage,
BeatmapSearchEngine search,
int limit = 100 /* 100 beatmaps can be updated at the same time per minute! */)
{
_factory = factory;
_apiAccess = apiAccess;
_bmDl = bmDl;
_cFactory = cFactory;
_store = store;
_storage = storage;
_search = search;
_rl = new RequestLimiter(limit, TimeSpan.FromMinutes(1));
}
public SetDownloader(Storage storage,
IAPIProvider apiProvider,
PisstaubeDbContext dbContext,
PisstaubeCacheDbContextFactory cacheFactory,
SmartStorage smartStorage,
RequestLimiter limiter,
IBeatmapSearchEngineProvider search,
IpfsCache ipfsCache
)
{
_storage = storage;
_apiProvider = apiProvider;
_dbContext = dbContext;
_cacheFactory = cacheFactory;
_smartStorage = smartStorage;
_limiter = limiter;
_search = search;
_ipfsCache = ipfsCache;
}
protected virtual void ThreadWorker()
{
var dbContext = _dbContextPool.Rent();
try {
LatestId = dbContext.BeatmapSet
.OrderByDescending(bs => bs.SetId)
.Take(1)
.ToList()
.FirstOrDefault()?.SetId + 1 ?? 1;
while (!CancellationToken.IsCancellationRequested)
{
if (_errorCount > 1024)
{
LatestId = dbContext.BeatmapSet
.OrderByDescending(bs => bs.SetId)
.Take(1)
.ToList()
.FirstOrDefault()?.SetId + 1 ?? 1;
_dbContextPool.Return(dbContext);
Logger.LogPrint("Error count too high! will continue tomorrow...", LoggingTarget.Network, LogLevel.Important);
Thread.Sleep(TimeSpan.FromDays(1));
dbContext = _dbContextPool.Rent();
}
if (Tasks.Count > 32)
{
Task.WaitAll(Tasks.ToArray(), CancellationToken); // wait for all tasks
Tasks.Clear(); // Remove all previous tasks.
}
RequestLimiter.Limit();
Tasks.Add(Crawl(LatestId++));
}
} finally {
_dbContextPool.Return(dbContext);
}
}
