public ActionResult GetBeatmap(int beatmapId)
{
DogStatsd.Increment("osu.beatmap.download");
var hash = _cache.Get()
.CacheBeatmaps.Where(bm => bm.BeatmapId == beatmapId)
.Select(bm => bm.Hash)
.FirstOrDefault();
osu.Game.IO.FileInfo info = null;
if (hash == null)
{
lock (_dbContextLock)
{
foreach (var map in _dbContext.Beatmaps.Where(bm => bm.BeatmapId == beatmapId))
{
var(fileInfo, fileMd5) = _downloader.Download(map);
map.FileMd5 = fileMd5;
info = fileInfo;
}
}
}
else
{
info = new osu.Game.IO.FileInfo {
Hash = hash
}
};
if (info == null)
{
return(NotFound("Beatmap not Found!"));
}
return(File(_fileStorage.GetStream(info.StoragePath), "application/octet-stream", hash));
}
public ActionResult GetBeatmap(int beatmapId)
{
var hash = _cache.Get().CacheBeatmaps.Where(bm => bm.BeatmapId == beatmapId).Select(bm => bm.Hash).FirstOrDefault();
if (hash == null)
{
foreach (var map in _contextFactory.Get().Beatmaps.Where(bm => bm.BeatmapId == beatmapId))
{
var fileInfo = _downloader.Download(map);
map.FileMd5 = _cache.Get()
.CacheBeatmaps
.Where(cmap => cmap.Hash == fileInfo.Hash)
.Select(cmap => cmap.FileMd5)
.FirstOrDefault();
}
}
var info = new osu.Game.IO.FileInfo {
Hash = hash
};
return(File(_fileStorage.GetStream(info.StoragePath), "application/octet-stream", hash));
}
public virtual async Task <bool> Crawl(int id)
{
var dbContext = _dbContextPool.Rent();
Logger.LogPrint($"Crawling BeatmapId {id}...", LoggingTarget.Network, LogLevel.Debug);
try
{
var beatmapSetRequest = new GetBeatmapSetRequest(id);
beatmapSetRequest.Perform(ApiProvider);
var beatmapSetInfo = beatmapSetRequest.Result;
if (beatmapSetInfo == null)
{
_errorCount++;
return(false);
}
var beatmapSet = BeatmapSet.FromBeatmapSetInfo(beatmapSetInfo.ToBeatmapSet());
if (beatmapSet == null)
{
_errorCount++;
return(false);
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
if (cacheStorage.Exists(beatmapSet.SetId.ToString("x8") + "_novid"))
{
cacheStorage.Delete(beatmapSet.SetId.ToString("x8") + "_novid");
}
if (cacheStorage.Exists(beatmapSet.SetId.ToString("x8")))
{
cacheStorage.Delete(beatmapSet.SetId.ToString("x8"));
}
foreach (var childrenBeatmap in beatmapSet.ChildrenBeatmaps)
{
var fileInfo = _beatmapDownloader.Download(childrenBeatmap);
childrenBeatmap.FileMd5 = fileInfo.Item2;
}
dbContext.BeatmapSet.AddOrUpdate(beatmapSet);
await dbContext.SaveChangesAsync();
SearchEngine.Index(new [] { beatmapSet });
_errorCount = 0;
return(true);
}
catch (WebException) // Don't worry about WebException exceptions, we can safely ignore those.
{
_errorCount++;
return(false);
}
catch (Exception e) // Everything else, redo the Crawl.
{
Logger.Error(e, "Unknown error during crawling occured!");
SentrySdk.CaptureException(e);
if (_errorCount > 1024)
{
Logger.LogPrint("Error count too high! canceling crawl...", LoggingTarget.Network, LogLevel.Important);
return(false);
}
_errorCount++;
return(await Crawl(id));
} finally {
_dbContextPool.Return(dbContext);
}
}
public bool Crawl(int id, PisstaubeDbContext _context)
{
try
{
while (!_apiAccess.IsLoggedIn)
{
Thread.Sleep(1000);
}
var setRequest = new GetBeatmapSetRequest(id);
lock (_lock)
_request_count++;
_rl.Limit();
setRequest.Perform(_apiAccess);
lock (_lock)
if (_request_count > int.Parse(Environment.GetEnvironmentVariable("CRAWLER_REQUESTS_PER_MINUTE")))
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
var apiSet = setRequest.Result;
var localSet = apiSet?.ToBeatmapSet(_store);
if (localSet == null)
{
return(false);
}
var dbSet = BeatmapSet.FromBeatmapSetInfo(localSet);
if (dbSet == null)
{
return(false);
}
foreach (var map in dbSet.ChildrenBeatmaps)
{
var fileInfo = _downloader.Download(map);
map.FileMd5 = _cache.Get()
.CacheBeatmaps
.Where(cmap => cmap.Hash == fileInfo.Hash)
.Select(cmap => cmap.FileMd5)
.FirstOrDefault();
}
lock (_lock)
{
_context.BeatmapSet.Add(dbSet);
}
_search.IndexBeatmap(dbSet);
}
catch (Exception ex)
{
Logger.Error(ex, $"Unknown Error occured while crawling Id {id}!");
lock (_lock)
Thread.Sleep(TimeSpan.FromMinutes(1));
return(false);
}
return(true);
}