public FileInfo Download(ChildrenBeatmap beatmap)
{
var req = new FileWebRequest(
_tmpStorage.GetFullPath(beatmap.BeatmapId.ToString(), true),
$"https://osu.ppy.sh/osu/{beatmap.BeatmapId}");
_limiter.Limit();
req.Perform();
FileInfo info;
using (var f = _tmpStorage.GetStream(beatmap.BeatmapId.ToString(), FileAccess.Read, FileMode.Open))
using (var db = _cache.GetForWrite())
{
info = _store.Add(f);
if (db.Context.CacheBeatmaps.Any(bm => bm.BeatmapId == beatmap.BeatmapId))
{
db.Context.CacheBeatmaps.Update(new Beatmap {
BeatmapId = beatmap.BeatmapId, Hash = info.Hash, FileMd5 = f.ComputeMD5Hash()
});
}
else
{
db.Context.CacheBeatmaps.Add(new Beatmap {
BeatmapId = beatmap.BeatmapId, Hash = info.Hash, FileMd5 = f.ComputeMD5Hash()
});
}
}
_tmpStorage.Delete(beatmap.BeatmapId.ToString());
return(info);
}
// ReSharper disable once UnusedParameter.Local
public Startup(IConfiguration configuration)
{
dataStorage = new NativeStorage("data");
osuContextFactory = new DatabaseContextFactory(dataStorage);
cacheContextFactory = new PisstaubeCacheDbContextFactory(dataStorage);
dbContextFactory = new PisstaubeDbContextFactory();
// copy paste of OsuGameBase.cs
try
{
using (var db = dbContextFactory.GetForWrite(false))
db.Context.Database.Migrate();
using (var db = osuContextFactory.GetForWrite(false))
db.Context.Migrate();
}
catch (Exception e)
{
Logger.Error(e.InnerException ?? e, "Migration failed! We'll be starting with a fresh database.", LoggingTarget.Database);
osuContextFactory.ResetDatabase();
Logger.Log("Database purged successfully.", LoggingTarget.Database);
using (var db = osuContextFactory.GetForWrite(false))
db.Context.Migrate();
}
// copy paste of OsuGameBase.cs
try
{
using (var db = cacheContextFactory.GetForWrite(false))
db.Context.Migrate();
}
catch (Exception e)
{
Logger.Error(e.InnerException ?? e, "Migration failed! We'll be starting with a fresh database.", LoggingTarget.Database);
cacheContextFactory.ResetDatabase();
Logger.Log("Database purged successfully.", LoggingTarget.Database);
using (var db = cacheContextFactory.GetForWrite(false))
db.Context.Migrate();
}
}
public bool FreeStorage()
{
for (var i = 0; i < 1000; i++)
{
Logger.LogPrint($"FreeStorage (DirectorySize: {DataDirectorySize} MaxSize: {_maxSize})");
DogStatsd.Set("cleaner.storage_usage", (ulong)DataDirectorySize / _maxSize);
if (IsFitting(0))
{
return(true);
}
Logger.LogPrint("Freeing Storage");
using (var db = _cache.GetForWrite())
{
var map = db.Context.CacheBeatmapSet.FirstOrDefault(cbs => (cbs.LastDownload - DateTime.Now).TotalDays < 7);
if (map != null)
{
db.Context.CacheBeatmapSet.Remove(map);
db.Context.SaveChanges();
if (!_cacheStorage.Exists(map.SetId.ToString("x8")))
{
continue;
}
DataDirectorySize -= new FileInfo(_cacheStorage.GetFullPath(map.SetId.ToString("x8"))).Length;
if (DataDirectorySize < 0)
{
DataDirectorySize = 0;
}
_cacheStorage.Delete(map.SetId.ToString("x8"));
db.Context.SaveChanges();
}
else
{
map = db.Context.CacheBeatmapSet.OrderByDescending(cbs => cbs.LastDownload)
.ThenByDescending(cbs => cbs.DownloadCount).FirstOrDefault();
if (map == null)
{
continue;
}
db.Context.CacheBeatmapSet.Remove(map);
db.Context.SaveChanges();
if (!_cacheStorage.Exists(map.SetId.ToString("x8")))
{
continue;
}
DataDirectorySize -= new FileInfo(_cacheStorage.GetFullPath(map.SetId.ToString("x8"))).Length;
if (DataDirectorySize < 0)
{
DataDirectorySize = 0;
}
_cacheStorage.Delete(map.SetId.ToString("x8"));
db.Context.SaveChanges();
}
}
}
return(false); // Failed to FreeStorage!
}
public Tuple <string, Stream> DownloadMap(int beatmapSetId, bool dlVideo = false)
{
if (_apiAccess.State == APIState.Offline)
{
Logger.Error(new NotSupportedException("API is not Authenticated!"),
"API is not Authenticated! check your Login Details!",
LoggingTarget.Network);
throw new UnauthorizedAccessException("API Is not Authorized!");
}
if (!_storage.ExistsDirectory("cache"))
{
_storage.GetFullPath("cache", true);
}
BeatmapSet set;
if ((set = _factory.Get().BeatmapSet
.FirstOrDefault(bmSet => bmSet.SetId == beatmapSetId && !bmSet.Disabled)) == null)
{
throw new LegacyScoreParser.BeatmapNotFoundException();
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
var bmFileId = beatmapSetId.ToString("x8") + (dlVideo ? "" : "_novid");
CacheBeatmapSet cachedMap;
if (!cacheStorage.Exists(bmFileId))
{
if (!_cleaner.FreeStorage())
{
Logger.Error(new Exception("Cache Storage is full!"),
"Please change the Cleaner Settings!",
LoggingTarget.Database);
throw new IOException("Storage Full!");
}
try
{
var req = new DownloadBeatmapSetRequest(new BeatmapSetInfo {
OnlineBeatmapSetID = beatmapSetId
}, !dlVideo);
// Video download is not supported, to much traffic. almost no one download videos anyways!
var tmpFile = string.Empty;
req.Success += c => tmpFile = c;
_limiter.Limit();
req.Perform(_apiAccess);
using (var f = cacheStorage.GetStream(bmFileId, FileAccess.Write))
using (var readStream = File.OpenRead(tmpFile))
readStream.CopyTo(f);
_cleaner.IncreaseSize(new FileInfo(tmpFile).Length);
File.Delete(tmpFile);
using var db = _cfactory.GetForWrite();
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
} catch (ObjectDisposedException)
{
// Cannot access a closed file.
// Beatmap got DMCA'd
_search.DeleteBeatmap(beatmapSetId);
using (var db = _factory.GetForWrite())
{
set.Disabled = true;
db.Context.BeatmapSet.Update(set);
}
throw new NotSupportedException("Beatmap got DMCA'd");
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)));
}
using (var db = _cfactory.GetForWrite())
{
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)
));
}
public ActionResult Recovery(
[FromQuery] string key,
[FromQuery] RecoveryAction action
)
{
if (key != Environment.GetEnvironmentVariable("PRIVATE_API_KEY"))
{
return(Unauthorized("Key is wrong!"));
}
switch (action)
{
case RecoveryAction.RepairElastic:
Logger.LogPrint("Repairing ElasticSearch");
_crawler.Stop();
_reibe.Stop();
_searchEngine.DeleteAllBeatmaps();
foreach (var beatmapSet in _contextFactory.Get().BeatmapSet)
{
beatmapSet.ChildrenBeatmaps = _contextFactory.Get().Beatmaps.Where(b => b.ParentSetId == beatmapSet.SetId).ToList();
_searchEngine.IndexBeatmap(beatmapSet);
}
if (Environment.GetEnvironmentVariable("CRAWLER_DISABLED") != "true")
{
_crawler.BeginCrawling();
}
if (Environment.GetEnvironmentVariable("CHEESEGULL_CRAWLER_DISABLED") != "true")
{
_reibe.BeginCrawling();
}
break;
case RecoveryAction.RecrawlEverything:
Logger.LogPrint("Recrawl Everything!");
_crawler.Stop();
_reibe.Stop();
_searchEngine.DeleteAllBeatmaps();
using (var db = _contextFactory.GetForWrite()) {
db.Context.Database.ExecuteSqlCommand("SET FOREIGN_KEY_CHECKS = 0;" +
"TRUNCATE TABLE `Beatmaps`;" +
"ALTER TABLE `Beatmaps` AUTO_INCREMENT = 1;" +
"TRUNCATE TABLE `BeatmapSet`;" +
"ALTER TABLE `BeatmapSet` AUTO_INCREMENT = 1;" +
"SET FOREIGN_KEY_CHECKS = 1;");
}
using (var cacheDb = _cache.GetForWrite())
{
cacheDb.Context.Database.ExecuteSqlCommand(
"DELETE FROM `CacheBeatmaps`;" +
"DELETE FROM `CacheBeatmapSet`;");
}
if (Environment.GetEnvironmentVariable("CRAWLER_DISABLED") != "true")
{
_crawler.BeginCrawling();
}
if (Environment.GetEnvironmentVariable("CHEESEGULL_CRAWLER_DISABLED") != "true")
{
_reibe.BeginCrawling();
}
break;
case RecoveryAction.RecrawlUnknown:
Logger.LogPrint("Recrawl All unknown maps!");
_crawler.Stop();
using (var db = _contextFactory.GetForWrite()) {
for (var i = 0; i < db.Context.BeatmapSet.Last().SetId; i++)
{
if (!db.Context.BeatmapSet.Any(set => set.SetId == i))
{
_crawler.Crawl(i, db.Context);
}
}
}
_crawler.BeginCrawling();
break;
default:
return(BadRequest("Unknown Action type!"));
}
return(Ok("Success!"));
}
public DownloadMapResponse DownloadMap(int beatmapSetId, bool dlVideo = false, bool ipfs = false)
{
if (_apiProvider.State == APIState.Offline)
{
Logger.Error(new NotSupportedException("API is not Authenticated!"),
"API is not Authenticated! check your Login Details!",
LoggingTarget.Network);
throw new UnauthorizedAccessException("API Is not Authorized!");
}
if (!_storage.ExistsDirectory("cache"))
{
_storage.GetFullPath("cache", true);
}
BeatmapSet set;
lock (_dbContextMutes)
{
if ((set = _dbContext.BeatmapSet
.FirstOrDefault(bmSet => bmSet.SetId == beatmapSetId && !bmSet.Disabled)) == null)
{
throw new LegacyScoreDecoder.BeatmapNotFoundException();
}
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
var bmFileId = beatmapSetId.ToString("x8") + (dlVideo ? "" : "_novid");
CacheBeatmapSet cachedMap;
if (!cacheStorage.Exists(bmFileId))
{
if (!_smartStorage.FreeStorage())
{
Logger.Error(new Exception("Cache Storage is full!"),
"Please change the Cleaner Settings!",
LoggingTarget.Database);
throw new IOException("Storage Full!");
}
var req = new DownloadBeatmapSetRequest(new BeatmapSetInfo {
OnlineBeatmapSetID = beatmapSetId
},
!dlVideo);
var tmpFile = string.Empty;
req.Success += c => tmpFile = c;
_limiter.Limit();
req.Perform(_apiProvider);
using (var f = cacheStorage.GetStream(bmFileId, FileAccess.Write))
{
using var readStream = File.OpenRead(tmpFile);
readStream.CopyTo(f);
}
_smartStorage.IncreaseSize(new FileInfo(tmpFile).Length);
File.Delete(tmpFile);
using var db = _cacheFactory.GetForWrite();
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) ==
null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
var cac = _ipfsCache.CacheFile("cache/" + bmFileId);
return(new DownloadMapResponse {
File = $"{set.SetId} {set.Artist} - {set.Title}.osz",
FileStream = !ipfs || cac.Result == "" ? cacheStorage.GetStream(bmFileId, FileAccess.Read, FileMode.Open) : null, // Don't even bother opening a stream.
IpfsHash = cac.Result,
});
}
using (var db = _cacheFactory.GetForWrite())
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
var cache = _ipfsCache.CacheFile("cache/" + bmFileId);
return(new DownloadMapResponse {
File = $"{set.SetId} {set.Artist} - {set.Title}.osz",
FileStream = !ipfs || cache.Result == "" ? cacheStorage.GetStream(bmFileId, FileAccess.Read, FileMode.Open) : null, // Don't even bother opening a stream.
IpfsHash = cache.Result,
});
}