private void _crawl()
{
lock (_lock)
if (LatestId == 0)
{
LatestId = _contextFactory.Get().BeatmapSet.LastOrDefault()?.SetId + 1 ?? 0;
}
while (!_should_stop)
{
int id;
lock (_lock)
id = LatestId++;
using (var db = _contextFactory.GetForWrite()) {
if (!Crawl(id, db.Context))
{
_fail_count++;
}
else
{
_fail_count = 0;
}
}
if (_fail_count > 50) // We failed 50 times, lets try tomorrow again! maybe there are new exciting beatmaps!
{
_should_stop = true;
}
}
}
private void _crawl()
{
lock (_lock)
if (LatestId == 0)
{
LatestId = _contextFactory.Get().BeatmapSet.LastOrDefault()?.SetId + 1 ?? 0;
}
while (!_should_stop)
{
int id;
lock (_lock)
id = LatestId++;
using (var db = _contextFactory.GetForWrite())
{
if (!Crawl(id, db.Context))
{
_fail_count++;
}
else
{
_fail_count = 0;
}
}
if (_fail_count > 1000)
{
_should_stop = true;
}
}
}
public ActionResult DumpDatabase(string key)
{
if (key != Environment.GetEnvironmentVariable("PRIVATE_API_KEY"))
{
return(Unauthorized("Key is wrong!"));
}
lock (_lock) {
var tmpStorage = _storage.GetStorageForDirectory("tmp");
if (tmpStorage.Exists("dump.piss"))
{
System.IO.File.Delete(tmpStorage.GetFullPath("dump.piss"));
}
using (var dumpStream = tmpStorage.GetStream("dump.piss", FileAccess.Write))
using (var sw = new MStreamWriter(dumpStream))
{
sw.Write(_contextFactory.Get().BeatmapSet.Count());
foreach (var bmSet in _contextFactory.Get().BeatmapSet)
{
bmSet.ChildrenBeatmaps = _contextFactory.Get().Beatmaps.Where(bm => bm.ParentSetId == bmSet.SetId).ToList();
sw.Write(bmSet);
}
}
return(File(tmpStorage.GetStream("dump.piss"),
"application/octet-stream",
"dump.piss"));
}
}
public ActionResult GetBeatmap(int beatmapId)
{
var hash = _cache.Get().CacheBeatmaps.Where(bm => bm.BeatmapId == beatmapId).Select(bm => bm.Hash).FirstOrDefault();
if (hash == null)
{
foreach (var map in _contextFactory.Get().Beatmaps.Where(bm => bm.BeatmapId == beatmapId))
{
var fileInfo = _downloader.Download(map);
map.FileMd5 = _cache.Get()
.CacheBeatmaps
.Where(cmap => cmap.Hash == fileInfo.Hash)
.Select(cmap => cmap.FileMd5)
.FirstOrDefault();
}
}
var info = new osu.Game.IO.FileInfo {
Hash = hash
};
return(File(_fileStorage.GetStream(info.StoragePath), "application/octet-stream", hash));
}
public void BeginUpdaterSync()
{
while (true)
{
var beatmapSets = _factory.Get().BeatmapSet.Where(x => !x.Disabled)
.Where(
x => x.LastChecked != null && ((
(
x.RankedStatus == BeatmapSetOnlineStatus.None ||
x.RankedStatus == BeatmapSetOnlineStatus.Graveyard ||
x.RankedStatus == BeatmapSetOnlineStatus.Pending ||
x.RankedStatus == BeatmapSetOnlineStatus.Ranked ||
x.RankedStatus == BeatmapSetOnlineStatus.Loved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(30))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Qualified ||
x.RankedStatus == BeatmapSetOnlineStatus.WIP
) &&
(x.LastChecked.Value + TimeSpan.FromDays(1))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Approved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(90))
.Subtract(DateTime.Now).TotalMilliseconds < 0
))
);
foreach (var bmSet in beatmapSets.ToList())
{
bmSet.ChildrenBeatmaps = _factory.Get().Beatmaps.Where(x => x.ParentSetId == bmSet.SetId).ToList();
var setRequest = new GetBeatmapSetRequest(bmSet.SetId);
_rl.Limit();
setRequest.Perform(_apiAccess);
Logger.LogPrint("Updating BeatmapSetId " + bmSet.SetId);
var setInfo = setRequest.Result.ToBeatmapSet(_store);
var newBm = BeatmapSet.FromBeatmapSetInfo(setInfo);
using var db = _factory.GetForWrite();
var hasChanged = false;
foreach (var cb in newBm.ChildrenBeatmaps)
{
var fInfo = _bmDl.Download(cb);
var ha = _cFactory.Get().CacheBeatmaps.Where(b => b.Hash == fInfo.Hash).Select(f => f.FileMd5).FirstOrDefault();
cb.FileMd5 = ha;
db.Context.Entry(cb).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.Beatmaps.Update(cb);
if (bmSet.ChildrenBeatmaps.Any(x => x.FileMd5 == ha))
{
continue;
}
hasChanged = true;
}
if (newBm.ChildrenBeatmaps.Count > bmSet.ChildrenBeatmaps.Count)
{
hasChanged = true;
}
var bmFileId = newBm.SetId.ToString("x8");
var bmFileIdNoVid = newBm.SetId.ToString("x8") + "_novid";
if (hasChanged)
{
_storage.GetStorageForDirectory("cache").Delete(bmFileId);
_storage.GetStorageForDirectory("cache").Delete(bmFileIdNoVid);
_search.DeleteBeatmap(newBm.SetId);
_search.IndexBeatmap(newBm);
}
db.Context.Entry(newBm).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.BeatmapSet.Update(newBm);
FileSafety.DeleteCleanupDirectory();
}
}
}
public Tuple <string, Stream> DownloadMap(int beatmapSetId, bool dlVideo = false)
{
if (_apiAccess.State == APIState.Offline)
{
Logger.Error(new NotSupportedException("API is not Authenticated!"),
"API is not Authenticated! check your Login Details!",
LoggingTarget.Network);
throw new UnauthorizedAccessException("API Is not Authorized!");
}
if (!_storage.ExistsDirectory("cache"))
{
_storage.GetFullPath("cache", true);
}
BeatmapSet set;
if ((set = _factory.Get().BeatmapSet
.FirstOrDefault(bmSet => bmSet.SetId == beatmapSetId && !bmSet.Disabled)) == null)
{
throw new LegacyScoreParser.BeatmapNotFoundException();
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
var bmFileId = beatmapSetId.ToString("x8") + (dlVideo ? "" : "_novid");
CacheBeatmapSet cachedMap;
if (!cacheStorage.Exists(bmFileId))
{
if (!_cleaner.FreeStorage())
{
Logger.Error(new Exception("Cache Storage is full!"),
"Please change the Cleaner Settings!",
LoggingTarget.Database);
throw new IOException("Storage Full!");
}
try
{
var req = new DownloadBeatmapSetRequest(new BeatmapSetInfo {
OnlineBeatmapSetID = beatmapSetId
}, !dlVideo);
// Video download is not supported, to much traffic. almost no one download videos anyways!
var tmpFile = string.Empty;
req.Success += c => tmpFile = c;
_limiter.Limit();
req.Perform(_apiAccess);
using (var f = cacheStorage.GetStream(bmFileId, FileAccess.Write))
using (var readStream = File.OpenRead(tmpFile))
readStream.CopyTo(f);
_cleaner.IncreaseSize(new FileInfo(tmpFile).Length);
File.Delete(tmpFile);
using var db = _cfactory.GetForWrite();
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
} catch (ObjectDisposedException)
{
// Cannot access a closed file.
// Beatmap got DMCA'd
_search.DeleteBeatmap(beatmapSetId);
using (var db = _factory.GetForWrite())
{
set.Disabled = true;
db.Context.BeatmapSet.Update(set);
}
throw new NotSupportedException("Beatmap got DMCA'd");
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)));
}
using (var db = _cfactory.GetForWrite())
{
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)
));
}