public FileInfo Download(ChildrenBeatmap beatmap)
{
var req = new FileWebRequest(
_tmpStorage.GetFullPath(beatmap.BeatmapId.ToString(), true),
$"https://osu.ppy.sh/osu/{beatmap.BeatmapId}");
_limiter.Limit();
req.Perform();
FileInfo info;
using (var f = _tmpStorage.GetStream(beatmap.BeatmapId.ToString(), FileAccess.Read, FileMode.Open))
using (var db = _cache.GetForWrite())
{
info = _store.Add(f);
if (db.Context.CacheBeatmaps.Any(bm => bm.BeatmapId == beatmap.BeatmapId))
{
db.Context.CacheBeatmaps.Update(new Beatmap {
BeatmapId = beatmap.BeatmapId, Hash = info.Hash, FileMd5 = f.ComputeMD5Hash()
});
}
else
{
db.Context.CacheBeatmaps.Add(new Beatmap {
BeatmapId = beatmap.BeatmapId, Hash = info.Hash, FileMd5 = f.ComputeMD5Hash()
});
}
}
_tmpStorage.Delete(beatmap.BeatmapId.ToString());
return(info);
}
protected virtual void ThreadWorker()
{
var dbContext = _dbContextPool.Rent();
try {
LatestId = dbContext.BeatmapSet
.OrderByDescending(bs => bs.SetId)
.Take(1)
.ToList()
.FirstOrDefault()?.SetId + 1 ?? 1;
while (!CancellationToken.IsCancellationRequested)
{
if (_errorCount > 1024)
{
LatestId = dbContext.BeatmapSet
.OrderByDescending(bs => bs.SetId)
.Take(1)
.ToList()
.FirstOrDefault()?.SetId + 1 ?? 1;
_dbContextPool.Return(dbContext);
Logger.LogPrint("Error count too high! will continue tomorrow...", LoggingTarget.Network, LogLevel.Important);
Thread.Sleep(TimeSpan.FromDays(1));
dbContext = _dbContextPool.Rent();
}
if (Tasks.Count > 32)
{
Task.WaitAll(Tasks.ToArray(), CancellationToken); // wait for all tasks
Tasks.Clear(); // Remove all previous tasks.
}
RequestLimiter.Limit();
Tasks.Add(Crawl(LatestId++));
}
} finally {
_dbContextPool.Return(dbContext);
}
}
protected override void ThreadWorker()
{
while (!CancellationToken.IsCancellationRequested)
{
var dbContext = _dbContextPool.Rent();
try
{
var beatmaps = dbContext.BeatmapSet
.Where(x => !x.Disabled)
.AsEnumerable()
.Where(x => x.LastChecked != null)
.Where(
x => (
(
(
x.RankedStatus == BeatmapSetOnlineStatus.None ||
x.RankedStatus == BeatmapSetOnlineStatus.Loved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(90)).Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Pending ||
x.RankedStatus == BeatmapSetOnlineStatus.Qualified ||
x.RankedStatus == BeatmapSetOnlineStatus.WIP
) &&
(x.LastChecked.Value + TimeSpan.FromDays(30)).Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Graveyard ||
x.RankedStatus == BeatmapSetOnlineStatus.Ranked ||
x.RankedStatus == BeatmapSetOnlineStatus.Approved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(365)).Subtract(DateTime.Now).TotalMilliseconds < 0
)
)
&& !x.Disabled
);
foreach (var beatmap in beatmaps)
{
if (Tasks.Count > 128)
{
Task.WaitAll(Tasks.ToArray(), CancellationToken); // wait for all tasks
Tasks.Clear(); // Remove all previous tasks.
}
_requestLimiter.Limit();
Tasks.Add(Crawl(beatmap.SetId));
}
}
catch (Exception e)
{
Logger.Error(e, "an Unknown error occured during HouseKeeping", LoggingTarget.Database);
SentrySdk.CaptureException(e);
} finally {
_dbContextPool.Return(dbContext);
}
Thread.Sleep(TimeSpan.FromHours(8)); // Update every 8 hours...
}
}
public void BeginUpdaterSync()
{
while (true)
{
var beatmapSets = _factory.Get().BeatmapSet.Where(x => !x.Disabled)
.Where(
x => x.LastChecked != null && ((
(
x.RankedStatus == BeatmapSetOnlineStatus.None ||
x.RankedStatus == BeatmapSetOnlineStatus.Graveyard ||
x.RankedStatus == BeatmapSetOnlineStatus.Pending ||
x.RankedStatus == BeatmapSetOnlineStatus.Ranked ||
x.RankedStatus == BeatmapSetOnlineStatus.Loved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(30))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Qualified ||
x.RankedStatus == BeatmapSetOnlineStatus.WIP
) &&
(x.LastChecked.Value + TimeSpan.FromDays(1))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Approved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(90))
.Subtract(DateTime.Now).TotalMilliseconds < 0
))
);
foreach (var bmSet in beatmapSets.ToList())
{
bmSet.ChildrenBeatmaps = _factory.Get().Beatmaps.Where(x => x.ParentSetId == bmSet.SetId).ToList();
var setRequest = new GetBeatmapSetRequest(bmSet.SetId);
_rl.Limit();
setRequest.Perform(_apiAccess);
Logger.LogPrint("Updating BeatmapSetId " + bmSet.SetId);
var setInfo = setRequest.Result.ToBeatmapSet(_store);
var newBm = BeatmapSet.FromBeatmapSetInfo(setInfo);
using var db = _factory.GetForWrite();
var hasChanged = false;
foreach (var cb in newBm.ChildrenBeatmaps)
{
var fInfo = _bmDl.Download(cb);
var ha = _cFactory.Get().CacheBeatmaps.Where(b => b.Hash == fInfo.Hash).Select(f => f.FileMd5).FirstOrDefault();
cb.FileMd5 = ha;
db.Context.Entry(cb).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.Beatmaps.Update(cb);
if (bmSet.ChildrenBeatmaps.Any(x => x.FileMd5 == ha))
{
continue;
}
hasChanged = true;
}
if (newBm.ChildrenBeatmaps.Count > bmSet.ChildrenBeatmaps.Count)
{
hasChanged = true;
}
var bmFileId = newBm.SetId.ToString("x8");
var bmFileIdNoVid = newBm.SetId.ToString("x8") + "_novid";
if (hasChanged)
{
_storage.GetStorageForDirectory("cache").Delete(bmFileId);
_storage.GetStorageForDirectory("cache").Delete(bmFileIdNoVid);
_search.DeleteBeatmap(newBm.SetId);
_search.IndexBeatmap(newBm);
}
db.Context.Entry(newBm).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.BeatmapSet.Update(newBm);
FileSafety.DeleteCleanupDirectory();
}
}
}
public Tuple <string, Stream> DownloadMap(int beatmapSetId, bool dlVideo = false)
{
if (_apiAccess.State == APIState.Offline)
{
Logger.Error(new NotSupportedException("API is not Authenticated!"),
"API is not Authenticated! check your Login Details!",
LoggingTarget.Network);
throw new UnauthorizedAccessException("API Is not Authorized!");
}
if (!_storage.ExistsDirectory("cache"))
{
_storage.GetFullPath("cache", true);
}
BeatmapSet set;
if ((set = _factory.Get().BeatmapSet
.FirstOrDefault(bmSet => bmSet.SetId == beatmapSetId && !bmSet.Disabled)) == null)
{
throw new LegacyScoreParser.BeatmapNotFoundException();
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
var bmFileId = beatmapSetId.ToString("x8") + (dlVideo ? "" : "_novid");
CacheBeatmapSet cachedMap;
if (!cacheStorage.Exists(bmFileId))
{
if (!_cleaner.FreeStorage())
{
Logger.Error(new Exception("Cache Storage is full!"),
"Please change the Cleaner Settings!",
LoggingTarget.Database);
throw new IOException("Storage Full!");
}
try
{
var req = new DownloadBeatmapSetRequest(new BeatmapSetInfo {
OnlineBeatmapSetID = beatmapSetId
}, !dlVideo);
// Video download is not supported, to much traffic. almost no one download videos anyways!
var tmpFile = string.Empty;
req.Success += c => tmpFile = c;
_limiter.Limit();
req.Perform(_apiAccess);
using (var f = cacheStorage.GetStream(bmFileId, FileAccess.Write))
using (var readStream = File.OpenRead(tmpFile))
readStream.CopyTo(f);
_cleaner.IncreaseSize(new FileInfo(tmpFile).Length);
File.Delete(tmpFile);
using var db = _cfactory.GetForWrite();
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
} catch (ObjectDisposedException)
{
// Cannot access a closed file.
// Beatmap got DMCA'd
_search.DeleteBeatmap(beatmapSetId);
using (var db = _factory.GetForWrite())
{
set.Disabled = true;
db.Context.BeatmapSet.Update(set);
}
throw new NotSupportedException("Beatmap got DMCA'd");
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)));
}
using (var db = _cfactory.GetForWrite())
{
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
}
DogStatsd.Increment("beatmap.downloads");
return(Tuple.Create(
$"{set.SetId} {set.Artist} - {set.Title}.osz",
cacheStorage.GetStream(bmFileId)
));
}
public DownloadMapResponse DownloadMap(int beatmapSetId, bool dlVideo = false, bool ipfs = false)
{
if (_apiProvider.State == APIState.Offline)
{
Logger.Error(new NotSupportedException("API is not Authenticated!"),
"API is not Authenticated! check your Login Details!",
LoggingTarget.Network);
throw new UnauthorizedAccessException("API Is not Authorized!");
}
if (!_storage.ExistsDirectory("cache"))
{
_storage.GetFullPath("cache", true);
}
BeatmapSet set;
lock (_dbContextMutes)
{
if ((set = _dbContext.BeatmapSet
.FirstOrDefault(bmSet => bmSet.SetId == beatmapSetId && !bmSet.Disabled)) == null)
{
throw new LegacyScoreDecoder.BeatmapNotFoundException();
}
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
var bmFileId = beatmapSetId.ToString("x8") + (dlVideo ? "" : "_novid");
CacheBeatmapSet cachedMap;
if (!cacheStorage.Exists(bmFileId))
{
if (!_smartStorage.FreeStorage())
{
Logger.Error(new Exception("Cache Storage is full!"),
"Please change the Cleaner Settings!",
LoggingTarget.Database);
throw new IOException("Storage Full!");
}
var req = new DownloadBeatmapSetRequest(new BeatmapSetInfo {
OnlineBeatmapSetID = beatmapSetId
},
!dlVideo);
var tmpFile = string.Empty;
req.Success += c => tmpFile = c;
_limiter.Limit();
req.Perform(_apiProvider);
using (var f = cacheStorage.GetStream(bmFileId, FileAccess.Write))
{
using var readStream = File.OpenRead(tmpFile);
readStream.CopyTo(f);
}
_smartStorage.IncreaseSize(new FileInfo(tmpFile).Length);
File.Delete(tmpFile);
using var db = _cacheFactory.GetForWrite();
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) ==
null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
var cac = _ipfsCache.CacheFile("cache/" + bmFileId);
return(new DownloadMapResponse {
File = $"{set.SetId} {set.Artist} - {set.Title}.osz",
FileStream = !ipfs || cac.Result == "" ? cacheStorage.GetStream(bmFileId, FileAccess.Read, FileMode.Open) : null, // Don't even bother opening a stream.
IpfsHash = cac.Result,
});
}
using (var db = _cacheFactory.GetForWrite())
if ((cachedMap = db.Context.CacheBeatmapSet.FirstOrDefault(cbm => cbm.SetId == set.SetId)) == null)
{
db.Context.CacheBeatmapSet.Add(new CacheBeatmapSet
{
SetId = set.SetId,
DownloadCount = 1,
LastDownload = DateTime.Now
});
}
else
{
cachedMap.DownloadCount++;
cachedMap.LastDownload = DateTime.Now;
db.Context.CacheBeatmapSet.Update(cachedMap);
}
var cache = _ipfsCache.CacheFile("cache/" + bmFileId);
return(new DownloadMapResponse {
File = $"{set.SetId} {set.Artist} - {set.Title}.osz",
FileStream = !ipfs || cache.Result == "" ? cacheStorage.GetStream(bmFileId, FileAccess.Read, FileMode.Open) : null, // Don't even bother opening a stream.
IpfsHash = cache.Result,
});
}