public void BeginUpdaterSync()
{
while (true)
{
var beatmapSets = _factory.Get().BeatmapSet.Where(x => !x.Disabled)
.Where(
x => x.LastChecked != null && ((
(
x.RankedStatus == BeatmapSetOnlineStatus.None ||
x.RankedStatus == BeatmapSetOnlineStatus.Graveyard ||
x.RankedStatus == BeatmapSetOnlineStatus.Pending ||
x.RankedStatus == BeatmapSetOnlineStatus.Ranked ||
x.RankedStatus == BeatmapSetOnlineStatus.Loved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(30))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Qualified ||
x.RankedStatus == BeatmapSetOnlineStatus.WIP
) &&
(x.LastChecked.Value + TimeSpan.FromDays(1))
.Subtract(DateTime.Now).TotalMilliseconds < 0
) ||
(
(
x.RankedStatus == BeatmapSetOnlineStatus.Approved
) &&
(x.LastChecked.Value + TimeSpan.FromDays(90))
.Subtract(DateTime.Now).TotalMilliseconds < 0
))
);
foreach (var bmSet in beatmapSets.ToList())
{
bmSet.ChildrenBeatmaps = _factory.Get().Beatmaps.Where(x => x.ParentSetId == bmSet.SetId).ToList();
var setRequest = new GetBeatmapSetRequest(bmSet.SetId);
_rl.Limit();
setRequest.Perform(_apiAccess);
Logger.LogPrint("Updating BeatmapSetId " + bmSet.SetId);
var setInfo = setRequest.Result.ToBeatmapSet(_store);
var newBm = BeatmapSet.FromBeatmapSetInfo(setInfo);
using var db = _factory.GetForWrite();
var hasChanged = false;
foreach (var cb in newBm.ChildrenBeatmaps)
{
var fInfo = _bmDl.Download(cb);
var ha = _cFactory.Get().CacheBeatmaps.Where(b => b.Hash == fInfo.Hash).Select(f => f.FileMd5).FirstOrDefault();
cb.FileMd5 = ha;
db.Context.Entry(cb).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.Beatmaps.Update(cb);
if (bmSet.ChildrenBeatmaps.Any(x => x.FileMd5 == ha))
{
continue;
}
hasChanged = true;
}
if (newBm.ChildrenBeatmaps.Count > bmSet.ChildrenBeatmaps.Count)
{
hasChanged = true;
}
var bmFileId = newBm.SetId.ToString("x8");
var bmFileIdNoVid = newBm.SetId.ToString("x8") + "_novid";
if (hasChanged)
{
_storage.GetStorageForDirectory("cache").Delete(bmFileId);
_storage.GetStorageForDirectory("cache").Delete(bmFileIdNoVid);
_search.DeleteBeatmap(newBm.SetId);
_search.IndexBeatmap(newBm);
}
db.Context.Entry(newBm).State = Microsoft.EntityFrameworkCore.EntityState.Modified;
db.Context.BeatmapSet.Update(newBm);
FileSafety.DeleteCleanupDirectory();
}
}
}
public virtual async Task <bool> Crawl(int id)
{
var dbContext = _dbContextPool.Rent();
Logger.LogPrint($"Crawling BeatmapId {id}...", LoggingTarget.Network, LogLevel.Debug);
try
{
var beatmapSetRequest = new GetBeatmapSetRequest(id);
beatmapSetRequest.Perform(ApiProvider);
var beatmapSetInfo = beatmapSetRequest.Result;
if (beatmapSetInfo == null)
{
_errorCount++;
return(false);
}
var beatmapSet = BeatmapSet.FromBeatmapSetInfo(beatmapSetInfo.ToBeatmapSet());
if (beatmapSet == null)
{
_errorCount++;
return(false);
}
var cacheStorage = _storage.GetStorageForDirectory("cache");
if (cacheStorage.Exists(beatmapSet.SetId.ToString("x8") + "_novid"))
{
cacheStorage.Delete(beatmapSet.SetId.ToString("x8") + "_novid");
}
if (cacheStorage.Exists(beatmapSet.SetId.ToString("x8")))
{
cacheStorage.Delete(beatmapSet.SetId.ToString("x8"));
}
foreach (var childrenBeatmap in beatmapSet.ChildrenBeatmaps)
{
var fileInfo = _beatmapDownloader.Download(childrenBeatmap);
childrenBeatmap.FileMd5 = fileInfo.Item2;
}
dbContext.BeatmapSet.AddOrUpdate(beatmapSet);
await dbContext.SaveChangesAsync();
SearchEngine.Index(new [] { beatmapSet });
_errorCount = 0;
return(true);
}
catch (WebException) // Don't worry about WebException exceptions, we can safely ignore those.
{
_errorCount++;
return(false);
}
catch (Exception e) // Everything else, redo the Crawl.
{
Logger.Error(e, "Unknown error during crawling occured!");
SentrySdk.CaptureException(e);
if (_errorCount > 1024)
{
Logger.LogPrint("Error count too high! canceling crawl...", LoggingTarget.Network, LogLevel.Important);
return(false);
}
_errorCount++;
return(await Crawl(id));
} finally {
_dbContextPool.Return(dbContext);
}
}
public bool Crawl(int id, PisstaubeDbContext _context)
{
try
{
while (!_apiAccess.IsLoggedIn)
{
Thread.Sleep(1000);
}
var setRequest = new GetBeatmapSetRequest(id);
lock (_lock)
_request_count++;
_rl.Limit();
setRequest.Perform(_apiAccess);
lock (_lock)
if (_request_count > int.Parse(Environment.GetEnvironmentVariable("CRAWLER_REQUESTS_PER_MINUTE")))
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
var apiSet = setRequest.Result;
var localSet = apiSet?.ToBeatmapSet(_store);
if (localSet == null)
{
return(false);
}
var dbSet = BeatmapSet.FromBeatmapSetInfo(localSet);
if (dbSet == null)
{
return(false);
}
foreach (var map in dbSet.ChildrenBeatmaps)
{
var fileInfo = _downloader.Download(map);
map.FileMd5 = _cache.Get()
.CacheBeatmaps
.Where(cmap => cmap.Hash == fileInfo.Hash)
.Select(cmap => cmap.FileMd5)
.FirstOrDefault();
}
lock (_lock)
{
_context.BeatmapSet.Add(dbSet);
}
_search.IndexBeatmap(dbSet);
}
catch (Exception ex)
{
Logger.Error(ex, $"Unknown Error occured while crawling Id {id}!");
lock (_lock)
Thread.Sleep(TimeSpan.FromMinutes(1));
return(false);
}
return(true);
}