public BeatmapDownloader(FileStore store, Storage dataStorage, PisstaubeCacheDbContextFactory cache,
RequestLimiter limiter, PisstaubeDbContext dbContext)
{
_store = store;
_cache = cache;
_limiter = limiter;
_tmpStorage = dataStorage.GetStorageForDirectory("tmp");
_dbContext = dbContext;
}
public IndexController(IAPIProvider apiProvider,
Storage storage,
PisstaubeCacheDbContextFactory cache,
BeatmapDownloader downloader,
SetDownloader setDownloader,
PisstaubeDbContext dbContext)
{
_apiProvider = apiProvider;
_cache = cache;
_downloader = downloader;
_setDownloader = setDownloader;
_dbContext = dbContext;
_fileStorage = storage.GetStorageForDirectory("files");
}
public BeatmapSearchEngine(PisstaubeDbContext dbContext)
{
_dbContext = dbContext;
var settings = new ConnectionSettings(
new Uri(
$"http://{Environment.GetEnvironmentVariable("ELASTIC_HOSTNAME")}:{Environment.GetEnvironmentVariable("ELASTIC_PORT")}"
)
)
.DefaultMappingFor <ElasticBeatmap>(m => m
.IdProperty(p => p.Id)
)
.EnableHttpCompression()
.RequestTimeout(new TimeSpan(0, 10, 0))
.DefaultIndex("pisstaube");
_elasticClient = new ElasticClient(settings);
}
public SetDownloader(Storage storage,
IAPIProvider apiProvider,
PisstaubeDbContext dbContext,
PisstaubeCacheDbContextFactory cacheFactory,
SmartStorage smartStorage,
RequestLimiter limiter,
IBeatmapSearchEngineProvider search,
IpfsCache ipfsCache
)
{
_storage = storage;
_apiProvider = apiProvider;
_dbContext = dbContext;
_cacheFactory = cacheFactory;
_smartStorage = smartStorage;
_limiter = limiter;
_search = search;
_ipfsCache = ipfsCache;
}
public void Configure(IAPIProvider apiProvider,
PisstaubeCacheDbContextFactory cacheDbContextFactory, IBeatmapSearchEngineProvider searchEngine,
PisstaubeDbContext dbContext)
{
Logger.Enabled = true;
Logger.Level = LogLevel.Debug;
Logger.GameIdentifier = "Pisstaube";
Logger.Storage = _dataStorage.GetStorageForDirectory("logs");
dbContext.Database.Migrate();
while (!searchEngine.IsConnected)
{
Logger.LogPrint("Search Engine is not yet Connected!", LoggingTarget.Database, LogLevel.Important);
Thread.Sleep(1000);
}
cacheDbContextFactory.Get().Migrate();
_osuContextFactory.Get().Migrate();
JsonUtil.Initialize();
apiProvider.Login(Environment.GetEnvironmentVariable("OSU_USERNAME"),
Environment.GetEnvironmentVariable("OSU_PASSWORD"));
while (true)
{
if (!apiProvider.IsLoggedIn)
{
Logger.LogPrint("Not Logged in yet...");
Thread.Sleep(1000);
continue;
}
if (apiProvider.State == APIState.Failing)
{
Logger.LogPrint($"Failed to Login using Username {Environment.GetEnvironmentVariable("OSU_USERNAME")}", LoggingTarget.Network, LogLevel.Error);
Environment.Exit(1);
}
break;
}
}
public bool Crawl(int id, PisstaubeDbContext _context)
{
try
{
var setRequest = new JsonWebRequest <BeatmapSet>($"https://{Environment.GetEnvironmentVariable("CHEESEGULL_API")}/api/s/{id}");
try
{
setRequest.Perform();
}
catch
{
if (!setRequest.ResponseString.StartsWith("null"))
{
throw;
}
}
var apiSet = setRequest.ResponseObject;
if (apiSet == null)
{
return(false);
}
_searchEngine.IndexBeatmap(apiSet);
_context.BeatmapSet.Add(apiSet);
}
catch (Exception ex)
{
Logger.Error(ex, $"Unknown Error occured while crawling Id {id}!");
// lock (_lock)
// Thread.Sleep(TimeSpan.FromMinutes(1));
return(false);
}
return(true);
}
private static async Task Main(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder();
var host = Environment.GetEnvironmentVariable("MARIADB_HOST");
var port = Environment.GetEnvironmentVariable("MARIADB_PORT");
var username = Environment.GetEnvironmentVariable("MARIADB_USERNAME");
var password = Environment.GetEnvironmentVariable("MARIADB_PASSWORD");
var db = Environment.GetEnvironmentVariable("MARIADB_DATABASE");
optionsBuilder.UseMySql(
$"Server={host};Database={db};User={username};Password={password};Port={port};CharSet=utf8mb4;SslMode=none;",
mysqlOptions =>
{
mysqlOptions.ServerVersion(new Version(10, 4, 12), ServerType.MariaDb);
mysqlOptions.CharSet(CharSet.Utf8Mb4);
}
);
_dbContext = new PisstaubeDbContext(optionsBuilder.Options);
Logger.Level = LogLevel.Debug;
while (!_searchEngine.IsConnected)
{
Logger.LogPrint("Search Engine is not yet Connected!", LoggingTarget.Database, LogLevel.Important);
Thread.Sleep(1000);
}
Logger.LogPrint("Fetching all beatmap sets...");
var beatmapSets = await _dbContext.BeatmapSet
.Include(o => o.ChildrenBeatmaps)
.ToListAsync();
Logger.LogPrint($"{beatmapSets.Count} Beatmap sets to index.", LoggingTarget.Database);
_searchEngine.Index(beatmapSets);
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IAPIProvider apiProvider,
PisstaubeCacheDbContextFactory cacheDbContextFactory, IBeatmapSearchEngineProvider searchEngine,
PisstaubeDbContext dbContext)
{
Logger.Enabled = true;
Logger.Level = LogLevel.Debug;
Logger.GameIdentifier = "Pisstaube";
Logger.Storage = _dataStorage.GetStorageForDirectory("logs");
dbContext.Database.Migrate();
while (!searchEngine.IsConnected)
{
Logger.LogPrint("Search Engine is not yet Connected!", LoggingTarget.Database, LogLevel.Important);
Thread.Sleep(1000);
}
cacheDbContextFactory.Get().Migrate();
_osuContextFactory.Get().Migrate();
DogStatsd.Configure(new StatsdConfig {
Prefix = "pisstaube"
});
JsonUtil.Initialize();
apiProvider.Login(Environment.GetEnvironmentVariable("OSU_USERNAME"),
Environment.GetEnvironmentVariable("OSU_PASSWORD"));
GlobalConfig.EnableCrawling = Environment.GetEnvironmentVariable("CRAWLER_DISABLED")?.ToLowerInvariant() == "false";
GlobalConfig.EnableUpdating = Environment.GetEnvironmentVariable("UPDATER_DISABLED")?.ToLowerInvariant() == "false";
while (true)
{
if (!apiProvider.IsLoggedIn)
{
Logger.LogPrint("Not Logged in yet...");
Thread.Sleep(1000);
continue;
}
if (apiProvider.State == APIState.Failing)
{
Logger.LogPrint($"Failed to Login using Username {Environment.GetEnvironmentVariable("OSU_USERNAME")}", LoggingTarget.Network, LogLevel.Error);
Environment.Exit(1);
}
break;
}
if (DebugUtils.IsDebugBuild)
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseHsts();
}
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(Path.Join(Directory.GetCurrentDirectory(), "data/wwwroot")),
EnableDirectoryBrowsing = true,
});
app.UseMvc(routes => routes.MapRoute("default", "{controller=Home}/{action=Index}/{id?}"));
}
public HashController(PisstaubeDbContext dbContext) => _dbContext = dbContext;
public bool Crawl(int id, PisstaubeDbContext _context)
{
try
{
while (!_apiAccess.IsLoggedIn)
{
Thread.Sleep(1000);
}
var setRequest = new GetBeatmapSetRequest(id);
lock (_lock)
_request_count++;
_rl.Limit();
setRequest.Perform(_apiAccess);
lock (_lock)
if (_request_count > int.Parse(Environment.GetEnvironmentVariable("CRAWLER_REQUESTS_PER_MINUTE")))
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
var apiSet = setRequest.Result;
var localSet = apiSet?.ToBeatmapSet(_store);
if (localSet == null)
{
return(false);
}
var dbSet = BeatmapSet.FromBeatmapSetInfo(localSet);
if (dbSet == null)
{
return(false);
}
foreach (var map in dbSet.ChildrenBeatmaps)
{
var fileInfo = _downloader.Download(map);
map.FileMd5 = _cache.Get()
.CacheBeatmaps
.Where(cmap => cmap.Hash == fileInfo.Hash)
.Select(cmap => cmap.FileMd5)
.FirstOrDefault();
}
lock (_lock)
{
_context.BeatmapSet.Add(dbSet);
}
_search.IndexBeatmap(dbSet);
}
catch (Exception ex)
{
Logger.Error(ex, $"Unknown Error occured while crawling Id {id}!");
lock (_lock)
Thread.Sleep(TimeSpan.FromMinutes(1));
return(false);
}
return(true);
}
public BeatmapController(PisstaubeDbContext dbContext) => _dbContext = dbContext;
