public BeatmapDownloader(FileStore store, Storage dataStorage, PisstaubeCacheDbContextFactory cache, RequestLimiter limiter)
{
_store = store;
_cache = cache;
_limiter = limiter;
_tmpStorage = dataStorage.GetStorageForDirectory("tmp");
}
public SmartStorage(Storage storage, PisstaubeCacheDbContextFactory cache)
{
_cache = cache;
var maximumSize = Environment.GetEnvironmentVariable("CLEANER_MAX_SIZE");
Debug.Assert(maximumSize != null, nameof(maximumSize) + " != null");
switch (maximumSize[^ 1])
public PrivateAPIController(PisstaubeDbContextFactory contextFactory,
Storage storage, BeatmapSearchEngine searchEngine, Crawler crawler, Cleaner cleaner,
PisstaubeCacheDbContextFactory cache, Kaesereibe reibe)
{
_contextFactory = contextFactory;
_storage = storage;
_searchEngine = searchEngine;
_crawler = crawler;
_cleaner = cleaner;
_cache = cache;
_reibe = reibe;
}
public IndexController(IAPIProvider apiProvider,
Storage storage,
PisstaubeCacheDbContextFactory cache,
BeatmapDownloader downloader,
SetDownloader setDownloader,
PisstaubeDbContext dbContext)
{
_apiProvider = apiProvider;
_cache = cache;
_downloader = downloader;
_setDownloader = setDownloader;
_dbContext = dbContext;
_fileStorage = storage.GetStorageForDirectory("files");
}
public IndexController(APIAccess apiAccess,
Storage storage,
PisstaubeCacheDbContextFactory cache,
BeatmapDownloader downloader,
SetDownloader setDownloader,
PisstaubeDbContextFactory contextFactory)
{
_apiAccess = apiAccess;
_storage = storage;
_cache = cache;
_downloader = downloader;
_setDownloader = setDownloader;
_contextFactory = contextFactory;
_fileStorage = storage.GetStorageForDirectory("files");
}
public SetDownloader(Storage storage,
APIAccess apiAccess,
PisstaubeDbContextFactory factory,
PisstaubeCacheDbContextFactory cfactory,
Cleaner cleaner,
RequestLimiter limiter,
BeatmapSearchEngine search
)
{
_storage = storage;
_apiAccess = apiAccess;
_factory = factory;
_cfactory = cfactory;
_cleaner = cleaner;
_limiter = limiter;
_search = search;
}
public Crawler(BeatmapSearchEngine search,
APIAccess apiAccess,
RulesetStore store,
BeatmapDownloader downloader,
PisstaubeCacheDbContextFactory cache,
RequestLimiter rl,
PisstaubeDbContextFactory contextFactory)
{
_pool = new List <Thread>();
_search = search;
_apiAccess = apiAccess;
_store = store;
_downloader = downloader;
_cache = cache;
_rl = rl;
_contextFactory = contextFactory;
_workerThreads = int.Parse(Environment.GetEnvironmentVariable("CRAWLER_THREADS"));
}
public BMUpdater(PisstaubeDbContextFactory factory,
APIAccess apiAccess,
BeatmapDownloader bmDl,
PisstaubeCacheDbContextFactory cFactory,
RulesetStore store,
Storage storage,
BeatmapSearchEngine search,
int limit = 100 /* 100 beatmaps can be updated at the same time per minute! */)
{
_factory = factory;
_apiAccess = apiAccess;
_bmDl = bmDl;
_cFactory = cFactory;
_store = store;
_storage = storage;
_search = search;
_rl = new RequestLimiter(limit, TimeSpan.FromMinutes(1));
}
public SetDownloader(Storage storage,
IAPIProvider apiProvider,
PisstaubeDbContext dbContext,
PisstaubeCacheDbContextFactory cacheFactory,
SmartStorage smartStorage,
RequestLimiter limiter,
IBeatmapSearchEngineProvider search,
IpfsCache ipfsCache
)
{
_storage = storage;
_apiProvider = apiProvider;
_dbContext = dbContext;
_cacheFactory = cacheFactory;
_smartStorage = smartStorage;
_limiter = limiter;
_search = search;
_ipfsCache = ipfsCache;
}
public void Configure(IAPIProvider apiProvider,
PisstaubeCacheDbContextFactory cacheDbContextFactory, IBeatmapSearchEngineProvider searchEngine,
PisstaubeDbContext dbContext)
{
Logger.Enabled = true;
Logger.Level = LogLevel.Debug;
Logger.GameIdentifier = "Pisstaube";
Logger.Storage = _dataStorage.GetStorageForDirectory("logs");
dbContext.Database.Migrate();
while (!searchEngine.IsConnected)
{
Logger.LogPrint("Search Engine is not yet Connected!", LoggingTarget.Database, LogLevel.Important);
Thread.Sleep(1000);
}
cacheDbContextFactory.Get().Migrate();
_osuContextFactory.Get().Migrate();
JsonUtil.Initialize();
apiProvider.Login(Environment.GetEnvironmentVariable("OSU_USERNAME"),
Environment.GetEnvironmentVariable("OSU_PASSWORD"));
while (true)
{
if (!apiProvider.IsLoggedIn)
{
Logger.LogPrint("Not Logged in yet...");
Thread.Sleep(1000);
continue;
}
if (apiProvider.State == APIState.Failing)
{
Logger.LogPrint($"Failed to Login using Username {Environment.GetEnvironmentVariable("OSU_USERNAME")}", LoggingTarget.Network, LogLevel.Error);
Environment.Exit(1);
}
break;
}
}
// ReSharper disable once UnusedParameter.Local
public Startup(IConfiguration configuration)
{
dataStorage = new NativeStorage("data");
osuContextFactory = new DatabaseContextFactory(dataStorage);
cacheContextFactory = new PisstaubeCacheDbContextFactory(dataStorage);
dbContextFactory = new PisstaubeDbContextFactory();
// copy paste of OsuGameBase.cs
try
{
using (var db = dbContextFactory.GetForWrite(false))
db.Context.Database.Migrate();
using (var db = osuContextFactory.GetForWrite(false))
db.Context.Migrate();
}
catch (Exception e)
{
Logger.Error(e.InnerException ?? e, "Migration failed! We'll be starting with a fresh database.", LoggingTarget.Database);
osuContextFactory.ResetDatabase();
Logger.Log("Database purged successfully.", LoggingTarget.Database);
using (var db = osuContextFactory.GetForWrite(false))
db.Context.Migrate();
}
// copy paste of OsuGameBase.cs
try
{
using (var db = cacheContextFactory.GetForWrite(false))
db.Context.Migrate();
}
catch (Exception e)
{
Logger.Error(e.InnerException ?? e, "Migration failed! We'll be starting with a fresh database.", LoggingTarget.Database);
cacheContextFactory.ResetDatabase();
Logger.Log("Database purged successfully.", LoggingTarget.Database);
using (var db = cacheContextFactory.GetForWrite(false))
db.Context.Migrate();
}
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IAPIProvider apiProvider,
PisstaubeCacheDbContextFactory cacheDbContextFactory, IBeatmapSearchEngineProvider searchEngine,
PisstaubeDbContext dbContext)
{
Logger.Enabled = true;
Logger.Level = LogLevel.Debug;
Logger.GameIdentifier = "Pisstaube";
Logger.Storage = _dataStorage.GetStorageForDirectory("logs");
dbContext.Database.Migrate();
while (!searchEngine.IsConnected)
{
Logger.LogPrint("Search Engine is not yet Connected!", LoggingTarget.Database, LogLevel.Important);
Thread.Sleep(1000);
}
cacheDbContextFactory.Get().Migrate();
_osuContextFactory.Get().Migrate();
DogStatsd.Configure(new StatsdConfig {
Prefix = "pisstaube"
});
JsonUtil.Initialize();
apiProvider.Login(Environment.GetEnvironmentVariable("OSU_USERNAME"),
Environment.GetEnvironmentVariable("OSU_PASSWORD"));
GlobalConfig.EnableCrawling = Environment.GetEnvironmentVariable("CRAWLER_DISABLED")?.ToLowerInvariant() == "false";
GlobalConfig.EnableUpdating = Environment.GetEnvironmentVariable("UPDATER_DISABLED")?.ToLowerInvariant() == "false";
while (true)
{
if (!apiProvider.IsLoggedIn)
{
Logger.LogPrint("Not Logged in yet...");
Thread.Sleep(1000);
continue;
}
if (apiProvider.State == APIState.Failing)
{
Logger.LogPrint($"Failed to Login using Username {Environment.GetEnvironmentVariable("OSU_USERNAME")}", LoggingTarget.Network, LogLevel.Error);
Environment.Exit(1);
}
break;
}
if (DebugUtils.IsDebugBuild)
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseHsts();
}
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(Path.Join(Directory.GetCurrentDirectory(), "data/wwwroot")),
EnableDirectoryBrowsing = true,
});
app.UseMvc(routes => routes.MapRoute("default", "{controller=Home}/{action=Index}/{id?}"));
}
public Cleaner(Storage storage, PisstaubeCacheDbContextFactory cache)
{
_cache = cache;
var maxSize = Environment.GetEnvironmentVariable("CLEANER_MAX_SIZE");
Debug.Assert(maxSize != null, nameof(maxSize) + " != null");
switch (maxSize[maxSize.Length - 1])
{
case 'b':
case 'B':
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
break;
case 'k':
case 'K':
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
else
{
_maxSize *= 1024;
}
break;
case 'm':
case 'M':
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
else
{
_maxSize *= 1048576;
}
break;
case 'g':
case 'G':
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
else
{
_maxSize *= 1073741824;
}
break;
case 't':
case 'T':
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
else
{
_maxSize *= 1099511627776;
}
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
long.TryParse(maxSize, out var x);
if (x == 0)
{
_maxSize = 536870912000; // 500 gb
}
break;
default:
ulong.TryParse(maxSize.Remove(maxSize.Length - 1), out _maxSize);
if (_maxSize == 0)
{
_maxSize = 536870912000; // 500 gb
}
break;
}
_cacheStorage = storage.GetStorageForDirectory("cache");
var info = new DirectoryInfo(_cacheStorage.GetFullPath("./"));
DataDirectorySize = info.EnumerateFiles().Sum(file => file.Length);
DogStatsd.Set("cleaner.storage_usage", (ulong)DataDirectorySize / _maxSize);
}
