private async Task PerformServerCleanUp(BaseServer server, IBaseSSHAccess sshAccess)
{
var start = DateTime.UtcNow;
// This deletes everything (maybe sometimes it would be better to leave the downloaded images alone)
// "-f" is very important here to prevent this just getting locked up forever
var cleanupResult =
sshAccess.RunCommand(
"sudo rm -rf /executor_cache/* && rm -rf ~/images && podman system reset -f");
if (!cleanupResult.Success)
{
Logger.LogError("Failed to cleanup on server: {Result}, error: {Error}", cleanupResult.Result,
cleanupResult.Error);
await Database.LogEntries.AddAsync(new LogEntry()
{
Message = $"Failed to cleanup server {server.Id}"
});
}
var elapsed = DateTime.UtcNow - start;
Logger.LogInformation("Completed cleanup for server {Id}, elapsed: {Elapsed}", server.Id, elapsed);
}
private async Task PerformServerCleanUpIfNeeded(BaseServer server, IBaseSSHAccess sshAccess)
{
int?detectedUsedPercentage = null;
// Check server remaining disk space here
int attempts = 0;
BaseSSHAccess.CommandResult diskSpaceResult;
while (true)
{
try
{
diskSpaceResult = sshAccess.RunCommand(DiskUsageCheckCommand);
break;
}
catch (SshOperationTimeoutException e)
{
Logger.LogInformation(e, "Disk space check command timed out");
if (++attempts >= AppInfo.SshServerCommandAttempts)
{
Logger.LogError("Disk space check command over SSH ran out of attempts");
throw;
}
}
}
if (!diskSpaceResult.Success || string.IsNullOrEmpty(diskSpaceResult.Result))
{
throw new Exception(
$"Failed to check server disk space: {diskSpaceResult.Result}, error: {diskSpaceResult.Error}");
}
foreach (var line in diskSpaceResult.Result.Split('\n'))
{
var match = Regex.Match(line, @"(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)%\s+(\S+)\s*");
if (!match.Success)
{
continue;
}
var groups = match.Groups;
// Skip if doesn't start with /dev
if (!groups[1].Captures[0].Value.StartsWith("/dev/"))
{
continue;
}
var percentage = Convert.ToInt32(groups[5].Captures[0].Value);
// Immediately return if we found the root file system
if (groups[6].Captures[0].Value == "/")
{
detectedUsedPercentage = percentage;
break;
}
detectedUsedPercentage = percentage;
}
if (detectedUsedPercentage == null)
{
Logger.LogWarning("Could not detect free disk space from: {Result}", diskSpaceResult.Result);
detectedUsedPercentage = -1;
}
server.UsedDiskSpace = detectedUsedPercentage.Value;
if (server.CleanUpQueued || server.UsedDiskSpace > cleanThreshold)
{
if (server.CleanUpQueued)
{
Logger.LogInformation("Cleaning server {Id} because clean up was manually requested", server.Id);
}
else
{
Logger.LogInformation(
"Cleaning server {Id} because it's disk use is: {UsedDiskSpace}% (> {CleanThreshold}%)",
server.Id, server.UsedDiskSpace, cleanThreshold);
}
await PerformServerCleanUp(server, sshAccess);
server.CleanUpQueued = false;
}
await Database.SaveChangesAsync();
}
