private async Task PerformServerCleanUp(BaseServer server, IBaseSSHAccess sshAccess) { var start = DateTime.UtcNow; // This deletes everything (maybe sometimes it would be better to leave the downloaded images alone) // "-f" is very important here to prevent this just getting locked up forever var cleanupResult = sshAccess.RunCommand( "sudo rm -rf /executor_cache/* && rm -rf ~/images && podman system reset -f"); if (!cleanupResult.Success) { Logger.LogError("Failed to cleanup on server: {Result}, error: {Error}", cleanupResult.Result, cleanupResult.Error); await Database.LogEntries.AddAsync(new LogEntry() { Message = $"Failed to cleanup server {server.Id}" }); } var elapsed = DateTime.UtcNow - start; Logger.LogInformation("Completed cleanup for server {Id}, elapsed: {Elapsed}", server.Id, elapsed); }
private async Task PerformServerCleanUpIfNeeded(BaseServer server, IBaseSSHAccess sshAccess) { int?detectedUsedPercentage = null; // Check server remaining disk space here int attempts = 0; BaseSSHAccess.CommandResult diskSpaceResult; while (true) { try { diskSpaceResult = sshAccess.RunCommand(DiskUsageCheckCommand); break; } catch (SshOperationTimeoutException e) { Logger.LogInformation(e, "Disk space check command timed out"); if (++attempts >= AppInfo.SshServerCommandAttempts) { Logger.LogError("Disk space check command over SSH ran out of attempts"); throw; } } } if (!diskSpaceResult.Success || string.IsNullOrEmpty(diskSpaceResult.Result)) { throw new Exception( $"Failed to check server disk space: {diskSpaceResult.Result}, error: {diskSpaceResult.Error}"); } foreach (var line in diskSpaceResult.Result.Split('\n')) { var match = Regex.Match(line, @"(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)%\s+(\S+)\s*"); if (!match.Success) { continue; } var groups = match.Groups; // Skip if doesn't start with /dev if (!groups[1].Captures[0].Value.StartsWith("/dev/")) { continue; } var percentage = Convert.ToInt32(groups[5].Captures[0].Value); // Immediately return if we found the root file system if (groups[6].Captures[0].Value == "/") { detectedUsedPercentage = percentage; break; } detectedUsedPercentage = percentage; } if (detectedUsedPercentage == null) { Logger.LogWarning("Could not detect free disk space from: {Result}", diskSpaceResult.Result); detectedUsedPercentage = -1; } server.UsedDiskSpace = detectedUsedPercentage.Value; if (server.CleanUpQueued || server.UsedDiskSpace > cleanThreshold) { if (server.CleanUpQueued) { Logger.LogInformation("Cleaning server {Id} because clean up was manually requested", server.Id); } else { Logger.LogInformation( "Cleaning server {Id} because it's disk use is: {UsedDiskSpace}% (> {CleanThreshold}%)", server.Id, server.UsedDiskSpace, cleanThreshold); } await PerformServerCleanUp(server, sshAccess); server.CleanUpQueued = false; } await Database.SaveChangesAsync(); }