public static async Task BuildFileTree(ILocalTempFileLocks tempFiles, ApplicationDbContext database,
LfsProject project, ILogger logger, CancellationToken cancellationToken)
{
var semaphore = tempFiles.GetTempFilePath($"gitFileTrees/{project.Slug}", out string tempPath);
await semaphore.WaitAsync(TimeSpan.FromMinutes(10), cancellationToken);
try
{
await GitRunHelpers.EnsureRepoIsCloned(project.CloneUrl, tempPath, true, cancellationToken);
try
{
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
catch (Exception)
{
// In case the branch refers to a new branch
await GitRunHelpers.Fetch(tempPath, true, cancellationToken);
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
await GitRunHelpers.Pull(tempPath, true, cancellationToken, true);
// Skip if commit has not changed
var newCommit = await GitRunHelpers.GetCurrentCommit(tempPath, cancellationToken);
if (newCommit == project.FileTreeCommit)
{
logger.LogDebug("Commit is still the same ({FileTreeCommit}), skipping tree update " +
"for {Id}",
project.FileTreeCommit, project.Id);
return;
}
logger.LogInformation("New commit {NewCommit} to build file tree from (previous: {FileTreeCommit}) " +
"for project {Id}", newCommit, project.FileTreeCommit, project.Id);
project.FileTreeCommit = newCommit;
// Make sure we don't have any extra files locally
await GitRunHelpers.Clean(tempPath, cancellationToken);
// And then make sure the DB file tree entries are fine
await UpdateFileTreeForProject(database, tempPath, project, cancellationToken);
}
finally
{
semaphore.Release();
}
project.FileTreeUpdated = DateTime.UtcNow;
await database.SaveChangesAsync(cancellationToken);
}
public static async Task DeleteReportTempFile(CrashReport report, ILocalTempFileLocks fileLocks, ILogger logger,
CancellationToken cancellationToken)
{
var semaphore =
fileLocks.GetTempFilePath(CrashReport.CrashReportTempStorageFolderName, out string baseFolder);
if (string.IsNullOrEmpty(report.DumpLocalFileName))
{
logger.LogInformation("Crash report doesn't have a dump file set, skip deleting it");
return;
}
var filePath = Path.Combine(baseFolder, report.DumpLocalFileName);
await semaphore.WaitAsync(cancellationToken);
try
{
if (!Directory.Exists(baseFolder))
{
logger.LogInformation("Crash report dump folder doesn't exist, skip deleting a dump file");
return;
}
if (!File.Exists(filePath))
{
logger.LogInformation(
"Crash report dump file with name {DumpLocalFileName} doesn't exist, skip trying to delete it",
report.DumpLocalFileName);
return;
}
File.Delete(filePath);
}
finally
{
semaphore.Release();
}
logger.LogInformation("Deleted crash dump file {DumpLocalFileName}", report.DumpLocalFileName);
}
public async Task <ActionResult <Guid> > Submit([Required][FromForm] IFormFile file)
{
if (!enabled)
{
return(BadRequest("This tool is not enabled on the server"));
}
if (file.Length > AppInfo.MaxCrashDumpUploadSize)
{
return(BadRequest("Uploaded crash dump is too large"));
}
if (file.Length < 1)
{
return(BadRequest("Uploaded crash dump file is empty"));
}
var address = HttpContext.Connection.RemoteIpAddress;
if (address == null)
{
return(Problem("Remote IP address could not be read"));
}
logger.LogInformation("Starting stackwalk tool run for a request from: {Address}", address);
var semaphore =
localTempFileLocks.GetTempFilePath(StackwalkTask.CrashDumpToolTempStorageFolderName,
out string baseFolder);
var task = new StackwalkTask()
{
DumpTempCategory = StackwalkTask.CrashDumpToolTempStorageFolderName,
DumpFileName = Guid.NewGuid() + ".dmp",
DeleteDumpAfterRunning = true,
// TODO: allow the user to specify this
// https://github.com/Revolutionary-Games/ThriveDevCenter/issues/247
StackwalkPlatform = ThrivePlatform.Windows,
};
await database.StackwalkTasks.AddAsync(task);
var filePath = Path.Combine(baseFolder, task.DumpFileName);
await semaphore.WaitAsync();
try
{
Directory.CreateDirectory(baseFolder);
// TODO: we don't necessarily have to hold the semaphore while in here, but our files are so small
// it probably won't cause any issue even with multiple requests being uploaded in a few seconds
await using var stream = System.IO.File.Create(filePath);
await file.CopyToAsync(stream);
}
finally
{
semaphore.Release();
}
try
{
await database.SaveChangesAsync();
}
catch (Exception e)
{
logger.LogError(e,
"Failed to save crash dump tool request, attempting to delete the temp file for it {FilePath}",
filePath);
// We don't need the path semaphore here as we have randomly generated file name
System.IO.File.Delete(filePath);
return(Problem("Failed to write to database"));
}
jobClient.Enqueue <RunStackwalkTaskJob>(x => x.Execute(task.Id, CancellationToken.None));
return(task.Id);
}
public async Task Execute(Guid taskId, CancellationToken cancellationToken)
{
if (!stackwalk.Configured)
{
throw new Exception("Stackwalk is not configured");
}
var task = await database.StackwalkTasks.FindAsync(new object[] { taskId }, cancellationToken);
if (task == null)
{
logger.LogError("Can't stackwalk on non-existent task: {TaskId}", taskId);
return;
}
var symbolPrepareTask = symbolPreparer.PrepareSymbolsInFolder(symbolFolder, cancellationToken);
logger.LogInformation("Starting stackwalk on task {TaskId}", taskId);
var semaphore =
localTempFileLocks.GetTempFilePath(task.DumpTempCategory, out string baseFolder);
var filePath = Path.Combine(baseFolder, task.DumpFileName);
FileStream?dump = null;
// On Linux an open file should not impact deleting etc. so I'm pretty sure this is pretty safe
await semaphore.WaitAsync(cancellationToken);
try
{
if (File.Exists(filePath))
{
dump = File.OpenRead(filePath);
}
}
finally
{
semaphore.Release();
}
await symbolPrepareTask;
if (string.IsNullOrEmpty(task.DumpFileName) || dump == null)
{
logger.LogError("Can't stackwalk for task with missing dump file: {FilePath}", filePath);
return;
}
var startTime = DateTime.UtcNow;
// TODO: implement an async API in the stackwalk service and swap to using that here
// TODO: also then combine this with StartStackwalkOnReportJob class
string result;
try
{
result = await stackwalk.PerformBlockingStackwalk(dump, task.StackwalkPlatform, cancellationToken);
task.Succeeded = true;
}
catch (Exception e)
{
// TODO: probably wants to retry at least once or twice here instead of immediately failing
logger.LogError(e, "Failed to run stackwalk task");
result = "Failed to run stackwalk";
task.Succeeded = false;
}
var duration = DateTime.UtcNow - startTime;
logger.LogInformation("Stackwalking (task) took: {Duration}", duration);
if (task.DeleteDumpAfterRunning)
{
await semaphore.WaitAsync(cancellationToken);
try
{
File.Delete(filePath);
logger.LogInformation("Deleted processed file for stackwalk task: {FilePath}", filePath);
}
finally
{
semaphore.Release();
}
}
if (string.IsNullOrWhiteSpace(result))
{
result = "Resulting decoded crash dump is empty";
}
task.Result = result;
task.FinishedAt = DateTime.UtcNow;
// Don't want to cancel here as we can no longer undelete the file
// ReSharper disable once MethodSupportsCancellation
await database.SaveChangesAsync();
}
public async Task <ActionResult <CreateCrashReportResponse> > CreateReport(
[Required][FromForm] CreateCrashReportData request, [Required] IFormFile dump)
{
if (!uploadEnabled)
{
return(Problem("Crash uploading is not enabled on the server"));
}
if (dump.Length > AppInfo.MaxCrashDumpUploadSize)
{
return(BadRequest("Uploaded crash dump file is too big"));
}
if (dump.Length < 1)
{
return(BadRequest("Uploaded crash dump file is empty"));
}
request.LogFiles ??= "No logs provided";
if (request.LogFiles.Length > AppInfo.MaxCrashLogsLength)
{
return(BadRequest("Crash related logs are too big"));
}
ThrivePlatform parsedPlatform;
try
{
parsedPlatform = ParsePlatform(request.Platform);
}
catch (ArgumentException)
{
return(BadRequest($"Unknown platform value: {request.Platform}"));
}
var crashTime = DateTime.UnixEpoch + TimeSpan.FromSeconds(request.CrashTime);
var address = HttpContext.Connection.RemoteIpAddress;
if (address == null)
{
return(Problem("Remote IP address could not be read"));
}
var fileName = Guid.NewGuid() + ".dmp";
var semaphore =
localTempFileLocks.GetTempFilePath(CrashReport.CrashReportTempStorageFolderName, out string baseFolder);
var filePath = Path.Combine(baseFolder, fileName);
var report = new CrashReport
{
Public = request.Public,
ExitCodeOrSignal = request.ExitCode,
Platform = parsedPlatform,
Logs = request.LogFiles,
Store = request.Store,
Version = request.GameVersion,
HappenedAt = crashTime,
UploadedFrom = address,
DumpLocalFileName = fileName,
ReporterEmail = request.Email,
};
if (report.ReporterEmail != null && !report.ReporterEmail.Contains("@"))
{
logger.LogWarning("Ignoring provided crash reporter email that seems invalid: {ReporterEmail}",
report.ReporterEmail);
report.ReporterEmail = null;
}
if (!string.IsNullOrWhiteSpace(request.ExtraDescription))
{
report.Description = $"Reporter provided description:\n{request.ExtraDescription}\n";
}
await database.CrashReports.AddAsync(report);
var saveTask = database.SaveChangesAsync();
await semaphore.WaitAsync();
try
{
Directory.CreateDirectory(baseFolder);
// TODO: we don't necessarily have to hold the semaphore while in here, but our files are so small
// it probably won't cause any issue even with multiple crash reports being uploaded in a few seconds
await using var stream = System.IO.File.Create(filePath);
await dump.CopyToAsync(stream);
}
finally
{
semaphore.Release();
}
try
{
await saveTask;
}
catch (Exception e)
{
logger.LogError(e, "Failed to save crash report, attempting to delete the temp file for it {FilePath}",
filePath);
// We don't need the path semaphore here as we have randomly generated file name
System.IO.File.Delete(filePath);
return(Problem("Failed to save the crash report to the database"));
}
logger.LogInformation("New crash report ({Id}) created, with crash dump at: {FilePath} from: {Address}",
report.Id, filePath, address);
await database.LogEntries.AddAsync(new LogEntry()
{
Message =
$"New crash report {report.Id} created for {report.StoreOrVersion} on platform: {report.Platform}",
});
saveTask = database.SaveChangesAsync();
try
{
jobClient.Enqueue <StartStackwalkOnReportJob>(x => x.Execute(report.Id, CancellationToken.None));
jobClient.Schedule <DeleteCrashReportDumpJob>(x => x.Execute(report.Id, CancellationToken.None),
TimeSpan.FromDays(AppInfo.CrashDumpDumpFileRetentionDays));
discordNotifications.NotifyAboutNewCrashReport(report, baseUrl);
// TODO: verify the reporter wants to receive email notifications with a confirmation email
await saveTask;
}
catch (Exception e)
{
logger.LogError(e, "Failed to save log entry or create jobs for crash report");
}
var response = new CreateCrashReportResponse()
{
CreatedId = report.Id,
DeleteKey = report.DeleteKey.ToString(),
};
return(Created($"reports/{response.CreatedId}", response));
}
public async Task Execute(long reportId, CancellationToken cancellationToken)
{
if (!stackwalk.Configured)
{
throw new Exception("Stackwalk is not configured");
}
var report = await database.CrashReports.FindAsync(new object[] { reportId }, cancellationToken);
if (report == null)
{
logger.LogError("Can't stackwalk on non-existent report: {ReportId}", reportId);
return;
}
if (string.IsNullOrEmpty(report.DumpLocalFileName))
{
logger.LogError("Can't stackwalk on report that no longer has local dump: {ReportId}", reportId);
return;
}
var symbolPrepareTask = symbolPreparer.PrepareSymbolsInFolder(symbolFolder, cancellationToken);
logger.LogInformation("Starting stackwalk on report {ReportId}", reportId);
var semaphore =
localTempFileLocks.GetTempFilePath(CrashReport.CrashReportTempStorageFolderName, out string baseFolder);
var filePath = Path.Combine(baseFolder, report.DumpLocalFileName);
FileStream?dump = null;
// On Linux an open file should not impact deleting etc. so I'm pretty sure this is pretty safe
await semaphore.WaitAsync(cancellationToken);
try
{
if (File.Exists(filePath))
{
dump = File.OpenRead(filePath);
}
}
finally
{
semaphore.Release();
}
await symbolPrepareTask;
if (report.DumpLocalFileName == null || dump == null)
{
logger.LogError("Can't stackwalk on report with missing dump file: {ReportId}", reportId);
return;
}
var startTime = DateTime.UtcNow;
// TODO: implement an async API in the stackwalk service and swap to using that here
var result = await stackwalk.PerformBlockingStackwalk(dump, report.Platform, cancellationToken);
var primaryCallstack = stackwalk.FindPrimaryCallstack(result);
var condensedCallstack = stackwalk.CondenseCallstack(primaryCallstack);
cancellationToken.ThrowIfCancellationRequested();
var duration = DateTime.UtcNow - startTime;
logger.LogInformation("Stackwalking took: {Duration}", duration);
await database.LogEntries.AddAsync(new LogEntry()
{
Message = $"Stackwalking performed on report {report.Id}, result length: {result.Length}, " +
$"duration: {duration}",
}, cancellationToken);
if (string.IsNullOrWhiteSpace(result))
{
result = "Resulting decoded crash dump is empty";
}
report.UpdateProcessedDumpIfChanged(result, primaryCallstack, condensedCallstack);
await database.SaveChangesWithConflictResolvingAsync(
conflictEntries =>
{
DatabaseConcurrencyHelpers.ResolveSingleEntityConcurrencyConflict(conflictEntries, report);
report.UpdateProcessedDumpIfChanged(result, primaryCallstack, condensedCallstack);
}, cancellationToken);
jobClient.Schedule <CheckCrashReportDuplicatesJob>(x => x.Execute(report.Id, CancellationToken.None),
TimeSpan.FromSeconds(10));
}
public async Task Execute(long ciProjectId, long ciBuildId, CancellationToken cancellationToken)
{
var build = await database.CiBuilds.Include(c => c.CiProject)
.FirstOrDefaultAsync(c => c.CiProjectId == ciProjectId && c.CiBuildId == ciBuildId, cancellationToken);
if (build == null)
{
logger.LogError("Failed to find CIBuild to start");
return;
}
// Update our local repo copy and see what the wanted build config is
var semaphore = localTempFileLocks.GetTempFilePath($"ciRepos/{ciProjectId}", out string tempPath);
await semaphore.WaitAsync(cancellationToken);
var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
if (build.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
CiBuildConfiguration?configuration;
try
{
await GitRunHelpers.EnsureRepoIsCloned(build.CiProject.RepositoryCloneUrl, tempPath, true,
cancellationToken);
// Fetch the ref
await GitRunHelpers.FetchRef(tempPath, build.RemoteRef, cancellationToken);
// Then checkout the commit this build is actually for
await GitRunHelpers.Checkout(tempPath, build.CommitHash, true, cancellationToken, true);
// Clean out non-ignored files
await GitRunHelpers.Clean(tempPath, cancellationToken);
// Read the build configuration
var text = await File.ReadAllTextAsync(Path.Join(tempPath, AppInfo.CIConfigurationFile), Encoding.UTF8,
cancellationToken);
configuration = deserializer.Deserialize <CiBuildConfiguration>(text);
}
catch (Exception e)
{
configuration = null;
logger.LogError("Error when trying to read repository for starting jobs: {@E}", e);
}
finally
{
semaphore.Release();
}
if (configuration == null)
{
await CreateFailedJob(build, "Failed to read repository or build configuration", cancellationToken);
return;
}
// Check that configuration is valid
var errors = new List <ValidationResult>();
if (!Validator.TryValidateObject(configuration, new ValidationContext(configuration), errors))
{
logger.LogError("Build configuration object didn't pass validations, see following errors:");
foreach (var error in errors)
{
logger.LogError("Failure: {Error}", error);
}
// TODO: pass validation errors to the build output
await CreateFailedJob(build, "Invalid configuration yaml", cancellationToken);
return;
}
// TODO: refactor these checks to be cleaner
if (configuration.Jobs.Select(j => j.Value.Cache).Any(c =>
c.LoadFrom.Any(p => p.Contains("..") || p.StartsWith("/")) || c.WriteTo.Contains("..") ||
c.WriteTo.Contains('/') ||
(c.Shared != null && c.Shared.Any(s =>
s.Key.Contains("..") || s.Key.StartsWith("/") || s.Value.Contains("..") ||
s.Value.Contains('/'))) ||
(c.System != null && c.System.Any(s =>
s.Key.Contains("..") || s.Key.Contains('\'') || !s.Key.StartsWith("/") ||
s.Value.Contains("..") || s.Value.Contains('/') || s.Value.Contains('\'')))))
{
logger.LogError("Build configuration cache paths have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden cache path", cancellationToken);
return;
}
if (configuration.Jobs.Select(j => j.Value.Image).Any(i => i.Contains("..") || i.StartsWith("/")))
{
logger.LogError("Build configuration image names have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden image name", cancellationToken);
return;
}
if (configuration.Jobs.SelectMany(j => j.Value.Artifacts.Paths).Any(p =>
p.Length < 3 || p.Length > 250 || p.StartsWith("/") || p.Contains("..")))
{
logger.LogError("Build has a too long, short, or non-relative artifact path");
await CreateFailedJob(build, "Invalid configuration yaml, invalid artifact path(s)", cancellationToken);
return;
}
if (configuration.Jobs.Any(j => j.Key == "CLA"))
{
logger.LogError("Build configuration job contains 'CLA' in it");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden job name", cancellationToken);
return;
}
// TODO: do something with the version number here...
// Then queue the jobs we found in the configuration
var jobs = new List <CiJob>();
long jobId = 0;
foreach (var jobEntry in configuration.Jobs)
{
if (string.IsNullOrWhiteSpace(jobEntry.Key) || jobEntry.Key.Length > 80)
{
await CreateFailedJob(build, "Invalid job name in configuration", cancellationToken);
return;
}
var job = new CiJob()
{
CiProjectId = ciProjectId,
CiBuildId = ciBuildId,
CiJobId = ++jobId,
JobName = jobEntry.Key,
Image = jobEntry.Value.Image,
CacheSettingsJson = JsonSerializer.Serialize(jobEntry.Value.Cache),
};
await database.CiJobs.AddAsync(job, cancellationToken);
jobs.Add(job);
}
await database.SaveChangesAsync(cancellationToken);
// Send statuses to github
foreach (var job in jobs)
{
if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
GithubAPI.CommitStatus.Pending, statusReporter.CreateStatusUrlForJob(job), "CI checks starting",
job.JobName))
{
logger.LogError("Failed to set commit status for a build's job: {JobName}", job.JobName);
}
}
// Queue remote executor check task which will allocate a server to run the job(s) on
jobClient.Enqueue <HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
}
