protected async Task OnJobEnded(BaseServer server, CiJob job)
{
ReleaseServerReservation(server);
job.RunningOnServerId = -1;
job.RunningOnServerIsExternal = null;
// After running the job, the changes saving should not be skipped
await Database.SaveChangesAsync();
// Send status to github
var status = GithubAPI.CommitStatus.Success;
string statusDescription = "Checks succeeded";
if (!job.Succeeded)
{
status = GithubAPI.CommitStatus.Failure;
statusDescription = "Some checks failed";
}
if (job.Build?.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
if (!await StatusReporter.SetCommitStatus(job.Build.CiProject.RepositoryFullName, job.Build.CommitHash,
status, StatusReporter.CreateStatusUrlForJob(job), statusDescription,
job.JobName))
{
Logger.LogError("Failed to set commit status for build's job: {JobName}", job.JobName);
}
JobClient.Enqueue <CheckOverallBuildStatusJob>(x =>
x.Execute(job.CiProjectId, job.CiBuildId, CancellationToken.None));
}
public string CreateStatusUrlForJob(CiJob job)
{
if (baseUrl == null)
{
throw new InvalidOperationException("Base URL is not set");
}
return(new Uri(baseUrl, $"/ci/{job.CiProjectId}/build/{job.CiBuildId}/jobs/{job.CiJobId}").ToString());
}
private async Task CreateFailedJob(CiBuild build, string failure, CancellationToken cancellationToken)
{
var outputSection = new CiJobOutputSection()
{
CiProjectId = build.CiProjectId,
CiBuildId = build.CiBuildId,
CiJobId = 1,
CiJobOutputSectionId = 1,
Name = "Invalid configuration",
Status = CIJobSectionStatus.Failed,
Output = failure
};
outputSection.CalculateOutputLength();
var job = new CiJob
{
CiProjectId = build.CiProjectId,
CiBuildId = build.CiBuildId,
CiJobId = 1,
JobName = "configuration_error",
FinishedAt = DateTime.UtcNow,
Succeeded = false,
State = CIJobState.Finished,
CiJobOutputSections = new List <CiJobOutputSection>()
{
outputSection
}
};
if (build.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
await database.CiJobs.AddAsync(job, cancellationToken);
await database.CiJobOutputSections.AddAsync(outputSection, cancellationToken);
await database.SaveChangesAsync(cancellationToken);
if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
GithubAPI.CommitStatus.Failure, statusReporter.CreateStatusUrlForJob(job), failure,
job.JobName))
{
logger.LogError("Failed to report serious failed commit status with context {JobName}", job.JobName);
}
jobClient.Enqueue <CheckOverallBuildStatusJob>(x =>
x.Execute(build.CiProjectId, build.CiBuildId, CancellationToken.None));
}
private async Task Requeue(CiJob job, int retries, BaseServer server, bool serverIsExternal)
{
if (retries < 1)
{
Logger.LogError("CI build ran out of tries to try starting on the server");
job.State = CIJobState.Finished;
job.FinishedAt = DateTime.UtcNow;
job.Succeeded = false;
await job.CreateFailureSection(Database, "Connection to build server failed after multiple retries",
"Server Failure");
// Stop hogging the server and mark the build as failed
await OnJobEnded(server, job);
return;
}
JobClient.Schedule <RunJobOnServerJob>(x =>
x.Execute(job.CiProjectId, job.CiBuildId, job.CiJobId, server.Id, serverIsExternal, retries,
CancellationToken.None),
TimeSpan.FromSeconds(10));
}
private string GetConnectToUrl(CiJob job)
{
return(new Uri(configuration.GetBaseUrl(), $"/ciBuildConnection?key={job.BuildOutputConnectKey}")
.ToString());
}
public async Task<IActionResult> ReRunBuild([Required] long projectId,
[Required] long buildId, bool onlyFailed = true)
{
var build = await database.CiBuilds.Include(b => b.CiProject).Include(b => b.CiJobs)
.FirstOrDefaultAsync(j => j.CiProjectId == projectId && j.CiBuildId == buildId);
if (build == null)
return NotFound();
if (build.CiProject == null)
throw new NotLoadedModelNavigationException();
if (build.CiProject.Deleted)
return NotFound();
if (build.Status == BuildStatus.Running || build.Status == BuildStatus.GoingToFail)
return BadRequest("Build can be rerun only after it is complete");
var user = HttpContext.AuthenticatedUser()!;
logger.LogInformation("CI build {ProjectId}-{BuildId} reran by {Email}", projectId, buildId, user.Email);
// TODO: would be nice to have that non-admin action log type
await database.LogEntries.AddAsync(new LogEntry()
{
Message = $"CI build reran {projectId}-{buildId} by a user",
TargetUserId = user.Id,
});
build.Status = BuildStatus.Running;
build.FinishedAt = null;
// TODO: add a rerun counter to the build model
// If there are no jobs, then the repo scan / result failed, so we might as well reset this and rerun
// the repo scan
if (build.CiJobs.Count < 1)
{
await database.SaveChangesAsync();
jobClient.Enqueue<CheckAndStartCIBuild>(x => x.Execute(projectId, buildId, CancellationToken.None));
return Ok("No jobs in this build, trying to re-run repo scan");
}
// Delete the jobs we are going to rerun
// But first we need to grab the information from them needed to rerun the jobs (as we don't want to
// re-check the repo here)
long nextJobId = build.CiJobs.Max(j => j.CiJobId);
var toRerun = build.CiJobs
.Where(j => j.State == CIJobState.Finished && (onlyFailed && !j.Succeeded || !onlyFailed)).ToList();
if (toRerun.Count < 1)
return BadRequest("Nothing needs to rerun");
foreach (var jobToRerun in toRerun)
{
var newJob = new CiJob()
{
CiProjectId = jobToRerun.CiProjectId,
CiBuildId = jobToRerun.CiBuildId,
CiJobId = ++nextJobId,
JobName = jobToRerun.JobName,
Image = jobToRerun.Image,
CacheSettingsJson = jobToRerun.CacheSettingsJson,
};
await database.CiJobs.AddAsync(newJob);
}
// TODO: implement deleting job artifacts if those exist
database.CiJobs.RemoveRange(toRerun);
await database.SaveChangesAsync();
jobClient.Enqueue<HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
return Ok();
}
public async Task Execute(long ciProjectId, long ciBuildId, CancellationToken cancellationToken)
{
var build = await database.CiBuilds.Include(c => c.CiProject)
.FirstOrDefaultAsync(c => c.CiProjectId == ciProjectId && c.CiBuildId == ciBuildId, cancellationToken);
if (build == null)
{
logger.LogError("Failed to find CIBuild to start");
return;
}
// Update our local repo copy and see what the wanted build config is
var semaphore = localTempFileLocks.GetTempFilePath($"ciRepos/{ciProjectId}", out string tempPath);
await semaphore.WaitAsync(cancellationToken);
var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
if (build.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
CiBuildConfiguration?configuration;
try
{
await GitRunHelpers.EnsureRepoIsCloned(build.CiProject.RepositoryCloneUrl, tempPath, true,
cancellationToken);
// Fetch the ref
await GitRunHelpers.FetchRef(tempPath, build.RemoteRef, cancellationToken);
// Then checkout the commit this build is actually for
await GitRunHelpers.Checkout(tempPath, build.CommitHash, true, cancellationToken, true);
// Clean out non-ignored files
await GitRunHelpers.Clean(tempPath, cancellationToken);
// Read the build configuration
var text = await File.ReadAllTextAsync(Path.Join(tempPath, AppInfo.CIConfigurationFile), Encoding.UTF8,
cancellationToken);
configuration = deserializer.Deserialize <CiBuildConfiguration>(text);
}
catch (Exception e)
{
configuration = null;
logger.LogError("Error when trying to read repository for starting jobs: {@E}", e);
}
finally
{
semaphore.Release();
}
if (configuration == null)
{
await CreateFailedJob(build, "Failed to read repository or build configuration", cancellationToken);
return;
}
// Check that configuration is valid
var errors = new List <ValidationResult>();
if (!Validator.TryValidateObject(configuration, new ValidationContext(configuration), errors))
{
logger.LogError("Build configuration object didn't pass validations, see following errors:");
foreach (var error in errors)
{
logger.LogError("Failure: {Error}", error);
}
// TODO: pass validation errors to the build output
await CreateFailedJob(build, "Invalid configuration yaml", cancellationToken);
return;
}
// TODO: refactor these checks to be cleaner
if (configuration.Jobs.Select(j => j.Value.Cache).Any(c =>
c.LoadFrom.Any(p => p.Contains("..") || p.StartsWith("/")) || c.WriteTo.Contains("..") ||
c.WriteTo.Contains('/') ||
(c.Shared != null && c.Shared.Any(s =>
s.Key.Contains("..") || s.Key.StartsWith("/") || s.Value.Contains("..") ||
s.Value.Contains('/'))) ||
(c.System != null && c.System.Any(s =>
s.Key.Contains("..") || s.Key.Contains('\'') || !s.Key.StartsWith("/") ||
s.Value.Contains("..") || s.Value.Contains('/') || s.Value.Contains('\'')))))
{
logger.LogError("Build configuration cache paths have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden cache path", cancellationToken);
return;
}
if (configuration.Jobs.Select(j => j.Value.Image).Any(i => i.Contains("..") || i.StartsWith("/")))
{
logger.LogError("Build configuration image names have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden image name", cancellationToken);
return;
}
if (configuration.Jobs.SelectMany(j => j.Value.Artifacts.Paths).Any(p =>
p.Length < 3 || p.Length > 250 || p.StartsWith("/") || p.Contains("..")))
{
logger.LogError("Build has a too long, short, or non-relative artifact path");
await CreateFailedJob(build, "Invalid configuration yaml, invalid artifact path(s)", cancellationToken);
return;
}
if (configuration.Jobs.Any(j => j.Key == "CLA"))
{
logger.LogError("Build configuration job contains 'CLA' in it");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden job name", cancellationToken);
return;
}
// TODO: do something with the version number here...
// Then queue the jobs we found in the configuration
var jobs = new List <CiJob>();
long jobId = 0;
foreach (var jobEntry in configuration.Jobs)
{
if (string.IsNullOrWhiteSpace(jobEntry.Key) || jobEntry.Key.Length > 80)
{
await CreateFailedJob(build, "Invalid job name in configuration", cancellationToken);
return;
}
var job = new CiJob()
{
CiProjectId = ciProjectId,
CiBuildId = ciBuildId,
CiJobId = ++jobId,
JobName = jobEntry.Key,
Image = jobEntry.Value.Image,
CacheSettingsJson = JsonSerializer.Serialize(jobEntry.Value.Cache),
};
await database.CiJobs.AddAsync(job, cancellationToken);
jobs.Add(job);
}
await database.SaveChangesAsync(cancellationToken);
// Send statuses to github
foreach (var job in jobs)
{
if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
GithubAPI.CommitStatus.Pending, statusReporter.CreateStatusUrlForJob(job), "CI checks starting",
job.JobName))
{
logger.LogError("Failed to set commit status for a build's job: {JobName}", job.JobName);
}
}
// Queue remote executor check task which will allocate a server to run the job(s) on
jobClient.Enqueue <HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
}
