public static async Task BuildFileTree(ILocalTempFileLocks tempFiles, ApplicationDbContext database,
LfsProject project, ILogger logger, CancellationToken cancellationToken)
{
var semaphore = tempFiles.GetTempFilePath($"gitFileTrees/{project.Slug}", out string tempPath);
await semaphore.WaitAsync(TimeSpan.FromMinutes(10), cancellationToken);
try
{
await GitRunHelpers.EnsureRepoIsCloned(project.CloneUrl, tempPath, true, cancellationToken);
try
{
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
catch (Exception)
{
// In case the branch refers to a new branch
await GitRunHelpers.Fetch(tempPath, true, cancellationToken);
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
await GitRunHelpers.Pull(tempPath, true, cancellationToken, true);
// Skip if commit has not changed
var newCommit = await GitRunHelpers.GetCurrentCommit(tempPath, cancellationToken);
if (newCommit == project.FileTreeCommit)
{
logger.LogDebug("Commit is still the same ({FileTreeCommit}), skipping tree update " +
"for {Id}",
project.FileTreeCommit, project.Id);
return;
}
logger.LogInformation("New commit {NewCommit} to build file tree from (previous: {FileTreeCommit}) " +
"for project {Id}", newCommit, project.FileTreeCommit, project.Id);
project.FileTreeCommit = newCommit;
// Make sure we don't have any extra files locally
await GitRunHelpers.Clean(tempPath, cancellationToken);
// And then make sure the DB file tree entries are fine
await UpdateFileTreeForProject(database, tempPath, project, cancellationToken);
}
finally
{
semaphore.Release();
}
project.FileTreeUpdated = DateTime.UtcNow;
await database.SaveChangesAsync(cancellationToken);
}
public async Task Execute(long ciProjectId, long ciBuildId, CancellationToken cancellationToken)
{
var build = await database.CiBuilds.Include(c => c.CiProject)
.FirstOrDefaultAsync(c => c.CiProjectId == ciProjectId && c.CiBuildId == ciBuildId, cancellationToken);
if (build == null)
{
logger.LogError("Failed to find CIBuild to start");
return;
}
// Update our local repo copy and see what the wanted build config is
var semaphore = localTempFileLocks.GetTempFilePath($"ciRepos/{ciProjectId}", out string tempPath);
await semaphore.WaitAsync(cancellationToken);
var deserializer = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
if (build.CiProject == null)
{
throw new NotLoadedModelNavigationException();
}
CiBuildConfiguration?configuration;
try
{
await GitRunHelpers.EnsureRepoIsCloned(build.CiProject.RepositoryCloneUrl, tempPath, true,
cancellationToken);
// Fetch the ref
await GitRunHelpers.FetchRef(tempPath, build.RemoteRef, cancellationToken);
// Then checkout the commit this build is actually for
await GitRunHelpers.Checkout(tempPath, build.CommitHash, true, cancellationToken, true);
// Clean out non-ignored files
await GitRunHelpers.Clean(tempPath, cancellationToken);
// Read the build configuration
var text = await File.ReadAllTextAsync(Path.Join(tempPath, AppInfo.CIConfigurationFile), Encoding.UTF8,
cancellationToken);
configuration = deserializer.Deserialize <CiBuildConfiguration>(text);
}
catch (Exception e)
{
configuration = null;
logger.LogError("Error when trying to read repository for starting jobs: {@E}", e);
}
finally
{
semaphore.Release();
}
if (configuration == null)
{
await CreateFailedJob(build, "Failed to read repository or build configuration", cancellationToken);
return;
}
// Check that configuration is valid
var errors = new List <ValidationResult>();
if (!Validator.TryValidateObject(configuration, new ValidationContext(configuration), errors))
{
logger.LogError("Build configuration object didn't pass validations, see following errors:");
foreach (var error in errors)
{
logger.LogError("Failure: {Error}", error);
}
// TODO: pass validation errors to the build output
await CreateFailedJob(build, "Invalid configuration yaml", cancellationToken);
return;
}
// TODO: refactor these checks to be cleaner
if (configuration.Jobs.Select(j => j.Value.Cache).Any(c =>
c.LoadFrom.Any(p => p.Contains("..") || p.StartsWith("/")) || c.WriteTo.Contains("..") ||
c.WriteTo.Contains('/') ||
(c.Shared != null && c.Shared.Any(s =>
s.Key.Contains("..") || s.Key.StartsWith("/") || s.Value.Contains("..") ||
s.Value.Contains('/'))) ||
(c.System != null && c.System.Any(s =>
s.Key.Contains("..") || s.Key.Contains('\'') || !s.Key.StartsWith("/") ||
s.Value.Contains("..") || s.Value.Contains('/') || s.Value.Contains('\'')))))
{
logger.LogError("Build configuration cache paths have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden cache path", cancellationToken);
return;
}
if (configuration.Jobs.Select(j => j.Value.Image).Any(i => i.Contains("..") || i.StartsWith("/")))
{
logger.LogError("Build configuration image names have \"..\" in them or starts with a slash");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden image name", cancellationToken);
return;
}
if (configuration.Jobs.SelectMany(j => j.Value.Artifacts.Paths).Any(p =>
p.Length < 3 || p.Length > 250 || p.StartsWith("/") || p.Contains("..")))
{
logger.LogError("Build has a too long, short, or non-relative artifact path");
await CreateFailedJob(build, "Invalid configuration yaml, invalid artifact path(s)", cancellationToken);
return;
}
if (configuration.Jobs.Any(j => j.Key == "CLA"))
{
logger.LogError("Build configuration job contains 'CLA' in it");
await CreateFailedJob(build, "Invalid configuration yaml, forbidden job name", cancellationToken);
return;
}
// TODO: do something with the version number here...
// Then queue the jobs we found in the configuration
var jobs = new List <CiJob>();
long jobId = 0;
foreach (var jobEntry in configuration.Jobs)
{
if (string.IsNullOrWhiteSpace(jobEntry.Key) || jobEntry.Key.Length > 80)
{
await CreateFailedJob(build, "Invalid job name in configuration", cancellationToken);
return;
}
var job = new CiJob()
{
CiProjectId = ciProjectId,
CiBuildId = ciBuildId,
CiJobId = ++jobId,
JobName = jobEntry.Key,
Image = jobEntry.Value.Image,
CacheSettingsJson = JsonSerializer.Serialize(jobEntry.Value.Cache),
};
await database.CiJobs.AddAsync(job, cancellationToken);
jobs.Add(job);
}
await database.SaveChangesAsync(cancellationToken);
// Send statuses to github
foreach (var job in jobs)
{
if (!await statusReporter.SetCommitStatus(build.CiProject.RepositoryFullName, build.CommitHash,
GithubAPI.CommitStatus.Pending, statusReporter.CreateStatusUrlForJob(job), "CI checks starting",
job.JobName))
{
logger.LogError("Failed to set commit status for a build's job: {JobName}", job.JobName);
}
}
// Queue remote executor check task which will allocate a server to run the job(s) on
jobClient.Enqueue <HandleControlledServerJobsJob>(x => x.Execute(CancellationToken.None));
}
