private async ValueTask <LFSResponse.LFSObject> HandleDownload(LfsProject project, LFSRequest.LFSObject obj)
{
var existingObject = await database.LfsObjects
.Where(o => o.LfsOid == obj.Oid && o.LfsProjectId == project.Id).Include(o => o.LfsProject)
.FirstOrDefaultAsync();
if (existingObject == null)
{
logger.LogWarning("Non-existing OID requested: {Oid}", obj.Oid);
return(new LFSResponse.LFSObject(obj.Oid, obj.Size,
new LFSResponse.LFSObject.ErrorInfo(StatusCodes.Status404NotFound, "OID not found")));
}
var createdUrl = downloadUrls.CreateDownloadFor(existingObject, DownloadUrlExpireTime);
return(new LFSResponse.LFSObject(obj.Oid, obj.Size)
{
Actions = new Dictionary <string, LFSResponse.LFSObject.Action>()
{
{
"download", new LFSResponse.LFSObject.DownloadAction()
{
Href = createdUrl,
ExpiresIn = (int)DownloadExpireTime.TotalSeconds
}
}
}
});
}
public static async Task BuildFileTree(ILocalTempFileLocks tempFiles, ApplicationDbContext database,
LfsProject project, ILogger logger, CancellationToken cancellationToken)
{
var semaphore = tempFiles.GetTempFilePath($"gitFileTrees/{project.Slug}", out string tempPath);
await semaphore.WaitAsync(TimeSpan.FromMinutes(10), cancellationToken);
try
{
await GitRunHelpers.EnsureRepoIsCloned(project.CloneUrl, tempPath, true, cancellationToken);
try
{
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
catch (Exception)
{
// In case the branch refers to a new branch
await GitRunHelpers.Fetch(tempPath, true, cancellationToken);
await GitRunHelpers.Checkout(tempPath, project.BranchToBuildFileTreeFor, true, cancellationToken);
}
await GitRunHelpers.Pull(tempPath, true, cancellationToken, true);
// Skip if commit has not changed
var newCommit = await GitRunHelpers.GetCurrentCommit(tempPath, cancellationToken);
if (newCommit == project.FileTreeCommit)
{
logger.LogDebug("Commit is still the same ({FileTreeCommit}), skipping tree update " +
"for {Id}",
project.FileTreeCommit, project.Id);
return;
}
logger.LogInformation("New commit {NewCommit} to build file tree from (previous: {FileTreeCommit}) " +
"for project {Id}", newCommit, project.FileTreeCommit, project.Id);
project.FileTreeCommit = newCommit;
// Make sure we don't have any extra files locally
await GitRunHelpers.Clean(tempPath, cancellationToken);
// And then make sure the DB file tree entries are fine
await UpdateFileTreeForProject(database, tempPath, project, cancellationToken);
}
finally
{
semaphore.Release();
}
project.FileTreeUpdated = DateTime.UtcNow;
await database.SaveChangesAsync(cancellationToken);
}
private async Task <List <LFSResponse.LFSObject> > HandleLFSObjectRequests(LfsProject project,
LFSRequest.OperationType operation,
IEnumerable <LFSRequest.LFSObject> objects)
{
switch (operation)
{
case LFSRequest.OperationType.Download:
return(await objects.ToAsyncEnumerable().SelectAwait(o => HandleDownload(project, o)).ToListAsync());
case LFSRequest.OperationType.Upload:
return(await objects.ToAsyncEnumerable().SelectAwait(o => HandleUpload(project, o)).ToListAsync());
default:
throw new ArgumentOutOfRangeException();
}
}
private static async Task UpdateFileTreeForProject(ApplicationDbContext database, string folder,
LfsProject project, CancellationToken cancellationToken)
{
// Folders that should exist and how many items they contain
var foldersThatShouldExist = new Dictionary <string, int>();
var existingEntries = await database.ProjectGitFiles.Where(f => f.LfsProjectId == project.Id)
.ToListAsync(cancellationToken);
var itemsThatShouldExist = new HashSet <ProjectGitFile>();
// Note that all paths need to start with a "/"
// Create new files
foreach (var entry in Directory.EnumerateFileSystemEntries(folder, "*", SearchOption.AllDirectories))
{
// Skip .git folder
if (entry.Contains(".git"))
{
continue;
}
var justRepoPath = entry.Substring(folder.Length);
if (!justRepoPath.StartsWith('/'))
{
throw new Exception("Generated file path doesn't start with a slash");
}
var parentFolder = GetParentPath(justRepoPath);
// Don't create the root folder item
if (parentFolder != "/")
{
foldersThatShouldExist.TryGetValue(parentFolder, out int existingItemCount);
foldersThatShouldExist[parentFolder] = existingItemCount + 1;
}
// Only add folders to their parent folder's count of items
if (Directory.Exists(entry))
{
continue;
}
var name = Path.GetFileName(justRepoPath);
// Detect if this is an LFS file
int size = (int)new FileInfo(entry).Length;
string?oid = null;
var(detectedOid, detectedSize) = await DetectLFSFile(entry, cancellationToken);
if (detectedOid != null)
{
oid = detectedOid;
size = detectedSize !.Value;
}
// For files there needs to be an entry
var existing = existingEntries.FirstOrDefault(f =>
f.FType == FileType.File && f.Name == name && f.Path == justRepoPath);
if (existing != null)
{
existing.Size = size;
existing.LfsOid = oid;
itemsThatShouldExist.Add(existing);
}
else
{
await database.ProjectGitFiles.AddAsync(new ProjectGitFile()
{
FType = FileType.File,
LfsProjectId = project.Id,
Name = name,
Path = parentFolder,
Size = size,
LfsOid = oid,
}, cancellationToken);
}
}
// Create / update folders
foreach (var(processedFolder, size) in foldersThatShouldExist)
{
var name = Path.GetFileName(processedFolder);
var path = GetParentPath(processedFolder);
var existing =
existingEntries.FirstOrDefault(f => f.FType == FileType.Folder && f.Name == name && f.Path == path);
if (existing != null)
{
existing.Size = size;
itemsThatShouldExist.Add(existing);
}
else
{
await database.ProjectGitFiles.AddAsync(new ProjectGitFile()
{
FType = FileType.Folder,
LfsProjectId = project.Id,
Name = name,
Path = path,
Size = size,
}, cancellationToken);
}
}
// Delete files and folders that shouldn't exist
database.ProjectGitFiles.RemoveRange(existingEntries.Except(itemsThatShouldExist));
}
private async ValueTask <LFSResponse.LFSObject> HandleUpload(LfsProject project, LFSRequest.LFSObject obj)
{
var existingObject = await database.LfsObjects
.Where(o => o.LfsOid == obj.Oid && o.LfsProjectId == project.Id).Include(o => o.LfsProject)
.FirstOrDefaultAsync();
if (existingObject != null)
{
// We already have this object
return(new LFSResponse.LFSObject(obj.Oid, obj.Size)
{
Actions = null,
Authenticated = null
});
}
if (obj.Size > AppInfo.MaxLfsUploadSize)
{
return(new LFSResponse.LFSObject(obj.Oid, obj.Size,
new LFSResponse.LFSObject.ErrorInfo(StatusCodes.Status422UnprocessableEntity, "File is too large")));
}
logger.LogTrace("Requesting auth because new object is to be uploaded {Oid} for project {Name}", obj.Oid,
project.Name);
// New object. User must have write access
if (!RequireWriteAccess(out var result))
{
throw new InvalidAccessException(result !);
}
// We don't yet create the LfsObject here to guard against upload failures
// instead the verify callback does that
// The uploads prefix is used here to ensure the user can't overwrite the file after uploading and
// verification
var storagePath = "uploads/" + LfsDownloadUrls.OidStoragePath(project, obj.Oid);
if (!bucketChecked)
{
try
{
if (!await remoteStorage.BucketExists())
{
throw new Exception("bucket doesn't exist");
}
}
catch (Exception e)
{
logger.LogWarning("Bucket check failed: {@E}", e);
var error = "remote storage is inaccessible";
throw new HttpResponseException()
{
Status = StatusCodes.Status500InternalServerError,
ContentType = AppInfo.GitLfsContentType,
Value = new GitLFSErrorResponse()
{
Message = error
}.ToString()
};
}
bucketChecked = true;
}
var verifyUrl = QueryHelpers.AddQueryString(
new Uri(configuration.GetBaseUrl(), $"api/v1/lfs/{project.Slug}/verify").ToString(),
"token", GenerateUploadVerifyToken(obj));
return(new LFSResponse.LFSObject(obj.Oid, obj.Size)
{
Actions = new Dictionary <string, LFSResponse.LFSObject.Action>()
{
{
"upload", new LFSResponse.LFSObject.UploadAction()
{
Href = remoteStorage.CreatePresignedUploadURL(storagePath, S3UploadValidTime),
ExpiresIn = (int)UploadValidTime.TotalSeconds
}
},
{
"verify", new LFSResponse.LFSObject.UploadAction()
{
Href = verifyUrl,
ExpiresIn = (int)UploadValidTime.TotalSeconds
}
}
}
});
}
public static string OidStoragePath(LfsProject project, string oid)
{
return($"{project.Slug}/objs/{oid[0..1]}/{oid[2..3]}/{oid[4..]}");
