private async ValueTask <LFSResponse.LFSObject> HandleUpload(LfsProject project, LFSRequest.LFSObject obj) { var existingObject = await database.LfsObjects .Where(o => o.LfsOid == obj.Oid && o.LfsProjectId == project.Id).Include(o => o.LfsProject) .FirstOrDefaultAsync(); if (existingObject != null) { // We already have this object return(new LFSResponse.LFSObject(obj.Oid, obj.Size) { Actions = null, Authenticated = null }); } if (obj.Size > AppInfo.MaxLfsUploadSize) { return(new LFSResponse.LFSObject(obj.Oid, obj.Size, new LFSResponse.LFSObject.ErrorInfo(StatusCodes.Status422UnprocessableEntity, "File is too large"))); } logger.LogTrace("Requesting auth because new object is to be uploaded {Oid} for project {Name}", obj.Oid, project.Name); // New object. User must have write access if (!RequireWriteAccess(out var result)) { throw new InvalidAccessException(result !); } // We don't yet create the LfsObject here to guard against upload failures // instead the verify callback does that // The uploads prefix is used here to ensure the user can't overwrite the file after uploading and // verification var storagePath = "uploads/" + LfsDownloadUrls.OidStoragePath(project, obj.Oid); if (!bucketChecked) { try { if (!await remoteStorage.BucketExists()) { throw new Exception("bucket doesn't exist"); } } catch (Exception e) { logger.LogWarning("Bucket check failed: {@E}", e); var error = "remote storage is inaccessible"; throw new HttpResponseException() { Status = StatusCodes.Status500InternalServerError, ContentType = AppInfo.GitLfsContentType, Value = new GitLFSErrorResponse() { Message = error }.ToString() }; } bucketChecked = true; } var verifyUrl = QueryHelpers.AddQueryString( new Uri(configuration.GetBaseUrl(), $"api/v1/lfs/{project.Slug}/verify").ToString(), "token", GenerateUploadVerifyToken(obj)); return(new LFSResponse.LFSObject(obj.Oid, obj.Size) { Actions = new Dictionary <string, LFSResponse.LFSObject.Action>() { { "upload", new LFSResponse.LFSObject.UploadAction() { Href = remoteStorage.CreatePresignedUploadURL(storagePath, S3UploadValidTime), ExpiresIn = (int)UploadValidTime.TotalSeconds } }, { "verify", new LFSResponse.LFSObject.UploadAction() { Href = verifyUrl, ExpiresIn = (int)UploadValidTime.TotalSeconds } } } }); }
public async Task <IActionResult> VerifyUpload([Required] string slug, [Required] string token) { SetContentType(); // Verify token first as there is no other protection on this endpoint UploadVerifyToken verifiedToken; try { verifiedToken = JsonSerializer.Deserialize <UploadVerifyToken>(dataProtector.Unprotect(token)) ?? throw new NullDecodedJsonException(); } catch (Exception e) { logger.LogWarning("Failed to verify LFS upload token: {@E}", e); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Invalid upload verify token provided" } .ToString()) { StatusCode = StatusCodes.Status400BadRequest, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } var project = await database.LfsProjects.Where(p => p.Slug == slug && p.Deleted != true) .FirstOrDefaultAsync(); if (project == null) { return(NotFound()); } var existingObject = await database.LfsObjects .Where(o => o.LfsProjectId == project.Id && o.LfsOid == verifiedToken.Oid).FirstOrDefaultAsync(); if (existingObject != null) { logger.LogWarning("Duplicate LFS oid attempted to be verified: {Oid}", verifiedToken.Oid); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Object with the given OID has already been verified" } .ToString()) { StatusCode = StatusCodes.Status400BadRequest, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } var finalStoragePath = LfsDownloadUrls.OidStoragePath(project, verifiedToken.Oid); var uploadStoragePath = "uploads/" + finalStoragePath; try { var actualSize = await remoteStorage.GetObjectSize(uploadStoragePath); if (actualSize != verifiedToken.Size) { logger.LogWarning("Detected partial upload to remote storage"); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Verification failed: the object size in remote storage is different than it should be" } .ToString()) { StatusCode = StatusCodes.Status400BadRequest, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } } catch (Exception e) { logger.LogWarning("Failed to check object size in storage: {@E}", e); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Verification failed: failed to retrieve the object size" } .ToString()) { StatusCode = StatusCodes.Status400BadRequest, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } try { // Move the uploaded file to a path the user can't anymore access to overwrite it await remoteStorage.MoveObject(uploadStoragePath, finalStoragePath); // Check the stored file hash var actualHash = await remoteStorage.ComputeSha256OfObject(finalStoragePath); if (actualHash != verifiedToken.Oid) { logger.LogWarning("Uploaded file OID doesn't match: {Oid}, actual: {ActualHash}", verifiedToken.Oid, actualHash); logger.LogInformation("Attempting to delete the copied invalid file"); await remoteStorage.DeleteObject(finalStoragePath); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Verification failed: the file you uploaded doesn't match the oid you claimed it to be" } .ToString()) { StatusCode = StatusCodes.Status400BadRequest, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } } catch (Exception e) { logger.LogError("Upload verify storage operation failed: {@E}", e); return(new ObjectResult(new GitLFSErrorResponse() { Message = "Internal storage operation failed" } .ToString()) { StatusCode = StatusCodes.Status500InternalServerError, ContentTypes = new MediaTypeCollection() { AppInfo.GitLfsContentType } }); } // Everything has been verified now so we can save the object // TODO: store the user Id who uploaded the object / if this was anonymous (for PRs) await database.LfsObjects.AddAsync(new LfsObject() { LfsOid = verifiedToken.Oid, Size = verifiedToken.Size, LfsProjectId = project.Id, StoragePath = finalStoragePath }); await database.SaveChangesAsync(); // TODO: queue one more hash check to happen in 30 minutes to ensure the file is right to avoid any possible // timing attack against managing to replace the file with incorrect content logger.LogInformation("New LFS object uploaded: {Oid} for project: {Name}", verifiedToken.Oid, project.Name); return(Ok()); }