public LFSController(ILogger <LFSController> logger,
ApplicationDbContext database, LfsDownloadUrls downloadUrls, IDataProtectionProvider dataProtectionProvider,
LfsRemoteStorage remoteStorage, IConfiguration configuration)
{
this.logger = logger;
this.database = database;
this.downloadUrls = downloadUrls;
this.remoteStorage = remoteStorage;
this.configuration = configuration;
dataProtector = dataProtectionProvider.CreateProtector(LfsUploadProtectionPurposeString)
.ToTimeLimitedDataProtector();
}
public LFSFileDownloadController(ApplicationDbContext database, LfsDownloadUrls downloadUrls)
{
this.database = database;
this.downloadUrls = downloadUrls;
}
private async ValueTask <LFSResponse.LFSObject> HandleUpload(LfsProject project, LFSRequest.LFSObject obj)
{
var existingObject = await database.LfsObjects
.Where(o => o.LfsOid == obj.Oid && o.LfsProjectId == project.Id).Include(o => o.LfsProject)
.FirstOrDefaultAsync();
if (existingObject != null)
{
// We already have this object
return(new LFSResponse.LFSObject(obj.Oid, obj.Size)
{
Actions = null,
Authenticated = null
});
}
if (obj.Size > AppInfo.MaxLfsUploadSize)
{
return(new LFSResponse.LFSObject(obj.Oid, obj.Size,
new LFSResponse.LFSObject.ErrorInfo(StatusCodes.Status422UnprocessableEntity, "File is too large")));
}
logger.LogTrace("Requesting auth because new object is to be uploaded {Oid} for project {Name}", obj.Oid,
project.Name);
// New object. User must have write access
if (!RequireWriteAccess(out var result))
{
throw new InvalidAccessException(result !);
}
// We don't yet create the LfsObject here to guard against upload failures
// instead the verify callback does that
// The uploads prefix is used here to ensure the user can't overwrite the file after uploading and
// verification
var storagePath = "uploads/" + LfsDownloadUrls.OidStoragePath(project, obj.Oid);
if (!bucketChecked)
{
try
{
if (!await remoteStorage.BucketExists())
{
throw new Exception("bucket doesn't exist");
}
}
catch (Exception e)
{
logger.LogWarning("Bucket check failed: {@E}", e);
var error = "remote storage is inaccessible";
throw new HttpResponseException()
{
Status = StatusCodes.Status500InternalServerError,
ContentType = AppInfo.GitLfsContentType,
Value = new GitLFSErrorResponse()
{
Message = error
}.ToString()
};
}
bucketChecked = true;
}
var verifyUrl = QueryHelpers.AddQueryString(
new Uri(configuration.GetBaseUrl(), $"api/v1/lfs/{project.Slug}/verify").ToString(),
"token", GenerateUploadVerifyToken(obj));
return(new LFSResponse.LFSObject(obj.Oid, obj.Size)
{
Actions = new Dictionary <string, LFSResponse.LFSObject.Action>()
{
{
"upload", new LFSResponse.LFSObject.UploadAction()
{
Href = remoteStorage.CreatePresignedUploadURL(storagePath, S3UploadValidTime),
ExpiresIn = (int)UploadValidTime.TotalSeconds
}
},
{
"verify", new LFSResponse.LFSObject.UploadAction()
{
Href = verifyUrl,
ExpiresIn = (int)UploadValidTime.TotalSeconds
}
}
}
});
}
public async Task <IActionResult> VerifyUpload([Required] string slug, [Required] string token)
{
SetContentType();
// Verify token first as there is no other protection on this endpoint
UploadVerifyToken verifiedToken;
try
{
verifiedToken = JsonSerializer.Deserialize <UploadVerifyToken>(dataProtector.Unprotect(token)) ??
throw new NullDecodedJsonException();
}
catch (Exception e)
{
logger.LogWarning("Failed to verify LFS upload token: {@E}", e);
return(new ObjectResult(new GitLFSErrorResponse()
{
Message = "Invalid upload verify token provided"
}
.ToString())
{
StatusCode = StatusCodes.Status400BadRequest,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
var project = await database.LfsProjects.Where(p => p.Slug == slug && p.Deleted != true)
.FirstOrDefaultAsync();
if (project == null)
{
return(NotFound());
}
var existingObject = await database.LfsObjects
.Where(o => o.LfsProjectId == project.Id && o.LfsOid == verifiedToken.Oid).FirstOrDefaultAsync();
if (existingObject != null)
{
logger.LogWarning("Duplicate LFS oid attempted to be verified: {Oid}", verifiedToken.Oid);
return(new ObjectResult(new GitLFSErrorResponse()
{
Message = "Object with the given OID has already been verified"
}
.ToString())
{
StatusCode = StatusCodes.Status400BadRequest,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
var finalStoragePath = LfsDownloadUrls.OidStoragePath(project, verifiedToken.Oid);
var uploadStoragePath = "uploads/" + finalStoragePath;
try
{
var actualSize = await remoteStorage.GetObjectSize(uploadStoragePath);
if (actualSize != verifiedToken.Size)
{
logger.LogWarning("Detected partial upload to remote storage");
return(new ObjectResult(new GitLFSErrorResponse()
{
Message =
"Verification failed: the object size in remote storage is different than it should be"
}
.ToString())
{
StatusCode = StatusCodes.Status400BadRequest,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
}
catch (Exception e)
{
logger.LogWarning("Failed to check object size in storage: {@E}", e);
return(new ObjectResult(new GitLFSErrorResponse()
{
Message = "Verification failed: failed to retrieve the object size"
}
.ToString())
{
StatusCode = StatusCodes.Status400BadRequest,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
try
{
// Move the uploaded file to a path the user can't anymore access to overwrite it
await remoteStorage.MoveObject(uploadStoragePath, finalStoragePath);
// Check the stored file hash
var actualHash = await remoteStorage.ComputeSha256OfObject(finalStoragePath);
if (actualHash != verifiedToken.Oid)
{
logger.LogWarning("Uploaded file OID doesn't match: {Oid}, actual: {ActualHash}", verifiedToken.Oid,
actualHash);
logger.LogInformation("Attempting to delete the copied invalid file");
await remoteStorage.DeleteObject(finalStoragePath);
return(new ObjectResult(new GitLFSErrorResponse()
{
Message =
"Verification failed: the file you uploaded doesn't match the oid you claimed it to be"
}
.ToString())
{
StatusCode = StatusCodes.Status400BadRequest,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
}
catch (Exception e)
{
logger.LogError("Upload verify storage operation failed: {@E}", e);
return(new ObjectResult(new GitLFSErrorResponse()
{
Message = "Internal storage operation failed"
}
.ToString())
{
StatusCode = StatusCodes.Status500InternalServerError,
ContentTypes = new MediaTypeCollection()
{
AppInfo.GitLfsContentType
}
});
}
// Everything has been verified now so we can save the object
// TODO: store the user Id who uploaded the object / if this was anonymous (for PRs)
await database.LfsObjects.AddAsync(new LfsObject()
{
LfsOid = verifiedToken.Oid,
Size = verifiedToken.Size,
LfsProjectId = project.Id,
StoragePath = finalStoragePath
});
await database.SaveChangesAsync();
// TODO: queue one more hash check to happen in 30 minutes to ensure the file is right to avoid any possible
// timing attack against managing to replace the file with incorrect content
logger.LogInformation("New LFS object uploaded: {Oid} for project: {Name}", verifiedToken.Oid,
project.Name);
return(Ok());
}
