public async Task Execute(string uploadId, string path, CancellationToken cancellationToken)
{
logger.LogInformation("Making sure no multipart parts exist for {UploadId}", uploadId);
bool hadParts = false;
try
{
var parts = await remoteStorage.ListMultipartUploadParts(path, uploadId);
if (parts.Count < 1)
{
logger.LogInformation("No parts detected for the multipart upload");
// But still try to delete it
}
else
{
hadParts = true;
}
}
catch (Exception e)
{
logger.LogInformation(e, "Couldn't list parts for upload {UploadId}, assuming no parts exist",
uploadId);
return;
}
try
{
await remoteStorage.AbortMultipartUpload(path, uploadId);
if (hadParts)
{
// Try to delete it one more time just to be extra safe
jobClient.Schedule <MakeSureNoMultipartPartsExistJob>(
x => x.Execute(uploadId, path, CancellationToken.None),
TimeSpan.FromDays(1));
}
}
catch (Exception e)
{
logger.LogWarning(e, "Failed to issue abort call for multipart upload {UploadId}", uploadId);
}
}
public async Task Execute(string uploadId, CancellationToken cancellationToken)
{
var upload = await database.InProgressMultipartUploads.FirstOrDefaultAsync(u => u.UploadId == uploadId,
cancellationToken);
if (upload == null)
{
logger.LogError("Failed to check if multipart upload is stuck, not found in db: {UploadId}", uploadId);
return;
}
if (!upload.Finished)
{
// Seems stuck
logger.LogWarning("Detected abandoned multipart upload ({Path}) {UploadId}, attempting to delete",
upload.Path, upload.UploadId);
try
{
await remoteStorage.AbortMultipartUpload(upload.Path, upload.UploadId);
}
catch (Exception e)
{
logger.LogError(e, "Failed to abort the multipart upload. Still going to try to cancel it later");
}
jobClient.Schedule <MakeSureNoMultipartPartsExistJob>(
x => x.Execute(uploadId, upload.Path, CancellationToken.None),
TimeSpan.FromDays(1));
}
// We can now remove this item as this is handled
database.InProgressMultipartUploads.Remove(upload);
// This is not cancellable as we have potentially made destructive changes already
// ReSharper disable once MethodSupportsCancellation
await database.SaveChangesAsync();
}
private async Task QueryAndCancelMultipartUploadsFromRemoteStorage(CancellationToken cancellationToken)
{
var activeUploads = await database.InProgressMultipartUploads.Where(i => !i.Finished)
.ToListAsync(cancellationToken);
var uploads = await remoteStorage.ListMultipartUploads(cancellationToken);
// Find uploads that don't match any active ones, those we want to cancel
foreach (var upload in uploads)
{
bool match = false;
foreach (var activeUpload in activeUploads)
{
if (activeUpload.Path == upload.key && activeUpload.UploadId == upload.uploadId)
{
match = true;
break;
}
}
if (match)
{
continue;
}
logger.LogError("Detected multipart upload that we have no record of {UploadId} for path: {Key}"
+ " will attempt to terminate it",
upload.uploadId, upload.key);
await remoteStorage.AbortMultipartUpload(upload.key, upload.uploadId);
jobClient.Schedule <MakeSureNoMultipartPartsExistJob>(
x => x.Execute(upload.uploadId, upload.key, CancellationToken.None),
TimeSpan.FromDays(1));
}
}
