public async Task Execute(string uploadId, string path, CancellationToken cancellationToken) { logger.LogInformation("Making sure no multipart parts exist for {UploadId}", uploadId); bool hadParts = false; try { var parts = await remoteStorage.ListMultipartUploadParts(path, uploadId); if (parts.Count < 1) { logger.LogInformation("No parts detected for the multipart upload"); // But still try to delete it } else { hadParts = true; } } catch (Exception e) { logger.LogInformation(e, "Couldn't list parts for upload {UploadId}, assuming no parts exist", uploadId); return; } try { await remoteStorage.AbortMultipartUpload(path, uploadId); if (hadParts) { // Try to delete it one more time just to be extra safe jobClient.Schedule <MakeSureNoMultipartPartsExistJob>( x => x.Execute(uploadId, path, CancellationToken.None), TimeSpan.FromDays(1)); } } catch (Exception e) { logger.LogWarning(e, "Failed to issue abort call for multipart upload {UploadId}", uploadId); } }
public async Task Execute(string uploadId, CancellationToken cancellationToken) { var upload = await database.InProgressMultipartUploads.FirstOrDefaultAsync(u => u.UploadId == uploadId, cancellationToken); if (upload == null) { logger.LogError("Failed to check if multipart upload is stuck, not found in db: {UploadId}", uploadId); return; } if (!upload.Finished) { // Seems stuck logger.LogWarning("Detected abandoned multipart upload ({Path}) {UploadId}, attempting to delete", upload.Path, upload.UploadId); try { await remoteStorage.AbortMultipartUpload(upload.Path, upload.UploadId); } catch (Exception e) { logger.LogError(e, "Failed to abort the multipart upload. Still going to try to cancel it later"); } jobClient.Schedule <MakeSureNoMultipartPartsExistJob>( x => x.Execute(uploadId, upload.Path, CancellationToken.None), TimeSpan.FromDays(1)); } // We can now remove this item as this is handled database.InProgressMultipartUploads.Remove(upload); // This is not cancellable as we have potentially made destructive changes already // ReSharper disable once MethodSupportsCancellation await database.SaveChangesAsync(); }
private async Task QueryAndCancelMultipartUploadsFromRemoteStorage(CancellationToken cancellationToken) { var activeUploads = await database.InProgressMultipartUploads.Where(i => !i.Finished) .ToListAsync(cancellationToken); var uploads = await remoteStorage.ListMultipartUploads(cancellationToken); // Find uploads that don't match any active ones, those we want to cancel foreach (var upload in uploads) { bool match = false; foreach (var activeUpload in activeUploads) { if (activeUpload.Path == upload.key && activeUpload.UploadId == upload.uploadId) { match = true; break; } } if (match) { continue; } logger.LogError("Detected multipart upload that we have no record of {UploadId} for path: {Key}" + " will attempt to terminate it", upload.uploadId, upload.key); await remoteStorage.AbortMultipartUpload(upload.key, upload.uploadId); jobClient.Schedule <MakeSureNoMultipartPartsExistJob>( x => x.Execute(upload.uploadId, upload.key, CancellationToken.None), TimeSpan.FromDays(1)); } }