private async Task DoUploadAsync(CancellationToken token) { await InitAsync(); try { while (!token.IsCancellationRequested) { var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync(); if (toUpload != null) { try { var access = new Access(toUpload.AccessGrant); var bucketService = new BucketService(access); var bucket = await bucketService.GetBucketAsync(toUpload.BucketName); var multipartUploadService = new MultipartUploadService(access); //If the upload has not UploadId, begin it if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId)) { var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()); toUpload.UploadId = uploadInfo.UploadId; //Save the UploadId await _connection.UpdateAsync(toUpload); } if (!token.IsCancellationRequested) { var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync(); if (toUploadData != null) { while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes) { //Now upload batches of 256 KiB (262144 bytes) var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(262144).ToArray(); var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload); //Refresh the uploaded bytes counter and define the next part number toUpload.BytesCompleted += (int)upload.BytesWritten; toUpload.CurrentPartNumber++; //Save the current state await _connection.UpdateAsync(toUpload); UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload); } } } //If all bytes are uploaded, commit the upload. if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes) { var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, new CommitUploadOptions()); if (string.IsNullOrEmpty(commitResult.Error)) { await RemoveEntry(toUpload); } else { toUpload.Failed = true; toUpload.FailedMessage = commitResult.Error; //Save the current state await _connection.UpdateAsync(toUpload); } } } catch (Exception ex) { toUpload.Failed = true; toUpload.FailedMessage = ex.Message; //Save the current state await _connection.UpdateAsync(toUpload); } } else { return; } } } catch { } //That's ok, simply quit. The next run should fix it. }
private async Task DoUploadAsync(CancellationToken token) { await InitAsync().ConfigureAwait(false); try { while (!token.IsCancellationRequested) { var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync().ConfigureAwait(false); if (toUpload != null) { try { var access = new Access(toUpload.AccessGrant); if (toUpload.TotalBytes <= 5242880) { //Upload the file with a single upload-operation var bucketService = new BucketService(access); var bucket = await bucketService.GetBucketAsync(toUpload.BucketName).ConfigureAwait(false); var objectService = new ObjectService(access); var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false); if (toUploadData != null) { UploadOperation upload; if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson)) { var customMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson); upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, customMetadata, false); } else { upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, false); } await upload.StartUploadAsync().ConfigureAwait(false); if (upload.Failed) { toUpload.Failed = true; toUpload.FailedMessage = upload.ErrorMessage; } else { await RemoveEntry(toUpload).ConfigureAwait(false); } } } else { //Use Multipart-Upload var multipartUploadService = new MultipartUploadService(access); //If the upload has not UploadId, begin it if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId)) { var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()).ConfigureAwait(false); toUpload.UploadId = uploadInfo.UploadId; //Save the UploadId await _connection.UpdateAsync(toUpload).ConfigureAwait(false); } if (!token.IsCancellationRequested) { var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false); if (toUploadData != null) { while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes) { //Now upload batches of 5 MiB (5242880 bytes) var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(5242880).ToArray(); var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload).ConfigureAwait(false); if (!string.IsNullOrEmpty(upload.Error)) { toUpload.Failed = true; toUpload.FailedMessage = upload.Error; } else { //Refresh the uploaded bytes counter and define the next part number toUpload.BytesCompleted += (int)upload.BytesWritten; toUpload.CurrentPartNumber++; } //Save the current state await _connection.UpdateAsync(toUpload).ConfigureAwait(false); UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload); } } else { toUpload.Failed = true; toUpload.FailedMessage = "No data to upload found"; //Save the current state await _connection.UpdateAsync(toUpload).ConfigureAwait(false); UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload); } } //If all bytes are uploaded, commit the upload. if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes) { var commitOptions = new CommitUploadOptions(); if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson)) { commitOptions.CustomMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson); } var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, commitOptions).ConfigureAwait(false); if (string.IsNullOrEmpty(commitResult.Error)) { await RemoveEntry(toUpload).ConfigureAwait(false); } else { toUpload.Failed = true; toUpload.FailedMessage = commitResult.Error; //Save the current state await _connection.UpdateAsync(toUpload).ConfigureAwait(false); } } } } catch (Exception ex) { toUpload.Failed = true; toUpload.FailedMessage = ex.Message; toUpload.UploadId = String.Empty; //Otherwise the upload-queue might get stuck if the upload cannot be succeeded, as on the next run it would hit the same error again //Save the current state await _connection.UpdateAsync(toUpload).ConfigureAwait(false); } } else { return; } } } catch { //That's ok, simply quit. The next run should fix it. } }