public async Task RetryAsync(string key)
{
await InitAsync();
var entry = await _connection.Table <UploadQueueEntry>().Where(e => e.Key == key).FirstOrDefaultAsync();
if (entry != null)
{
try
{
var access = new Access(entry.AccessGrant);
var multipartUploadService = new MultipartUploadService(access);
await multipartUploadService.AbortUploadAsync(entry.BucketName, entry.Key, entry.UploadId);
}
catch { }
entry.BytesCompleted = 0;
entry.Failed = false;
entry.FailedMessage = string.Empty;
entry.CurrentPartNumber = 0;
entry.UploadId = string.Empty;
await _connection.UpdateAsync(entry);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, entry);
}
}
public async Task RetryAsync(string key)
{
await InitAsync().ConfigureAwait(false);
var entry = await _connection.Table <UploadQueueEntry>().Where(e => e.Key == key).FirstOrDefaultAsync().ConfigureAwait(false);
if (entry != null)
{
try
{
var access = new Access(entry.AccessGrant);
var multipartUploadService = new MultipartUploadService(access);
await multipartUploadService.AbortUploadAsync(entry.BucketName, entry.Key, entry.UploadId).ConfigureAwait(false);
}
catch
{
//Ignore errors - we restart that upload anyways
}
var entryData = await _connection.GetAsync <UploadQueueEntryData>(e => e.UploadQueueEntryId == entry.Id);
if (entryData != null)
{
entry.BytesCompleted = 0;
entry.Failed = false;
entry.FailedMessage = string.Empty;
entry.CurrentPartNumber = 0;
entry.UploadId = string.Empty;
await _connection.RunInTransactionAsync((SQLiteConnection transaction) =>
{
transaction.Update(entry);
transaction.Update(entryData);
});
}
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, entry);
}
}
private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync();
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync();
if (toUpload != null)
{
try
{
var access = new Access(toUpload.AccessGrant);
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName);
var multipartUploadService = new MultipartUploadService(access);
//If the upload has not UploadId, begin it
if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
{
var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions());
toUpload.UploadId = uploadInfo.UploadId;
//Save the UploadId
await _connection.UpdateAsync(toUpload);
}
if (!token.IsCancellationRequested)
{
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync();
if (toUploadData != null)
{
while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
{
//Now upload batches of 256 KiB (262144 bytes)
var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(262144).ToArray();
var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload);
//Refresh the uploaded bytes counter and define the next part number
toUpload.BytesCompleted += (int)upload.BytesWritten;
toUpload.CurrentPartNumber++;
//Save the current state
await _connection.UpdateAsync(toUpload);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
}
//If all bytes are uploaded, commit the upload.
if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
{
var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, new CommitUploadOptions());
if (string.IsNullOrEmpty(commitResult.Error))
{
await RemoveEntry(toUpload);
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = commitResult.Error;
//Save the current state
await _connection.UpdateAsync(toUpload);
}
}
}
catch (Exception ex)
{
toUpload.Failed = true;
toUpload.FailedMessage = ex.Message;
//Save the current state
await _connection.UpdateAsync(toUpload);
}
}
else
{
return;
}
}
}
catch { } //That's ok, simply quit. The next run should fix it.
}
private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync().ConfigureAwait(false);
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUpload != null)
{
try
{
var access = new Access(toUpload.AccessGrant);
if (toUpload.TotalBytes <= 5242880)
{
//Upload the file with a single upload-operation
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName).ConfigureAwait(false);
var objectService = new ObjectService(access);
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
UploadOperation upload;
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
var customMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, customMetadata, false);
}
else
{
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, false);
}
await upload.StartUploadAsync().ConfigureAwait(false);
if (upload.Failed)
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.ErrorMessage;
}
else
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
}
}
else
{
//Use Multipart-Upload
var multipartUploadService = new MultipartUploadService(access);
//If the upload has not UploadId, begin it
if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
{
var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()).ConfigureAwait(false);
toUpload.UploadId = uploadInfo.UploadId;
//Save the UploadId
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
if (!token.IsCancellationRequested)
{
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
{
//Now upload batches of 5 MiB (5242880 bytes)
var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(5242880).ToArray();
var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload).ConfigureAwait(false);
if (!string.IsNullOrEmpty(upload.Error))
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.Error;
}
else
{
//Refresh the uploaded bytes counter and define the next part number
toUpload.BytesCompleted += (int)upload.BytesWritten;
toUpload.CurrentPartNumber++;
}
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = "No data to upload found";
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
//If all bytes are uploaded, commit the upload.
if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
{
var commitOptions = new CommitUploadOptions();
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
commitOptions.CustomMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
}
var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, commitOptions).ConfigureAwait(false);
if (string.IsNullOrEmpty(commitResult.Error))
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = commitResult.Error;
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
}
}
catch (Exception ex)
{
toUpload.Failed = true;
toUpload.FailedMessage = ex.Message;
toUpload.UploadId = String.Empty; //Otherwise the upload-queue might get stuck if the upload cannot be succeeded, as on the next run it would hit the same error again
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
else
{
return;
}
}
}
catch
{
//That's ok, simply quit. The next run should fix it.
}
}
