private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync();
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().FirstAsync();
try
{
var access = new Access(toUpload.AccessGrant);
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName);
var objectService = new ObjectService(access);
var upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUpload.Bytes, false);
await upload.StartUploadAsync();
if (upload.Completed)
{
await _connection.Table <UploadQueueEntry>().DeleteAsync(e => e.Id == toUpload.Id);
}
else
{
//ToDo: Error handling
}
}
catch (Exception ex)
{
}
}
}
catch { } //That's ok, nothing more to do.
}
private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync();
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync();
if (toUpload != null)
{
try
{
var access = new Access(toUpload.AccessGrant);
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName);
var multipartUploadService = new MultipartUploadService(access);
//If the upload has not UploadId, begin it
if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
{
var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions());
toUpload.UploadId = uploadInfo.UploadId;
//Save the UploadId
await _connection.UpdateAsync(toUpload);
}
if (!token.IsCancellationRequested)
{
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync();
if (toUploadData != null)
{
while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
{
//Now upload batches of 256 KiB (262144 bytes)
var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(262144).ToArray();
var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload);
//Refresh the uploaded bytes counter and define the next part number
toUpload.BytesCompleted += (int)upload.BytesWritten;
toUpload.CurrentPartNumber++;
//Save the current state
await _connection.UpdateAsync(toUpload);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
}
//If all bytes are uploaded, commit the upload.
if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
{
var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, new CommitUploadOptions());
if (string.IsNullOrEmpty(commitResult.Error))
{
await RemoveEntry(toUpload);
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = commitResult.Error;
//Save the current state
await _connection.UpdateAsync(toUpload);
}
}
}
catch (Exception ex)
{
toUpload.Failed = true;
toUpload.FailedMessage = ex.Message;
//Save the current state
await _connection.UpdateAsync(toUpload);
}
}
else
{
return;
}
}
}
catch { } //That's ok, simply quit. The next run should fix it.
}
private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync().ConfigureAwait(false);
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUpload != null)
{
try
{
var access = new Access(toUpload.AccessGrant);
if (toUpload.TotalBytes <= 5242880)
{
//Upload the file with a single upload-operation
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName).ConfigureAwait(false);
var objectService = new ObjectService(access);
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
UploadOperation upload;
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
var customMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, customMetadata, false);
}
else
{
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, false);
}
await upload.StartUploadAsync().ConfigureAwait(false);
if (upload.Failed)
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.ErrorMessage;
}
else
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
}
}
else
{
//Use Multipart-Upload
var multipartUploadService = new MultipartUploadService(access);
//If the upload has not UploadId, begin it
if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
{
var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()).ConfigureAwait(false);
toUpload.UploadId = uploadInfo.UploadId;
//Save the UploadId
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
if (!token.IsCancellationRequested)
{
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
{
//Now upload batches of 5 MiB (5242880 bytes)
var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(5242880).ToArray();
var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload).ConfigureAwait(false);
if (!string.IsNullOrEmpty(upload.Error))
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.Error;
}
else
{
//Refresh the uploaded bytes counter and define the next part number
toUpload.BytesCompleted += (int)upload.BytesWritten;
toUpload.CurrentPartNumber++;
}
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = "No data to upload found";
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
//If all bytes are uploaded, commit the upload.
if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
{
var commitOptions = new CommitUploadOptions();
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
commitOptions.CustomMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
}
var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, commitOptions).ConfigureAwait(false);
if (string.IsNullOrEmpty(commitResult.Error))
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = commitResult.Error;
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
}
}
catch (Exception ex)
{
toUpload.Failed = true;
toUpload.FailedMessage = ex.Message;
toUpload.UploadId = String.Empty; //Otherwise the upload-queue might get stuck if the upload cannot be succeeded, as on the next run it would hit the same error again
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
else
{
return;
}
}
}
catch
{
//That's ok, simply quit. The next run should fix it.
}
}