private async Task DoUploadAsync(CancellationToken token)
{
await InitAsync().ConfigureAwait(false);
try
{
while (!token.IsCancellationRequested)
{
var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUpload != null)
{
try
{
var access = new Access(toUpload.AccessGrant);
if (toUpload.TotalBytes <= 5242880)
{
//Upload the file with a single upload-operation
var bucketService = new BucketService(access);
var bucket = await bucketService.GetBucketAsync(toUpload.BucketName).ConfigureAwait(false);
var objectService = new ObjectService(access);
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
UploadOperation upload;
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
var customMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, customMetadata, false);
}
else
{
upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, false);
}
await upload.StartUploadAsync().ConfigureAwait(false);
if (upload.Failed)
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.ErrorMessage;
}
else
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
}
}
else
{
//Use Multipart-Upload
var multipartUploadService = new MultipartUploadService(access);
//If the upload has not UploadId, begin it
if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
{
var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()).ConfigureAwait(false);
toUpload.UploadId = uploadInfo.UploadId;
//Save the UploadId
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
if (!token.IsCancellationRequested)
{
var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);
if (toUploadData != null)
{
while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
{
//Now upload batches of 5 MiB (5242880 bytes)
var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(5242880).ToArray();
var upload = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload).ConfigureAwait(false);
if (!string.IsNullOrEmpty(upload.Error))
{
toUpload.Failed = true;
toUpload.FailedMessage = upload.Error;
}
else
{
//Refresh the uploaded bytes counter and define the next part number
toUpload.BytesCompleted += (int)upload.BytesWritten;
toUpload.CurrentPartNumber++;
}
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = "No data to upload found";
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
}
}
//If all bytes are uploaded, commit the upload.
if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
{
var commitOptions = new CommitUploadOptions();
if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
{
commitOptions.CustomMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
}
var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, commitOptions).ConfigureAwait(false);
if (string.IsNullOrEmpty(commitResult.Error))
{
await RemoveEntry(toUpload).ConfigureAwait(false);
}
else
{
toUpload.Failed = true;
toUpload.FailedMessage = commitResult.Error;
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
}
}
catch (Exception ex)
{
toUpload.Failed = true;
toUpload.FailedMessage = ex.Message;
toUpload.UploadId = String.Empty; //Otherwise the upload-queue might get stuck if the upload cannot be succeeded, as on the next run it would hit the same error again
//Save the current state
await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
}
}
else
{
return;
}
}
}
catch
{
//That's ok, simply quit. The next run should fix it.
}
}
public async Task <CommitUploadResult> CommitUploadAsync(string bucketName, string objectKey, string uploadId, CommitUploadOptions commitUploadOptions)
{
CommitUploadResult result = new CommitUploadResult();
await Task.Run(() =>
{
UploadOperation.customMetadataSemaphore.Wait();
try
{
if (commitUploadOptions.CustomMetadata != null)
{
commitUploadOptions.CustomMetadata.ToSWIG(); //Appends the customMetadata in the go-layer to a global field
}
using (var commitUploadResult = SWIG.storj_uplink.uplink_commit_upload2(_access._project, bucketName, objectKey, uploadId))
{
if (commitUploadResult.error != null && !string.IsNullOrEmpty(commitUploadResult.error.message))
{
result.Error = commitUploadResult.error.message;
}
else
{
result.Object = Models.Object.FromSWIG(commitUploadResult.object_);
}
}
}
finally
{
UploadOperation.customMetadataSemaphore.Release();
}
}).ConfigureAwait(false);
return(result);
}
public async Task <CommitUploadResult> CommitUploadAsync(string bucketName, string objectKey, string uploadId, CommitUploadOptions commitUploadOptions)
{
CommitUploadResult result = new CommitUploadResult();
var uplinkCommitUploadOptions = new SWIG.UplinkCommitUploadOptions();
using (var commitUploadResult = await Task.Run(() => SWIG.storj_uplink.uplink_commit_upload(_access._project, bucketName, objectKey, uploadId, uplinkCommitUploadOptions)))
{
if (commitUploadResult.error != null && !string.IsNullOrEmpty(commitUploadResult.error.message))
{
result.Error = commitUploadResult.error.message;
}
else
{
result.Object = Models.Object.FromSWIG(commitUploadResult.object_);
}
}
return(result);
}
