Esempio n. 1
0
        private async Task DoUploadAsync(CancellationToken token)
        {
            await InitAsync();

            try
            {
                while (!token.IsCancellationRequested)
                {
                    var toUpload = await _connection.Table <UploadQueueEntry>().FirstAsync();

                    try
                    {
                        var access        = new Access(toUpload.AccessGrant);
                        var bucketService = new BucketService(access);
                        var bucket        = await bucketService.GetBucketAsync(toUpload.BucketName);

                        var objectService = new ObjectService(access);
                        var upload        = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUpload.Bytes, false);

                        await upload.StartUploadAsync();

                        if (upload.Completed)
                        {
                            await _connection.Table <UploadQueueEntry>().DeleteAsync(e => e.Id == toUpload.Id);
                        }
                        else
                        {
                            //ToDo: Error handling
                        }
                    }
                    catch (Exception ex)
                    {
                    }
                }
            }
            catch { } //That's ok, nothing more to do.
        }
Esempio n. 2
0
        private async Task DoUploadAsync(CancellationToken token)
        {
            await InitAsync();

            try
            {
                while (!token.IsCancellationRequested)
                {
                    var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync();

                    if (toUpload != null)
                    {
                        try
                        {
                            var access        = new Access(toUpload.AccessGrant);
                            var bucketService = new BucketService(access);
                            var bucket        = await bucketService.GetBucketAsync(toUpload.BucketName);

                            var multipartUploadService = new MultipartUploadService(access);

                            //If the upload has not UploadId, begin it
                            if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
                            {
                                var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions());

                                toUpload.UploadId = uploadInfo.UploadId;

                                //Save the UploadId
                                await _connection.UpdateAsync(toUpload);
                            }

                            if (!token.IsCancellationRequested)
                            {
                                var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync();

                                if (toUploadData != null)
                                {
                                    while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
                                    {
                                        //Now upload batches of 256 KiB (262144 bytes)
                                        var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(262144).ToArray();
                                        var upload        = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload);

                                        //Refresh the uploaded bytes counter and define the next part number
                                        toUpload.BytesCompleted += (int)upload.BytesWritten;
                                        toUpload.CurrentPartNumber++;

                                        //Save the current state
                                        await _connection.UpdateAsync(toUpload);

                                        UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
                                    }
                                }
                            }

                            //If all bytes are uploaded, commit the upload.
                            if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
                            {
                                var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, new CommitUploadOptions());

                                if (string.IsNullOrEmpty(commitResult.Error))
                                {
                                    await RemoveEntry(toUpload);
                                }
                                else
                                {
                                    toUpload.Failed        = true;
                                    toUpload.FailedMessage = commitResult.Error;

                                    //Save the current state
                                    await _connection.UpdateAsync(toUpload);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            toUpload.Failed        = true;
                            toUpload.FailedMessage = ex.Message;

                            //Save the current state
                            await _connection.UpdateAsync(toUpload);
                        }
                    }
                    else
                    {
                        return;
                    }
                }
            }
            catch { } //That's ok, simply quit. The next run should fix it.
        }
Esempio n. 3
0
        private async Task DoUploadAsync(CancellationToken token)
        {
            await InitAsync().ConfigureAwait(false);

            try
            {
                while (!token.IsCancellationRequested)
                {
                    var toUpload = await _connection.Table <UploadQueueEntry>().Where(e => !e.Failed).FirstOrDefaultAsync().ConfigureAwait(false);

                    if (toUpload != null)
                    {
                        try
                        {
                            var access = new Access(toUpload.AccessGrant);

                            if (toUpload.TotalBytes <= 5242880)
                            {
                                //Upload the file with a single upload-operation
                                var bucketService = new BucketService(access);
                                var bucket        = await bucketService.GetBucketAsync(toUpload.BucketName).ConfigureAwait(false);

                                var objectService = new ObjectService(access);
                                var toUploadData  = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);

                                if (toUploadData != null)
                                {
                                    UploadOperation upload;
                                    if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
                                    {
                                        var customMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
                                        upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, customMetadata, false);
                                    }
                                    else
                                    {
                                        upload = await objectService.UploadObjectAsync(bucket, toUpload.Key, new UploadOptions(), toUploadData.Bytes, false);
                                    }
                                    await upload.StartUploadAsync().ConfigureAwait(false);

                                    if (upload.Failed)
                                    {
                                        toUpload.Failed        = true;
                                        toUpload.FailedMessage = upload.ErrorMessage;
                                    }
                                    else
                                    {
                                        await RemoveEntry(toUpload).ConfigureAwait(false);
                                    }
                                }
                            }
                            else
                            {
                                //Use Multipart-Upload

                                var multipartUploadService = new MultipartUploadService(access);

                                //If the upload has not UploadId, begin it
                                if (!token.IsCancellationRequested && string.IsNullOrEmpty(toUpload.UploadId))
                                {
                                    var uploadInfo = await multipartUploadService.BeginUploadAsync(toUpload.BucketName, toUpload.Key, new UploadOptions()).ConfigureAwait(false);

                                    toUpload.UploadId = uploadInfo.UploadId;

                                    //Save the UploadId
                                    await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
                                }

                                if (!token.IsCancellationRequested)
                                {
                                    var toUploadData = await _connection.Table <UploadQueueEntryData>().Where(d => d.UploadQueueEntryId == toUpload.Id).FirstOrDefaultAsync().ConfigureAwait(false);

                                    if (toUploadData != null)
                                    {
                                        while (!token.IsCancellationRequested && toUpload.BytesCompleted != toUpload.TotalBytes)
                                        {
                                            //Now upload batches of 5 MiB (5242880 bytes)
                                            var bytesToUpload = toUploadData.Bytes.Skip(toUpload.BytesCompleted).Take(5242880).ToArray();
                                            var upload        = await multipartUploadService.UploadPartAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, toUpload.CurrentPartNumber, bytesToUpload).ConfigureAwait(false);

                                            if (!string.IsNullOrEmpty(upload.Error))
                                            {
                                                toUpload.Failed        = true;
                                                toUpload.FailedMessage = upload.Error;
                                            }
                                            else
                                            {
                                                //Refresh the uploaded bytes counter and define the next part number
                                                toUpload.BytesCompleted += (int)upload.BytesWritten;
                                                toUpload.CurrentPartNumber++;
                                            }

                                            //Save the current state
                                            await _connection.UpdateAsync(toUpload).ConfigureAwait(false);

                                            UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
                                        }
                                    }
                                    else
                                    {
                                        toUpload.Failed        = true;
                                        toUpload.FailedMessage = "No data to upload found";

                                        //Save the current state
                                        await _connection.UpdateAsync(toUpload).ConfigureAwait(false);

                                        UploadQueueChangedEvent?.Invoke(QueueChangeType.EntryUpdated, toUpload);
                                    }
                                }

                                //If all bytes are uploaded, commit the upload.
                                if (!token.IsCancellationRequested && toUpload.BytesCompleted == toUpload.TotalBytes)
                                {
                                    var commitOptions = new CommitUploadOptions();
                                    if (!string.IsNullOrEmpty(toUpload.CustomMetadataJson))
                                    {
                                        commitOptions.CustomMetadata = JsonSerializer.Deserialize <CustomMetadata>(toUpload.CustomMetadataJson);
                                    }
                                    var commitResult = await multipartUploadService.CommitUploadAsync(toUpload.BucketName, toUpload.Key, toUpload.UploadId, commitOptions).ConfigureAwait(false);

                                    if (string.IsNullOrEmpty(commitResult.Error))
                                    {
                                        await RemoveEntry(toUpload).ConfigureAwait(false);
                                    }
                                    else
                                    {
                                        toUpload.Failed        = true;
                                        toUpload.FailedMessage = commitResult.Error;

                                        //Save the current state
                                        await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            toUpload.Failed        = true;
                            toUpload.FailedMessage = ex.Message;
                            toUpload.UploadId      = String.Empty; //Otherwise the upload-queue might get stuck if the upload cannot be succeeded, as on the next run it would hit the same error again

                            //Save the current state
                            await _connection.UpdateAsync(toUpload).ConfigureAwait(false);
                        }
                    }
                    else
                    {
                        return;
                    }
                }
            }
            catch
            {
                //That's ok, simply quit. The next run should fix it.
            }
        }