private void AbortMultipartUpload(MultipartUpload u, AmazonS3Client client)
{
var request = new AbortMultipartUploadRequest
{
BucketName = bucket,
Key = u.Key,
UploadId = u.UploadId,
};
client.AbortMultipartUploadAsync(request).Wait();
}
public async Task Abort()
{
// No need to cancel if the uploadid call is the thing that failed
if (_uploadId.IsFaulted || !_uploadId.IsCompleted)
{
return;
}
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = _bucket,
Key = _key,
UploadId = _uploadId.Result
};
await _client.AbortMultipartUploadAsync(abortMPURequest).ConfigureAwait(false);
}
public async Task AbortBlobWriteAsync(string correlationId, string token)
{
CheckOpened(correlationId);
var tokens = (token ?? "").Split(';');
if (tokens.Length < 2)
{
throw new BadRequestException(
correlationId,
"BAD_TOKEN",
"Token " + token + " is invalid"
).WithDetails("token", token);
}
await _client.AbortMultipartUploadAsync(new AbortMultipartUploadRequest
{
BucketName = _bucket,
Key = tokens[0],
UploadId = tokens[1]
});
}
private static async Task UploadObjectAsync(AmazonS3Client client, string filePath)
{
Console.WriteLine($"Uploading {filePath}");
var keyName = filePath.Substring(3, filePath.Length - 3);
keyName = keyName.Replace(@"\", "/");
// Create list to store upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
};
// Initiate the upload.
InitiateMultipartUploadResponse initResponse =
await client.InitiateMultipartUploadAsync(initiateRequest);
// Upload parts.
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath
};
// Track upload progress.
uploadRequest.StreamTransferProgress +=
new EventHandler <StreamTransferProgressArgs>(UploadPartProgressEventCallback);
// Upload a part and add the response to our list.
uploadResponses.Add(await client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
// Setup to complete the upload.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
CompleteMultipartUploadResponse completeUploadResponse =
await client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await client.AbortMultipartUploadAsync(abortMPURequest);
}
}
public void SSEMultipartUploadTest()
{
var testKey = "test.tst";
var size = 1 * 1024 * 1024;
var data = new byte[size];
new Random().NextBytes(data);
Aes aesEncryption = Aes.Create();
aesEncryption.KeySize = 256;
aesEncryption.GenerateKey();
var encKey = Convert.ToBase64String(aesEncryption.Key);
Assert.DoesNotThrowAsync(async() =>
{
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = TestBucket,
Key = testKey,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = encKey
};
var initResponse = await _s3Client.InitiateMultipartUploadAsync(initiateRequest);
// Assert.IsNotNull(initResponse.ServerSideEncryptionCustomerMethod);
// Assert.IsNotNull(initResponse.ServerSideEncryptionCustomerProvidedKeyMD5);
try
{
UploadPartResponse uploadResponse;
await using (var stream = new MemoryStream(data))
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = TestBucket,
Key = testKey,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = size,
FilePosition = 0,
InputStream = stream,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = encKey,
};
uploadResponse = await _s3Client.UploadPartAsync(uploadRequest);
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = TestBucket,
Key = testKey,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponse);
var completeUploadResponse = await _s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch
{
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = TestBucket,
Key = testKey,
UploadId = initResponse.UploadId
};
await _s3Client.AbortMultipartUploadAsync(abortMPURequest);
throw;
}
});
}
private async Task MultipartUploadAsync(AmazonS3Client s3, Stream stream, string key, IOperationExecutionContext context)
{
var uploadResponse = await s3.InitiateMultipartUploadAsync(
new InitiateMultipartUploadRequest
{
BucketName = this.BucketName,
Key = key,
CannedACL = this.CannedACL,
StorageClass = this.StorageClass,
ServerSideEncryptionMethod = this.EncryptionMethod
},
context.CancellationToken
);
try
{
var parts = this.GetParts(stream.Length);
var completedParts = new List <PartETag>(parts.Count);
for (int i = 0; i < parts.Count; i++)
{
var partResponse = await s3.UploadPartAsync(
new UploadPartRequest
{
BucketName = this.BucketName,
Key = key,
InputStream = stream,
UploadId = uploadResponse.UploadId,
PartSize = parts[i].Length,
FilePosition = parts[i].StartOffset,
PartNumber = i + 1,
StreamTransferProgress = (s, e) => Interlocked.Add(ref this.uploadedBytes, e.IncrementTransferred)
},
context.CancellationToken
);
completedParts.Add(new PartETag(i + 1, partResponse.ETag));
}
await s3.CompleteMultipartUploadAsync(
new CompleteMultipartUploadRequest
{
BucketName = this.BucketName,
Key = key,
UploadId = uploadResponse.UploadId,
PartETags = completedParts
},
context.CancellationToken
);
}
catch
{
await s3.AbortMultipartUploadAsync(
new AbortMultipartUploadRequest
{
BucketName = this.BucketName,
Key = key,
UploadId = uploadResponse.UploadId
}
);
throw;
}
}
private async Task <bool> UploadObjectBandWidthThrottlingAsync(string bucketName, string localFullFilename, string key, CancellationToken token, int sleepMiliSec, bool publicReadTrueOrFalse = false)
{
bool bReturn = true;
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = key
};
if (publicReadTrueOrFalse)
{
initiateRequest.CannedACL = S3CannedACL.PublicRead;
}
IAmazonS3 s3Client = new AmazonS3Client(AccessKey, SecretKey, new AmazonS3Config
{
ServiceURL = ServiceUrl,
BufferSize = 64 * (int)Math.Pow(2, 10),
ProgressUpdateInterval = this.ProgressUpdateInterval,
Timeout = new TimeSpan(1, 0, 0, 0, 0)
});
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest, token);
long contentLength = new FileInfo(localFullFilename).Length;
ContentTotalBytes = contentLength;
long partSize = 5 * (long)Math.Pow(2, 20);
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
if (token.IsCancellationRequested)
{
bReturn = false;
break;
}
var task = Task.Run(() => Thread.Sleep(sleepMiliSec));
await task;
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = localFullFilename
};
uploadRequest.StreamTransferProgress += ProgressEventCallback;
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest, token));
filePosition += partSize;
CurrentTransferredBytes = filePosition;
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest, token);
}
catch (Exception)
{
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
throw;
}
finally
{
s3Client.Dispose();
}
return(bReturn);
}
public async static Task MainAsync(string[] args)
{
//Welcome user
Console.BackgroundColor = ConsoleColor.Blue;
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Digital Ocean Spaces Manager");
Console.ResetColor();
#region Keys
Console.Write("Enter DO Access Key: ");
ConsoleKeyInfo key;
KeyManager.ACCESS_KEY.Clear();
do
{
key = Console.ReadKey(true);
if (key.Key != ConsoleKey.Backspace && key.Key != ConsoleKey.Enter)
{
KeyManager.ACCESS_KEY.AppendChar(key.KeyChar);
Console.Write("*");
}
else
{
if (key.Key == ConsoleKey.Backspace && KeyManager.ACCESS_KEY.Length > 0)
{
KeyManager.ACCESS_KEY.RemoveAt(KeyManager.ACCESS_KEY.Length - 1);
Console.Write("\b \b");
}
}
} while (key.Key != ConsoleKey.Enter);
Console.Write("\n");
Console.Write("Enter DO Secrey Key: ");
KeyManager.SECRET_KEY.Clear();
do
{
key = Console.ReadKey(true);
if (key.Key != ConsoleKey.Backspace && key.Key != ConsoleKey.Enter)
{
KeyManager.SECRET_KEY.AppendChar(key.KeyChar);
Console.Write("*");
}
else
{
if (key.Key == ConsoleKey.Backspace && KeyManager.ACCESS_KEY.Length > 0)
{
KeyManager.ACCESS_KEY.RemoveAt(KeyManager.ACCESS_KEY.Length - 1);
Console.Write("\b \b");
}
}
} while (key.Key != ConsoleKey.Enter);
Console.Write("\n");
#endregion
var client = new AmazonS3Client(KeyManager.SecureStringToString(KeyManager.ACCESS_KEY), KeyManager.SecureStringToString(KeyManager.SECRET_KEY), new AmazonS3Config()
{
ServiceURL = "https://nyc3.digitaloceanspaces.com"
});
client.ExceptionEvent += Client_ExceptionEvent;
string filePath, uploadName, spaceName, contentType = string.Empty;
Console.Write("Enter Space name to use: ");
spaceName = Console.ReadLine();
bool fileExists = false;
do
{
Console.Write("Enter file location: ");
filePath = Console.ReadLine();
if (File.Exists(filePath))
{
contentType = MimeGuesser.GuessFileType(filePath).MimeType;
fileExists = true;
}
else
{
fileExists = false;
Console.WriteLine("File does not exist. Please enter again.");
}
} while (!fileExists);
Console.Write("Enter name to use when uploaded: ");
uploadName = Console.ReadLine();
Console.Write("Wipe away previous attempts? (Y/n): ");
var wipeAway = Console.ReadLine();
if (wipeAway == "Y")
{
var currentMultiParts = await client.ListMultipartUploadsAsync(spaceName);
foreach (var multiPart in currentMultiParts.MultipartUploads)
{
try
{
await client.AbortMultipartUploadAsync(currentMultiParts.BucketName, multiPart.Key, multiPart.UploadId);
}
catch (Exception) { }
}
Console.WriteLine("Wiped away previous upload attempts");
}
var fileInfo = new FileInfo(filePath);
var multiPartStart = await client.InitiateMultipartUploadAsync(new Amazon.S3.Model.InitiateMultipartUploadRequest()
{
BucketName = spaceName,
ContentType = contentType,
Key = uploadName
});
try
{
var i = 0L;
var n = 1;
Dictionary <string, int> parts = new Dictionary <string, int>();
while (i < fileInfo.Length)
{
Console.WriteLine($"Starting upload for Part #{n}");
long partSize = 6000000;
var lastPart = (i + partSize) >= fileInfo.Length;
if (lastPart)
{
partSize = fileInfo.Length - i;
}
bool complete = false;
int retry = 0;
Amazon.S3.Model.UploadPartResponse partResp = null;
do
{
retry++;
try
{
partResp = await client.UploadPartAsync(new Amazon.S3.Model.UploadPartRequest()
{
BucketName = spaceName,
FilePath = filePath,
FilePosition = i,
IsLastPart = lastPart,
PartSize = partSize,
PartNumber = n,
UploadId = multiPartStart.UploadId,
Key = uploadName
});
complete = true;
}
catch (Exception)
{
Console.WriteLine($"Failed to upload part {n} on try #{retry}...");
}
} while (!complete && retry <= 3);
if (!complete || partResp == null)
{
throw new Exception($"Unable to upload part {n}... Failing");
}
parts.Add(partResp.ETag, n);
i += partSize;
n++;
Console.WriteLine($"Uploading {(((float)i/(float)fileInfo.Length) * 100).ToString("N2")} ({i}/{fileInfo.Length})");
}
Console.WriteLine("Done uploading! Completing upload");
var completePart = await client.CompleteMultipartUploadAsync(new Amazon.S3.Model.CompleteMultipartUploadRequest()
{
UploadId = multiPartStart.UploadId,
BucketName = spaceName,
Key = uploadName,
PartETags = parts.Select(p => new Amazon.S3.Model.PartETag(p.Value, p.Key)).ToList()
});
Console.WriteLine("Successfully uploaded!");
}
catch (Exception ex)
{
var abortPart = await client.AbortMultipartUploadAsync(spaceName, uploadName, multiPartStart.UploadId);
Console.WriteLine("Error while uploading! " + ex.Message);
await Task.Delay(10000);
}
Console.WriteLine("Press enter to exit");
Console.ReadLine();
}
/// <summary>
/// Copies large file as chunks from Azure BLOB to Amazon S3.
/// </summary>
/// <returns></returns>
public async Task CopyLargeFileFromAzureBlobToAwsS3()
{
AmazonS3Client s3Client = new AmazonS3Client(AwsAccessKeyId, AwsSecretKey, Amazon.RegionEndpoint.APSouth1);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageAccount); //Create Storage account reference.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Create the blob client.
CloudBlobContainer container = blobClient.GetContainerReference(ContainerName); // Retrieve reference to a container.
container.CreateIfNotExists();
CloudBlockBlob blob = container.GetBlockBlobReference(BlobFileName); // Create Blob reference.
blob.FetchAttributes(); // Prepare blob instance To get the file length.
var remainingBytes = blob.Properties.Length;
long readPosition = 0; // To be used offset / position from where to start reading from BLOB.
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName
};
// Will use UploadId from this response.
InitiateMultipartUploadResponse initiateMultipartUploadResponse = s3Client.InitiateMultipartUpload(initiateMultipartUploadRequest);
List <UploadPartResponse> uploadPartResponses = new List <UploadPartResponse>();
Stopwatch stopwatch = Stopwatch.StartNew();
try
{
int partCounter = 0; // To increment on each read of parts and use it as part number.
while (remainingBytes > 0)
{
// Determine the size when final block reached as it might be less than Part size.
// Will be PartSize except final block.
long bytesToCopy = Math.Min(PartSize, remainingBytes);
using (MemoryStream memoryStream = new MemoryStream())
{
// To download part from BLOB.
await blob.DownloadRangeToStreamAsync(memoryStream, readPosition, bytesToCopy).ConfigureAwait(false);
memoryStream.Position = 0;
partCounter++;
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId,
PartNumber = partCounter,
PartSize = bytesToCopy,
InputStream = memoryStream
};
UploadPartResponse uploadPartResponse = s3Client.UploadPart(uploadRequest);
uploadPartResponses.Add(uploadPartResponse);
remainingBytes -= bytesToCopy;
readPosition += bytesToCopy;
this.logger.WriteLine($"Uploaded part with part number {partCounter}, size {bytesToCopy}bytes and remaining {remainingBytes}bytes to read.");
}
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId
};
completeMultipartUploadRequest.AddPartETags(uploadPartResponses);
CompleteMultipartUploadResponse completeMultipartUploadResponse = await s3Client.CompleteMultipartUploadAsync(completeMultipartUploadRequest).ConfigureAwait(false);
}
catch (Exception exception)
{
this.logger.WriteLine($"Exception : {exception.Message}");
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMultipartUploadRequest).ConfigureAwait(false);
}
finally
{
stopwatch.Stop();
this.logger.WriteLine($"Execution time in mins: {stopwatch.Elapsed.TotalMinutes}");
}
}