/// <summary>
/// Sample code to contrast uploading a file using Amazon S3's Multi-Part Upload API
/// </summary>
/// <param name="s3Client"></param>
/// <param name="bucketName"></param>
/// <param name="fileName"></param>
static void UploadUsingMultiPartAPI(IAmazonS3 s3Client, string bucketName, string fileName)
{
const string objectKey = "multipart/myobject";
// tell S3 we're going to upload an object in multiple parts and receive an upload ID
// in return
var initializeUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey
};
var initializeUploadResponse = s3Client.InitiateMultipartUpload(initializeUploadRequest);
// this ID must accompany all parts and the final 'completed' call
var uploadID = initializeUploadResponse.UploadId;
// Send the file (synchronously) using 4*5MB parts - note we pass the upload id
// with each call. For each part we need to log the returned etag value to pass
// to the completion call
var partETags = new List <PartETag>();
var partSize = 5 * ONE_MEG; // this is the minimum part size allowed
for (var partNumber = 0; partNumber < 4; partNumber++)
{
// part numbers must be between 1 and 1000
var logicalPartNumber = partNumber + 1;
var uploadPartRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartNumber = logicalPartNumber,
PartSize = partSize,
FilePosition = partNumber * partSize,
FilePath = fileName
};
var partUploadResponse = s3Client.UploadPart(uploadPartRequest);
partETags.Add(new PartETag {
PartNumber = logicalPartNumber, ETag = partUploadResponse.ETag
});
}
var completeUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartETags = partETags
};
s3Client.CompleteMultipartUpload(completeUploadRequest);
}
/// <summary>
/// Sample code to contrast uploading a file using Amazon S3's Multi-Part Upload API
/// </summary>
/// <param name="s3Client"></param>
/// <param name="bucketName"></param>
/// <param name="fileName"></param>
static void UploadUsingMultiPartAPI(IAmazonS3 s3Client, string bucketName, string fileName)
{
const string objectKey = "multipart/myobject";
// tell S3 we're going to upload an object in multiple parts and receive an upload ID
// in return
var initializeUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey
};
var initializeUploadResponse = s3Client.InitiateMultipartUpload(initializeUploadRequest);
// this ID must accompany all parts and the final 'completed' call
var uploadID = initializeUploadResponse.UploadId;
// Send the file (synchronously) using 4*5MB parts - note we pass the upload id
// with each call. For each part we need to log the returned etag value to pass
// to the completion call
var partETags = new List<PartETag>();
var partSize = 5 * ONE_MEG; // this is the minimum part size allowed
for (var partNumber = 0; partNumber < 4; partNumber++)
{
// part numbers must be between 1 and 1000
var logicalPartNumber = partNumber + 1;
var uploadPartRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartNumber = logicalPartNumber,
PartSize = partSize,
FilePosition = partNumber * partSize,
FilePath = fileName
};
var partUploadResponse = s3Client.UploadPart(uploadPartRequest);
partETags.Add(new PartETag { PartNumber = logicalPartNumber, ETag = partUploadResponse.ETag });
}
var completeUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartETags = partETags
};
s3Client.CompleteMultipartUpload(completeUploadRequest);
}
private static void CopyTo(string src, string dst)
{
var uri = new Uri(dst);
var dstBucket = uri.Host;
var dstPath = uri.PathAndQuery.Substring(1);
Log.InfoFormat("Bucket: {0}, Path: {1}", dstBucket, dstPath);
var md5File = Path.Combine(src, md5fileName);
var md5Dictionary = LoadMd5File(md5File);
var files = FetchFiles(src)
.Where(x => x.Length > 0)
.Where(x => !x.Attributes.HasFlag(FileAttributes.Hidden))
.Where(x => !x.Name.StartsWith("."))
.Where(x => x.FullName != md5File)
.Where(x => (x.Attributes & FileAttributes.Archive) == FileAttributes.Archive)
.Where(x => (x.Attributes & FileAttributes.Temporary) != FileAttributes.Temporary)
.Select(x => Map(src, x));
using (var md5FileWriter = File.AppendText(md5File))
{
files = files.Select(x =>
{
Md5Item item;
if (!md5Dictionary.TryGetValue(x.Path, out item) ||
x.Length != item.Length ||
x.ModifiedUtc != item.ModifiedUtc
)
{
x.Hash = Md5Hash.Calculate(x.Path).ToHex();
item = new Md5Item(x.Hash, x.Path, x.Length, x.ModifiedUtc);
md5Dictionary[x.Path] = item;
lock (md5FileWriter)
{
md5FileWriter.WriteLine(item);
}
}
x.Hash = item.Md5;
return(x);
}).ToList();
}
WriteMd5Dictionary(md5Dictionary, md5File);
var objects = FetchObjects(dstBucket, dstPath)
.Select(x => Map(dstPath, x))
.ToDictionary(x => x.Relative, x => x);
int chunkSize = 1024 * 1024 * 8; //8MB
Log.DebugFormat("Chunk size: {0} ({1} bytes)", Pretty(chunkSize), chunkSize);
/*
* Alleen files die:
* - nog niet bestaan
* of een andere lengte hebben
* of een andere hash md5 hebben
* of een andere hash s3md5 hebben
*/
var items = files
.Where(x =>
{
var destinationNotExists = !objects.ContainsKey(x.Relative);
if (destinationNotExists)
{
return(true);
}
var obj = objects[x.Relative];
var differentSize = x.Length != obj.Length;
if (differentSize)
{
return(true);
}
var etag = obj.S3Object.ETag;
var isMultiPart = etag.Contains("-");
if (isMultiPart)
{
var parts = ExtractParts(etag);
var guestimateChunkSize = GuessChunkSize(obj.Length, parts);
var s3md5mismatch = !obj.Hash.Contains(S3Md5.Calculate(x.Path, guestimateChunkSize));
return(s3md5mismatch);
}
else
{
var md5mismatch = x.Hash != obj.Hash;
return(md5mismatch);
}
}
)
.ToList();
Log.InfoFormat("Items to be mirrored: {0}", items.Count);
using (IAmazonS3 client = CreateS3Client())
foreach (var item in items)
{
var key = dstPath + item.Relative;
Log.DebugFormat("Uploading {0} => {1}", item.Relative, key);
if (item.Length < chunkSize)
{
client.UploadObjectFromFilePath(dstBucket, key, item.Path, null);
var isMatch = client.GetObject(dstBucket, key).ETag.Contains(item.Hash);
if (!isMatch)
{
Log.ErrorFormat("Upload failed: {0}", item.Relative);
}
}
else
{
Log.Debug("Multi-part");
var response = client.InitiateMultipartUpload(dstBucket, key);
try
{
long index = 0;
var md5s = new List <PartETag>();
for (int part = 1; index < item.Length; part++)
{
var md5 = Md5Hash.Calculate(item.Path, index, chunkSize);
var partSize = Math.Min(chunkSize, item.Length - index);
Log.DebugFormat("\tPart {0} ({1:N0}): {2}", part, partSize, md5.ToHex());
client.UploadPart(new UploadPartRequest
{
Key = key,
BucketName = dstBucket,
FilePath = item.Path,
FilePosition = index,
PartNumber = part,
PartSize = chunkSize,
UploadId = response.UploadId,
MD5Digest = System.Convert.ToBase64String(md5),
});
md5s.Add(new PartETag(part, md5.ToHex()));
index += partSize;
}
client.CompleteMultipartUpload(new CompleteMultipartUploadRequest
{
Key = key,
BucketName = dstBucket,
PartETags = md5s,
UploadId = response.UploadId,
});
}
catch (Exception ex)
{
Log.Error(item.Relative, ex);
client.AbortMultipartUpload(dstBucket, key, response.UploadId);
}
}
File.SetAttributes(item.Path, File.GetAttributes(item.Path) & ~FileAttributes.Archive);
}
}
public static async Task PutObjectAsMultiPartAsync()
{
Stopwatch sw = Stopwatch.StartNew();
s3Client = new AmazonS3Client(
"AKIA6PYYJMASLJEFTI6E",
"IJPo9Ys58iAb35dKw4kcW/SkOU2J+iI9IOA5Wpl6",
Amazon.RegionEndpoint.APSoutheast1
);
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// List to store upload part responses.
// 1. Initialize.
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
var initResponse = s3Client.InitiateMultipartUpload(initiateRequest);
// 2. Upload Parts.
var partNumber = 5;
var contentLength = new FileInfo(filePath).Length;
var partSize = contentLength / partNumber;
long filePosition = 0;
try
{
for (var i = 1; filePosition < contentLength; ++i)
{
// Create request to upload a part.
Console.WriteLine(filePosition);
var uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath
};
// Upload part and add response to our list.
uploadResponses.Add(s3Client.UploadPart(uploadRequest));
filePosition += partSize;
}
// Step 3: complete.
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
};
// add ETags for uploaded files
completeRequest.AddPartETags(uploadResponses);
var completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest);
sw.Stop();
Console.WriteLine("Upload completed : {0}", sw.Elapsed.TotalMilliseconds);
}
catch (Exception exception)
{
Console.WriteLine("Exception occurred: {0}", exception.ToString());
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
s3Client.AbortMultipartUpload(abortMPURequest);
}
}
public string UploadFileFromStreamWithChunking(MemoryStream stream, string s3Key)
{
string uploadedFileUrl = string.Empty;
Dictionary <int, string> partNumbersAndETags = new Dictionary <int, string>();
int chunkSize = 5500000;
try
{
stream.Seek(0, SeekOrigin.Begin);
string uploadId = string.Empty;
int numChunks = (int)Math.Floor((double)stream.Length / chunkSize);
if (stream.Length % chunkSize > 0)
{
numChunks += 1;
}
byte[] chunkBytes = new byte[5500000];
int bytesRead = 0;
int offset = 0;
int partNumber = 0;
for (int i = 0; i < numChunks; i++)
{
bytesRead = stream.Read(chunkBytes, 0, 5500000);
offset = bytesRead;
partNumber = i + 1;
if (partNumber == 1)
{
// if first chunk, initiate the upload
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest
{
BucketName = _bucketName,
Key = s3Key
};
InitiateMultipartUploadResponse initResponse = _awsS3Client.InitiateMultipartUpload(initRequest);
if (initResponse.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
throw new ApplicationException("S3 UploadFileChunk: could not initiate multipart upload.");
}
uploadId = initResponse.UploadId;
}
Array.Resize(ref chunkBytes, bytesRead);
MemoryStream chunkMemoryStream = new MemoryStream(chunkBytes);
// pload the chunk
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = _bucketName,
Key = s3Key,
UploadId = uploadId,
PartNumber = partNumber,
PartSize = chunkBytes.Length,
FilePosition = offset,
InputStream = chunkMemoryStream
};
UploadPartResponse uploadResponse = _awsS3Client.UploadPart(uploadRequest);
if (uploadResponse.HttpStatusCode != System.Net.HttpStatusCode.OK)
{
AbortChunkUpload(uploadId, s3Key);
}
// get ETag and Part # from response and store it in the file information
partNumbersAndETags.Add(uploadResponse.PartNumber, uploadResponse.ETag);
// update file position
offset = chunkBytes.Length;
}
List <PartETag> partETags = new List <PartETag>();
foreach (KeyValuePair <int, string> kvp in partNumbersAndETags)
{
partETags.Add(new PartETag {
ETag = kvp.Value, PartNumber = kvp.Key
});
}
CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest
{
Key = s3Key,
BucketName = _bucketName,
PartETags = partETags,
UploadId = uploadId
};
CompleteMultipartUploadResponse response = _awsS3Client.CompleteMultipartUpload(request);
uploadedFileUrl = bucketUrl + s3Key;
}
catch (Exception ex)
{
throw new ApplicationException("UploadFileFromStream - unexpected exception.", ex);
}
return(uploadedFileUrl);
}
