public static async Task CreateDirectory(this S3Helper s3, string bucketName, string path, CancellationToken cancellationToken = default(CancellationToken)) { var key = $"{path.Trim('/')}/"; if (await s3.ObjectExistsAsync(bucketName: bucketName, key: key)) { return; //already exists } var stream = new MemoryStream(new byte[0]); var obj = await s3.UploadStreamAsync(bucketName : bucketName, key : key, inputStream : stream, cancellationToken : cancellationToken); }
public static async Task <Stream> DownloadObjectAsync(this S3Helper s3, string bucketName, string key, string version = null, string eTag = null, bool throwIfNotFound = true, CancellationToken cancellationToken = default(CancellationToken)) { if (!throwIfNotFound && !await s3.ObjectExistsAsync(bucketName: bucketName, key: key)) { return(null); } var obj = await s3.GetObjectAsync( bucketName : bucketName, key : key, versionId : version, eTag : eTag, cancellationToken : cancellationToken); return(obj.ResponseStream); }
public static async Task <string> UploadStreamAsync(this S3Helper s3, string bucketName, string key, Stream inputStream, string keyId = null, string contentType = "application/octet-stream", bool throwIfAlreadyExists = false, int msTimeout = int.MaxValue, CancellationToken cancellationToken = default(CancellationToken)) { CancellationToken ct; void UpdateCancellationToken() { if (cancellationToken != null) { ct = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken).Token; } else { ct = new CancellationTokenSource().Token; } } UpdateCancellationToken(); if (throwIfAlreadyExists && await s3.ObjectExistsAsync(bucketName: bucketName, key: key, cancellationToken: ct) .TryCancelAfter(ct, msTimeout: msTimeout)) { throw new Exception($"Object {key} in bucket {bucketName} already exists."); } if (keyId == "") { keyId = null; } if (keyId != null && !keyId.IsGuid()) { UpdateCancellationToken(); var alias = await(new KMSHelper(s3._credentials)).GetKeyAliasByNameAsync(name: keyId, cancellationToken: ct) .TryCancelAfter(ct, msTimeout: msTimeout); keyId = alias.TargetKeyId; } var bufferSize = 128 * 1024; var blob = inputStream.ToMemoryBlob(maxLength: s3.MaxSinglePartSize, bufferSize: bufferSize); var ih = IncrementalHash.CreateHash(HashAlgorithmName.MD5); string md5, etag; if (blob.Length < s3.MaxSinglePartSize) { UpdateCancellationToken(); using (var ms = blob.CopyToMemoryStream(bufferSize: (int)blob.Length)) { var spResult = s3.PutObjectAsync(bucketName: bucketName, key: key, inputStream: ms, keyId: keyId, cancellationToken: ct, contentType: contentType) .TryCancelAfter(ct, msTimeout: msTimeout); blob.Seek(0, SeekOrigin.Begin); ih.AppendData(blob.ToArray()); md5 = ih.GetHashAndReset().ToHexString(); etag = (await spResult).ETag.Trim('"'); return(md5); } } UpdateCancellationToken(); var init = await s3.InitiateMultipartUploadAsync(bucketName, key, contentType : contentType, keyId : keyId, cancellationToken : ct) .TryCancelAfter(ct, msTimeout: msTimeout); var partNumber = 0; var tags = new List <PartETag>(); while (blob.Length > 0) { partNumber = ++partNumber; UpdateCancellationToken(); //copy so new part can be read at the same time using (var ms = blob.CopyToMemoryStream(bufferSize: (int)blob.Length)) { var tUpload = s3.UploadPartAsync( bucketName: bucketName, key: key, uploadId: init.UploadId, partNumber: partNumber, partSize: (int)ms.Length, inputStream: ms, progress: null, cancellationToken: ct).TryCancelAfter(ct, msTimeout: msTimeout); if (ct.IsCancellationRequested) { throw new OperationCanceledException("Operation was cancelled or timed out."); } if (blob.Length <= s3.DefaultPartSize) //read next part from input before stream gets uploaded { blob = inputStream.ToMemoryBlob(maxLength: s3.DefaultPartSize, bufferSize: bufferSize); } tags.Add(new PartETag(partNumber, (await tUpload).ETag)); ms.Seek(0, SeekOrigin.Begin); ih.AppendData(ms.ToArray()); } } UpdateCancellationToken(); var mpResult = await s3.CompleteMultipartUploadAsync( bucketName : bucketName, key : key, uploadId : init.UploadId, partETags : tags, cancellationToken : ct).TryCancelAfter(ct, msTimeout: msTimeout); md5 = ih.GetHashAndReset().ToHexString(); etag = mpResult.ETag.Trim('"'); return(md5); }
public static async Task <bool> DeleteVersionedObjectAsync(this S3Helper s3, string bucketName, string key, bool throwOnFailure = true, CancellationToken cancellationToken = default(CancellationToken)) { var versions = await s3.ListVersionsAsync( bucketName : bucketName, prefix : key, cancellationToken : cancellationToken); var keyVersions = versions .Where(v => !v.IsLatest) .Select(ver => new KeyVersion() { Key = key, VersionId = ver.VersionId }) .ToArray(); if (keyVersions.Length > 0) { var response = await s3.DeleteObjectsAsync( bucketName : bucketName, objects : keyVersions, cancellationToken : cancellationToken); if (response.DeleteErrors.Count > 0) { if (throwOnFailure) { throw new Exception($"Failed to delete all object versions of key '{key}' in bucket '{bucketName}', {response.DeletedObjects.Count} Deleted {response.DeleteErrors.Count} Errors, Delete Errors: {response.DeleteErrors.JsonSerialize(Newtonsoft.Json.Formatting.Indented)}"); } else { return(false); } } return(await s3.DeleteVersionedObjectAsync(bucketName, key, throwOnFailure, cancellationToken)); } try { var latest = versions.Single(x => x.IsLatest); await s3.DeleteObjectAsync(bucketName : bucketName, key : key, versionId : latest.VersionId, cancellationToken : cancellationToken); return(true); } catch { if (!await s3.ObjectExistsAsync(bucketName, key, cancellationToken)) { return(true); } if (throwOnFailure) { throw; } else { return(false); } } }