示例#1
0
        public static async Task <GetObjectMetadataResponse> ObjectMetadataAsync(this S3Helper s3,
                                                                                 string bucketName,
                                                                                 string key,
                                                                                 bool throwIfNotFound = true,
                                                                                 CancellationToken cancellationToken = default(CancellationToken))
        {
            try
            {
                var metadata = await s3.GetObjectMetadata(bucketName : bucketName, key : key, cancellationToken : cancellationToken);

                return(metadata);
            }
            catch (Exception ex)
            {
                if (!throwIfNotFound)
                {
                    if (ex is Amazon.S3.AmazonS3Exception &&
                        (ex as Amazon.S3.AmazonS3Exception).StatusCode == System.Net.HttpStatusCode.NotFound)
                    {
                        return(null);
                    }
                }

                throw;
            }
        }
示例#2
0
        public static async Task <StatusFile> GetStatusFile(S3Helper s3h, SyncTarget st, long minTimestamp, long maxTimestamp)
        {
            var bkp    = st.status.ToBucketKeyPair();
            var prefix = $"{bkp.key}/{UploadStatusFilePrefix}";
            var list   = await GetStatusList(s3h, st, UploadStatusFilePrefix);

            if (list.IsNullOrEmpty())
            {
                return(null);
            }

            list.Reverse();
            foreach (var f in list) //find non obsolete files
            {
                var timestamp = f.Key.TrimStart(prefix).TrimEnd(".json").ToLongOrDefault(0);

                if (timestamp < minTimestamp || timestamp > maxTimestamp)
                {
                    continue;
                }

                var s = await s3h.DownloadJsonAsync <StatusFile>(bkp.bucket, f.Key, throwIfNotFound : false)
                        .Timeout(msTimeout: st.timeout)
                        .TryCatchRetryAsync(maxRepeats: st.retry);

                if (s?.finalized == true)
                {
                    return(s);
                }
            }

            return(null);
        }
示例#3
0
        public static async Task <T> DownloadJsonAsync <T>(this S3Helper s3,
                                                           string bucketName,
                                                           string key,
                                                           string version       = null,
                                                           string eTag          = null,
                                                           bool throwIfNotFound = true,
                                                           Encoding encoding    = null,
                                                           CancellationToken cancellationToken = default(CancellationToken))
        {
            var stream = await s3.DownloadObjectAsync(
                bucketName : bucketName,
                key : key,
                version : version,
                eTag : eTag,
                throwIfNotFound : throwIfNotFound,
                cancellationToken : cancellationToken);

            if (!throwIfNotFound && stream == null)
            {
                return(default(T));
            }

            return(await stream
                   .ToStringAsync(encoding ?? Encoding.UTF8)
                   .JsonDeserializeAsync <T>());
        }
示例#4
0
        public S3HashStore(SyncTarget st)
        {
            parallelism = st?.parallelism ?? 1;
            this.st     = st;
            this.si     = null;

            this.s3h = (st?.profile).IsNullOrEmpty() ?
                       new S3Helper() :
                       new S3Helper(Extensions.Helper.GetAWSCredentials(st.profile));
        }
示例#5
0
 public static Task <string> UploadTextAsync(this S3Helper s3,
                                             string bucketName,
                                             string key,
                                             string text,
                                             string keyId      = null,
                                             Encoding encoding = null,
                                             CancellationToken cancellationToken = default(CancellationToken))
 => s3.UploadStreamAsync(bucketName: bucketName,
                         key: key,
                         inputStream: text.ToMemoryStream(encoding),
                         contentType: "text/plain",
                         cancellationToken: cancellationToken);
示例#6
0
 public static Task <string> UploadJsonAsync <T>(this S3Helper s3,
                                                 string bucketName,
                                                 string key,
                                                 T content,
                                                 Newtonsoft.Json.Formatting formatting = Newtonsoft.Json.Formatting.Indented,
                                                 string keyId      = null,
                                                 Encoding encoding = null,
                                                 CancellationToken cancellationToken = default(CancellationToken))
 => s3.UploadStreamAsync(bucketName: bucketName,
                         key: key,
                         inputStream: content.JsonSerialize(formatting).ToMemoryStream(encoding),
                         contentType: "text/plain",
                         cancellationToken: cancellationToken);
示例#7
0
        /// <summary>
        /// Returns list of s3 status files in ascending order (from oldest to latest)
        /// </summary>
        /// <param name="st"></param>
        /// <returns></returns>
        public static async Task <List <S3Object> > GetStatusList(S3Helper s3h, SyncTarget st, string statusPrefix)
        {
            var cts = new CancellationTokenSource();

            var bkp    = st.status.ToBucketKeyPair();
            var prefix = $"{bkp.key}/{statusPrefix}";
            var list   = (await s3h.ListObjectsAsync(bkp.bucket, prefix, msTimeout: st.timeout, cancellationToken: cts.Token)
                          .TryCatchRetryAsync(maxRepeats: st.retry))
                         .SortAscending(x => x.Key.TrimStart(prefix).TrimEnd(".json").ToLongOrDefault(0)).ToList();

            if (cts.IsCancellationRequested)
            {
                return(null);
            }

            return(list);
        }
示例#8
0
        public static async Task CreateDirectory(this S3Helper s3,
                                                 string bucketName,
                                                 string path,
                                                 CancellationToken cancellationToken = default(CancellationToken))
        {
            var key = $"{path.Trim('/')}/";

            if (await s3.ObjectExistsAsync(bucketName: bucketName, key: key))
            {
                return; //already exists
            }
            var stream = new MemoryStream(new byte[0]);

            var obj = await s3.UploadStreamAsync(bucketName : bucketName,
                                                 key : key,
                                                 inputStream : stream,
                                                 cancellationToken : cancellationToken);
        }
示例#9
0
        public static async Task <FileInfo> DownloadObjectAsync(this S3Helper s3,
                                                                string bucketName,
                                                                string key,
                                                                string outputFile,
                                                                string version,
                                                                string eTag,
                                                                bool @override,
                                                                CancellationToken cancellationToken = default(CancellationToken))
        {
            var fi = new FileInfo(outputFile);

            if (fi.Exists)
            {
                if (@override)
                {
                    fi.Delete();
                }
                else
                {
                    throw new Exception($"Can't download '{bucketName}/{key}', becuause output file '{fi.FullName}' already exists.");
                }
            }

            var stream = await DownloadObjectAsync(
                s3 : s3,
                bucketName : bucketName,
                key : key,
                version : version,
                eTag : eTag,
                cancellationToken : cancellationToken);

            var buffSize = 1024 * 1024;

            using (var fw = File.Create(outputFile, buffSize))
                await stream.CopyToAsync(fw, buffSize);

            fi.Refresh();

            return(fi);
        }
示例#10
0
        public static async Task <bool> ObjectExistsAsync(this S3Helper s3,
                                                          string bucketName,
                                                          string key,
                                                          CancellationToken cancellationToken = default(CancellationToken))
        {
            try
            {
                var metadata = await s3.GetObjectMetadata(bucketName : bucketName, key : key, cancellationToken : cancellationToken);

                return(true);
            }
            catch (Exception ex)
            {
                if (ex is Amazon.S3.AmazonS3Exception &&
                    (ex as Amazon.S3.AmazonS3Exception).StatusCode == System.Net.HttpStatusCode.NotFound)
                {
                    return(false);
                }

                throw;
            }
        }
示例#11
0
        public static async Task <Stream> DownloadObjectAsync(this S3Helper s3,
                                                              string bucketName,
                                                              string key,
                                                              string version       = null,
                                                              string eTag          = null,
                                                              bool throwIfNotFound = true,
                                                              CancellationToken cancellationToken = default(CancellationToken))
        {
            if (!throwIfNotFound && !await s3.ObjectExistsAsync(bucketName: bucketName, key: key))
            {
                return(null);
            }

            var obj = await s3.GetObjectAsync(
                bucketName : bucketName,
                key : key,
                versionId : version,
                eTag : eTag,
                cancellationToken : cancellationToken);

            return(obj.ResponseStream);
        }
示例#12
0
        public static async Task <string> DownloadTextAsync(this S3Helper s3,
                                                            string bucketName,
                                                            string key,
                                                            string version       = null,
                                                            string eTag          = null,
                                                            bool throwIfNotFound = true,
                                                            Encoding encoding    = null,
                                                            CancellationToken cancellationToken = default(CancellationToken))
        {
            var stream = await s3.DownloadObjectAsync(
                bucketName : bucketName,
                key : key,
                version : version,
                eTag : eTag,
                throwIfNotFound : throwIfNotFound,
                cancellationToken : cancellationToken);

            if (!throwIfNotFound && stream == null)
            {
                return(null);
            }

            return((encoding ?? Encoding.UTF8).GetString(stream.ToArray(bufferSize: 256 * 1024)));
        }
示例#13
0
        public static async Task <StatusFile> GetStatusFile(S3Helper s3h, SyncTarget st, string statusPrefix)
        {
            var bkp    = st.status.ToBucketKeyPair();
            var prefix = $"{bkp.key}/{statusPrefix}";
            var list   = await GetStatusList(s3h, st, statusPrefix);

            var id = list.IsNullOrEmpty() ? 0 : list.Last().Key.TrimStart(prefix).TrimEnd(".json").ToLongOrDefault(0); //latest staus id

            id = id <= 0 ? DateTimeEx.TimestampNow() : id;
            var key = $"{prefix}{id}.json";

            if (list.IsNullOrEmpty() || id <= 0)
            {
                return(new StatusFile()
                {
                    id = id.ToString(),
                    timestamp = DateTimeEx.UnixTimestampNow(),
                    bucket = bkp.bucket,
                    key = key,
                    location = $"{bkp.bucket}/{key}",
                    finalized = false,
                    version = 0
                });
            }

            var status = await s3h.DownloadJsonAsync <StatusFile>(bkp.bucket, key, throwIfNotFound : true)
                         .Timeout(msTimeout: st.timeout)
                         .TryCatchRetryAsync(maxRepeats: st.retry);

            var elapsed = (DateTime.UtcNow - long.Parse(status?.id ?? "0").ToDateTimeFromTimestamp()).TotalSeconds;

            if (status == null || (status.finalized == true && st.retention > 0 && elapsed > st.retention))
            {
                id     = DateTimeEx.TimestampNow();
                key    = $"{prefix}{id}.json";
                status = new Models.StatusFile()
                {
                    id        = id.ToString(),
                    timestamp = DateTimeEx.UnixTimestampNow(),
                    bucket    = bkp.bucket,
                    key       = key,
                    location  = $"{bkp.bucket}/{key}",
                    finalized = false,
                    version   = 0
                };
            }

            if (st.cleanup && st.rotation > 0 && list.Count > st.rotation)
            {
                var validStatus = new List <StatusFile>();
                list.Reverse();
                foreach (var f in list) //find non obsolete files
                {
                    var s = await s3h.DownloadJsonAsync <StatusFile>(bkp.bucket, f.Key, throwIfNotFound : false)
                            .Timeout(msTimeout: st.timeout)
                            .TryCatchRetryAsync(maxRepeats: st.retry);

                    if (s == null)
                    {
                        continue;
                    }

                    if (s.finalized && s.id.ToLongOrDefault(0) > 0)
                    {
                        validStatus.Add(s);
                    }
                    else if (!s.finalized || status.id == id.ToString())
                    {
                        validStatus.Add(s);
                    }

                    if (validStatus.Count > st.rotation)
                    {
                        break;
                    }
                }

                status.obsoletes = list.Where(x => !validStatus.Any(v => v.key.ToLower().Trim() == x.Key.ToLower().Trim()))
                                   .Select(x => x.Key).ToArray(); //status files that are obsolete
            }

            return(status);
        }
示例#14
0
        public static async Task <DeleteObjectsResponse> DeleteObjectsModifiedBeforeDateAsync(this S3Helper s3,
                                                                                              string bucketName,
                                                                                              string prefix,
                                                                                              DateTime dt,
                                                                                              bool throwIfNotFound,
                                                                                              CancellationToken cancellationToken = default(CancellationToken))
        {
            var list = await s3.ListObjectsAsync(bucketName : bucketName, prefix : prefix, cancellationToken : cancellationToken);

            var toDelete = list.Where(x => x.LastModified.Ticks < dt.Ticks).Select(x => new KeyVersion()
            {
                Key = x.Key,
            }).ToArray();

            if (toDelete.IsNullOrEmpty())
            {
                if (throwIfNotFound)
                {
                    throw new Exception($"Found {list?.Length ?? 0} objects with prefix {prefix} in bucket {bucketName}, but none were marked for deletion.");
                }
                else
                {
                    return(null);
                }
            }

            var deleted = await s3.DeleteObjectsAsync(bucketName : bucketName, objects : toDelete, cancellationToken : cancellationToken);

            if (!deleted.DeleteErrors.IsNullOrEmpty())
            {
                throw new Exception($"Deleted {deleted.DeletedObjects.Count} objects, but failed {deleted.DeleteErrors.Count}, due to following errors: {deleted.DeleteErrors.JsonSerialize()}");
            }

            return(deleted);
        }
示例#15
0
        public static async Task <string> UploadStreamAsync(this S3Helper s3,
                                                            string bucketName,
                                                            string key,
                                                            Stream inputStream,
                                                            string keyId                        = null,
                                                            string contentType                  = "application/octet-stream",
                                                            bool throwIfAlreadyExists           = false,
                                                            int msTimeout                       = int.MaxValue,
                                                            CancellationToken cancellationToken = default(CancellationToken))
        {
            CancellationToken ct;

            void UpdateCancellationToken()
            {
                if (cancellationToken != null)
                {
                    ct = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken).Token;
                }
                else
                {
                    ct = new CancellationTokenSource().Token;
                }
            }

            UpdateCancellationToken();

            if (throwIfAlreadyExists &&
                await s3.ObjectExistsAsync(bucketName: bucketName, key: key, cancellationToken: ct)
                .TryCancelAfter(ct, msTimeout: msTimeout))
            {
                throw new Exception($"Object {key} in bucket {bucketName} already exists.");
            }

            if (keyId == "")
            {
                keyId = null;
            }

            if (keyId != null && !keyId.IsGuid())
            {
                UpdateCancellationToken();
                var alias = await(new KMSHelper(s3._credentials)).GetKeyAliasByNameAsync(name: keyId, cancellationToken: ct)
                            .TryCancelAfter(ct, msTimeout: msTimeout);
                keyId = alias.TargetKeyId;
            }

            var    bufferSize = 128 * 1024;
            var    blob = inputStream.ToMemoryBlob(maxLength: s3.MaxSinglePartSize, bufferSize: bufferSize);
            var    ih = IncrementalHash.CreateHash(HashAlgorithmName.MD5);
            string md5, etag;

            if (blob.Length < s3.MaxSinglePartSize)
            {
                UpdateCancellationToken();
                using (var ms = blob.CopyToMemoryStream(bufferSize: (int)blob.Length))
                {
                    var spResult = s3.PutObjectAsync(bucketName: bucketName, key: key, inputStream: ms, keyId: keyId, cancellationToken: ct, contentType: contentType)
                                   .TryCancelAfter(ct, msTimeout: msTimeout);

                    blob.Seek(0, SeekOrigin.Begin);
                    ih.AppendData(blob.ToArray());

                    md5  = ih.GetHashAndReset().ToHexString();
                    etag = (await spResult).ETag.Trim('"');
                    return(md5);
                }
            }

            UpdateCancellationToken();
            var init = await s3.InitiateMultipartUploadAsync(bucketName, key, contentType : contentType, keyId : keyId, cancellationToken : ct)
                       .TryCancelAfter(ct, msTimeout: msTimeout);

            var partNumber = 0;
            var tags       = new List <PartETag>();

            while (blob.Length > 0)
            {
                partNumber = ++partNumber;
                UpdateCancellationToken();

                //copy so new part can be read at the same time
                using (var ms = blob.CopyToMemoryStream(bufferSize: (int)blob.Length))
                {
                    var tUpload = s3.UploadPartAsync(
                        bucketName: bucketName,
                        key: key,
                        uploadId: init.UploadId,
                        partNumber: partNumber,
                        partSize: (int)ms.Length,
                        inputStream: ms,
                        progress: null,
                        cancellationToken: ct).TryCancelAfter(ct, msTimeout: msTimeout);

                    if (ct.IsCancellationRequested)
                    {
                        throw new OperationCanceledException("Operation was cancelled or timed out.");
                    }

                    if (blob.Length <= s3.DefaultPartSize) //read next part from input before stream gets uploaded
                    {
                        blob = inputStream.ToMemoryBlob(maxLength: s3.DefaultPartSize, bufferSize: bufferSize);
                    }

                    tags.Add(new PartETag(partNumber, (await tUpload).ETag));

                    ms.Seek(0, SeekOrigin.Begin);
                    ih.AppendData(ms.ToArray());
                }
            }

            UpdateCancellationToken();
            var mpResult = await s3.CompleteMultipartUploadAsync(
                bucketName : bucketName,
                key : key,
                uploadId : init.UploadId,
                partETags : tags,
                cancellationToken : ct).TryCancelAfter(ct, msTimeout: msTimeout);

            md5  = ih.GetHashAndReset().ToHexString();
            etag = mpResult.ETag.Trim('"');
            return(md5);
        }
示例#16
0
        public static async Task <bool> DeleteVersionedObjectAsync(this S3Helper s3,
                                                                   string bucketName,
                                                                   string key,
                                                                   bool throwOnFailure = true,
                                                                   CancellationToken cancellationToken = default(CancellationToken))
        {
            var versions = await s3.ListVersionsAsync(
                bucketName : bucketName,
                prefix : key,
                cancellationToken : cancellationToken);

            var keyVersions = versions
                              .Where(v => !v.IsLatest)
                              .Select(ver => new KeyVersion()
            {
                Key = key, VersionId = ver.VersionId
            })
                              .ToArray();

            if (keyVersions.Length > 0)
            {
                var response = await s3.DeleteObjectsAsync(
                    bucketName : bucketName,
                    objects : keyVersions,
                    cancellationToken : cancellationToken);

                if (response.DeleteErrors.Count > 0)
                {
                    if (throwOnFailure)
                    {
                        throw new Exception($"Failed to delete all object versions of key '{key}' in bucket '{bucketName}', {response.DeletedObjects.Count} Deleted {response.DeleteErrors.Count} Errors, Delete Errors: {response.DeleteErrors.JsonSerialize(Newtonsoft.Json.Formatting.Indented)}");
                    }
                    else
                    {
                        return(false);
                    }
                }

                return(await s3.DeleteVersionedObjectAsync(bucketName, key, throwOnFailure, cancellationToken));
            }

            try
            {
                var latest = versions.Single(x => x.IsLatest);
                await s3.DeleteObjectAsync(bucketName : bucketName, key : key, versionId : latest.VersionId, cancellationToken : cancellationToken);

                return(true);
            }
            catch
            {
                if (!await s3.ObjectExistsAsync(bucketName, key, cancellationToken))
                {
                    return(true);
                }

                if (throwOnFailure)
                {
                    throw;
                }
                else
                {
                    return(false);
                }
            }
        }