protected void Page_Load(object sender, EventArgs e)
    {
        AWSAuthConnection conn = new AWSAuthConnection(accessKey, secretKey);

        XmlTextReader r = new XmlTextReader(Request.InputStream);

        r.MoveToContent();
        string xml = r.ReadOuterXml();

        XmlDocument documentToSave = new XmlDocument();

        documentToSave.LoadXml(xml);

        SortedList metadata = new SortedList();

        metadata.Add("title", bucket);
        metadata.Add("Content-Type", "application/xml");
        S3Object titledObject =
            new S3Object(documentToSave.OuterXml, metadata);

        SortedList headers = new SortedList();

        headers.Add("Content-Type", "application/xml");
        headers.Add("x-amz-acl", "public-read");

        conn.put(bucket, documentKey, titledObject, headers);

        Response.Write("saved: " + documentToSave.OuterXml);
    }
Exemple #2
0
            public WasabiRemoteItem(S3Object sourceItem, S3Updater source, string rootFolder)
            {
                _source     = source ?? throw new ArgumentNullException(nameof(source));
                _sourceItem = sourceItem ?? throw new ArgumentNullException(nameof(sourceItem));

                _fullPath = _sourceItem.Key.TrimEnd('/');

                if (rootFolder != null)
                {
                    _rootFolder = rootFolder.TrimEnd('/');
                    if (!_fullPath.StartsWith(_rootFolder))
                    {
                        throw new IOException($"Remote item full path {_fullPath} doesn't start with the specified root path {_rootFolder}");
                    }
                    ClientRelativeFileName = _fullPath.Substring(_rootFolder.Length).Trim('/');
                }

                IsDirectory = IsDirectory(sourceItem);
                if (IsDirectory)
                {
                    throw new ArgumentException("Directory object received in wrong overload");
                }
                IsFile = true;

                ItemSize     = _sourceItem.Size;
                ModifiedTime = _sourceItem.LastModified;
                Name         = Path.GetFileName(_fullPath);
            }
Exemple #3
0
        private void ProcessFile(S3Object file)
        {
            Trace.WriteLine(string.Format("Processing item {0}", file.Key));

            var lines = ReadS3File(file);

            var trimmedDate = file.Key.Substring(DATE_EXTRACTION_PREFIX.Length, 11);
            var date        = DateTime.ParseExact(trimmedDate, FORMAT, PROVIDER);

            var fileMod   = file.GetHashCode() % ComputeNode.GlobalBucketCount;
            var buckets   = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName == CATALOG).First().Buckets;
            var bucketMod = buckets.First(b => b.Value.BucketMod == fileMod).Value;

            Trace.WriteLine(string.Format("Adding data items from {0}", file.Key));
            lines.AsParallel().ForAll(line =>
            {
                var items = line.Split(' ');
                Debug.Assert(items.Length == 4);

                var projectCode = HttpUtility.UrlDecode(items[0]);
                var pageName    = HttpUtility.UrlDecode(items[1]);
                var pageViews   = int.Parse(items[2]);
                var pageSizeKB  = long.Parse(items[3]);

                var wikiStat = new WikipediaHourlyPageStats(date, projectCode, pageName, pageViews, pageSizeKB);
                bucketMod.BucketDataTables[TABLE].AddItem(wikiStat);
            });

            Trace.WriteLine(string.Format("Added data items from {0}", file.Key));
        }
Exemple #4
0
        private static void WriteToBlobStorage(BucketBlobPackage package)
        {
            foreach (var item in package.FileInfo)
            {
                using (client = new AmazonS3Client(package.AccessKey, package.SecretAccessKey, GetAwsRegionFromString(package.BucketInfo.Region)))
                {
                    GetObjectRequest request = new GetObjectRequest
                    {
                        BucketName = package.BucketInfo.Name,
                        Key        = item.Key
                    };

                    using (GetObjectResponse response = client.GetObject(request))
                    {
                        using (Stream responseStream = response.ResponseStream)

                        {
                            string fileExtension = item.Key.Substring(Math.Max(0, item.Key.Length - 4)).ToUpper();
                            switch (fileExtension)
                            {
                            case ".ZIP":
                                CopyZIPtoBlob(responseStream);
                                break;

                            default:
                                S3Object o = item.Value as S3Object;
                                CopySingleFileToBlob(responseStream, item.Key, o.Size);
                                break;
                            }
                        }
                    }
                }
            }
        }
Exemple #5
0
        /// <summary>
        /// Download a specific  file from a S3 bucket
        /// </summary>
        /// <param name="path"></param>
        public void DownloadFile(string path)
        {
            TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(accessKey, secretKey, Amazon.RegionEndpoint.USEast1));

            BasicAWSCredentials basicCredentials = new BasicAWSCredentials(accessKey, secretKey);
            AmazonS3Client      s3Client         = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), Amazon.RegionEndpoint.USEast1);
            ListObjectsRequest  request          = new ListObjectsRequest();

            request.BucketName = bucketName;

            ListObjectsResponse response = s3Client.ListObjects(request.BucketName = bucketName, request.Prefix = path);

            try
            {
                S3Object obj = response.S3Objects[0];

                string     filename = filepath + "\\" + obj.Key;
                FileStream fs       = File.Create(filename);
                fs.Close();
                fileTransferUtility.Download(filename, bucketName, obj.Key);
                log.Info("File: " + obj.Key + " downloaded");
            }
            catch (Exception ex) {
                log.Info(ex.Message + ex.StackTrace);
            }
        }
Exemple #6
0
        //path format (band/16/profile.jpg) or (profile.jpg)
        public async Task <bool> DeleteFile(string filepath)
        {
            try
            {
                bool status = false;
                //int pos = filepath.LastIndexOf("/") + 1;
                //string path = filepath.Substring(pos, filepath.Length)

                var list = await s3Client.ListObjectsAsync(new ListObjectsRequest()
                {
                    BucketName = BucketName, Prefix = filepath
                });

                S3Object s3Object = list.S3Objects.FirstOrDefault();
                if (s3Object != null)
                {
                    var res = await s3Client.DeleteObjectAsync(new DeleteObjectRequest()
                    {
                        BucketName = BucketName, Key = s3Object.Key
                    });

                    status = true;
                }
                return(status);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return(false);
            }
        }
        private bool ShouldDownload(S3Object s3o)
        {
            // skip objects based on ModifiedSinceDateUtc
            if (this._request.IsSetModifiedSinceDateUtc() && s3o.LastModified.ToUniversalTime() <= this._request.ModifiedSinceDateUtc.ToUniversalTime())
            {
                return(false);
            }
            // skip objects based on UnmodifiedSinceDateUtc
            if (this._request.IsSetUnmodifiedSinceDateUtc() && s3o.LastModified.ToUniversalTime() > this._request.UnmodifiedSinceDateUtc.ToUniversalTime())
            {
                return(false);
            }
#pragma warning disable CS0618 //A class member was marked with the Obsolete attribute
            // skip objects based on ModifiedSinceDate
            if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate)
            {
                return(false);
            }
            // skip objects based on UnmodifiedSinceDate
            if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate)
            {
                return(false);
            }
#pragma warning restore CS0618 //A class member was marked with the Obsolete attribute
            // skip objects which are instruction files and we're using encryption client
            if (IsInstructionFile(s3o.Key))
            {
                return(false);
            }

            return(true);
        }
        private SourceImage ProcessObject(
            Bucket tracked,
            S3Object obj,
            IDictionary<string, ImageEntry> entries,
            IList<ImageEntry> toAdd)
        {
            if (entries.TryGetValue(obj.Key, out var entry))
            {
                return this.GetFromCacheEntry(entry);
            }
            else
            {
                var image = this.CreateSourceImage(
                    key: obj.Key,
                    lastModified: obj.LastModified,
                    totalBytes: obj.Size);

                toAdd.Add(new ImageEntry()
                {
                    Bucket = tracked,
                    Key = obj.Key,
                    LastModified = obj.LastModified,
                    FileSize = obj.Size,
                    Status = image.Status == ImageStatus.Skipped ?
                        ImageStatus.Pending : image.Status
                });

                return image;
            }
        } 
        // Download object to local file
        // Returns true on success, false on error.

        public override bool DownloadFile(CloudFile file, String outputFilePath)
        {
            try
            {
                this.Exception = null;
                int pos = outputFilePath.LastIndexOf("\\");
                if (pos != -1)
                {
                    outputFilePath = outputFilePath.Substring(0, pos);
                }

                S3Object        obj = file.StorageObject as S3Object;
                TransferUtility fileTransferUtility = new TransferUtility(new AmazonS3Client(this.AccessKey, this.SecretKey, this.RegionEndpoint));
                fileTransferUtility.Download(outputFilePath, obj.BucketName, file.Name);

                return(true);
            }
            catch (Exception ex)
            {
                this.Exception = ex;
                if (!this.HandleErrors)
                {
                    throw ex;
                }
                return(false);
            }
        }
Exemple #10
0
        //path format (band/16) or (band)
        public async Task <bool> DeleteFolder(string path)
        {
            try
            {
                bool status = false;
                var  list   = await s3Client.ListObjectsAsync(new ListObjectsRequest()
                {
                    BucketName = BucketName, Prefix = path
                });

                S3Object s3Object = list.S3Objects.FirstOrDefault();
                if (s3Object == null)
                {
                    var res = await s3Client.DeleteObjectAsync(new DeleteObjectRequest()
                    {
                        BucketName = BucketName, Key = s3Object.Key
                    });

                    status = true;
                }

                return(status);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return(false);
            }
        }
Exemple #11
0
 public S3FileSystemItem(S3Object file, string key)
 {
     this.Name           = PathEx.GetFileName(key);
     this.Size           = file.Size;
     this.IsDirectory    = false;
     this.LastModifyTime = new DateTimeOffset(file.LastModified.ToUniversalTime(), TimeSpan.Zero);
 }
        private bool Backup(S3Object obj, FileObject fo)
        {
            string displayName = Helpers.FormatDisplayFileName(_wideDisplay, Helpers.RemoveSecExt(obj.Key));

            Logger.WriteLog(ErrorCodes.S3BucketSource_SyncFile,
                            $"cpy src {displayName} [{Helpers.FormatByteSize(fo.DataContent.Length)}]", Severity.Information, VerboseLevel.User);

            // Transforms the file if a transformation is provided
            FileObject fo2 = Transform != null?Transform.Process(fo) : fo;

            bool result = Destination.Write(fo2);

            if (result)
            {
                Interlocked.Increment(ref _fileSynchronized);
                Interlocked.Add(ref _writeSize, fo2.DataContent.Length);
            }

            if (result)
            {
                Interlocked.Increment(ref _synchronizedCount);
            }
            else
            {
                Interlocked.Increment(ref _errorCount);
            }

            return(result);
        }
        private S3Object getFile()
        {
            AmazonS3Client       client  = getConfig();
            ListObjectsV2Request request = new ListObjectsV2Request()
            {
                BucketName = _bucketName
            };
            ListObjectsV2Response list = client.ListObjectsV2(request);
            long     data = 0;
            S3Object file = new S3Object();

            foreach (var item in list.S3Objects)
            {
                if (data < item.Size)
                {
                    file = item;
                    data = item.Size;
                }
            }
            // Convert Bytes to MB
            double t = (file.Size / 1024f) / 1024f;

            // Check how many files we need to achieve x quantity of GB (This case 4.9GB)
            t = 4900 / t;
            // Round number and converted to Int
            cantidad = Convert.ToInt32(Math.Round(t));
            // Return S3 Object
            return(file);
        }
        public void Remove_previously_saved_S3_object(string s3ObjectKey, S3Object s3Object = null)
        {
            "And a S3 Object with given key is saved"
            .x(async() =>
            {
                s3Object = new S3Object
                {
                    Bucket       = BucketId,
                    Key          = s3ObjectKey,
                    CreationDate = DateTime.UtcNow,
                    Content      = new byte[] { 0x01, 0x02, 0x03 },
                    ContentType  = "application/octet-stream"
                };
                await _storage.AddObjectAsync(s3Object);
            });

            "When method RemoveS3ObjectAsync() will be called with given key"
            .x(async() => await _storage.DeleteObjectAsync(BucketId, s3ObjectKey));

            "Then the corresponding reliable dictionaries are removed"
            .x(async() =>
            {
                using (var tx = _stateManager.CreateTransaction())
                {
                    var uriPrefix  = $"fabric://mocks/{S3EntriesKey}:{s3Object.Bucket}:";
                    var enumerator = _stateManager.GetAsyncEnumerator();
                    while (await enumerator.MoveNextAsync(CancellationToken.None))
                    {
                        var uriStr = enumerator.Current.Name.ToString();
                        Assert.False(uriStr.StartsWith(uriPrefix));
                    }
                }
            });
        }
Exemple #15
0
        public S3Object GetS3File()
        {
            S3Object s3Object;

            try
            {
                var request = new GetObjectRequest
                {
                    BucketName = this.BucketName,
                    Key        = this.Key
                };

                var response = this.S3Client.GetObject(request);

                s3Object = new S3Object
                {
                    ETag         = response.ETag,
                    Key          = response.Key,
                    LastModified = response.LastModified.ToUniversalTime()
                };
            }
            catch (AmazonS3Exception aex)
            {
                const string FORMAT  = "Couldn't get file ({0}:/{1}).";
                var          message = string.Format(FORMAT, this.BucketName, this.Key);
                throw new Exception(message, aex);
            }

            return(s3Object);
        }
Exemple #16
0
    public async Task ListObjects(S3Provider provider, string _, ISimpleClient client)
    {
        await CreateTempBucketAsync(provider, client, async tempBucket =>
        {
            string tempObjName        = "object-" + Guid.NewGuid();
            PutObjectResponse putResp = await client.PutObjectStringAsync(tempBucket, tempObjName, "hello").ConfigureAwait(false);
            Assert.Equal(200, putResp.StatusCode);

            ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket).ConfigureAwait(false);
            Assert.Equal(200, listResp.StatusCode);

            Assert.Equal(tempBucket, listResp.BucketName);
            Assert.Equal(1, listResp.KeyCount);

            if (provider == S3Provider.AmazonS3)
            {
                Assert.Equal(1000, listResp.MaxKeys);
            }

            Assert.False(listResp.IsTruncated);

            S3Object obj = listResp.Objects.First();
            Assert.Equal(tempObjName, obj.ObjectKey);
            Assert.Equal("\"5d41402abc4b2a76b9719d911017c592\"", obj.ETag);
            Assert.Equal(5, obj.Size);
            Assert.Equal(DateTime.UtcNow, obj.LastModifiedOn.UtcDateTime, TimeSpan.FromSeconds(5));

            if (provider == S3Provider.AmazonS3)
            {
                Assert.Equal(StorageClass.Standard, obj.StorageClass);
            }
        }).ConfigureAwait(false);
    }
Exemple #17
0
 /// <summary>
 /// Tracks the specified <see cref="S3Object"/>.
 /// </summary>
 /// <param name="o">The <see cref="S3Object"/> being synchronized.</param>
 /// <param name="copied">Whether or not the item needed to be copied.</param>
 /// <param name="updated">Whether or not the item needed to be udpated.</param>
 public void TrackObject(string prefix, S3Object o, bool copied, bool updated)
 {
     if (copied)
     {
         Interlocked.Exchange(ref _lastCopyCompleted, o.Key);
         Interlocked.Exchange(ref _lastCopyDate, o.LastModified.Ticks);
     }
     if (updated)
     {
         Interlocked.Exchange(ref _lastUpdateCompleted, o.Key);
         Interlocked.Exchange(ref _lastUpdateDate, o.LastModified.Ticks);
     }
     InterlockedMax(ref _greatestKeyCompleted, o.Key.Substring(prefix.Length));
     InterlockedMin(ref _earliestDate, o.LastModified.Ticks);
     InterlockedMax(ref _latestDate, o.LastModified.Ticks);
     Interlocked.Add(ref _bytesProcessed, o.Size);
     Interlocked.Add(ref _objectsProcessed, 1);
     if (copied)
     {
         InterlockedMin(ref _earliestCopiedDate, o.LastModified.Ticks);
         InterlockedMax(ref _latestCopiedDate, o.LastModified.Ticks);
         Interlocked.Add(ref _bytesCopied, o.Size);
         Interlocked.Add(ref _objectsCopied, 1);
     }
     if (updated)
     {
         InterlockedMin(ref _earliestUpdatedDate, o.LastModified.Ticks);
         InterlockedMax(ref _latestUpdatedDate, o.LastModified.Ticks);
         Interlocked.Add(ref _bytesUpdated, o.Size);
         Interlocked.Add(ref _objectsUpdated, 1);
     }
 }
Exemple #18
0
        private Response AddObject(string bucket, string key, Stream stream)
        {
            if (Request.Url.Query == "?acl")
            {
                return(new Response {
                    StatusCode = HttpStatusCode.OK
                });
            }

            var content = stream.Copy(configuration.MaxBytesPerSecond);

            var s3Object = new S3Object
            {
                Bucket       = bucket,
                Key          = key,
                ContentType  = Request.Headers.ContentType,
                CreationDate = DateTime.UtcNow,
                Content      = () => content,
                ContentMD5   = content.GenerateMD5CheckSum(),
                Size         = content.Length
            };

            storage.AddObject(s3Object);

            var response = new Response {
                StatusCode = HttpStatusCode.OK
            };

            response.WithHeader("ETag", string.Format("\"{0}\"", s3Object.ContentMD5));
            return(response);
        }
Exemple #19
0
        private async Task UpdateItem(S3Object sourceItem, FileInfo targetPath, IProgress <double> progressCallback,
                                      CancellationToken cancellationToken)
        {
            progressCallback.Report(0d);

            //_s3Client.DownloadToFilePathAsync() doesn't support progress callbacks, need to do all this instead
            var objectResponse = await _s3Client.GetObjectAsync(sourceItem.BucketName, sourceItem.Key, cancellationToken);

            using (var input = objectResponse.ResponseStream)
                using (var output = targetPath.OpenWrite())
                {
                    // buffer based on file size to have decent rate of progress reporting
                    const int  preferredUpdateSteps = 100;
                    const long minBufferSize        = 128 * 1024;      //kb
                    const long maxBufferSize        = 5 * 1024 * 1024; //mb
                    var        buffer = new byte[Math.Min(maxBufferSize, Math.Max(minBufferSize, sourceItem.Size / preferredUpdateSteps))];
                    int        bytesRead;
                    while ((bytesRead = await input.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
                    {
                        var task = output.WriteAsync(buffer, 0, bytesRead, cancellationToken);

                        progressCallback.Report(100d * Math.Min(1d, (double)output.Position / (double)sourceItem.Size));

                        await task;
                    }

                    if (output.Position != sourceItem.Size)
                    {
                        throw new InvalidDataException("The downloaded file was not the correct size");
                    }
                }
        }
Exemple #20
0
        /// <summary>
        /// Write files to desired destination path.
        /// </summary>
        /// <param name="parameters"></param>
        /// <param name="fileObject"></param>
        /// <param name="s3Client"></param>
        /// <param name="destinationFolder"></param>
        /// <param name="fullPath"></param>
        /// <returns>string</returns>
        private static async Task <string> WriteToFile(
            Parameters parameters,
            S3Object fileObject,
            AmazonS3Client s3Client,
            string destinationFolder,
            string fullPath
            )
        {
            string responseBody;
            var    request = new GetObjectRequest
            {
                BucketName = parameters.BucketName,
                Key        = fileObject.Key
            };

            using (var response = await s3Client.GetObjectAsync(request))
                using (var responseStream = response.ResponseStream)
                    using (var reader = new StreamReader(responseStream)) responseBody = await reader.ReadToEndAsync();
            if (File.Exists(fullPath))
            {
                File.Delete(fullPath);
                File.WriteAllText(fullPath, responseBody);
                return(fullPath);
            }
            else
            {
                if (!Directory.Exists(destinationFolder))
                {
                    Directory.CreateDirectory(destinationFolder);
                }
                File.WriteAllText(fullPath, responseBody);
                return(fullPath);
            }
        }
Exemple #21
0
        protected override async Task ExecuteAsync(CommandLineApplication app, CancellationToken token)
        {
            IAsyncEnumerator <S3Object> list = Manager.ObjectManager.ListAsync(BucketName, IncludeOwner).GetAsyncEnumerator(token);

            bool hasData = await list.MoveNextAsync().ConfigureAwait(false);

            if (!hasData)
            {
                Console.WriteLine();
                Console.WriteLine("There were no objects.");
            }
            else
            {
                Console.WriteLine();

                if (IncludeDetails)
                {
                    if (IncludeOwner)
                    {
                        Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4,-20}{5}", "Modified on", "Size", "Storage class", "ETag", "Owner", "Name");
                    }
                    else
                    {
                        Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4}", "Modified on", "Size", "Storage class", "ETag", "Name");
                    }
                }
                else
                {
                    Console.WriteLine("{0,-20}{1,-12}{2}", "Modified on", "Size", "Name");
                }

                do
                {
                    S3Object obj = list.Current;

                    if (IncludeDetails)
                    {
                        if (IncludeOwner)
                        {
                            string ownerInfo = string.Empty;

                            if (obj.Owner != null)
                            {
                                ownerInfo = obj.Owner.Name;
                            }

                            Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4,-20}{5}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.StorageClass, obj.ETag !, ownerInfo, obj.ObjectKey);
                        }
                        else
                        {
                            Console.WriteLine("{0,-20}{1,-12}{2,-18}{3,-38}{4}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.StorageClass, obj.ETag !, obj.ObjectKey);
                        }
                    }
                    else
                    {
                        Console.WriteLine("{0,-20}{1,-12}{2}", obj.LastModifiedOn.ToString("yyy-MM-dd hh:mm:ss", DateTimeFormatInfo.InvariantInfo), obj.Size, obj.ObjectKey);
                    }
                } while (await list.MoveNextAsync().ConfigureAwait(false));
            }
        }
        public async Task <IActionResult> Put(string bucket, string key, CancellationToken cancellationToken)
        {
            if (Request.QueryString.HasValue && Request.QueryString.Value == "?acl")
            {
                return(Ok());
            }

            var s3Object = new S3Object
            {
                Bucket       = bucket,
                Key          = key,
                ContentType  = string.IsNullOrEmpty(Request.ContentType) ? "application/octet-stream" : Request.ContentType,
                CreationDate = DateTime.UtcNow,
                Size         = 0
            };

            if (Request.Body != null)
            {
                s3Object.Content = await Request.Body.ReadAllBytesAsync();
            }

            await _storage.AddObjectAsync(s3Object, cancellationToken);

            var response = Ok();

            Response.Headers.Add("ETag", $"\"{s3Object.ContentMD5}\"");
            return(response);
        }
Exemple #23
0
        /// <summary>
        /// Downloads the latest backup set from the Aws service.
        /// </summary>
        /// <param name="compressor">The compresor to use when decompressing the downloaded file.</param>
        /// <returns>The path to the downloaded and decompressed backup file.</returns>
        public string DownloadBackup(ICompressor compressor)
        {
            S3Object latest = this.GetLatestBackupItem();
            string   path   = latest.Key;

            if (!String.IsNullOrEmpty(this.Target.AwsPrefix) && path.StartsWith(this.Target.AwsPrefix, StringComparison.OrdinalIgnoreCase))
            {
                path = path.Substring(this.Target.AwsPrefix.Length);

                if (path.StartsWith("/", StringComparison.Ordinal))
                {
                    path = path.Substring(1);
                }
            }

            path = Path.Combine(TempDir, path);
            string fileName = Path.GetFileName(path);

            if (File.Exists(path))
            {
                File.Delete(path);
            }

            TransferInfo info = new TransferInfo()
            {
                BytesTransferred = 0,
                FileName         = fileName,
                FileSize         = 0
            };

            using (TransferUtility transfer = new TransferUtility(S3Client))
            {
                TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
                                                         .WithBucketName(AwsConfig.BucketName)
                                                         .WithFilePath(path)
                                                         .WithKey(latest.Key);

                request.WriteObjectProgressEvent += (sender, e) =>
                {
                    info.BytesTransferred = e.TransferredBytes;
                    info.FileSize         = e.TotalBytes;
                    this.Fire(this.TransferProgress, new TransferInfo(info));
                };

                this.Fire(this.TransferStart, new TransferInfo(info));
                transfer.Download(request);
            }

            this.Fire(this.TransferComplete, new TransferInfo(info));

            this.Fire(this.DecompressStart);
            string decompressedPath = compressor.Decompress(path);

            this.Fire(this.DecompressComplete);

            File.Delete(path);

            return(decompressedPath);
        }
Exemple #24
0
 /// <summary>
 /// Pick up raw ETag string without "" defined in RFC
 /// </summary>
 /// <param name="source"></param>
 /// <returns></returns>
 public static string GetETag(this S3Object source)
 {
     if (source == null)
     {
         throw new ArgumentNullException();
     }
     return(source.ETag.Replace("\"", ""));
 }
        private static void UnmarshallResult(XmlUnmarshallerContext context, ListObjectsResponse response)
        {
            int currentDepth = context.get_CurrentDepth();
            int num          = currentDepth + 1;

            if (context.get_IsStartOfDocument())
            {
                num += 2;
            }
            while (context.Read())
            {
                if (context.get_IsStartElement() || context.get_IsAttribute())
                {
                    if (context.TestExpression("IsTruncated", num))
                    {
                        response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("NextMarker", num))
                    {
                        response.NextMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("Contents", num))
                    {
                        S3Object s3Object = ContentsItemUnmarshaller.Instance.Unmarshall(context);
                        s3Object.BucketName = response.Name;
                        response.S3Objects.Add(s3Object);
                    }
                    else if (context.TestExpression("Name", num))
                    {
                        response.Name = StringUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("Prefix", num))
                    {
                        response.Prefix = StringUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("Delimiter", num))
                    {
                        response.Delimiter = StringUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("MaxKeys", num))
                    {
                        response.MaxKeys = IntUnmarshaller.GetInstance().Unmarshall(context);
                    }
                    else if (context.TestExpression("CommonPrefixes", num))
                    {
                        string text = CommonPrefixesItemUnmarshaller.Instance.Unmarshall(context);
                        if (text != null)
                        {
                            response.CommonPrefixes.Add(text);
                        }
                    }
                }
                else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth)
                {
                    break;
                }
            }
        }
Exemple #26
0
        public async Task <S3Response> DownloadFileAsync(string bucketName, string folderName, string FileName)
        {
            string target = Path.GetTempPath();

            folderName = folderName.Replace(@"\", "/");
            try {
                if (await AmazonS3Util.DoesS3BucketExistV2Async(_client, bucketName) == false)
                {
                    ListObjectsRequest request = new ListObjectsRequest()
                    {
                        BucketName = bucketName
                    };
                    do
                    {
                        ListObjectsResponse response = await _client.ListObjectsAsync(request);

                        for (int i = 1; i <= response.S3Objects.Count - 1; i++)
                        {
                            S3Object entry = response.S3Objects[i];
                            if (entry.Key.Replace(folderName + "/", "") == FileName)
                            {
                                GetObjectRequest objRequest = new GetObjectRequest()
                                {
                                    BucketName = bucketName, Key = entry.Key
                                };
                                GetObjectResponse objResponse = await _client.GetObjectAsync(objRequest);

                                await objResponse.WriteResponseStreamToFileAsync(target + FileName, true, CancellationToken.None);

                                break;
                            }
                        }
                        if ((response.IsTruncated))
                        {
                            request.Marker = response.NextMarker;
                        }
                        else
                        {
                            request = null /* TODO Change to default(_) if this is not a reference type */;
                        }
                    }while (request != null);
                }
            } catch (AmazonS3Exception e) {
                return(new S3Response {
                    Message = e.Message,
                    Status = e.StatusCode
                });
            } catch (Exception ex) {
                return(new S3Response {
                    Message = ex.Message,
                    Status = HttpStatusCode.InternalServerError
                });
            }
            return(new S3Response {
                Message = "Something went wrong...",
                Status = HttpStatusCode.InternalServerError
            });
        }
Exemple #27
0
        public static string ToString(S3Object s)
        {
            if (s == null)
            {
                return("");
            }

            return(Path.Combine(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString() + s.BucketName, s.Key.Replace('/', Path.DirectorySeparatorChar)).TrimEnd(Path.DirectorySeparatorChar));
        }
Exemple #28
0
 public static async Task supprime(S3Object obj, string bucketName, AmazonS3Client amazonS3Client)
 {
     DeleteObjectRequest requestDelete = new DeleteObjectRequest
     {
         BucketName = bucketName,
         Key        = obj.Key,
     };
     await amazonS3Client.DeleteObjectAsync(requestDelete);
 }
        // return a file from S3
        private static S3Object getObject(string filename)
        {
            S3Object o = new S3Object()
            {
                Bucket = "licenselatesfx",
                Name   = filename
            };

            return(o);
        }
Exemple #30
0
        private static async Task <string> GetFileContent(IAmazonS3 client, S3Object s3Object)
        {
            var response = await client.GetObjectAsync(s3Object.BucketName, s3Object.Key);

            await using var responseStream = response.ResponseStream;

            using var reader = new StreamReader(responseStream);

            return(await reader.ReadToEndAsync());
        }
        //
        // GET: /S3Object/Edit/5
        public ActionResult Edit(string id = null, string prefix = "", int maxKeys = 100)
        {
            ViewBag.prefix = prefix;
              ViewBag.maxKeys = maxKeys;

              var list = new Amazon.S3.Model.ListObjectsRequest();
              list.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]);
              list.WithPrefix(id);

              var s3Objects = s3.ListObjects(list).S3Objects;
              if (s3Objects.Count == 0)
              {
            return HttpNotFound();
              }

              var get = new Amazon.S3.Model.GetObjectRequest();
              get.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]);
              get.WithKey(s3Objects[0].Key);

              var response = s3.GetObject(get);

              S3Object modelObject = new S3Object
              {
            Key = s3Objects[0].Key,
            Size = s3Objects[0].Size,
            LastModified = s3Objects[0].LastModified,
            ContentType = response.ContentType
              };

              return View(modelObject);
        }
 private void DownloadFolder(S3Object s3Obj, string folderPath)
 {
     string fullPath = s3Obj.Shared ? folderPath : Path.Combine(folderPath, s3Obj.Key.Replace("/", "\\"));
     try
     {
         // need to import complete directory
         _downloadObjects.Add(fullPath);
         if (!Directory.Exists(fullPath))
             Directory.CreateDirectory(fullPath);
         // get the folder structure from application database
         string url = Utilities.DevelopmentMode ? "http://localhost:3000" : "http://versavault.com";
         url += "/api/child_files?bucket_key=" + Utilities.MyConfig.BucketKey + "&parent_uid=" + s3Obj.Uid + "&machine_key=" + Utilities.MyConfig.MachineKey + "&shared=" + s3Obj.Shared;
         string result = GetResponse(url);
         if (!string.IsNullOrEmpty(result))
         {
             var res = JsonConvert.DeserializeObject<s3_object>(result);
             foreach (var s3ObjSub in res.S3Object)
             {
                 if (s3ObjSub != null)
                 {
                     string fullPathSub = Path.Combine(s3ObjSub.Shared ? Utilities.SharedFolderPath + "\\" + s3ObjSub.Username : Utilities.Path, s3ObjSub.Key.Replace("/", "\\"));
                     if (s3ObjSub.Shared)
                     {
                         if (!Directory.Exists(Utilities.SharedFolderPath + "\\" + s3ObjSub.Username))
                             Directory.CreateDirectory(Utilities.SharedFolderPath + "\\" + s3ObjSub.Username);
                     }
                     if (s3ObjSub.Folder)
                     {
                         if (!Directory.Exists(fullPathSub))
                         {
                             DownloadFolder(s3ObjSub, fullPathSub);
                         }
                         else
                         {
                             try
                             {
                                 TimeSpan timeSpan = new DirectoryInfo(fullPathSub).LastWriteTime.ToUniversalTime().Subtract(s3ObjSub.LastModified);
                                 var seconds = Math.Floor(timeSpan.TotalSeconds);
                                 if (seconds != 0)
                                 {
                                     if (seconds < 0)
                                     {
                                         _downloadObjects.Add(fullPathSub);
                                         DownloadFolder(s3ObjSub, fullPathSub);
                                         string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/");
                                         if (s3ObjSub.Shared)
                                         {
                                             var folders = relativePath.Split('\\');
                                             ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]);
                                         }
                                         else
                                             ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty);
                                         //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update });
                                     }
                                     //else
                                     //   Uploadfiles(fullPathSub); // need to figure it out how to do this beacause we cannot upload the entire folder here
                                 }
                             }
                             catch (Exception)
                             {
                                 // Todo
                                 // need to fix this when an error occured
                                 continue;
                             }
                         }
                     }
                     else
                     {
                         if (!File.Exists(fullPathSub))
                         {
                             _downloadObjects.Add(fullPathSub);
                             _service.GetObject(s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey, s3ObjSub.Key, fullPathSub);
                             string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/");
                             if (s3ObjSub.Shared)
                             {
                                 var folders = relativePath.Split('\\');
                                 ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]);
                             }
                             else
                                 ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty);
                             //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update });
                         }
                         else
                         {
                             try
                             {
                                 if (!Utilities.IsFileUsedbyAnotherProcess(fullPathSub))
                                 {
                                     TimeSpan timeSpan = new FileInfo(fullPathSub).LastWriteTime.ToUniversalTime().Subtract(s3ObjSub.LastModified);
                                     var seconds = Math.Floor(timeSpan.TotalSeconds);
                                     if (seconds != 0)
                                     {
                                         if (seconds < 0)
                                         {
                                             _downloadObjects.Add(fullPathSub);
                                             _service.GetObject(s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey, s3ObjSub.Key, fullPathSub);
                                             string relativePath = fullPathSub.Replace(Utilities.Path + "\\", "").Replace("\\", "/");
                                             if (s3ObjSub.Shared)
                                             {
                                                 var folders = relativePath.Split('\\');
                                                 ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, s3ObjSub.Shared, folders.Length > 1 ? folders[1] : folders[0]);
                                             }
                                             else
                                                 ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty);
                                             //_applicationUpates.Enqueue(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPathSub).LastWriteTime, Status = UpdateStatus.Update });
                                         }
                                         else
                                             Addobject(fullPathSub, s3ObjSub.Shared ? s3ObjSub.BucketKey : Utilities.MyConfig.BucketKey);
                                     }
                                 }
                             }
                             catch (Exception)
                             {
                                 // Todo
                                 // need to fix this when an error occured
                                 continue;
                             }
                         }
                     }
                 }
             }
         }
     }
     catch (Exception)
     {
         // Todo
         // need to figure it out why it is happening
     }
     finally
     {
         string relativePath = fullPath.Replace(Utilities.Path + "\\", "").Replace("\\", "/");
         if (relativePath.ToLower().IndexOf("shared") == 0)
         {
             var folders = relativePath.Split('\\');
             ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPath).LastWriteTime, Status = UpdateStatus.Add }, true, folders.Length > 1 ? folders[1] : folders[0]);
         }
         else
             ProcessApplicationUpdates(new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(fullPath).LastWriteTime, Status = UpdateStatus.Add }, false, string.Empty);
         //_applicationUpates.Enqueue(new AppUpdateInfo { Key = s3Obj.Key, LastModifiedTime = new DirectoryInfo(fullPath).LastWriteTime, Status = UpdateStatus.Update });
     }
 }
        public ActionResult Edit(S3Object s3object, string prefix = "", int maxKeys = 100)
        {
            ViewBag.prefix = prefix;
              ViewBag.maxKeys = maxKeys;

              if (ModelState.IsValid && Request.Files.Count > 0)
              {
            var put = new Amazon.S3.Model.PutObjectRequest();
            put.WithBucketName(WebConfigurationManager.AppSettings["UploadBucket"]);
            put.WithKey(s3object.Key);

            var file = Request.Files[0];
            put.WithContentType(file.ContentType);
            put.WithInputStream(file.InputStream);
            put.WithCannedACL(Amazon.S3.Model.S3CannedACL.PublicRead);
            s3.PutObject(put);

            return RedirectToAction("Index", new { prefix = prefix, maxKeys = maxKeys });
              }

              return View(s3object);
        }