private TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S3Object s3Object, int prefixLength)
        {
            var downloadRequest = new TransferUtilityDownloadRequest();

            downloadRequest.BucketName = this._request.BucketName;
            downloadRequest.Key        = s3Object.Key;
            var file = s3Object.Key.Substring(prefixLength).Replace('/', Path.DirectorySeparatorChar);

            downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, file);
            downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;

            return(downloadRequest);
        }
Пример #2
0
        public bool testDownload()
        {
            TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest();

            request.BucketName = "eecs393minesweeper";
            String key = "test.map";

            request.Key      = key;
            request.FilePath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\Minesweeper\\" + key;
            utility.Download(request);
            ReadLocalFiles();
            PopulateLocalList();
            return(File.Exists(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + "\\Minesweeper\\" + key));
        }
Пример #3
0
        public async Task GetImageAsync(string bucketName, string objectKey, string filePath)
        {
            using (var transferUtility = new TransferUtility(_s3Client, _config))
            {
                var request = new TransferUtilityDownloadRequest
                {
                    BucketName = bucketName,
                    Key        = objectKey,
                    FilePath   = filePath
                };

                await transferUtility.DownloadAsync(request);
            }
        }
Пример #4
0
        public void DownloadFileAsync(string fileName, string targetPath)
        {
            var fileTransferUtility =
                new TransferUtility(client);

            var fileTransferUtilityRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                FilePath   = Path.Combine(targetPath, Path.GetFileName(fileName)),
                Key        = $"{fileName}"
            };

            fileTransferUtility.Download(fileTransferUtilityRequest);
            logger.Info($"Object {fileName} downloaded.");
        }
Пример #5
0
        public async Task Should_TransferUtitlity()
        {
            var utility = new TransferUtility(client);

            var request = new TransferUtilityDownloadRequest
            {
                BucketName = "ecap-falcon",
                Key        = "03CB552E-A3FE-4EF9-A600-909FBA63E900/000c4fcd95e34d90958a5605f2976b6e/461d8d9c-cd33-425e-bb11-3f78b5f98089/000c4fcd95e34d90958a5605f2976b6e.png",
                FilePath   = @"d:\abcd.png",
            };

            await utility.DownloadAsync(request);

            //client.CopyObjectAsync();
        }
Пример #6
0
        public async Task DownloadFile(string bucketName, string fileName)
        {
            var pathAndFileName = $"C:\\S3Temp\\{fileName}";
            var downloadRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                Key        = fileName,
                FilePath   = pathAndFileName
            };

            using (var transferUtility = new TransferUtility(_s3Client))
            {
                await transferUtility.DownloadAsync(downloadRequest);
            }
        }
Пример #7
0
        public async Task DownloadFile(string bucketName, string fileName)
        {
            var pathAndFileName = $"C:\\Users\\magef\\AppData\\Local\\CSReporterV3\\{fileName}";

            var downloadRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                Key        = fileName,
                FilePath   = pathAndFileName
            };

            using (var transferUtility = new TransferUtility(_s3Client))
            {
                await transferUtility.DownloadAsync(downloadRequest);
            };
        }
Пример #8
0
        public void DownloadFile(string remoteFilename, string localSavePath)
        {
            TransferUtilityDownloadRequest req = new TransferUtilityDownloadRequest();

            req.BucketName = CurrentBucketName;
            req.FilePath   = localSavePath;

            req.Key = remoteFilename;
            req.WriteObjectProgressEvent += (s, e) =>
            {
                if (FileProgressChanged != null)
                {
                    FileProgressChanged(e.Key, e.TransferredBytes, e.TotalBytes, e.PercentDone);
                }
            };
            _transfer.Download(req);
        }
Пример #9
0
        public async Task DownloadFile(string bucketName, string fileName)
        {
            var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "temp");

            Directory.CreateDirectory(path);
            var pathAndFileName = Path.Combine(path, fileName);
            var downloadRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                Key        = fileName,
                FilePath   = pathAndFileName,
            };

            using (var transferUtility = new TransferUtility(_s3Client))
            {
                await transferUtility.DownloadAsync(downloadRequest);
            }
        }
Пример #10
0
        /// <summary>
        /// Downloads the S3 object and saves it the specified  destination asynchronously.
        /// </summary>
        /// <param name="path">The path.</param>
        /// <param name="destinationPath">The destination path.</param>
        /// <returns></returns>
        public async Task CopyToLocalAsync(string path, string destinationPath)
        {
            var bucketName = ExtractBucketNameFromPath(path);
            var subPath    = path.SubstringAfterChar(bucketName + "/");

            var client = InitialiseClient();

            var utlity = new TransferUtility(client);

            var request = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                Key        = subPath,
                FilePath   = destinationPath
            };

            await utlity.DownloadAsync(request);
        }
Пример #11
0
        /// <summary>
        /// Download file from s3 to local temp folder
        /// </summary>
        /// <param name="bucketName">bucket name</param>
        /// <param name="key">key = path + file name</param>
        /// <param name="temporalPath">temporal path to download + file name</param>
        /// <returns>task just to download file</returns>
        public async Task DownloadFile(string bucketName, string key, string temporalPath)
        {
            GetObjectRequest request = new GetObjectRequest
            {
                BucketName = bucketName,
                Key        = key
            };

            var downloadRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                Key        = key,
                FilePath   = temporalPath,
            };

            using var transferUtility = new TransferUtility(_s3Client);
            await transferUtility.DownloadAsync(downloadRequest);
        }
Пример #12
0
        public string DownloadFile(string fileName, string targetPath)
        {
            var targetFileName = Path.Combine(targetPath, fileName.Replace(":", ""));

            var fileTransferUtility =
                new TransferUtility(client);

            var fileTransferUtilityRequest = new TransferUtilityDownloadRequest
            {
                BucketName = bucketName,
                FilePath   = targetFileName,
                Key        = $"{fileName}"
            };

            fileTransferUtility.Download(fileTransferUtilityRequest);
            logger.Info($"Object {fileName} downloaded.");

            return(targetFileName);
        }
Пример #13
0
        public async Task <GetFileResponse> DownloadFile(string bucketName, string fileName)
        {
            var response = new List <KeyValuePair <string, string> >();

            try
            {
                var pathAndFileName = $"C:\\S3Test\\{fileName}";

                var downloadRequest = new TransferUtilityDownloadRequest
                {
                    BucketName = bucketName,
                    Key        = fileName,
                    FilePath   = pathAndFileName
                };

                using (var transferUtility = new TransferUtility(_s3Client))
                {
                    await transferUtility.DownloadAsync(downloadRequest);

                    var metadataRequest = new GetObjectMetadataRequest
                    {
                        BucketName = bucketName,
                        Key        = fileName
                    };

                    var metadata = await _s3Client.GetObjectMetadataAsync(metadataRequest);

                    foreach (string key in metadata.Metadata.Keys)
                    {
                        response.Add(new KeyValuePair <string, string>(key, metadata.Metadata[key]));
                    }

                    return(new GetFileResponse
                    {
                        Metadata = response
                    });
                }
            }
            catch (AmazonS3Exception ex)
            {
                throw ex;
            }
        }
Пример #14
0
        private TransferUtilityDownloadRequest CreateDownloadRequest(DownloadSettings settings)
        {
            TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest();

            request.BucketName = settings.BucketName;

            request.ServerSideEncryptionCustomerProvidedKey    = settings.EncryptionKey;
            request.ServerSideEncryptionCustomerProvidedKeyMD5 = settings.EncryptionKeyMD5;
            request.ServerSideEncryptionCustomerMethod         = settings.EncryptionMethod;

            if (!String.IsNullOrEmpty(settings.EncryptionKey))
            {
                request.ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256;
            }

            request.ModifiedSinceDate = settings.ModifiedDate;

            return(request);
        }
Пример #15
0
        private string bucketName = "missions69";      //папка в интернете (корзина)
        //private string keyName = "mission69.xml";        // название файла в папке в инете
        //private string filePath = "App1.dmissions.xml"; //путь файла для отправления/загрузки
        //private string dfilePath = "App1.missions.xml";



        public async void DownloadFile(string path, string key)
        {
            AmazonS3Config config = new AmazonS3Config();

            //config.ServiceURL = "https://missions69.s3.eu-west-3.amazonaws.com/missions.xml" ;
            config.UseHttp        = true;
            config.RegionEndpoint = Amazon.RegionEndpoint.EUWest3;

            var client = new AmazonS3Client("AKIAJO3YO2ATRVV5UQYA", "4+SelI41UmF0oO8IiXazTizmLy60C82Eaem2nonH", config);

            Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.Personal), path);
            TransferUtility transferUtility        = new TransferUtility(client);
            TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest();

            request.Key        = key;
            request.BucketName = bucketName;
            request.FilePath   = path;
            System.Threading.CancellationToken cancellationToken = new System.Threading.CancellationToken();
            await transferUtility.DownloadAsync(request, cancellationToken);
        }
Пример #16
0
        public async Task <bool> DownloadObjectAsync(string bucketName, string localFullFilename, string key, CancellationToken token, int sleepMiliSec = 0)
        {
            bool bReturn = true;

            try
            {
                using (IAmazonS3 s3Client = new AmazonS3Client(AccessKey, SecretKey, new AmazonS3Config
                {
                    ServiceURL = ServiceUrl,
                    BufferSize = 64 * (int)Math.Pow(2, 10),
                    ProgressUpdateInterval = this.ProgressUpdateInterval,
                    Timeout = new TimeSpan(1, 0, 0, 0, 0)
                }))
                {
                    //ProgressUpdateInterval setting error....why?
                    TransferUtility t = new TransferUtility(s3Client);
                    TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest
                    {
                        BucketName = bucketName,
                        Key        = key,
                        FilePath   = localFullFilename.Replace(@"/", @"\")
                    };
                    request.WriteObjectProgressEvent += ProgressEventCallback;
                    Task  task = t.DownloadAsync(request, token);
                    await task;
                }
            }
            catch (Exception)
            {
                if (token.IsCancellationRequested)
                {
                    bReturn = false;
                }
                else
                {
                    throw;
                }
            }

            return(bReturn);
        }
Пример #17
0
        /// <summary>
        /// Downloads the content from Amazon S3 and writes it to the specified file.
        /// </summary>
        /// <param name="filePath">The file path of the file to upload.</param>
        /// <param name="key">The key under which the Amazon S3 object is stored.</param>
        /// <param name="version">The identifier for the specific version of the object to be downloaded, if required.</param>
        /// <param name="settings">The <see cref="DownloadSettings"/> required to download from Amazon S3.</param>
        public void Download(FilePath filePath, string key, string version, DownloadSettings settings)
        {
            TransferUtility utility = this.GetUtility(settings);
            TransferUtilityDownloadRequest request = this.CreateDownloadRequest(settings);

            this.SetWorkingDirectory(settings);
            string fullPath = filePath.MakeAbsolute(settings.WorkingDirectory).FullPath;

            request.FilePath = fullPath;
            request.Key      = key;

            if (!String.IsNullOrEmpty(version))
            {
                request.VersionId = version;
            }

            request.WriteObjectProgressEvent += new EventHandler <WriteObjectProgressArgs>(this.WriteObjectProgressEvent);

            _Log.Verbose("Downloading file {0} from bucket {1}...", key, settings.BucketName);
            utility.Download(request);
        }
Пример #18
0
        public async Task <string> StreamS3ObjectToString(JobDetail jobDetail)
        {
            string result = string.Empty;

            //GetObjectRequest request = new GetObjectRequest()
            string filePath = $"/tmp/{jobDetail.SourceKeyName}";
            TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
            {
                FilePath   = filePath,
                BucketName = jobDetail.SourceBucketName,
                Key        = jobDetail.SourceKeyName
            };
            TransferUtility transferUtility = new TransferUtility(s3Client);

            await transferUtility.DownloadAsync(request);

            Console.WriteLine($"S3 Request Downloaded");
            using (StreamReader reader = new StreamReader(filePath))
            {
                result = await reader.ReadToEndAsync();
            }
            return(result);
        }
Пример #19
0
 public string Download_FileAsync(string emailId)
 {
     //DirectoryExample.getAccess(mydocumentsPath, WindowsIdentity.GetCurrent().Name);
     //getAccessToCurrentUsersMyDocuments(mydocumentsPath);
     try
     {
         var    currentDirectory = Directory.GetParent(Environment.CurrentDirectory).Parent.FullName;
         var    directoryInfo    = Directory.CreateDirectory(currentDirectory + "\\Download");
         string path             = currentDirectory + "\\Download";
         GrantAccess(path);
         path = path + "\\" + keyName;
         //checks if file is already available in local and retuns the path if not proceeds
         //if (File.Exists(path)) return path;
         var downloadRequest = new TransferUtilityDownloadRequest
         {
             FilePath   = path,
             BucketName = bucketName,
             Key        = keyName
         };
         var fileTransferUtility = new TransferUtility(s3Client);
         fileTransferUtility.Download(downloadRequest);
         using (FileStream fileDownloaded = new FileStream(path, FileMode.Open, FileAccess.Read))
         {
             return(path);
         }
     }
     catch (AmazonS3Exception e)
     {
         Trace.WriteLine(String.Format("Error encountered on server. Message: '{0}' when writing an object", e.Message));
         return("");
     }
     catch (Exception e)
     {
         Trace.WriteLine(String.Format("Unknown encountered on server. Message: '{0}' when writing an object", e.Message));
         return("");
     }
 }
Пример #20
0
 public void DownloadFileS3Tranfer(string AccessKey, string SerectKey, string bucketName, string keyName, string endPoint, string DestPath, EventHandler <WriteObjectProgressArgs> WriteObjectProgressEvent)
 {
     try
     {
         using (var client = new AmazonS3Client(AccessKey, SerectKey, RegionEndpointS3.ParseRegion(endPoint)))
         {
             var downloadRequest = new TransferUtilityDownloadRequest
             {
                 Key        = keyName,
                 BucketName = bucketName,
                 FilePath   = DestPath,
             };
             downloadRequest.WriteObjectProgressEvent += WriteObjectProgressEvent;
             using (var Utility = new TransferUtility(client))
             {
                 Utility.Download(downloadRequest);
             }
         }
     }
     catch (AmazonS3Exception ex)
     {
         throw new AmazonS3Exception(string.Format("Message {0} Error Code {1} Status Code {2}", ex.Message, ex.ErrorCode, ex.StatusCode));
     }
 }
Пример #21
0
        public async Task <Stream> DownloadFileAsync(string savedName, int workspaceId)
        {
            var downloadFromSubFolder = iChatConstants.AwsBucketWorkspaceFileFolderPrefix + workspaceId;
            var downloadPath          = Path.GetTempFileName();

            using (var client = new AmazonS3Client(_appSettings.AwsAccessKeyId, _appSettings.AwsSecretAccessKey, RegionEndpoint.APSoutheast2))
            {
                var downloadRequest = new TransferUtilityDownloadRequest
                {
                    Key        = savedName,
                    BucketName = $"{_appSettings.AwsFileBucketName}/{downloadFromSubFolder}",
                    FilePath   = downloadPath
                };

                var fileTransferUtility = new TransferUtility(client);
                await fileTransferUtility.DownloadAsync(downloadRequest);
            }

            var bufferSize = 4096;
            var stream     = new FileStream(downloadPath, FileMode.Open, FileAccess.Read, FileShare.None, bufferSize,
                                            FileOptions.DeleteOnClose);

            return(stream);
        }
Пример #22
0
        /// <summary>
        /// Transfers a file from S3. This method uses the "TransferUtility" class in order to download the file from S3
        /// </summary>
        public bool FileDownload(string bucketname, string dataname, string filepath)
        {
            // Reset error info
            ClearErrorInfo();

            // Load data
            try
            {
                TransferUtilityDownloadRequest fileTransferUtilityRequest = new TransferUtilityDownloadRequest
                {
                    BucketName = bucketname,
                    FilePath   = filepath,
                    Key        = dataname,
                };
                fileTransferUtility.Download(fileTransferUtilityRequest);
            }
            catch (Exception ex)
            {
                ErrorCode    = e_Exception;
                ErrorMessage = ex.Message + "::" + ex.InnerException;
            }

            return(ErrorCode == 0);
        }
        public override void Execute()
        {
            if (!this._request.IsSetBucketName())
            {
                throw new ArgumentNullException("BucketName", "The bucketName Specified is null or empty!");
            }
            if (!this._request.IsSetS3Directory())
            {
                throw new ArgumentNullException("S3Directory", "The S3Directory Specified is null or empty!");
            }
            if (!this._request.IsSetLocalDirectory())
            {
                throw new ArgumentNullException("LocalDirectory", "The LocalDirectory Specified is null or empty!");
            }

            if (File.Exists(this._request.S3Directory))
            {
                throw new ArgumentNullException("LocalDirectory", "A file already exists with the same name indicated by LocalDirectory!");
            }

            EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));

            ListObjectsRequest listRequest = new ListObjectsRequest();

            listRequest.BucketName = this._request.BucketName;
            listRequest.Prefix     = this._request.S3Directory;

            listRequest.Prefix = listRequest.Prefix.Replace('\\', '/');
            if (!listRequest.Prefix.EndsWith("/"))
            {
                listRequest.Prefix += "/";
            }

            if (listRequest.Prefix.StartsWith("/"))
            {
                if (listRequest.Prefix.Length == 1)
                {
                    listRequest.Prefix = "";
                }
                else
                {
                    listRequest.Prefix = listRequest.Prefix.Substring(1);
                }
            }

            List <S3Object> objs = new List <S3Object>();

            do
            {
                ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
                foreach (S3Object s3o in listResponse.S3Objects)
                {
                    DateTime lastModified = DateTime.Parse(s3o.LastModified);
                    if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < lastModified)
                    {
                        continue;
                    }
                    if (this._request.IsSetUnmodifiedSinceDate() && lastModified < this._request.UnmodifiedSinceDate)
                    {
                        continue;
                    }

                    objs.Add(s3o);
                }
                listRequest.Marker = listResponse.NextMarker;
            } while (!string.IsNullOrEmpty(listRequest.Marker));

            this._totalNumberOfFilesToDownload = objs.Count;

            foreach (S3Object s3o in objs)
            {
                if (s3o.Key.EndsWith("/"))
                {
                    continue;
                }

                this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);

                TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();
                downloadRequest.BucketName = this._request.BucketName;
                downloadRequest.Key        = s3o.Key;
                downloadRequest.FilePath   = Path.Combine(this._request.LocalDirectory, this._currentFile);
                downloadRequest.Timeout    = this._request.Timeout;
                downloadRequest.WriteObjectProgressEvent += new EventHandler <WriteObjectProgressArgs>(downloadedProgressEventCallback);


                DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
                command.Execute();

                this._numberOfFilesDownloaded++;
            }
        }
Пример #24
0
        static void Main(string[] args)
        {
            Debug modoDebug = Debug.Desligado;

            ModalidadeTransferencia modalidadeTransferencia = ModalidadeTransferencia.Download;

            file = new System.IO.StreamWriter(@"C:\TEMP\aws-getfroms3.log", true);

            string Sintaxe = "";

            Sintaxe = " " + Environment.NewLine +
                      "SINTAX:" + Environment.NewLine +
                      " " + Environment.NewLine +
                      " > aws-getfroms3.exe <mode>,<bucket>,<key>,<secret>,<full-path> " + Environment.NewLine +
                      " " + Environment.NewLine +
                      "PS.: IF <mode> = 'A' or null, config file will be considered." + Environment.NewLine +
                      " " + Environment.NewLine;

            PutObjectResponse resposta = null;

            file.WriteLine("Loading configs...");
            Console.WriteLine("Loading configs...");

            // Carrega os parâmetros ou exibe mensagem sobre sinxaxe
            if (!carregouParametros(args))
            {
                if (args.Length.Equals(0))
                {
                    Console.WriteLine(Sintaxe);
                    return;
                }
            }
            else
            {
                if (!MODO.Equals("A"))
                {
                    Console.WriteLine(Sintaxe);
                    return;
                }
            }

            if (!File.Exists(caminho_arquivo))
            {
                Console.WriteLine("File not found.");
                file.WriteLine("File not found.");
                return;
            }

            try
            {
                try
                {
                    transfer = new TransferUtility(Chave, Segredo);

                    switch (modalidadeTransferencia)
                    {
                    case ModalidadeTransferencia.Download:
                        file.WriteLine("Creating request to download from bucket " + meuBucket + "...");
                        Console.WriteLine("Creating request to download from bucket " + meuBucket + "...");

                        TransferUtilityDownloadRequest requestDownload = new TransferUtilityDownloadRequest()
                        {
                            BucketName = meuBucket,
                            FilePath   = caminho_arquivo
                        };

                        Console.WriteLine("Starting download...");
                        file.WriteLine("Starting download...");
                        transfer.Download(requestDownload);
                        Console.WriteLine("... download done.");
                        file.WriteLine("... download done.");

                        Console.WriteLine("/nFile received.");
                        file.WriteLine("/nFile received.");

                        break;

                    case ModalidadeTransferencia.Upload:

                        file.WriteLine("Creating request to upload to bucket " + meuBucket + "...");
                        Console.WriteLine("Creating request to upload to bucket " + meuBucket + "...");

                        TransferUtilityUploadRequest requestUpload = new TransferUtilityUploadRequest()
                        {
                            BucketName = meuBucket,
                            FilePath   = caminho_arquivo
                        };

                        Console.WriteLine("Starting sending...");
                        file.WriteLine("Starting sending...");
                        transfer.Upload(requestUpload);
                        Console.WriteLine("... file sent.");
                        file.WriteLine("... file sent.");
                        break;
                    }
                }
                catch (Exception ex)
                {
                    throw ex;
                }
                finally
                {
                    transfer.Dispose();
                }
            }
            catch (Exception ex)
            {
                string msgErro = "An error occourred." + Environment.NewLine +
                                 "[INTERNAL_MESSAGE=" + ex.Message.ToString() + Environment.NewLine;
                if (ex.InnerException != null)
                {
                    msgErro += "] & [INNER_MESSAGE=" + ex.InnerException.Message.ToString() + Environment.NewLine;
                }

                if (ex.StackTrace != null)
                {
                    msgErro += "] & [STACK_TRACE=" + ex.StackTrace.ToString() + "]";
                }

                if (ex.InnerException == null && ex.StackTrace == null)
                {
                    msgErro += "]";
                }

                Console.WriteLine(msgErro);
                file.WriteLine(msgErro);
            }
            finally
            {
                if (modoDebug == Debug.Ligado)
                {
                    Console.WriteLine("Awswer from S3...");
                    file.WriteLine("Awswer from S3...");
                    if (resposta == null)
                    {
                        Console.WriteLine("Awswer from S3 was null.");
                        file.WriteLine("Awswer from S3 was null.");
                    }
                    else
                    {
                        Console.WriteLine(resposta.ResponseMetadata.ToString());
                        file.WriteLine(resposta.ResponseMetadata.ToString());
                    }
                }
                file.Flush();
                file.Dispose();
            }
        }
Пример #25
0
 internal DownloadCommand(AmazonS3 s3Client, TransferUtilityDownloadRequest request)
 {
     this._s3Client = s3Client;
     this._request  = request;
 }
Пример #26
0
        protected override void _Rollback()
        {
            if (ExecutionMode == ExecuteOn.ForwardExecution)
            {
                foreach (string d in _FilesActioned.Keys)
                {
                    try
                    {
                        string s = _FilesActioned[d];

                        if (Action == ActionType.Move)
                        {
                            if (Direction == S3Direction.ToS3Bucket)
                            {
                                string bucket = Authentication.BucketFromPath(s);
                                string prefix = Authentication.PrefixFromPath(s);

                                FDCFileInfo fi = Authentication.GetFileInfo(s);

                                if (fi != null)
                                {
                                    string tmp = "";

                                    try
                                    {
                                        tmp = Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(d), "TEMP");

                                        if (!Directory.Exists(tmp))
                                        {
                                            Directory.CreateDirectory(tmp);
                                        }

                                        tmp = Path.Combine(tmp, STEM.Sys.IO.Path.GetFileName(d));

                                        TransferUtility ftu = new TransferUtility(Authentication.Client);

                                        TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest
                                        {
                                            BucketName = bucket,
                                            Key        = prefix,
                                            FilePath   = tmp
                                        };

                                        ftu.DownloadAsync(request).Wait();

                                        try
                                        {
                                            File.SetLastWriteTimeUtc(tmp, fi.LastWriteTimeUtc);
                                        }
                                        catch { }

                                        File.Move(tmp, STEM.Sys.IO.Path.AdjustPath(d));
                                    }
                                    finally
                                    {
                                        try
                                        {
                                            if (File.Exists(tmp))
                                            {
                                                File.Delete(tmp);
                                            }
                                        }
                                        catch { }
                                    }
                                }
                                else
                                {
                                    throw new System.IO.FileNotFoundException(s);
                                }
                            }
                            else
                            {
                                string bucket = Authentication.BucketFromPath(d);
                                string prefix = Authentication.PrefixFromPath(d);

                                TransferUtility ftu = new TransferUtility(Authentication.Client);

                                TransferUtilityUploadRequest request = new TransferUtilityUploadRequest
                                {
                                    BucketName = bucket,
                                    Key        = prefix,
                                    FilePath   = STEM.Sys.IO.Path.AdjustPath(s),
                                    PartSize   = 10485760
                                };

                                ftu.UploadAsync(request).Wait();
                            }
                        }

                        if (Direction == S3Direction.ToS3Bucket)
                        {
                            Authentication.DeleteFile(s);
                        }
                        else
                        {
                            STEM.Sys.IO.File.STEM_Delete(s, false, Retry, RetryDelaySeconds);
                        }
                    }
                    catch { }
                }
            }
            else
            {
                Execute();
            }
        }
        public override void Execute()
        {
            if (!this._request.IsSetBucketName())
            {
                throw new ArgumentNullException("BucketName", "The bucketName Specified is null or empty!");
            }
            if (!this._request.IsSetS3Directory())
            {
                throw new ArgumentNullException("S3Directory", "The S3Directory Specified is null or empty!");
            }
            if (!this._request.IsSetLocalDirectory())
            {
                throw new ArgumentNullException("LocalDirectory", "The LocalDirectory Specified is null or empty!");
            }

            if (File.Exists(this._request.S3Directory))
            {
                throw new ArgumentNullException("LocalDirectory", "A file already exists with the same name indicated by LocalDirectory!");
            }

            EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));

            ListObjectsRequest listRequest = new ListObjectsRequest();
            listRequest.BucketName = this._request.BucketName;
            listRequest.Prefix = this._request.S3Directory;

            listRequest.Prefix = listRequest.Prefix.Replace('\\', '/');
            if (!listRequest.Prefix.EndsWith("/"))
                listRequest.Prefix += "/";

            if (listRequest.Prefix.StartsWith("/"))
            {
                if (listRequest.Prefix.Length == 1)
                    listRequest.Prefix = "";
                else
                    listRequest.Prefix = listRequest.Prefix.Substring(1);
            }

            ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
            List<S3Object> objs = new List<S3Object>();
            foreach (S3Object s3o in listResponse.S3Objects)
            {
                DateTime lastModified = DateTime.Parse(s3o.LastModified);
                if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < lastModified)
                    continue;
                if (this._request.IsSetUnmodifiedSinceDate() && lastModified < this._request.UnmodifiedSinceDate)
                    continue;

                objs.Add(s3o);
            }

            this._totalNumberOfFilesToDownload = objs.Count;

            foreach (S3Object s3o in objs)
            {
                this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);

                TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();
                downloadRequest.BucketName = this._request.BucketName;
                downloadRequest.Key = s3o.Key;
                downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile);
                downloadRequest.Timeout = this._request.Timeout;
                downloadRequest.WithSubscriber(new EventHandler<WriteObjectProgressArgs>(downloadedProgressEventCallback));

                DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
                command.Execute();

                this._numberOfFilesDownloaded++;
            }
        }
Пример #28
0
        bool Execute()
        {
            try
            {
                if (Direction == S3Direction.ToS3Bucket)
                {
                    ExpandDestination = false;
                }
                else
                {
                    ExpandSource = false;
                }

                List <string> sources = new List <string>();
                if (ExpandSource)
                {
                    sources = STEM.Sys.IO.Path.ExpandRangedPath(SourcePath);
                }
                else
                {
                    sources.Add(SourcePath);
                }

                List <string> destinations = new List <string>();
                if (ExpandDestination)
                {
                    destinations = STEM.Sys.IO.Path.ExpandRangedPath(DestinationPath);
                }
                else
                {
                    destinations.Add(DestinationPath);
                }

                List <string> sourceFiles = new List <string>();

                int filesActioned = 0;

                foreach (string src in sources)
                {
                    List <S3Object> items = new List <S3Object>();

                    if (Direction == S3Direction.ToS3Bucket)
                    {
                        sourceFiles = STEM.Sys.IO.Directory.STEM_GetFiles(src, FileFilter, DirectoryFilter, (RecurseSource ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly), false);
                    }
                    else
                    {
                        string bucket = Authentication.BucketFromPath(src);
                        string prefix = Authentication.PrefixFromPath(src);

                        items = Authentication.ListObjects(bucket, prefix, S3ListType.File, RecurseSource, DirectoryFilter, FileFilter);

                        sourceFiles = items.Select(i => Authentication.ToString(i)).ToList();
                    }

                    foreach (string s in sourceFiles)
                    {
                        try
                        {
                            bool success = false;

                            Exception lastEX = null;

                            foreach (string d in destinations)
                            {
                                try
                                {
                                    string dFile = "";

                                    try
                                    {
                                        if (PopulatePostMortemMeta)
                                        {
                                            PostMortemMetaData["SourceIP"]      = STEM.Sys.IO.Path.IPFromPath(s);
                                            PostMortemMetaData["DestinationIP"] = STEM.Sys.IO.Path.IPFromPath(d);

                                            if (Direction == S3Direction.ToS3Bucket)
                                            {
                                                PostMortemMetaData["FileSize"] = new FileInfo(s).Length.ToString();
                                            }
                                            else
                                            {
                                                PostMortemMetaData["FileSize"] = Authentication.GetFileInfo(s).Size.ToString();
                                            }
                                        }
                                    }
                                    catch { }

                                    if (Direction == S3Direction.ToS3Bucket)
                                    {
                                        string dPath = STEM.Sys.IO.Path.AdjustPath(d);
                                        if (RecurseSource && RecreateTree)
                                        {
                                            dPath = System.IO.Path.Combine(dPath, STEM.Sys.IO.Path.GetDirectoryName(s).Replace(STEM.Sys.IO.Path.AdjustPath(src), "").Trim(System.IO.Path.DirectorySeparatorChar));
                                        }

                                        dPath = System.IO.Path.Combine(dPath, DestinationFilename);

                                        if (dPath.Contains("*.*"))
                                        {
                                            dPath = dPath.Replace("*.*", STEM.Sys.IO.Path.GetFileName(s));
                                        }

                                        if (dPath.Contains("*"))
                                        {
                                            dPath = dPath.Replace("*", STEM.Sys.IO.Path.GetFileNameWithoutExtension(s));
                                        }

                                        if (!Authentication.DirectoryExists(STEM.Sys.IO.Path.GetDirectoryName(dPath)))
                                        {
                                            Authentication.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(dPath));
                                        }

                                        if (Authentication.FileExists(dPath))
                                        {
                                            switch (ExistsAction)
                                            {
                                            case Sys.IO.FileExistsAction.Overwrite:
                                                Authentication.DeleteFile(dPath);
                                                dFile = dPath;
                                                break;

                                            case Sys.IO.FileExistsAction.OverwriteIfNewer:

                                                if (Authentication.GetFileInfo(dPath).LastWriteTimeUtc >= File.GetLastWriteTimeUtc(s))
                                                {
                                                    continue;
                                                }

                                                Authentication.DeleteFile(dPath);
                                                dFile = dPath;
                                                break;

                                            case Sys.IO.FileExistsAction.Skip:
                                                continue;

                                            case Sys.IO.FileExistsAction.Throw:
                                                throw new IOException("Destination file exists. (" + dPath + ")");

                                            case Sys.IO.FileExistsAction.MakeUnique:
                                                dFile = Authentication.UniqueFilename(dPath);
                                                break;
                                            }
                                        }
                                        else
                                        {
                                            dFile = dPath;
                                        }

                                        string bucket = Authentication.BucketFromPath(dFile);
                                        string prefix = Authentication.PrefixFromPath(dFile);

                                        TransferUtility ftu = new TransferUtility(Authentication.Client);

                                        TransferUtilityUploadRequest request = new TransferUtilityUploadRequest
                                        {
                                            BucketName = bucket,
                                            Key        = prefix,
                                            FilePath   = STEM.Sys.IO.Path.AdjustPath(s),
                                            PartSize   = 10485760
                                        };

                                        ftu.UploadAsync(request).Wait();
                                    }
                                    else
                                    {
                                        string dPath = STEM.Sys.IO.Path.AdjustPath(d);
                                        if (RecurseSource && RecreateTree)
                                        {
                                            dPath = System.IO.Path.Combine(dPath, STEM.Sys.IO.Path.GetDirectoryName(s).Replace(STEM.Sys.IO.Path.AdjustPath(src), "").Trim(System.IO.Path.DirectorySeparatorChar));
                                        }

                                        dPath = System.IO.Path.Combine(dPath, DestinationFilename);

                                        if (dPath.Contains("*.*"))
                                        {
                                            dPath = dPath.Replace("*.*", STEM.Sys.IO.Path.GetFileName(s));
                                        }

                                        if (dPath.Contains("*"))
                                        {
                                            dPath = dPath.Replace("*", STEM.Sys.IO.Path.GetFileNameWithoutExtension(s));
                                        }

                                        if (File.Exists(dPath))
                                        {
                                            switch (ExistsAction)
                                            {
                                            case Sys.IO.FileExistsAction.Overwrite:
                                                File.Delete(dPath);
                                                dFile = dPath;
                                                break;

                                            case Sys.IO.FileExistsAction.OverwriteIfNewer:

                                                if (File.GetLastWriteTimeUtc(dPath) >= Authentication.GetFileInfo(s).LastWriteTimeUtc)
                                                {
                                                    continue;
                                                }

                                                File.Delete(dPath);
                                                dFile = dPath;
                                                break;

                                            case Sys.IO.FileExistsAction.Skip:
                                                continue;

                                            case Sys.IO.FileExistsAction.Throw:
                                                throw new IOException("Destination file exists. (" + dPath + ")");

                                            case Sys.IO.FileExistsAction.MakeUnique:
                                                dFile = STEM.Sys.IO.File.UniqueFilename(dPath);
                                                break;
                                            }
                                        }
                                        else
                                        {
                                            dFile = dPath;
                                        }

                                        if (!Directory.Exists(STEM.Sys.IO.Path.GetDirectoryName(dFile)))
                                        {
                                            Directory.CreateDirectory(STEM.Sys.IO.Path.GetDirectoryName(dFile));
                                        }

                                        string bucket = Authentication.BucketFromPath(s);
                                        string prefix = Authentication.PrefixFromPath(s);

                                        FDCFileInfo fi = Authentication.GetFileInfo(s);

                                        if (fi != null)
                                        {
                                            string tmp = "";
                                            try
                                            {
                                                tmp = Path.Combine(STEM.Sys.IO.Path.GetDirectoryName(dFile), "TEMP");

                                                if (!Directory.Exists(tmp))
                                                {
                                                    Directory.CreateDirectory(tmp);
                                                }

                                                tmp = Path.Combine(tmp, STEM.Sys.IO.Path.GetFileName(dFile));

                                                TransferUtility ftu = new TransferUtility(Authentication.Client);

                                                TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest
                                                {
                                                    BucketName = bucket,
                                                    Key        = prefix,
                                                    FilePath   = tmp
                                                };

                                                ftu.DownloadAsync(request).Wait();

                                                try
                                                {
                                                    File.SetLastWriteTimeUtc(tmp, fi.LastWriteTimeUtc);
                                                }
                                                catch { }

                                                File.Move(tmp, STEM.Sys.IO.Path.AdjustPath(dFile));
                                            }
                                            finally
                                            {
                                                try
                                                {
                                                    if (File.Exists(tmp))
                                                    {
                                                        File.Delete(tmp);
                                                    }
                                                }
                                                catch { }
                                            }
                                        }
                                        else
                                        {
                                            throw new System.IO.FileNotFoundException(s);
                                        }
                                    }

                                    if (!String.IsNullOrEmpty(dFile))
                                    {
                                        filesActioned++;

                                        _FilesActioned[s] = dFile;

                                        if (Action == ActionType.Move)
                                        {
                                            AppendToMessage(s + " moved to " + dFile);
                                        }
                                        else
                                        {
                                            AppendToMessage(s + " copied to " + dFile);
                                        }
                                    }

                                    success = true;

                                    if (DestinationActionRule == DestinationRule.FirstSuccess)
                                    {
                                        break;
                                    }
                                }
                                catch (Exception ex)
                                {
                                    lastEX = ex;

                                    if (DestinationActionRule == DestinationRule.AllOrNone)
                                    {
                                        throw ex;
                                    }
                                }
                            }

                            if (!success)
                            {
                                throw new Exception("No successful actions taken for " + s, lastEX); // + "\r\n" + ((lastEX == null) ? "No additional information." : lastEX.ToString()));
                            }
                            if (Action == ActionType.Move)
                            {
                                if (Direction == S3Direction.ToS3Bucket)
                                {
                                    File.Delete(s);
                                }
                                else
                                {
                                    Authentication.DeleteFile(s);
                                }
                            }
                        }
                        catch (AggregateException ex)
                        {
                            foreach (Exception e in ex.InnerExceptions)
                            {
                                AppendToMessage(e.Message);
                                Exceptions.Add(e);
                            }
                        }
                        catch (Exception ex)
                        {
                            AppendToMessage(ex.Message);
                            Exceptions.Add(ex);
                        }
                    }
                }

                if (PopulatePostMortemMeta)
                {
                    PostMortemMetaData["FilesActioned"] = filesActioned.ToString();
                }
            }
            catch (AggregateException ex)
            {
                foreach (Exception e in ex.InnerExceptions)
                {
                    AppendToMessage(e.Message);
                    Exceptions.Add(e);
                }
            }
            catch (Exception ex)
            {
                AppendToMessage(ex.Message);
                Exceptions.Add(ex);
            }

            if (_FilesActioned.Count == 0)
            {
                switch (ZeroFilesAction)
                {
                case FailureAction.SkipRemaining:
                    SkipRemaining();
                    return(true);

                case FailureAction.SkipNext:
                    SkipNext();
                    return(true);

                case FailureAction.SkipToLabel:
                    SkipForwardToFlowControlLabel(FailureActionLabel);
                    return(true);

                case FailureAction.Rollback:
                    RollbackAllPreceedingAndSkipRemaining();
                    break;

                case FailureAction.Continue:
                    return(true);
                }

                Message = "0 Files Actioned\r\n" + Message;
            }

            return(Exceptions.Count == 0);
        }
Пример #29
0
        /// <summary>
        /// Warning, if the book already exists in the location, this is going to delete it an over-write it. So it's up to the caller to check the sanity of that.
        /// </summary>
        /// <param name="storageKeyOfBookFolder"></param>
        public string DownloadBook(string bucketName, string storageKeyOfBookFolder, string pathToDestinationParentDirectory,
                                   IProgressDialog downloadProgress = null)
        {
            //review: should we instead save to a newly created folder so that we don't have to worry about the
            //other folder existing already? Todo: add a test for that first.

            // We need to download individual files to avoid downloading unwanted files (PDFs and thumbs.db to
            // be specific).  See https://silbloom.myjetbrains.com/youtrack/issue/BL-2312.  So we need the list
            // of items, not just the count.
            var matching   = GetMatchingItems(bucketName, storageKeyOfBookFolder);
            var totalItems = CountDesiredFiles(matching);

            if (totalItems == 0)
            {
                throw new DirectoryNotFoundException("The book we tried to download is no longer in the BloomLibrary");
            }

            Debug.Assert(matching.S3Objects[0].Key.StartsWith(storageKeyOfBookFolder + "/"));

            // Get the top-level directory name of the book from the first object key.
            var bookFolderName = matching.S3Objects[0].Key.Substring(storageKeyOfBookFolder.Length + 1);

            while (bookFolderName.Contains("/"))
            {
                bookFolderName = Path.GetDirectoryName(bookFolderName);
            }

            // Amazon.S3 appears to truncate titles at 50 characters when building directory and filenames.  This means
            // that relative paths can be as long as 117 characters (2 * 50 + 2 for slashes + 15 for .BloomBookOrder).
            // So our temporary folder must be no more than 140 characters (allow some margin) since paths can be a
            // maximum of 260 characters in Windows.  (More margin than that may be needed because there's no guarantee
            // that image filenames are no longer than 65 characters.)  See https://jira.sil.org/browse/BL-1160.
            using (var tempDestination = new TemporaryFolder("BDS_" + Guid.NewGuid()))
            {
                var   tempDirectory = Path.Combine(tempDestination.FolderPath, bookFolderName);
                float progressStep  = 1.0F;
                float progress      = 0.0F;
                if (downloadProgress != null)
                {
                    downloadProgress.Invoke((Action)(() => {
                        // We cannot change ProgressRangeMaximum here because the worker thread is already active.
                        // So we calculate a step value instead.  See https://issues.bloomlibrary.org/youtrack/issue/BL-5443.
                        progressStep = (float)(downloadProgress.ProgressRangeMaximum - downloadProgress.Progress) / (float)totalItems;
                        progress = (float)downloadProgress.Progress;
                    }));
                }
                using (var transferUtility = new TransferUtility(_amazonS3))
                {
                    for (int i = 0; i < matching.S3Objects.Count; ++i)
                    {
                        var objKey = matching.S3Objects[i].Key;
                        if (AvoidThisFile(objKey))
                        {
                            continue;
                        }
                        // Removing the book's prefix from the object key, then using the remainder of the key
                        // in the filepath allows for nested subdirectories.
                        var filepath = objKey.Substring(storageKeyOfBookFolder.Length + 1);
                        // Download this file then bump progress.
                        var req = new TransferUtilityDownloadRequest()
                        {
                            BucketName = bucketName,
                            Key        = objKey,
                            FilePath   = Path.Combine(tempDestination.FolderPath, filepath)
                        };
                        transferUtility.Download(req);
                        if (downloadProgress != null)
                        {
                            downloadProgress.Invoke((Action)(() => {
                                progress += progressStep;
                                downloadProgress.Progress = (int)progress;
                            }));
                        }
                    }
                    var destinationPath = Path.Combine(pathToDestinationParentDirectory, bookFolderName);

                    //clear out anything existing on our target
                    var didDelete = false;
                    if (Directory.Exists(destinationPath))
                    {
                        try
                        {
                            SIL.IO.RobustIO.DeleteDirectory(destinationPath, true);
                            didDelete = true;
                        }
                        catch (IOException)
                        {
                            // can't delete it...see if we can copy into it.
                        }
                    }

                    //if we're on the same volume, we can just move it. Else copy it.
                    // It's important that books appear as nearly complete as possible, because a file watcher will very soon add the new
                    // book to the list of downloaded books the user can make new ones from, once it appears in the target directory.
                    bool done = false;
                    if (didDelete && PathUtilities.PathsAreOnSameVolume(pathToDestinationParentDirectory, tempDirectory))
                    {
                        try
                        {
                            SIL.IO.RobustIO.MoveDirectory(tempDirectory, destinationPath);
                            done = true;
                        }
                        catch (IOException)
                        {
                            // If moving didn't work we'll just try copying
                        }
                        catch (UnauthorizedAccessException)
                        {
                        }
                    }
                    if (!done)
                    {
                        done = CopyDirectory(tempDirectory, destinationPath);
                    }
                    if (!done)
                    {
                        var msg = LocalizationManager.GetString("Download.CopyFailed",
                                                                "Bloom downloaded the book but had problems making it available in Bloom. Please restart your computer and try again. If you get this message again, please click the 'Details' button and report the problem to the Bloom developers");
                        // The exception doesn't add much useful information but it triggers a version of the dialog with a Details button
                        // that leads to the yellow box and an easy way to send the report.
                        ErrorReport.NotifyUserOfProblem(new ApplicationException("File Copy problem"), msg);
                    }
                    return(destinationPath);
                }
            }
        }
Пример #30
0
        CmdletOutput DownloadFileFromS3(ExecutorContext context)
        {
            var cmdletContext = context as CmdletContext;
            var request       = new TransferUtilityDownloadRequest
            {
                BucketName = cmdletContext.BucketName,
                FilePath   = cmdletContext.File,
                Key        = cmdletContext.Key
            };

            if (!string.IsNullOrEmpty(cmdletContext.Version))
            {
                request.VersionId = cmdletContext.Version;
            }
            if (cmdletContext.UtcModifiedSinceDate.HasValue)
            {
                request.ModifiedSinceDateUtc = cmdletContext.UtcModifiedSinceDate.Value;
            }
            if (cmdletContext.UtcUnmodifiedSinceDate.HasValue)
            {
                request.UnmodifiedSinceDateUtc = cmdletContext.UtcUnmodifiedSinceDate.Value;
            }
#pragma warning disable CS0618, CS0612 //A class member was marked with the Obsolete attribute
            if (cmdletContext.ModifiedSinceDate.HasValue)
            {
                if (cmdletContext.UtcModifiedSinceDate != null)
                {
                    throw new ArgumentException("Parameters ModifiedSinceDate and UtcModifiedSinceDate are mutually exclusive.");
                }
                request.ModifiedSinceDate = cmdletContext.ModifiedSinceDate.Value;
            }
            if (cmdletContext.UnmodifiedSinceDate.HasValue)
            {
                if (cmdletContext.UtcUnmodifiedSinceDate != null)
                {
                    throw new ArgumentException("Parameters UnmodifiedSinceDate and UtcUnmodifiedSinceDate are mutually exclusive.");
                }
                request.UnmodifiedSinceDate = cmdletContext.UnmodifiedSinceDate.Value;
            }
#pragma warning restore CS0618, CS0612 //A class member was marked with the Obsolete attribute

            request.ServerSideEncryptionCustomerMethod         = cmdletContext.ServerSideEncryptionCustomerMethod;
            request.ServerSideEncryptionCustomerProvidedKey    = cmdletContext.ServerSideEncryptionCustomerProvidedKey;
            request.ServerSideEncryptionCustomerProvidedKeyMD5 = cmdletContext.ServerSideEncryptionCustomerProvidedKeyMD5;

            CmdletOutput output;
            using (var tu = new TransferUtility(Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint)))
            {
                Utils.Common.WriteVerboseEndpointMessage(this, Client.Config, "Amazon S3 object download APIs");

                var runner  = new ProgressRunner(this);
                var tracker = new DownloadFileProgressTracker(runner, handler => request.WriteObjectProgressEvent += handler, cmdletContext.Key);

                output = runner.SafeRun(() => tu.Download(request), tracker);
                if (output.ErrorResponse == null)
                {
                    output.PipelineOutput = new FileInfo(cmdletContext.File);
                }
            }
            return(output);
        }
Пример #31
0
        /*private static string s3AccessKey = Environment.GetEnvironmentVariable("S3AccessKey");
         * private static string s3SecretKey = Environment.GetEnvironmentVariable("S3SecretKey");
         * private static string s3Region = Environment.GetEnvironmentVariable("S3Region");
         * private AmazonS3Client s3Client = new AmazonS3Client(s3AccessKey,s3SecretKey, RegionEndpoint.GetBySystemName(s3Region));
         */

        // public async Task<string> CropImage(string sourceBucket, string key, string destBucket, string permission)
        //public async void CropImage(string sourceBucket, string key, string destBucket, string permission)
        public async Task <string> CropImage(string input, ILambdaContext context)
        {
            s3Client = new AmazonS3Client();

            string[] arr          = input.Split(',');
            string   sourceBucket = arr[0].Trim();
            string   key          = arr[1].Trim();
            string   destBucket   = arr[2].Trim();
            //string size = arr[3].Trim();
            //string crop = arr[3].Trim();
            string permission = arr[3].Trim();

            string path = Path.Combine("/tmp", key);

            try
            {
                Console.WriteLine("Checkpoint - 0");

                TransferUtility fileTransferUtility            = new TransferUtility(s3Client);
                TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();

                Console.WriteLine("path - " + path);

                if (File.Exists(path))
                {
                    File.Delete(path);
                }
                downloadRequest.FilePath   = path;
                downloadRequest.BucketName = sourceBucket;
                downloadRequest.Key        = key;
                fileTransferUtility.Download(downloadRequest);

                Console.WriteLine("Checkpoint - file transfer");

                orgImg = Image.FromFile(path);
                orgImg.RotateFlip(RotateFlipType.Rotate90FlipNone);
                //FluentImage.FromFile(path)
                //   .Resize.Width(200)
                //   .Save("/tmp/test_file.png", OutputFormat.Png);



                Console.WriteLine("Checkpoint - Img creation");

                TransferUtilityUploadRequest uploadRequest = new TransferUtilityUploadRequest();
                //uploadRequest.FilePath = "/tmp/test_file.png";
                uploadRequest.FilePath   = path;
                uploadRequest.BucketName = destBucket;
                uploadRequest.Key        = key;
                if (permission.ToUpper() == "PUBLIC")
                {
                    uploadRequest.CannedACL = S3CannedACL.PublicRead;
                }
                else if (permission.ToUpper() == "PRIVATE")
                {
                    uploadRequest.CannedACL = S3CannedACL.Private;
                }
                else if (permission.ToUpper() == "NONE")
                {
                    uploadRequest.CannedACL = S3CannedACL.NoACL;
                }
                fileTransferUtility.Upload(uploadRequest);

                Console.WriteLine("Checkpoint - Done");

                return(context.AwsRequestId.ToString());
            }
            catch (Exception ex)
            {
                Console.WriteLine("ex message - " + ex.Message);
                Console.WriteLine("ex stack - " + ex.StackTrace);
                return(ex.Message);
                //context.Logger.LogLine(ex.Message);
            }
            finally
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
                if (orgImg != null)
                {
                    orgImg.Dispose();
                }
            }
        }
Пример #32
0
 internal DownloadCommand(IAmazonS3 s3Client, TransferUtilityDownloadRequest request)
 {
     this._s3Client = s3Client;
     this._request = request;
 }
        private TransferUtilityDownloadRequest ConstructTransferUtilityDownloadRequest(S3Object s3Object, int prefixLength)
        {
            var downloadRequest = new TransferUtilityDownloadRequest();
            downloadRequest.BucketName = this._request.BucketName;
            downloadRequest.Key = s3Object.Key;
            var file = s3Object.Key.Substring(prefixLength).Replace('/','\\');
            downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, file);
            downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;

            return downloadRequest;
        }
        public override void Execute()
        {
            if (!this._request.IsSetBucketName())
            {
                throw new InvalidOperationException("The bucketName Specified is null or empty!");
            }
            if (!this._request.IsSetS3Directory())
            {
                throw new InvalidOperationException("The S3Directory Specified is null or empty!");
            }
            if (!this._request.IsSetLocalDirectory())
            {
                throw new InvalidOperationException("The LocalDirectory Specified is null or empty!");
            }

            if (File.Exists(this._request.S3Directory))
            {
                throw new InvalidOperationException("A file already exists with the same name indicated by LocalDirectory!");
            }

            EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));

            ListObjectsRequest listRequest = new ListObjectsRequest();
            listRequest.BucketName = this._request.BucketName;
            listRequest.Prefix = this._request.S3Directory;

            listRequest.Prefix = listRequest.Prefix.Replace('\\', '/');
            if (!listRequest.Prefix.EndsWith("/", StringComparison.Ordinal))
                listRequest.Prefix += "/";

            if (listRequest.Prefix.StartsWith("/", StringComparison.Ordinal))
            {
                if (listRequest.Prefix.Length == 1)
                    listRequest.Prefix = "";
                else
                    listRequest.Prefix = listRequest.Prefix.Substring(1);
            }

            List<S3Object> objs = new List<S3Object>();
            do
            {
                ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
                foreach (S3Object s3o in listResponse.S3Objects)
                {
					if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < s3o.LastModified)
						continue;
					if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified < this._request.UnmodifiedSinceDate)
						continue;

					objs.Add(s3o);
                }
                listRequest.Marker = listResponse.NextMarker;
            } while (!string.IsNullOrEmpty(listRequest.Marker));
			
            this._totalNumberOfFilesToDownload = objs.Count;

            foreach (S3Object s3o in objs)
            {
                if (s3o.Key.EndsWith("/", StringComparison.Ordinal))
                    continue;

                this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);

                TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();
                downloadRequest.BucketName = this._request.BucketName;
                downloadRequest.Key = s3o.Key;
                downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile);
                downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;


                DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
                command.Execute();

                this._numberOfFilesDownloaded++;
            }
        }