/// <summary>
        ///     Directory download operation.
        /// </summary>
        /// <param name="input"></param>
        /// <param name="option"></param>
        /// <param name="cToken"></param>
        /// <param name="dirInfo"></param>
        /// <returns>Task</returns>
        private static List <string> DownloadFiles(
            DownloadInput input,
            DownloadOptions option,
            S3DirectoryInfo dirInfo,
            CancellationToken cToken
            )
        {
            var files = dirInfo.GetFiles(input.SearchPattern,
                                         option.DownloadFromCurrentDirectoryOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories);

            if (option.ThrowErrorIfNoMatches && files.Length < 1)
            {
                throw new ArgumentException("Could not find any files matching pattern.");
            }

            var filelist = new List <string>();

            foreach (var file in files)
            {
                if (!file.Exists)
                {
                    continue;
                }

                cToken.ThrowIfCancellationRequested();

                var path = Path.Combine(input.DestinationPath, file.Name);

                try
                {
                    // Apparently MoveToLocal does not have overwrite as signature :(
                    var localFile = option.DeleteSourceFile
                        ? file.MoveToLocal(path, option.Overwrite)
                        : file.CopyToLocal(path, option.Overwrite);

                    if (!localFile.Exists)
                    {
                        throw new IOException($"Could not find {localFile.FullName} from local filesystem.");
                    }

                    filelist.Add(localFile.FullName);
                }
                catch (IOException ex)
                {
                    // normal exception does not give filename info, which would be nice.
                    throw new IOException($"{path} already exists or insufficient privileges to write file.", ex);
                }
            }

            return(filelist);
        }
Beispiel #2
0
 /// <summary>
 /// Amazon AWS S3 DownloadFiles task.
 /// </summary>
 /// <param name="input"></param>
 /// <param name="parameters"></param>
 /// <param name="option"></param>
 /// <param name="cToken"></param>
 /// <returns>List&lt;string&gt;</returns>
 public static List <string> DownloadFiles(
     [PropertyTab] DownloadInput input,
     [PropertyTab] Parameters parameters,
     [PropertyTab] DownloadOptions option,
     CancellationToken cToken
     )
 {
     if (!parameters.UseDefaultCredentials && parameters.AwsCredentials == null)
     {
         parameters.IsAnyNullOrWhiteSpaceThrow();
     }
     if (string.IsNullOrWhiteSpace(input.DestinationPath))
     {
         throw new ArgumentNullException(nameof(input.DestinationPath));
     }
     return(DownloadUtility(input, parameters, option, cToken).Result);
 }
 /// <summary>
 ///     Method to create client and call DownloadFiles.
 /// </summary>
 /// <param name="input"></param>
 /// <param name="parameters"></param>
 /// <param name="cancellationToken"></param>
 /// <param name="option"></param>
 /// <returns></returns>
 private static List <string> DownloadUtility(
     DownloadInput input,
     Parameters parameters,
     DownloadOptions option,
     CancellationToken cancellationToken
     )
 {
     using (var s3Client = Utilities.GetS3Client(parameters, cancellationToken))
     {
         var dirInfo = new S3DirectoryInfo(s3Client, parameters.BucketName, input.S3Directory);
         if (dirInfo.Exists)
         {
             return(DownloadFiles(input, option, dirInfo, cancellationToken));
         }
         throw new ArgumentException($"Cannot find {input.S3Directory} directory. {nameof(input.S3Directory)}");
     }
 }
Beispiel #4
0
        /// <summary>
        /// Prepare for download by checking options and finding files from S3.
        /// </summary>
        /// <param name="input"></param>
        /// <param name="parameters"></param>
        /// <param name="option"></param>
        /// <param name="cancellationToken"></param>
        /// <returns>List&lt;string&gt;</returns>
        private static async Task <List <string> > DownloadUtility(
            DownloadInput input,
            Parameters parameters,
            DownloadOptions option,
            CancellationToken cancellationToken
            )
        {
            var paths      = new List <string>();
            var targetPath = input.S3Directory + input.SearchPattern;
            var mask       = new Regex(input.SearchPattern.Replace(".", "[.]").Replace("*", ".*").Replace("?", "."));

            using (var s3Client = (AmazonS3Client)Utilities.GetS3Client(parameters))
            {
                var allObjectsResponse = await s3Client.ListObjectsAsync(parameters.BucketName, cancellationToken);

                var allObjectsInDirectory = allObjectsResponse.S3Objects;
                foreach (var fileObject in allObjectsInDirectory)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    if (mask.IsMatch(fileObject.Key.Split('/').Last()) && (targetPath.Split('/').Length == fileObject.Key.Split('/').Length || !option.DownloadFromCurrentDirectoryOnly) && !fileObject.Key.EndsWith("/") && fileObject.Key.StartsWith(input.S3Directory))
                    {
                        if (!input.DestinationPath.EndsWith(Path.DirectorySeparatorChar.ToString()))
                        {
                            input.DestinationPath += Path.DirectorySeparatorChar.ToString();
                        }

                        var fullPath = Path.Combine(input.DestinationPath, fileObject.Key.Split('/').Last());
                        if (File.Exists(fullPath) & !option.Overwrite)
                        {
                            throw new IOException($"File {fileObject.Key.Split('/').Last()} already exists at {fullPath}. Set Overwrite to true from options to overwrite the file.");
                        }
                        paths.Add(await WriteToFile(parameters, fileObject, s3Client, input.DestinationPath, fullPath));
                        if (option.DeleteSourceFile)
                        {
                            Utilities.DeleteSourceFile(s3Client, cancellationToken, parameters.BucketName, fileObject.Key, true);
                        }
                    }
                }
            }
            if (paths.Count == 0 && option.ThrowErrorIfNoMatches)
            {
                throw new ArgumentException($"No matches found with search pattern {input.SearchPattern}");
            }
            return(paths);
        }