/// <summary> /// Directory download operation. /// </summary> /// <param name="input"></param> /// <param name="option"></param> /// <param name="cToken"></param> /// <param name="dirInfo"></param> /// <returns>Task</returns> private static List <string> DownloadFiles( DownloadInput input, DownloadOptions option, S3DirectoryInfo dirInfo, CancellationToken cToken ) { var files = dirInfo.GetFiles(input.SearchPattern, option.DownloadFromCurrentDirectoryOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories); if (option.ThrowErrorIfNoMatches && files.Length < 1) { throw new ArgumentException("Could not find any files matching pattern."); } var filelist = new List <string>(); foreach (var file in files) { if (!file.Exists) { continue; } cToken.ThrowIfCancellationRequested(); var path = Path.Combine(input.DestinationPath, file.Name); try { // Apparently MoveToLocal does not have overwrite as signature :( var localFile = option.DeleteSourceFile ? file.MoveToLocal(path, option.Overwrite) : file.CopyToLocal(path, option.Overwrite); if (!localFile.Exists) { throw new IOException($"Could not find {localFile.FullName} from local filesystem."); } filelist.Add(localFile.FullName); } catch (IOException ex) { // normal exception does not give filename info, which would be nice. throw new IOException($"{path} already exists or insufficient privileges to write file.", ex); } } return(filelist); }
/// <summary> /// Amazon AWS S3 DownloadFiles task. /// </summary> /// <param name="input"></param> /// <param name="parameters"></param> /// <param name="option"></param> /// <param name="cToken"></param> /// <returns>List<string></returns> public static List <string> DownloadFiles( [PropertyTab] DownloadInput input, [PropertyTab] Parameters parameters, [PropertyTab] DownloadOptions option, CancellationToken cToken ) { if (!parameters.UseDefaultCredentials && parameters.AwsCredentials == null) { parameters.IsAnyNullOrWhiteSpaceThrow(); } if (string.IsNullOrWhiteSpace(input.DestinationPath)) { throw new ArgumentNullException(nameof(input.DestinationPath)); } return(DownloadUtility(input, parameters, option, cToken).Result); }
/// <summary> /// Method to create client and call DownloadFiles. /// </summary> /// <param name="input"></param> /// <param name="parameters"></param> /// <param name="cancellationToken"></param> /// <param name="option"></param> /// <returns></returns> private static List <string> DownloadUtility( DownloadInput input, Parameters parameters, DownloadOptions option, CancellationToken cancellationToken ) { using (var s3Client = Utilities.GetS3Client(parameters, cancellationToken)) { var dirInfo = new S3DirectoryInfo(s3Client, parameters.BucketName, input.S3Directory); if (dirInfo.Exists) { return(DownloadFiles(input, option, dirInfo, cancellationToken)); } throw new ArgumentException($"Cannot find {input.S3Directory} directory. {nameof(input.S3Directory)}"); } }
/// <summary> /// Prepare for download by checking options and finding files from S3. /// </summary> /// <param name="input"></param> /// <param name="parameters"></param> /// <param name="option"></param> /// <param name="cancellationToken"></param> /// <returns>List<string></returns> private static async Task <List <string> > DownloadUtility( DownloadInput input, Parameters parameters, DownloadOptions option, CancellationToken cancellationToken ) { var paths = new List <string>(); var targetPath = input.S3Directory + input.SearchPattern; var mask = new Regex(input.SearchPattern.Replace(".", "[.]").Replace("*", ".*").Replace("?", ".")); using (var s3Client = (AmazonS3Client)Utilities.GetS3Client(parameters)) { var allObjectsResponse = await s3Client.ListObjectsAsync(parameters.BucketName, cancellationToken); var allObjectsInDirectory = allObjectsResponse.S3Objects; foreach (var fileObject in allObjectsInDirectory) { cancellationToken.ThrowIfCancellationRequested(); if (mask.IsMatch(fileObject.Key.Split('/').Last()) && (targetPath.Split('/').Length == fileObject.Key.Split('/').Length || !option.DownloadFromCurrentDirectoryOnly) && !fileObject.Key.EndsWith("/") && fileObject.Key.StartsWith(input.S3Directory)) { if (!input.DestinationPath.EndsWith(Path.DirectorySeparatorChar.ToString())) { input.DestinationPath += Path.DirectorySeparatorChar.ToString(); } var fullPath = Path.Combine(input.DestinationPath, fileObject.Key.Split('/').Last()); if (File.Exists(fullPath) & !option.Overwrite) { throw new IOException($"File {fileObject.Key.Split('/').Last()} already exists at {fullPath}. Set Overwrite to true from options to overwrite the file."); } paths.Add(await WriteToFile(parameters, fileObject, s3Client, input.DestinationPath, fullPath)); if (option.DeleteSourceFile) { Utilities.DeleteSourceFile(s3Client, cancellationToken, parameters.BucketName, fileObject.Key, true); } } } } if (paths.Count == 0 && option.ThrowErrorIfNoMatches) { throw new ArgumentException($"No matches found with search pattern {input.SearchPattern}"); } return(paths); }