/// <summary>
/// Directory download operation.
/// </summary>
/// <param name="input"></param>
/// <param name="option"></param>
/// <param name="cToken"></param>
/// <param name="dirInfo"></param>
/// <returns>Task</returns>
private static List <string> DownloadFiles(
DownloadInput input,
DownloadOptions option,
S3DirectoryInfo dirInfo,
CancellationToken cToken
)
{
var files = dirInfo.GetFiles(input.SearchPattern,
option.DownloadFromCurrentDirectoryOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories);
if (option.ThrowErrorIfNoMatches && files.Length < 1)
{
throw new ArgumentException("Could not find any files matching pattern.");
}
var filelist = new List <string>();
foreach (var file in files)
{
if (!file.Exists)
{
continue;
}
cToken.ThrowIfCancellationRequested();
var path = Path.Combine(input.DestinationPath, file.Name);
try
{
// Apparently MoveToLocal does not have overwrite as signature :(
var localFile = option.DeleteSourceFile
? file.MoveToLocal(path, option.Overwrite)
: file.CopyToLocal(path, option.Overwrite);
if (!localFile.Exists)
{
throw new IOException($"Could not find {localFile.FullName} from local filesystem.");
}
filelist.Add(localFile.FullName);
}
catch (IOException ex)
{
// normal exception does not give filename info, which would be nice.
throw new IOException($"{path} already exists or insufficient privileges to write file.", ex);
}
}
return(filelist);
}
/// <summary>
/// Amazon AWS S3 DownloadFiles task.
/// </summary>
/// <param name="input"></param>
/// <param name="parameters"></param>
/// <param name="option"></param>
/// <param name="cToken"></param>
/// <returns>List<string></returns>
public static List <string> DownloadFiles(
[PropertyTab] DownloadInput input,
[PropertyTab] Parameters parameters,
[PropertyTab] DownloadOptions option,
CancellationToken cToken
)
{
if (!parameters.UseDefaultCredentials && parameters.AwsCredentials == null)
{
parameters.IsAnyNullOrWhiteSpaceThrow();
}
if (string.IsNullOrWhiteSpace(input.DestinationPath))
{
throw new ArgumentNullException(nameof(input.DestinationPath));
}
return(DownloadUtility(input, parameters, option, cToken).Result);
}
/// <summary>
/// Method to create client and call DownloadFiles.
/// </summary>
/// <param name="input"></param>
/// <param name="parameters"></param>
/// <param name="cancellationToken"></param>
/// <param name="option"></param>
/// <returns></returns>
private static List <string> DownloadUtility(
DownloadInput input,
Parameters parameters,
DownloadOptions option,
CancellationToken cancellationToken
)
{
using (var s3Client = Utilities.GetS3Client(parameters, cancellationToken))
{
var dirInfo = new S3DirectoryInfo(s3Client, parameters.BucketName, input.S3Directory);
if (dirInfo.Exists)
{
return(DownloadFiles(input, option, dirInfo, cancellationToken));
}
throw new ArgumentException($"Cannot find {input.S3Directory} directory. {nameof(input.S3Directory)}");
}
}
/// <summary>
/// Prepare for download by checking options and finding files from S3.
/// </summary>
/// <param name="input"></param>
/// <param name="parameters"></param>
/// <param name="option"></param>
/// <param name="cancellationToken"></param>
/// <returns>List<string></returns>
private static async Task <List <string> > DownloadUtility(
DownloadInput input,
Parameters parameters,
DownloadOptions option,
CancellationToken cancellationToken
)
{
var paths = new List <string>();
var targetPath = input.S3Directory + input.SearchPattern;
var mask = new Regex(input.SearchPattern.Replace(".", "[.]").Replace("*", ".*").Replace("?", "."));
using (var s3Client = (AmazonS3Client)Utilities.GetS3Client(parameters))
{
var allObjectsResponse = await s3Client.ListObjectsAsync(parameters.BucketName, cancellationToken);
var allObjectsInDirectory = allObjectsResponse.S3Objects;
foreach (var fileObject in allObjectsInDirectory)
{
cancellationToken.ThrowIfCancellationRequested();
if (mask.IsMatch(fileObject.Key.Split('/').Last()) && (targetPath.Split('/').Length == fileObject.Key.Split('/').Length || !option.DownloadFromCurrentDirectoryOnly) && !fileObject.Key.EndsWith("/") && fileObject.Key.StartsWith(input.S3Directory))
{
if (!input.DestinationPath.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
input.DestinationPath += Path.DirectorySeparatorChar.ToString();
}
var fullPath = Path.Combine(input.DestinationPath, fileObject.Key.Split('/').Last());
if (File.Exists(fullPath) & !option.Overwrite)
{
throw new IOException($"File {fileObject.Key.Split('/').Last()} already exists at {fullPath}. Set Overwrite to true from options to overwrite the file.");
}
paths.Add(await WriteToFile(parameters, fileObject, s3Client, input.DestinationPath, fullPath));
if (option.DeleteSourceFile)
{
Utilities.DeleteSourceFile(s3Client, cancellationToken, parameters.BucketName, fileObject.Key, true);
}
}
}
}
if (paths.Count == 0 && option.ThrowErrorIfNoMatches)
{
throw new ArgumentException($"No matches found with search pattern {input.SearchPattern}");
}
return(paths);
}