private static IEnumerable <GcsFileTransferOperation> CreateUploadOPerationsForPath(
string bucket,
string bucketPath,
string src)
{
var info = new FileInfo(src);
var isDirectory = (info.Attributes & FileAttributes.Directory) != 0;
if (isDirectory)
{
return(CreateUploadOperationsForDirectory(
sourceDir: info.FullName,
bucket: bucket,
baseGcsPath: GcsPathUtils.Combine(bucketPath, info.Name)));
}
else
{
return(new GcsFileTransferOperation[]
{
new GcsFileTransferOperation(
localPath: info.FullName,
gcsItem: new GcsItemRef(bucket: bucket, name: GcsPathUtils.Combine(bucketPath, info.Name)))
});
}
}
/// <summary>
/// Starts all of the necessary operations to rename the directory <paramref name="oldLeafName"/> to
/// the <paramref name="newLeafName"/> name. This includes renaming the directory and all subdirectories and files
/// under the directory being renamed.
/// </summary>
/// <param name="bucket">The bucket where the directory to rename resides.</param>
/// <param name="parentName">The parent directory of the directory being renamed.</param>
/// <param name="oldLeafName">The old (current) name of the directory.</param>
/// <param name="newLeafName">The new name for the directory.</param>
/// <param name="cancellationToken">The cancellation token for the operaitons.</param>
/// <returns></returns>
public async Task <OperationsQueue> StartDirectoryRenameOperationsAsync(
string bucket,
string parentName,
string oldLeafName,
string newLeafName,
CancellationToken cancellationToken)
{
var oldNamePrefix = GcsPathUtils.Combine(parentName, oldLeafName) + "/";
var newNamePrefix = GcsPathUtils.Combine(parentName, newLeafName) + "/";
var filesToMove = await GetGcsFilesFromPrefixAsync(bucket, oldNamePrefix);
var moveOperations = new List <GcsMoveFileOperation>();
foreach (var file in filesToMove)
{
var movedName = newNamePrefix + file.Name.Substring(oldNamePrefix.Length);
var movedItem = new GcsItemRef(bucket, movedName);
moveOperations.Add(new GcsMoveFileOperation(file, movedItem));
}
var operationsQueue = new OperationsQueue(cancellationToken);
operationsQueue.EnqueueOperations(moveOperations, _dataSource.StartMoveOperation);
operationsQueue.StartOperations();
return(operationsQueue);
}
/// <summary>
/// Starts a set of download operations for files and directories stored on GCS. It will enumerate
/// all of the directories and subdirectories specified as well as all of the files stored.
/// </summary>
/// <param name="sources">The list of files or directories to download.</param>
/// <param name="destinationDir">Where to store the downloaded files.</param>
/// <param name="cancellationToken">The cancellation token for the operation.</param>
/// <returns>The list of operations started.</returns>
public async Task <OperationsQueue> StartDownloadOperationsAsync(
IEnumerable <GcsItemRef> sources,
string destinationDir,
CancellationToken cancellationToken)
{
List <GcsFileTransferOperation> downloadOperations = new List <GcsFileTransferOperation>(
sources
.Where(x => x.IsFile)
.Select(x => new GcsFileTransferOperation(
localPath: Path.Combine(destinationDir, GcsPathUtils.GetFileName(x.Name)),
gcsItem: x)));
// Collect all of the files from all of the directories to download, collects all of the
// local file system directories that need to be created to mirror the source structure.
var subDirs = new HashSet <string>();
foreach (var dir in sources.Where(x => x.IsDirectory))
{
var files = await GetGcsFilesFromPrefixAsync(dir.Bucket, dir.Name);
foreach (var file in files)
{
var relativeFilePath = Path.Combine(
GcsPathUtils.GetFileName(dir.Name),
GetRelativeName(file.Name, dir.Name).Replace('/', '\\'));
var absoluteFilePath = Path.Combine(destinationDir, relativeFilePath);
// Create the file operation for this file.
downloadOperations.Add(new GcsFileTransferOperation(
localPath: absoluteFilePath,
gcsItem: file));
// Collects the list of directories to create.
subDirs.Add(Path.GetDirectoryName(absoluteFilePath));
}
}
// Create all of the directories.
foreach (var dir in subDirs)
{
Directory.CreateDirectory(dir);
}
var operationsQueue = new OperationsQueue(cancellationToken);
operationsQueue.EnqueueOperations(downloadOperations, _dataSource.StartFileDownloadOperation);
operationsQueue.StartOperations();
return(operationsQueue);
}
/// <summary>
/// Creates the <seealso cref="GcsFileTransferOperation"/> instances for all of the files in the given directory. The
/// target directory will be based on <paramref name="baseGcsPath"/>.
/// </summary>
/// <param name="sourceDir">The local dir to process.</param>
/// <param name="bucket">The name of the bucket.</param>
/// <param name="baseGcsPath">The base gcs path where to copy the files.</param>
private static IEnumerable <GcsFileTransferOperation> CreateUploadOperationsForDirectory(
string sourceDir,
string bucket,
string baseGcsPath)
{
var fileOperations = Directory.EnumerateFiles(sourceDir)
.Select(file => new GcsFileTransferOperation(
localPath: file,
gcsItem: new GcsItemRef(
bucket: bucket,
name: GcsPathUtils.Combine(baseGcsPath, Path.GetFileName(file)))));
var directoryOperations = Directory.EnumerateDirectories(sourceDir)
.Select(subDir => CreateUploadOperationsForDirectory(
subDir,
bucket: bucket,
baseGcsPath: GcsPathUtils.Combine(baseGcsPath, Path.GetFileName(subDir))))
.SelectMany(x => x);
return(Enumerable.Concat(fileOperations, directoryOperations));
}
