private void MountPath(string path, IBlobStorage storage, bool isMountPoint)
{
string containerPath = StoragePath.IsRootPath(path) ? path : StoragePath.GetParent(path);
if (!_pathToMountBlobs.TryGetValue(containerPath, out HashSet <Blob> blobs))
{
blobs = new HashSet <Blob>();
_pathToMountBlobs[containerPath] = blobs;
}
// this is the mount
if (isMountPoint)
{
var mountBlob = new Blob(path, BlobItemKind.Folder)
{
Tag = storage
};
mountBlob.TryAddProperties("IsMountPoint", true);
blobs.Add(mountBlob);
}
else
{
var intBlob = new Blob(path, BlobItemKind.Folder);
blobs.Add(intBlob);
}
}
protected override async Task <IReadOnlyCollection <Blob> > ListAtAsync(string path, ListOptions options, CancellationToken cancellationToken)
{
ObjectsResource.ListRequest request = _client.Service.Objects.List(_bucketName);
request.Prefix = StoragePath.IsRootPath(path) ? null : (NormalisePath(path) + "/");
request.Delimiter = "/";
var page = new List <Blob>();
do
{
Objects serviceObjects = await request.ExecuteAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
if (serviceObjects.Items != null)
{
page.AddRange(GConvert.ToBlobs(serviceObjects.Items, options));
}
if (serviceObjects.Prefixes != null)
{
//the only info we have about prefixes is it's name
page.AddRange(serviceObjects.Prefixes.Select(p => new Blob(p, BlobItemKind.Folder)));
}
request.PageToken = serviceObjects.NextPageToken;
}while(request.PageToken != null);
return(page);
}
private async Task <Blob> GetBlobAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
try
{
ApiResponse <string> response = await _restApi.GetFilesystemProperties(fs);
await response.EnsureSuccessStatusCodeAsync();
var fsProps = new PathProperties(response);
return(LConvert.ToBlob(fs, fsProps));
}
catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
}
PathProperties pp;
try
{
pp = await GetPathPropertiesAsync(fs, rp, "getProperties").ConfigureAwait(false);
}
catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
return(LConvert.ToBlob(fullPath, pp));
}
protected virtual async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, false).ConfigureAwait(false);
if (StoragePath.IsRootPath(path))
{
//deleting the entire container / filesystem
await container.DeleteIfExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
BlockBlobClient blob = string.IsNullOrEmpty(path)
? null
: container.GetBlockBlobClient(StoragePath.Normalize(path));
if (blob != null)
{
try
{
await blob.DeleteAsync(
DeleteSnapshotsOption.IncludeSnapshots, cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "BlobNotFound")
{
//this might be a folder reference, just try it
await foreach (BlobItem recursedFile in
container.GetBlobsAsync(prefix: path, cancellationToken: cancellationToken).ConfigureAwait(false))
{
BlobClient client = container.GetBlobClient(recursedFile.Name);
await client.DeleteIfExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
}
}
}
}
private async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
(CloudBlobContainer container, string path) = await GetPartsAsync(fullPath, false);
if (StoragePath.IsRootPath(path))
{
//deleting the entire container
await container.DeleteIfExistsAsync(cancellationToken).ConfigureAwait(false);
}
else
{
CloudBlockBlob blob = string.IsNullOrEmpty(path)
? null
: container.GetBlockBlobReference(StoragePath.Normalize(path, false));
if (blob != null && await blob.ExistsAsync().ConfigureAwait(false))
{
await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null, cancellationToken);
}
else
{
//try deleting as a folder
CloudBlobDirectory dir = container.GetDirectoryReference(StoragePath.Normalize(path, false));
using (var browser = new AzureBlobDirectoryBrowser(container, _containerName == null, 3))
{
await browser.RecursiveDeleteAsync(dir, cancellationToken).ConfigureAwait(false);
}
}
}
}
public async Task RefreshBlobsAsync()
{
if (Storage == null)
{
return;
}
SingleClickNavigation = GlobalSettings.Default.SingleClickNavigation;
IsLoading = true;
try
{
List <Blob> items = (await Storage.ListAsync(recurse: false, folderPath: FolderPath, includeAttributes: true)).ToList();
var finalItems = new List <Blob>(items.Count);
if (GlobalSettings.Default.FoldersFirst)
{
finalItems.AddRange(items.Where(i => i.IsFolder).OrderBy(f => f.Name));
finalItems.AddRange(items.Where(i => i.IsFile).OrderBy(f => f.Name));
}
else
{
finalItems = items;
}
if (!StoragePath.IsRootPath(FolderPath))
{
//can't use ".." as it has a special meaning
var parentBlob = new Blob(ParentFolderName, BlobItemKind.Folder);
if (finalItems.Count == 0)
{
finalItems.Add(parentBlob);
}
else
{
finalItems.Insert(0, parentBlob);
}
}
Blobs.AddAll(finalItems, true);
RefreshStatusBar();
FilterBlobs();
HasError = false;
BlobsReloaded?.Invoke(Blobs.Count);
}
catch (Exception ex)
{
HasError = true;
ErrorMessage = ex.Message;
ErrorDetails = ex.StackTrace;
}
finally
{
IsLoading = false;
}
}
public async Task <IReadOnlyCollection <Blob> > InternalListAsync(ListOptions options, CancellationToken cancellationToken)
{
if (StoragePath.IsRootPath(options.FolderPath))
{
//only filesystems are in the root path
var result = new List <Blob>(await ListFilesystemsAsBlobsAsync(cancellationToken).ConfigureAwait(false));
if (options.Recurse)
{
foreach (Blob folder in result.Where(b => b.IsFolder).ToList())
{
int?maxResults = options.MaxResults == null
? null
: (int?)(options.MaxResults.Value - result.Count);
result.AddRange(await ListPathAsync(folder, maxResults, options, cancellationToken).ConfigureAwait(false));
}
}
return(result);
}
else
{
return(await ListPathAsync(options.FolderPath, options.MaxResults, options, cancellationToken).ConfigureAwait(false));
}
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
if (options == null)
{
options = new ListOptions();
}
GenericValidation.CheckBlobPrefix(options.FilePrefix);
if (!StoragePath.IsRootPath(options.FolderPath))
{
return(new List <Blob>());
}
var secretNames = new List <Blob>();
IPage <SecretItem> page = await _vaultClient.GetSecretsAsync(_vaultUri).ConfigureAwait(false);
do
{
var ids = page
.Select((Func <SecretItem, Blob>)AzureKeyVaultBlobStorageProvider.ToBlobId)
.Where(options.IsMatch)
.Where(s => options.BrowseFilter == null || options.BrowseFilter(s))
.ToList();
secretNames.AddRange(ids);
if (options.MaxResults != null && secretNames.Count >= options.MaxResults.Value)
{
return(secretNames.Take(options.MaxResults.Value).ToList());
}
}while (page.NextPageLink != null && (page = await _vaultClient.GetSecretsNextAsync(page.NextPageLink).ConfigureAwait(false)) != null);
return(secretNames);
}
/// <summary>
/// Prepends path to this blob's path without modifying blob's properties
/// </summary>
/// <param name="path"></param>
public void PrependPath(string path)
{
if (path == null || StoragePath.IsRootPath(path))
{
return;
}
FolderPath = StoragePath.Combine(path, FolderPath);
}
public Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken = default)
{
lock (_lock)
{
if (!File.Exists(_filePath))
{
return(Task.FromResult <IReadOnlyCollection <Blob> >(new List <Blob>()));
}
ZipArchive archive = GetArchive(false);
if (options == null)
{
options = new ListOptions();
}
IEnumerable <Blob> blobs = archive.Entries.Select(ze => new Blob(ze.FullName, BlobItemKind.File));
if (options.FilePrefix != null)
{
blobs = blobs.Where(id => id.Name.StartsWith(options.FilePrefix));
}
//find ones that belong to this folder
if (!StoragePath.IsRootPath(options.FolderPath))
{
blobs = blobs.Where(id => id.FullPath.StartsWith(options.FolderPath));
}
blobs = AppendVirtualFolders(options.FolderPath ?? StoragePath.RootFolderPath, blobs.ToList());
//cut off sub-items
if (!options.Recurse)
{
blobs = blobs
.Select(b => new { rp = b.FullPath.Substring(options.FolderPath.Length + 1), b = b })
.Where(a => !a.rp.Contains(StoragePath.PathSeparator))
.Select(a => a.b);
//blobs = blobs.Where(id => !id.FullPath.Substring(0, options.FolderPath.Length).Contains(StoragePath.PathSeparator));
}
if (options.BrowseFilter != null)
{
blobs = blobs.Where(id => options.BrowseFilter(id));
}
if (options.MaxResults != null)
{
blobs = blobs.Take(options.MaxResults.Value);
}
return(Task.FromResult <IReadOnlyCollection <Blob> >(blobs.ToList()));
}
}
private async Task <IReadOnlyCollection <Blob> > LegacyListAtAsync(string path, ListOptions options, CancellationToken cancellationToken)
{
PagedAsyncEnumerable <Objects, Object> objects = _client.ListObjectsAsync(
_bucketName,
StoragePath.IsRootPath(options.FolderPath) ? null : options.FolderPath,
new ListObjectsOptions
{
Delimiter = options.Recurse ? null : "/"
});
return(await GConvert.ToBlobsAsync(objects, options).ConfigureAwait(false));
}
public async Task <IReadOnlyCollection <BlobId> > ListAsync(ListOptions options, CancellationToken cancellationToken = default)
{
if (options == null)
{
options = new ListOptions();
}
var result = new List <BlobId>();
var containers = new List <CloudBlobContainer>();
CloudBlobContainer fixedContainer = await GetFixedContainerAsync();
if (fixedContainer != null)
{
containers.Add(fixedContainer);
}
else if (StoragePath.IsRootPath(options.FolderPath))
{
// list all of the containers
containers.AddRange(await GetCloudBlobContainersAsync(cancellationToken));
//represent containers as folders in the result
result.AddRange(containers.Select(c => new BlobId(c.Name, BlobItemKind.Folder)));
if (!options.Recurse)
{
return(result);
}
}
else
{
(CloudBlobContainer container, string path) = await GetPartsAsync(options.FolderPath, false);
if (container == null)
{
return(new List <BlobId>());
}
options = options.Clone();
options.FolderPath = path; //scan from subpath now
containers.Add(container);
//add container as search result
//result.Add(new BlobId(container.Name, BlobItemKind.Folder));
}
await Task.WhenAll(containers.Select(c => ListAsync(c, fixedContainer, result, options, cancellationToken)));
if (options.MaxResults != null)
{
result = result.Take(options.MaxResults.Value).ToList();
}
return(result);
}
private static string?FormatFolderPrefix(string folderPath)
{
folderPath = StoragePath.Normalize(folderPath);
if (StoragePath.IsRootPath(folderPath))
{
return(null);
}
if (!folderPath.EndsWith("/"))
{
folderPath += "/";
}
return(folderPath.TrimStart('/'));
}
private void CreateCommands()
{
GoLevelUpCommand = new RelayCommand(() =>
{
GoLevelUp();
},
() => !StoragePath.IsRootPath(FolderPath));
ExternalOpenCommand = new RelayCommand(() =>
{
ExternalOpenAsync(SelectedBlob).Forget();
},
() => HasOneFileSelected);
}
private void AddVirtualFolderHierarchy(Blob fileBlob)
{
string path = fileBlob.FolderPath;
while (!StoragePath.IsRootPath(path))
{
var vf = new Blob(path, BlobItemKind.Folder);
_pathToTag[path] = new Tag {
blob = vf
};
path = StoragePath.GetParent(path);
}
}
private static void AssumeImplicitPrefixes(string absoluteRoot, List <Blob> blobs)
{
absoluteRoot = StoragePath.Normalize(absoluteRoot);
List <Blob> implicitFolders = blobs
.Select(b => b.FullPath)
.Select(p => p.Substring(absoluteRoot.Length))
.Select(p => StoragePath.GetParent(p))
.Where(p => !StoragePath.IsRootPath(p))
.Distinct()
.Select(p => new Blob(p, BlobItemKind.Folder))
.ToList();
blobs.AddRange(implicitFolders);
}
private async Task DiscoverBlobsWorkflowAsync(IReadOnlyCollection <Blob> sourceBlobs)
{
foreach (Blob blob in sourceBlobs)
{
if (_sourceRoot == null)
{
_sourceRoot = StoragePath.Normalize(blob.FolderPath);
if (StoragePath.IsRootPath(_sourceRoot))
{
_sourceRoot = string.Empty;
}
}
await DiscoverBlobAsync(blob);
}
}
/// <summary>
/// Changes full path of this blob without modifying any other property
/// </summary>
public void SetFullPath(string fullPath)
{
string path = StoragePath.Normalize(fullPath);
if (StoragePath.IsRootPath(path))
{
Name = StoragePath.RootFolderPath;
FolderPath = StoragePath.RootFolderPath;
}
else
{
string[] parts = StoragePath.Split(path);
Name = parts.Last();
FolderPath = StoragePath.GetParent(path);
}
}
public override async Task WriteAsync(
string fullPath, Stream dataStream, bool append = false, CancellationToken cancellationToken = default)
{
string path = StoragePath.Normalize(fullPath);
//import will fail unless all the parent folders exist, so make sure they do
string parent = StoragePath.GetParent(path);
if (!StoragePath.IsRootPath(parent))
{
await _api.Mkdirs(StoragePath.Normalize(parent)).ConfigureAwait(false);
}
GetImportParameters(path, out ExportFormat exportFormat, out Language? language);
await _api.Import(path, exportFormat, language, dataStream.ToByteArray(), true).ConfigureAwait(false);
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken = default)
{
if (options == null)
{
options = new ListOptions();
}
var result = new List <Blob>();
var containers = new List <CloudBlobContainer>();
if (StoragePath.IsRootPath(options.FolderPath))
{
// list all of the containers
containers.AddRange(await GetCloudBlobContainersAsync(cancellationToken));
//represent containers as folders in the result
result.AddRange(containers.Select(AzConvert.ToBlob));
if (!options.Recurse)
{
return(result);
}
}
else
{
(CloudBlobContainer container, string path) = await GetPartsAsync(options.FolderPath, false).ConfigureAwait(false);
if (container == null)
{
return(new List <Blob>());
}
options = options.Clone();
options.FolderPath = path; //scan from subpath now
containers.Add(container);
}
await Task.WhenAll(containers.Select(c => ListAsync(c, result, options, cancellationToken))).ConfigureAwait(false);
if (options.MaxResults != null)
{
result = result.Take(options.MaxResults.Value).ToList();
}
return(result);
}
private async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
await _restApi.DeleteFilesystemAsync(fs).ConfigureAwait(false);
}
else
{
try
{
await _restApi.DeletePathAsync(fs, rp, true).ConfigureAwait(false);
}
catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
// file not found, ignore
}
}
}
protected override async Task <IReadOnlyCollection <Blob> > ListAtAsync(
string path, ListOptions options, CancellationToken cancellationToken)
{
if (StoragePath.IsRootPath(path))
{
//list file shares
ShareResultSegment shares = await _client.ListSharesSegmentedAsync(null, cancellationToken).ConfigureAwait(false);
return(shares.Results.Select(AzConvert.ToBlob).ToList());
}
else
{
var chunk = new List <Blob>();
CloudFileDirectory dir = await GetDirectoryReferenceAsync(path, cancellationToken).ConfigureAwait(false);
FileContinuationToken token = null;
do
{
try
{
FileResultSegment segment = await dir.ListFilesAndDirectoriesSegmentedAsync(options.FilePrefix, token, cancellationToken).ConfigureAwait(false);
token = segment.ContinuationToken;
chunk.AddRange(segment.Results.Select(r => AzConvert.ToBlob(path, r)));
}
catch (AzStorageException ex) when(ex.RequestInformation.ErrorCode == "ShareNotFound")
{
break;
}
catch (AzStorageException ex) when(ex.RequestInformation.ErrorCode == "ResourceNotFound")
{
break;
}
}while(token != null);
return(chunk);
}
}
public Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
if (options == null)
{
options = new ListOptions();
}
IEnumerable <KeyValuePair <string, Tag> > query = _pathToTag;
//limit by folder path
if (options.Recurse)
{
if (!StoragePath.IsRootPath(options.FolderPath))
{
string prefix = options.FolderPath + StoragePath.PathSeparatorString;
query = query.Where(p => p.Key.StartsWith(prefix));
}
}
else
{
query = query.Where(p => StoragePath.ComparePath(p.Value.blob.FolderPath, options.FolderPath));
}
//prefix
query = query.Where(p => options.IsMatch(p.Value.blob));
//browser filter
query = query.Where(p => options.BrowseFilter == null || options.BrowseFilter(p.Value.blob));
//limit
if (options.MaxResults != null)
{
query = query.Take(options.MaxResults.Value);
}
IReadOnlyCollection <Blob> matches = query.Select(p => p.Value.blob).ToList();
return(Task.FromResult(matches));
}
protected override async Task <IReadOnlyCollection <Blob> > ListAtAsync(
string path, ListOptions options, CancellationToken cancellationToken)
{
var r = new List <Blob>();
// listing from "/secrets"
if (StoragePath.IsRootPath(path))
{
IEnumerable <SecretScope> scopes = await _api.ListScopes().ConfigureAwait(false);
foreach (SecretScope scope in scopes)
{
var scopeBlob = new Blob(scope.Name, BlobItemKind.Folder);
scopeBlob.TryAddProperties(
"Backend", scope.BackendType,
"ObjectType", "secretScope");
IEnumerable <AclItem> acl = await _api.ListSecretAcl(scope.Name).ConfigureAwait(false);
scopeBlob.Properties.Add("ACL", string.Join(";", acl.Select(a => $"{a.Principal}:{a.Permission}")));
r.Add(scopeBlob);
}
return(r);
}
// listing from "/secrets/[scope name]
string scopeName = StoragePath.Split(path)[0];
var secretNames = (await _api.ListSecrets(scopeName).ConfigureAwait(false)).ToList();
foreach (SecretMetadata sm in secretNames)
{
var sb = new Blob(scopeName, sm.Key, BlobItemKind.File);
sb.LastModificationTime = sm.LastUpdatedTimestamp;
sb.TryAddProperties(
"ObjectType", "secret");
r.Add(sb);
}
return(r);
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
if (options == null)
{
options = new ListOptions();
}
GenericValidation.CheckBlobPrefix(options.FilePrefix);
if (!StoragePath.IsRootPath(options.FolderPath))
{
return(new List <Blob>());
}
var secrets = new List <Blob>();
await foreach (SecretProperties secretProperties in _client.GetPropertiesOfSecretsAsync(cancellationToken).ConfigureAwait(false))
{
Blob blob = ToBlob(secretProperties);
if (!options.IsMatch(blob))
{
continue;
}
if (options.BrowseFilter != null && !options.BrowseFilter(blob))
{
continue;
}
secrets.Add(blob);
if (options.MaxResults != null && secrets.Count >= options.MaxResults.Value)
{
break;
}
}
return(secrets);
}
public async Task DeleteAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
await DeleteFilesystemAsync(fs, cancellationToken).ConfigureAwait(false);
}
else
{
try
{
await InvokeAsync <Void>(
$"{fs}/{rp}?recursive=true",
RequestMethod.Delete,
cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound")
{
// file not found, ignore
}
}
}
public static IReadOnlyCollection <Blob> ToBlobs(this ListObjectsV2Response response, ListOptions options)
{
var result = new List <Blob>();
//the files are listed as the S3Objects member, but they don't specifically contain folders,
//but even if they do, they need to be filtered out
result.AddRange(
response.S3Objects
.Where(b => !b.Key.EndsWith("/")) //check if this is "virtual folder" as S3 console creates them (rubbish)
.Select(b => b.ToBlob())
.Where(options.IsMatch)
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b)));
//subfolders are listed in another field (what a funny name!)
//prefix is absolute too
result.AddRange(
response.CommonPrefixes
.Where(p => !StoragePath.IsRootPath(p))
.Select(p => new Blob(p, BlobItemKind.Folder)));
return(result);
}
public async Task <Blob> GetBlobAsync(string fullPath, CancellationToken cancellationToken)
{
DecomposePath(fullPath, out string fs, out string rp, false);
if (StoragePath.IsRootPath(rp))
{
try
{
(Void _, IDictionary <string, string> headers) = await InvokeExtraAsync <Void>(
$"{fs}?resource=filesystem",
RequestMethod.Head,
cancellationToken).ConfigureAwait(false);
return(AzConvert.ToBlob(fullPath, headers, true));
}
catch (RequestFailedException ex) when(ex.ErrorCode == "FilesystemNotFound")
{
//filesystem doesn't exist
return(null);
}
}
try
{
(Void _, IDictionary <string, string> fheaders) = await InvokeExtraAsync <Void>(
$"{fs}/{rp.UrlEncode()}?action=getProperties",
RequestMethod.Head,
cancellationToken).ConfigureAwait(false);
return(AzConvert.ToBlob(fullPath, fheaders, false));
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound")
{
return(null);
}
}