public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
ValidateSecretName(fullPath);
SecretBundle secret;
try
{
secret = await _vaultClient.GetSecretAsync(_vaultUri, fullPath).ConfigureAwait(false);
}
catch (KeyVaultErrorException ex)
{
if (IsNotFound(ex))
{
return(null);
}
TryHandleException(ex);
throw;
}
string value = secret.Value;
return(value.ToMemoryStream());
}
private async Task <(BlobContainerClient, string)> GetPartsAsync(string fullPath, bool createContainer = true)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (fullPath == null)
{
throw new ArgumentNullException(nameof(fullPath));
}
string containerName, relativePath;
if (_containerName == null)
{
int idx = fullPath.IndexOf(StoragePath.PathSeparator);
if (idx == -1)
{
containerName = fullPath;
relativePath = string.Empty;
}
else
{
containerName = fullPath.Substring(0, idx);
relativePath = fullPath.Substring(idx + 1);
}
}
else
{
containerName = _containerName;
relativePath = fullPath;
}
if (!_containerNameToContainerClient.TryGetValue(containerName, out BlobContainerClient container))
{
container = _client.GetBlobContainerClient(containerName);
if (_containerName == null)
{
try
{
//check if container exists
await container.GetPropertiesAsync().ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "ContainerNotFound")
{
if (createContainer)
{
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
}
else
{
return(null, null);
}
}
}
_containerNameToContainerClient[containerName] = container;
}
return(container, relativePath);
}
public async Task WriteAsync(string fullPath, Stream dataStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (dataStream is null)
{
throw new ArgumentNullException(nameof(dataStream));
}
AdlsClient client = await GetAdlsClientAsync().ConfigureAwait(false);
AdlsOutputStream stream;
if (append)
{
stream = await client.GetAppendStreamAsync(fullPath, cancellationToken).ConfigureAwait(false);
}
else
{
stream = await client.CreateFileAsync(fullPath, IfExists.Overwrite,
createParent : true,
cancelToken : cancellationToken).ConfigureAwait(false);
}
using (stream)
{
await dataStream.CopyToAsync(stream).ConfigureAwait(false);
}
}
public Task WriteAsync(string fullPath, Stream sourceStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (sourceStream is null)
{
throw new ArgumentNullException(nameof(sourceStream));
}
if (append)
{
if (!Exists(fullPath))
{
Write(fullPath, sourceStream);
}
else
{
Tag tag = _pathToTag[fullPath];
byte[] data = tag.data.Concat(sourceStream.ToByteArray()).ToArray();
Write(fullPath, new MemoryStream(data));
}
}
else
{
Write(fullPath, sourceStream);
}
return(Task.FromResult(true));
}
public async Task WriteAsync(string fullPath, Stream dataStream,
bool append = false, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (dataStream == null)
{
throw new ArgumentNullException(nameof(dataStream));
}
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, true).ConfigureAwait(false);
BlockBlobClient client = container.GetBlockBlobClient(path);
try
{
await client.UploadAsync(
new StorageSourceStream(dataStream),
cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "OperationNotAllowedInCurrentState")
{
//happens when trying to write to a non-file object i.e. folder
}
}
private void Write(string fullPath, Stream sourceStream)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (sourceStream is MemoryStream ms)
{
ms.Position = 0;
}
byte[] data = sourceStream.ToByteArray();
if (!_pathToTag.TryGetValue(fullPath, out Tag tag))
{
tag = new Tag
{
data = data,
blob = new Blob(fullPath)
{
Size = data.Length,
LastModificationTime = DateTime.UtcNow,
MD5 = data.GetHash(HashType.Md5).ToHexString()
}
};
}
else
{
tag.data = data;
tag.blob.Size = data.Length;
tag.blob.LastModificationTime = DateTime.UtcNow;
tag.blob.MD5 = data.GetHash(HashType.Md5).ToHexString();
}
_pathToTag[fullPath] = tag;
AddVirtualFolderHierarchy(tag.blob);
}
private async Task <bool> ExistsAsync(string fullPath)
{
GenericValidation.CheckBlobFullPath(fullPath);
try
{
fullPath = StoragePath.Normalize(fullPath, false);
using (GetObjectResponse response = await GetObjectAsync(fullPath).ConfigureAwait(false))
{
if (response == null)
{
return(false);
}
}
}
catch (StorageException ex)
{
if (ex.ErrorCode == ErrorCode.NotFound)
{
return(false);
}
}
return(true);
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
(BlobContainerClient container, string path) = await GetPartsAsync(fullPath, false).ConfigureAwait(false);
BlockBlobClient client = container.GetBlockBlobClient(path);
try
{
//current SDK fails to download 0-sized files
Response <BlobProperties> p = await client.GetPropertiesAsync().ConfigureAwait(false);
if (p.Value.ContentLength == 0)
{
return(new MemoryStream());
}
Response <BlobDownloadInfo> response = await client.DownloadAsync(cancellationToken).ConfigureAwait(false);
return(response.Value.Content);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "BlobNotFound")
{
return(null);
}
}
private async Task <Blob> GetBlobAsync(string fullPath)
{
GenericValidation.CheckBlobFullPath(fullPath);
try
{
fullPath = StoragePath.Normalize(fullPath, false);
using (GetObjectResponse obj = await GetObjectAsync(fullPath).ConfigureAwait(false))
{
//ETag contains actual MD5 hash, not sure why!
if (obj != null)
{
var r = new Blob(fullPath);
r.MD5 = obj.ETag.Trim('\"');
r.Size = obj.ContentLength;
r.LastModificationTime = obj.LastModified.ToUniversalTime();
return(r);
}
}
}
catch (StorageException ex) when(ex.ErrorCode == ErrorCode.NotFound)
{
//if blob is not found, don't return any information
}
return(null);
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
(CloudBlobContainer container, string path) = await GetPartsAsync(fullPath, false);
if (container == null)
{
return(null);
}
CloudBlockBlob blob = container.GetBlockBlobReference(StoragePath.Normalize(path, false));
try
{
return(await blob.OpenReadAsync());
}
catch (AzureStorageException ex)
{
if (AzureStorageValidation.IsDoesntExist(ex))
{
return(null);
}
if (!AzureStorageValidation.TryHandleStorageException(ex))
{
throw;
}
}
throw new InvalidOperationException("must not be here");
}
public async Task <IReadOnlyCollection <bool> > ExistsAsync(IEnumerable <string> fullPaths, CancellationToken cancellationToken = default)
{
var result = new List <bool>();
foreach (string fullPath in fullPaths)
{
GenericValidation.CheckBlobFullPath(fullPath);
(CloudBlobContainer container, string path) = await GetPartsAsync(fullPath, false).ConfigureAwait(false);
if (container == null)
{
result.Add(false);
}
else
{
CloudBlockBlob blob = container.GetBlockBlobReference(StoragePath.Normalize(path, false));
bool exists = await blob.ExistsAsync().ConfigureAwait(false);
result.Add(exists);
}
}
return(result);
}
/// <summary>
/// Opens file and returns the open stream
/// </summary>
public Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
Stream result = OpenStream(fullPath);
return(Task.FromResult(result));
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
var info = new PathInformation(fullPath);
return(await TryGetPropertiesAsync(info.Filesystem, info.Path, cancellationToken) == null
? null
: Client.OpenRead(info.Filesystem, info.Path));
}
/// <summary>
///
/// </summary>
public Task <Stream> OpenWriteAsync(string fullPath, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
Stream stream = CreateStream(fullPath, !append);
return(Task.FromResult(stream));
}
private async Task DeleteAsync(string fullPath, AmazonS3Client client, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
await client.DeleteObjectAsync(_bucketName, fullPath, cancellationToken).ConfigureAwait(false);
await new AwsS3DirectoryBrowser(client, _bucketName).DeleteRecursiveAsync(fullPath, cancellationToken).ConfigureAwait(false);
}
private async Task <(CloudBlobContainer, string)> GetPartsAsync(string fullPath, bool createContainer = true)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (fullPath == null)
{
throw new ArgumentNullException(nameof(fullPath));
}
string containerName, relativePath;
if (_containerName == null)
{
int idx = fullPath.IndexOf(StoragePath.PathSeparator);
if (idx == -1)
{
containerName = fullPath;
relativePath = string.Empty;
}
else
{
containerName = fullPath.Substring(0, idx);
relativePath = fullPath.Substring(idx + 1);
}
}
else
{
containerName = _containerName;
relativePath = fullPath;
}
if (!_containerNameToContainer.TryGetValue(containerName, out CloudBlobContainer container))
{
container = _client.GetContainerReference(containerName);
if (_containerName == null)
{
if (!(await container.ExistsAsync().ConfigureAwait(false)))
{
if (createContainer)
{
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
}
else
{
return(null, null);
}
}
}
_containerNameToContainer[containerName] = container;
}
return(container, relativePath);
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options = null, CancellationToken cancellationToken = default)
{
if (options == null)
{
options = new ListOptions()
{
FolderPath = "/",
Recurse = true
};
}
GenericValidation.CheckBlobFullPath(options.FolderPath);
var info = new PathInformation(options.FolderPath);
int maxResults = options.MaxResults ?? ListBatchSize;
var blobs = new List <Blob>();
FilesystemList filesystemList = await Client.ListFilesystemsAsync(cancellationToken : cancellationToken);
IEnumerable <FilesystemItem> filesystems =
filesystemList.Filesystems
.Where(x => info.Filesystem == "" || x.Name == info.Filesystem)
.OrderBy(x => x.Name);
foreach (FilesystemItem filesystem in filesystems)
{
try
{
DirectoryList directoryList = await Client.ListDirectoryAsync(
filesystem.Name, info.Path, options.Recurse,
cancellationToken : cancellationToken);
IEnumerable <Blob> results = directoryList.Paths
.Where(x => options.FilePrefix == null || x.Name.StartsWith(options.FilePrefix))
.Select(x =>
new Blob($"{filesystem.Name}/{x.Name}",
x.IsDirectory ? BlobItemKind.Folder : BlobItemKind.File))
.Where(x => options.BrowseFilter == null || options.BrowseFilter(x))
.OrderBy(x => x.FullPath);
blobs.AddRange(results);
}
catch (DataLakeGen2Exception e) when(e.StatusCode == HttpStatusCode.NotFound)
{
}
if (blobs.Count >= maxResults)
{
return(blobs.Take(maxResults).ToList());
}
}
return(blobs.ToList());
}
private async Task SetBlobAsync(Blob blob, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(blob);
DecomposePath(blob, out string fs, out string rp);
string properties = string.Join(",", blob.Metadata.Select(kv => $"{kv.Key}={kv.Value.Base64Encode()}"));
await _restApi.UpdatePathAsync(fs, rp, "setProperties",
properties : properties,
body : EmptyStream);
}
public Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (!_pathToTag.TryGetValue(fullPath, out Tag tag) || tag.data == null)
{
return(Task.FromResult <Stream>(null));
}
return(Task.FromResult <Stream>(new NonCloseableStream(new MemoryStream(tag.data))));
}
public override async Task WriteAsync(string fullPath, Stream dataStream,
bool append = false, CancellationToken cancellationToken = default)
{
if (append)
{
throw new NotSupportedException();
}
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = NormalisePath(fullPath);
await _client.UploadObjectAsync(_bucketName, fullPath, null, dataStream, cancellationToken : cancellationToken).ConfigureAwait(false);
}
public async Task <Stream> OpenWriteAsync(string fullPath, bool append = false, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
var info = new PathInformation(fullPath);
if (!append)
{
await Client.CreateFileAsync(info.Filesystem, info.Path, cancellationToken);
}
return(await Client.OpenWriteAsync(info.Filesystem, info.Path, cancellationToken));
}
private Stream OpenStream(string fullPath)
{
GenericValidation.CheckBlobFullPath(fullPath);
string path = GetFilePath(fullPath);
if (!File.Exists(path))
{
return(null);
}
return(File.OpenRead(path));
}
public async Task <Stream?> OpenReadAsync(
string fullPath,
CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
GetObjectResponse?getObjectResponse = await this.GetObjectAsync(fullPath).ConfigureAwait(false);
return(getObjectResponse != null
? new FixedStream(getObjectResponse.ResponseStream, getObjectResponse.ContentLength,
(Action <FixedStream>)null !)
: (Stream)null !);
}
public async Task WriteAsync(string fullPath, Stream dataStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = NormaliseSecretName(fullPath);
if (append)
{
throw new ArgumentException("appending to secrets is not supported", nameof(append));
}
byte[] data = dataStream.ToByteArray();
string value = Encoding.UTF8.GetString(data);
await _client.SetSecretAsync(fullPath, value, cancellationToken).ConfigureAwait(false);
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, true);
GetObjectResponse response = await GetObjectAsync(fullPath).ConfigureAwait(false);
if (response == null)
{
return(null);
}
return(new FixedStream(response.ResponseStream, length: response.ContentLength, (Action <FixedStream>)null));
}
/// <summary>
/// S3 doesnt support this natively and will cache everything in MemoryStream until disposed.
/// </summary>
public async Task WriteAsync(string fullPath, Stream dataStream, bool append = false,
CancellationToken cancellationToken = default)
{
if (append)
{
throw new NotSupportedException();
}
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, true);
//http://docs.aws.amazon.com/AmazonS3/latest/dev/HLuploadFileDotNet.html
await _fileTransferUtility.UploadAsync(dataStream, _bucketName, fullPath, cancellationToken).ConfigureAwait(false);
}
private void DecomposePath(string path, out string filesystemName, out string relativePath)
{
GenericValidation.CheckBlobFullPath(path);
string[] parts = StoragePath.Split(path);
if (parts.Length == 1)
{
throw new ArgumentException($"path {path} must include filesystem name as root folder", nameof(path));
}
filesystemName = parts[0];
relativePath = StoragePath.Combine(parts.Skip(1));
}
public async Task WriteAsync(string fullPath, Stream dataStream,
bool append, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (append)
{
await AppendAsync(fullPath, dataStream, cancellationToken).ConfigureAwait(false);
}
else
{
await WriteAsync(fullPath, dataStream, cancellationToken).ConfigureAwait(false);
}
}
public Task <Stream> WriteAsync(string fullPath, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
var result = new FixedStream(new MemoryStream(), null, async fx =>
{
MemoryStream ms = (MemoryStream)fx.Parent;
ms.Position = 0;
await WriteAsync(fullPath, ms, append, cancellationToken).ConfigureAwait(false);
});
return(Task.FromResult <Stream>(result));
}
private Stream CreateStream(string fullPath, bool overwrite = true)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (!Directory.Exists(_directoryFullName))
{
Directory.CreateDirectory(_directoryFullName);
}
string path = GetFilePath(fullPath);
Stream s = overwrite ? File.Create(path) : File.OpenWrite(path);
s.Seek(0, SeekOrigin.End);
return(s);
}
