/// <summary>
///
/// </summary>
public Task <Stream> OpenWriteAsync(string id, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id, false);
return(Task.FromResult(CreateStream(id, !append)));
}
public async Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken = default)
{
fullPath = StoragePath.Normalize(fullPath, true);
var ms = new MemoryStream(0);
await _dbfs.Download(fullPath, ms);
return(ms);
}
public async Task <IEnumerable <BlobId> > Browse(ListOptions options, CancellationToken token)
{
string path = StoragePath.Normalize(options.FolderPath);
var result = new List <BlobId>();
await Browse(path, options, result, token);
return(result);
}
private async Task DeleteAsync(string id, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id, false);
AmazonS3Client client = await GetClientAsync();
await client.DeleteObjectAsync(_bucketName, id, cancellationToken);
}
/// <summary>
/// Opens file and returns the open stream
/// </summary>
public Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
Stream result = OpenStream(fullPath);
return(Task.FromResult(result));
}
public async Task <IReadOnlyCollection <Blob> > BrowseAsync(ListOptions options, CancellationToken token)
{
string path = StoragePath.Normalize(options.FolderPath);
var result = new List <Blob>();
await BrowseAsync(path, options, result, token).ConfigureAwait(false);
return(result);
}
private void Write(string id, Stream sourceStream)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id);
Tag tag = ToTag(sourceStream);
_idToData[id] = tag;
}
/// <summary>
/// Opens file and returns the open stream
/// </summary>
public Task <Stream> OpenReadAsync(string id, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id, false);
Stream result = OpenStream(id);
return(Task.FromResult(result));
}
public Task <Stream> OpenReadAsync(string id, CancellationToken cancellationToken = default(CancellationToken))
{
id = StoragePath.Normalize(id, false);
ZipArchive archive = GetArchive(false);
ZipArchiveEntry entry = archive.GetEntry(id);
return(Task.FromResult(entry.Open()));
}
public async Task <Stream> OpenReadAsync(string id, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id, false);
GetObjectResponse response = await GetObjectAsync(id);
return(new AwsS3BlobStorageExternalStream(response));
}
public async Task DeleteAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
foreach (string id in ids)
{
GenericValidation.CheckBlobId(id);
CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(StoragePath.Normalize(id, false));
await blob.DeleteIfExistsAsync();
}
}
/// <summary>
///
/// </summary>
public Task <Stream> OpenWriteAsync(string fullPath, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
Stream stream = CreateStream(fullPath, !append);
return(Task.FromResult(stream));
}
private async Task DeleteAsync(string fullPath, AmazonS3Client client, CancellationToken cancellationToken = default)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
await client.DeleteObjectAsync(_bucketName, fullPath, cancellationToken).ConfigureAwait(false);
await new AwsS3DirectoryBrowser(client, _bucketName).DeleteRecursiveAsync(fullPath, cancellationToken).ConfigureAwait(false);
}
private async Task <(CloudBlobContainer, string)> GetPartsAsync(string fullPath, bool createContainer = true)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (fullPath == null)
{
throw new ArgumentNullException(nameof(fullPath));
}
string containerName, relativePath;
if (_containerName == null)
{
int idx = fullPath.IndexOf(StoragePath.PathSeparator);
if (idx == -1)
{
containerName = fullPath;
relativePath = string.Empty;
}
else
{
containerName = fullPath.Substring(0, idx);
relativePath = fullPath.Substring(idx + 1);
}
}
else
{
containerName = _containerName;
relativePath = fullPath;
}
if (!_containerNameToContainer.TryGetValue(containerName, out CloudBlobContainer container))
{
container = _client.GetContainerReference(containerName);
if (_containerName == null)
{
if (!(await container.ExistsAsync().ConfigureAwait(false)))
{
if (createContainer)
{
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
}
else
{
return(null, null);
}
}
}
_containerNameToContainer[containerName] = container;
}
return(container, relativePath);
}
public Task <IReadOnlyCollection <bool> > ExistsAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
var result = new List <bool>();
foreach (string id in ids)
{
result.Add(_idToData.ContainsKey(StoragePath.Normalize(id)));
}
return(Task.FromResult <IReadOnlyCollection <bool> >(result));
}
public Task DeleteAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
foreach (string blobId in ids)
{
_idToData.Remove(StoragePath.Normalize(blobId));
}
return(Task.FromResult(true));
}
/// <summary>
/// Create a new instance
/// </summary>
/// <param name="fullPath"></param>
/// <param name="kind"></param>
public Blob(string fullPath, BlobItemKind kind = BlobItemKind.File)
{
string path = StoragePath.Normalize(fullPath);
string[] parts = StoragePath.Split(path);
Name = parts.Last();
FolderPath = StoragePath.GetParent(path);
Kind = kind;
}
public Task <IReadOnlyCollection <bool> > ExistsAsync(IEnumerable <string> fullPaths, CancellationToken cancellationToken)
{
var result = new List <bool>();
foreach (string fullPath in fullPaths)
{
result.Add(_pathToTag.ContainsKey(StoragePath.Normalize(fullPath)));
}
return(Task.FromResult <IReadOnlyCollection <bool> >(result));
}
private static string NormaliseSecretName(string fullPath)
{
fullPath = StoragePath.Normalize(fullPath).Substring(1);
if (!secretNameRegex.IsMatch(fullPath))
{
throw new NotSupportedException($"secret '{fullPath}' does not match expected pattern '^[0-9a-zA-Z-]+$'");
}
return(fullPath);
}
/// <summary>
/// Streams into file
/// </summary>
public async Task WriteAsync(string id, Stream sourceStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
GenericValidation.CheckSourceStream(sourceStream);
id = StoragePath.Normalize(id, false);
using (Stream dest = CreateStream(id, !append))
{
await sourceStream.CopyToAsync(dest);
}
}
public Task <Stream> OpenReadAsync(string id, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
id = StoragePath.Normalize(id);
if (!_idToData.TryGetValue(id, out Tag tag))
{
return(Task.FromResult <Stream>(null));
}
return(Task.FromResult <Stream>(new NonCloseableStream(new MemoryStream(tag.data))));
}
public Task <Stream> OpenReadAsync(string fullPath, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (!_pathToTag.TryGetValue(fullPath, out Tag tag) || tag.data == null)
{
return(Task.FromResult <Stream>(null));
}
return(Task.FromResult <Stream>(new NonCloseableStream(new MemoryStream(tag.data))));
}
public override async Task WriteAsync(string fullPath, Stream dataStream,
bool append = false, CancellationToken cancellationToken = default)
{
if (append)
{
throw new NotSupportedException();
}
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath, false);
await _client.UploadObjectAsync(_bucketName, fullPath, null, dataStream, cancellationToken : cancellationToken).ConfigureAwait(false);
}
public async Task WriteAsync(string id, Stream sourceStream, bool append = false, CancellationToken cancellationToken = default(CancellationToken))
{
id = StoragePath.Normalize(id, false);
ZipArchive archive = GetArchive(true);
ZipArchiveEntry entry = archive.CreateEntry(id, CompressionLevel.Optimal);
using (Stream dest = entry.Open())
{
await sourceStream.CopyToAsync(dest);
}
}
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken)
{
//get filesystem name and folder path
string[] parts = StoragePath.Split(path);
string fs = parts[0];
string relativePath = StoragePath.Normalize(StoragePath.Combine(parts.Skip(1)), true);
var list = new List <Gen2Path>();
try
{
string continuation = null;
do
{
string continuationParam = continuation == null ? null : $"&continuation={continuation.UrlEncode()}";
(PathList pl, IDictionary <string, string> responseHeaders) = await InvokeExtraAsync <PathList>(
$"{fs}?resource=filesystem&directory={relativePath.UrlEncode()}&recursive={options.Recurse}{continuationParam}",
RequestMethod.Get,
cancellationToken).ConfigureAwait(false);
list.AddRange(pl.Paths);
responseHeaders.TryGetValue("x-ms-continuation", out continuation);
} while(continuation != null);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound" || ex.ErrorCode == "FilesystemNotFound")
{
// trying to list a path which doesn't exist, just return an empty result
return(new List <Blob>());
}
IEnumerable <Blob> result = list.Select(p => AzConvert.ToBlob(fs, p));
if (options.FilePrefix != null)
{
result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix));
}
if (options.BrowseFilter != null)
{
result = result.Where(b => options.BrowseFilter(b));
}
if (maxResults != null)
{
result = result.Take(maxResults.Value);
}
return(result.ToList());
}
/// <summary>
/// Get presigned url for requested operation with Blob Storage.
/// </summary>
public async Task <string> GetPresignedUrlAsync(string fullPath, string mimeType, int expiresInSeconds, HttpVerb verb)
{
IAmazonS3 client = await GetClientAsync().ConfigureAwait(false);
return(client.GetPreSignedURL(new GetPreSignedUrlRequest()
{
BucketName = _bucketName,
ContentType = mimeType,
Expires = DateTime.UtcNow.AddSeconds(expiresInSeconds),
Key = StoragePath.Normalize(fullPath, true),
Verb = verb,
}));
}
private async Task ListFolderAsync(string path, List <Blob> container, ListOptions options)
{
IEnumerable <FileInfo> objects = await _dbfs.List(StoragePath.Normalize(options.FolderPath, true));
List <Blob> batch = objects.Select(DConvert.ToBlob).ToList();
container.AddRange(batch);
if (options.Recurse)
{
await Task.WhenAll(batch.Where(b => b.IsFolder).Select(f => ListFolderAsync(f.FullPath, container, options)));
}
}
public Task WriteAsync(string id, Stream sourceStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
GenericValidation.CheckSourceStream(sourceStream);
id = StoragePath.Normalize(id, false);
using (Stream dest = CreateStream(id, !append))
{
sourceStream.CopyTo(dest);
}
return(Task.FromResult(true));
}
private static string?FormatFolderPrefix(string folderPath)
{
folderPath = StoragePath.Normalize(folderPath);
if (StoragePath.IsRootPath(folderPath))
{
return(null);
}
if (!folderPath.EndsWith("/"))
{
folderPath += "/";
}
return(folderPath.TrimStart('/'));
}
/// <summary>
/// Create a new instance
/// </summary>
/// <param name="fullId"></param>
/// <param name="kind"></param>
public BlobId(string fullId, BlobItemKind kind = BlobItemKind.File)
{
string path = StoragePath.Normalize(fullId);
string[] parts = StoragePath.Split(path);
Id = parts.Last();
FolderPath = parts.Length > 1
? StoragePath.Combine(parts.Take(parts.Length - 1))
: StoragePath.PathStrSeparator;
Kind = kind;
}