/// <summary>
/// Creates a new folder in this storage. If storage supports hierarchy, the folder is created as is, otherwise a folder is created by putting a dummy zero size file in that folder.
/// </summary>
/// <param name="blobStorage"></param>
/// <param name="folderPath">Path to the folder</param>
/// <param name="dummyFileName">If storage doesn't support hierary, you can override the dummy file name created in that empty folder.</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task CreateFolderAsync(
this IBlobStorage blobStorage, string folderPath, string dummyFileName = null, CancellationToken cancellationToken = default)
{
if (blobStorage is IHierarchicalBlobStorage hierarchicalBlobStorage)
{
await hierarchicalBlobStorage.CreateFolderAsync(folderPath, cancellationToken);
}
else
{
string fullPath = StoragePath.Combine(folderPath, dummyFileName ?? ".empty");
await blobStorage.WriteTextAsync(
fullPath,
"created as a workaround by Storage.Net when creating an empty parent folder",
null,
cancellationToken);
}
}
public static Blob ToBlob(string containerName, string path, BlobProperties properties)
{
string GetFullName(string name) => containerName == null
? name
: StoragePath.Combine(containerName, name);
var blob = new Blob(GetFullName(path), BlobItemKind.File);
blob.MD5 = properties.ContentHash.ToHexString();
blob.Size = properties.ContentLength;
blob.LastModificationTime = properties.LastModified;
AddProperties(blob, properties);
blob.Metadata.MergeRange(properties.Metadata);
return(blob);
}
internal static string GetBlobPath(DbFile file)
{
var guidIinitialPart = file.Id.ToString().Split(new[] { '-' })[0];
var fileName = file.FilePath.Split(new[] { '/' }).Last();
var depth1Folder = guidIinitialPart.Substring(0, 2);
var depth2Folder = guidIinitialPart.Substring(2, 2);
string filenameExt = Path.GetExtension(fileName);
if (!string.IsNullOrWhiteSpace(filenameExt))
{
return(StoragePath.Combine(depth1Folder, depth2Folder, file.Id + filenameExt));
}
else
{
return(StoragePath.Combine(depth1Folder, depth2Folder, file.Id.ToString()));
}
}
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);
if (options.IsMatch(id) && (options.BrowseFilter == null || options.BrowseFilter(id)))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
await Task.WhenAll(
folderIds.Select(folderId => ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken)));
}
}
public async Task Sas_BlobPublicAccess()
{
string path = StoragePath.Combine("test", Guid.NewGuid().ToString() + ".txt");
await _native.WriteTextAsync(path, "read me!");
var policy = new BlobSasPolicy(DateTime.UtcNow, TimeSpan.FromHours(12))
{
Permissions = BlobSasPermission.Read | BlobSasPermission.Write
};
string publicUrl = await _native.GetBlobSasAsync(path);
Assert.NotNull(publicUrl);
string text = await new HttpClient().GetStringAsync(publicUrl);
Assert.Equal("read me!", text);
}
public async Task List_FilesInFolder_Recursive()
{
string id1 = RandomGenerator.RandomString;
string id2 = StoragePath.Combine(RandomGenerator.RandomString, RandomGenerator.RandomString);
string id3 = StoragePath.Combine(RandomGenerator.RandomString, RandomGenerator.RandomString, RandomGenerator.RandomString);
try
{
await _storage.WriteTextAsync(id1, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id2, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id3, RandomGenerator.RandomString);
IEnumerable<BlobId> items = await _storage.ListAsync(new ListOptions { Recurse = true });
}
catch(NotSupportedException)
{
//it ok for providers not to support hierarchy
}
}
public async Task Rename_File_Renames()
{
string prefix = RandomBlobPath();
string file = StoragePath.Combine(prefix, "1");
try
{
await _storage.WriteTextAsync(file, "test");
await _storage.RenameAsync(file, StoragePath.Combine(prefix, "2"));
IReadOnlyCollection <Blob> list = await _storage.ListAsync(prefix);
Assert.Single(list);
Assert.True(list.First().Name == "2");
}
catch (NotSupportedException)
{
}
}
private BlobId ToBlobId(IListBlobItem blob, bool attachMetadata)
{
BlobId id;
if (blob is CloudBlockBlob blockBlob)
{
string fullName = _prependContainerName
? StoragePath.Combine(_container.Name, blockBlob.Name)
: blockBlob.Name;
id = new BlobId(fullName, BlobItemKind.File);
}
else if (blob is CloudAppendBlob appendBlob)
{
string fullName = _prependContainerName
? StoragePath.Combine(_container.Name, appendBlob.Name)
: appendBlob.Name;
id = new BlobId(fullName, BlobItemKind.File);
}
else if (blob is CloudBlobDirectory dirBlob)
{
string fullName = _prependContainerName
? StoragePath.Combine(_container.Name, dirBlob.Prefix)
: dirBlob.Prefix;
id = new BlobId(fullName, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {blob.GetType()}");
}
//attach metadata if we can
if (attachMetadata && blob is CloudBlob cloudBlob)
{
id.Meta = AzureUniversalBlobStorageProvider.GetblobMeta(cloudBlob);
}
return(id);
}
public void Combining()
{
StoragePath path = new StoragePath();
path += "C:/";
Assert.AreEqual(@"C:", path.ToString());
path += (StoragePath)(new Uri("some/other", UriKind.RelativeOrAbsolute));
Assert.AreEqual(@"C:\some\other", path.ToString());
path = path.Combine("some///malformed\\\\other.some").SetExtension(".txt");
Assert.AreEqual(@"C:\some\other\some\malformed\other.txt", path.ToString());
path = path.SetExtension("txt");
Assert.AreEqual(@"C:\some\other\some\malformed\other.txt", path.ToString());
Assert.AreEqual("other.txt", path.ScopeToName().ToString());
Assert.AreEqual("other", path.ScopeToNameWithoutExtension().ToString());
Assert.AreEqual(@"C:\some\other\some\malformed", path.ParentDirectory.ToString());
Assert.AreEqual(@"C:\some\other\some", path.GetNthParentDirectory(2).ToString());
}
public async Task Acl_get_with_upn()
{
string path = StoragePath.Combine(Filesystem, Guid.NewGuid().ToString());
string userId = _settings.AzureDataLakeGen2TestObjectId;
//write something
await _storage.WriteTextAsync(path, "perm?");
//check that user has no permissions
AccessControl access = await _storage.GetAccessControlAsync(path, true);
Assert.DoesNotContain(access.Acl, x => x.Identity == userId);
//assign user a write permission
access.Acl.Add(new AclEntry(ObjectType.User, userId, false, true, false));
await _storage.SetAccessControlAsync(path, access);
//check user has permissions now
access = await _storage.GetAccessControlAsync(path, true);
Assert.True(access.Acl.First().Identity.Contains('@'));
}
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken)
{
//get filesystem name and folder path
string[] parts = StoragePath.Split(path);
string fs = parts[0];
string relativePath = StoragePath.Combine(parts.Skip(1));
PathList list;
try
{
list = await _api.ListPathAsync(fs, relativePath, recursive : options.Recurse).ConfigureAwait(false);
}
catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
// specified path is not found, nothing serious
return(new List <Blob>());
}
IEnumerable <Blob> result = list.Paths.Select(p => LConvert.ToBlob(fs, p));
if (options.FilePrefix != null)
{
result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix));
}
if (options.BrowseFilter != null)
{
result = result.Where(b => options.BrowseFilter(b));
}
if (maxResults != null)
{
result = result.Take(maxResults.Value);
}
return(result.ToList());
}
/// <summary>
/// Handle
/// </summary>
/// <param name="command"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <CommandResponse> Handle(UploadMultipleAttachmentCommand command, CancellationToken cancellationToken)
{
CommandResponse response = new CommandResponse()
{
UploadedAttachments = new List <AttachmentModel>()
};
foreach (var commandFile in command.Files)
{
AttachmentModel attachment = new
AttachmentModel();
// prepare the upload
string originalName = System.IO.Path.GetFileName(commandFile.FileName);
string extension = System.IO.Path.GetExtension(commandFile.FileName).ToLower();
string storedAsFileName = $"{attachment.Id}{extension}";
string fileFullPath = StoragePath.Combine("attachments",
storedAsFileName);
// upload
string fullPath = await _storageService.UploadFile(commandFile, fileFullPath);
attachment.FilePath = fullPath;
// prepare response
attachment.FileName = storedAsFileName;
attachment.FileType = extension;
attachment.ContentType = commandFile.ContentType;
attachment.OriginalFileName = originalName;
attachment.Name = originalName;
attachment.Description = $"Attachment {originalName}";
response.UploadedAttachments.Add(attachment);
}
return(response);
}
private Blob ToBlob(IListBlobItem nativeBlob)
{
Blob blob;
if (nativeBlob is CloudBlockBlob blockBlob)
{
string fullName = StoragePath.Combine(_container.Name, blockBlob.Name);
blob = new Blob(fullName, BlobItemKind.File);
}
else if (nativeBlob is CloudAppendBlob appendBlob)
{
string fullName = StoragePath.Combine(_container.Name, appendBlob.Name);
blob = new Blob(fullName, BlobItemKind.File);
}
else if (nativeBlob is CloudBlobDirectory dirBlob)
{
string fullName = StoragePath.Combine(_container.Name, dirBlob.Prefix);
blob = new Blob(fullName, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {nativeBlob.GetType()}");
}
//attach metadata if we can
if (nativeBlob is CloudBlob cloudBlob)
{
//no need to fetch attributes, parent request includes the details
//await cloudBlob.FetchAttributesAsync().ConfigureAwait(false);
AzureUniversalBlobStorageProvider.AttachBlobMeta(blob, cloudBlob);
}
return(blob);
}
public async Task List_FilesInFolder_Recursive()
{
string folderPath = RandomBlobPath();
string id1 = StoragePath.Combine(folderPath, "1.txt");
string id2 = StoragePath.Combine(folderPath, "sub", "2.txt");
string id3 = StoragePath.Combine(folderPath, "sub", "3.txt");
try
{
await _storage.WriteTextAsync(id1, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id2, RandomGenerator.RandomString);
await _storage.WriteTextAsync(id3, RandomGenerator.RandomString);
IReadOnlyCollection <Blob> items = await _storage.ListAsync(recurse : true, folderPath : folderPath);
Assert.Equal(4, items.Count); //1.txt + sub (folder) + 2.txt + 3.txt
}
catch (NotSupportedException)
{
//it ok for providers not to support hierarchy
}
}
public async Task Rename_Folder_Renames()
{
string prefix = RandomBlobPath();
string file1 = StoragePath.Combine(prefix, "old", "1.txt");
string file11 = StoragePath.Combine(prefix, "old", "1", "1.txt");
string file111 = StoragePath.Combine(prefix, "old", "1", "1", "1.txt");
try
{
await _storage.WriteTextAsync(file1, string.Empty);
}
catch (NotSupportedException)
{
return;
}
await _storage.WriteTextAsync(file11, string.Empty);
await _storage.WriteTextAsync(file111, string.Empty);
await _storage.RenameAsync(StoragePath.Combine(prefix, "old"), StoragePath.Combine(prefix, "new"));
IReadOnlyCollection <Blob> list = await _storage.ListAsync(prefix);
}
/// <summary>
/// Mounts a storage to virtual path
/// </summary>
/// <param name="path"></param>
/// <param name="storage"></param>
public void Mount(string path, IBlobStorage storage)
{
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
if (storage is null)
{
throw new ArgumentNullException(nameof(storage));
}
path = StoragePath.Normalize(path);
_mountPoints.Add(new Blob(path)
{
Tag = storage
});
string absPath = null;
string[] parts = StoragePath.Split(path);
if (parts.Length == 0) //mount at root
{
MountPath(path, storage, true);
}
else
{
for (int i = 0; i < parts.Length; i++)
{
absPath = StoragePath.Combine(absPath, parts[i]);
MountPath(absPath, storage, i == parts.Length - 1);
}
}
}
private async Task <(BlobContainerClient, string)> GetPartsAsync(string fullPath, bool createContainer = true)
{
GenericValidation.CheckBlobFullPath(fullPath);
fullPath = StoragePath.Normalize(fullPath);
if (fullPath == null)
{
throw new ArgumentNullException(nameof(fullPath));
}
string containerName, relativePath;
if (_containerName == null)
{
string[] parts = StoragePath.Split(fullPath);
if (parts.Length == 1)
{
containerName = parts[0];
relativePath = string.Empty;
}
else
{
containerName = parts[0];
relativePath = StoragePath.Combine(parts.Skip(1)).Substring(1);
}
}
else
{
containerName = _containerName;
relativePath = fullPath;
}
if (!_containerNameToContainerClient.TryGetValue(containerName, out BlobContainerClient container))
{
container = _client.GetBlobContainerClient(containerName);
if (_containerName == null)
{
try
{
//check if container exists
await container.GetPropertiesAsync().ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "ContainerNotFound")
{
if (createContainer)
{
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
}
else
{
return(null, null);
}
}
}
_containerNameToContainerClient[containerName] = container;
}
return(container, relativePath);
}
public void Combine_theory(string expected, string[] parts)
{
Assert.Equal(expected, StoragePath.Combine(parts));
}
/// <summary>
/// moves file from source to destination location
/// </summary>
/// <param name="sourceFilepath"></param>
/// <param name="destinationFilepath"></param>
/// <param name="overwrite"></param>
/// <returns></returns>
public DbFile Move(string sourceFilepath, string destinationFilepath, bool overwrite = false)
{
if (string.IsNullOrWhiteSpace(sourceFilepath))
{
throw new ArgumentException("sourceFilepath cannot be null or empty");
}
if (string.IsNullOrWhiteSpace(destinationFilepath))
{
throw new ArgumentException("destinationFilepath cannot be null or empty");
}
sourceFilepath = sourceFilepath.ToLowerInvariant();
destinationFilepath = destinationFilepath.ToLowerInvariant();
if (!sourceFilepath.StartsWith(FOLDER_SEPARATOR))
{
sourceFilepath = FOLDER_SEPARATOR + sourceFilepath;
}
if (!destinationFilepath.StartsWith(FOLDER_SEPARATOR))
{
destinationFilepath = FOLDER_SEPARATOR + destinationFilepath;
}
var srcFile = Find(sourceFilepath);
var destFile = Find(destinationFilepath);
if (srcFile == null)
{
throw new Exception("Source file cannot be found.");
}
if (destFile != null && overwrite == false)
{
throw new Exception("Destination file already exists and no overwrite specified.");
}
using (var connection = CurrentContext.CreateConnection())
{
try
{
connection.BeginTransaction();
if (destFile != null && overwrite)
{
Delete(destFile.FilePath);
}
var command = connection.CreateCommand(@"UPDATE files SET filepath = @filepath WHERE id = @id");
command.Parameters.Add(new NpgsqlParameter("@id", srcFile.Id));
command.Parameters.Add(new NpgsqlParameter("@filepath", destinationFilepath));
command.ExecuteNonQuery();
if (ErpSettings.EnableCloudBlobStorage)
{
var srcPath = StoragePath.Combine(StoragePath.RootFolderPath, sourceFilepath);
var destinationPath = StoragePath.Combine(StoragePath.RootFolderPath, destinationFilepath);
using (IBlobStorage storage = GetBlobStorage())
{
using (Stream original = storage.OpenReadAsync(srcPath).Result)
{
if (original != null)
{
storage.WriteAsync(destinationPath, original).Wait();
storage.DeleteAsync(sourceFilepath).Wait();
}
}
}
}
else if (ErpSettings.EnableFileSystemStorage)
{
var srcFileName = Path.GetFileName(sourceFilepath);
var destFileName = Path.GetFileName(destinationFilepath);
if (srcFileName != destFileName)
{
var fsSrcFilePath = GetFileSystemPath(srcFile);
srcFile.FilePath = destinationFilepath;
var fsDestFilePath = GetFileSystemPath(srcFile);
File.Move(fsSrcFilePath, fsDestFilePath);
}
}
connection.CommitTransaction();
return(Find(destinationFilepath));
}
catch
{
connection.RollbackTransaction();
throw;
}
}
}
private string GenerateBlobPath(QueueMessage message)
{
return(StoragePath.Combine("message", Guid.NewGuid().ToString()));
}
private static string GetBlobName(string partitionId)
{
return(StoragePath.Combine("partition", $"{partitionId}.json"));
}
private string GetFullName(string name)
{
return(_prependContainerName
? StoragePath.Combine(_container.Name, name)
: name);
}
public async Task ListFolder(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id;
if (blob is CloudBlockBlob blockBlob)
{
id = new BlobId(blockBlob.Name, BlobItemKind.File);
}
else if (blob is CloudAppendBlob appendBlob)
{
id = new BlobId(appendBlob.Name, BlobItemKind.File);
}
else if (blob is CloudBlobDirectory dirBlob)
{
id = new BlobId(dirBlob.Prefix, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {blob.GetType()}");
}
if (options.IsMatch(id))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
foreach (BlobId folderId in folderIds)
{
await ListFolder(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken);
}
}
}
public async Task <IReadOnlyCollection <BlobId> > ListAsync(ListOptions options, CancellationToken cancellationToken = default)
{
if (options == null)
{
options = new ListOptions();
}
var result = new List <BlobId>();
var containers = new List <CloudBlobContainer>();
if (_fixedContainer != null)
{
containers.Add(_fixedContainer);
}
else if (options.FolderPath == null)
{
// list all of the containers
containers.AddRange(await GetCloudBlobContainersAsync(cancellationToken));
//represent containers as folders in the result
result.AddRange(containers.Select(c => new BlobId(c.Name, BlobItemKind.Folder)));
}
else
{
(CloudBlobContainer container, string path) = await GetPartsAsync(options.FolderPath, false);
if (container == null)
{
return(new List <BlobId>());
}
options.FolderPath = path; //scan from subpath now
containers.Add(container);
//add container as search result
result.Add(new BlobId(container.Name, BlobItemKind.Folder));
}
foreach (CloudBlobContainer container in containers)
{
var browser = new AzureBlobDirectoryBrowser(container);
IReadOnlyCollection <BlobId> containerBlobs = await browser.ListFolderAsync(options, cancellationToken);
if (containerBlobs.Count > 0)
{
if (_fixedContainer == null)
{
result.AddRange(containerBlobs.Select(bid => new BlobId(StoragePath.Combine(container.Name, bid.FullPath), bid.Kind)));
}
else
{
result.AddRange(containerBlobs);
}
}
if (options.MaxResults != null && result.Count >= options.MaxResults.Value)
{
break;
}
}
return(result);
}