public async Task ErrorsAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a container named "sample-filesystem-errorsasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-errors"));
await filesystem.CreateAsync();
try
{
// Try to create the filesystem again
await filesystem.CreateAsync();
}
catch (RequestFailedException ex)
when(ex.ErrorCode == "ContainerAlreadyExists")
{
// Ignore any errors if the filesystem already exists
}
catch (RequestFailedException ex)
{
Assert.Fail($"Unexpected error: {ex}");
}
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
public async Task CreateFileClientAsync_Directory()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Create a DataLake Filesystem
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
await filesystem.CreateAsync();
//Create a DataLake Directory
DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory"));
await directory.CreateAsync();
// Create a DataLake File using a DataLake Directory
DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Cleanup
await filesystem.DeleteAsync();
}
/// <summary>
/// The <see cref="CreateFileSystemAsync"/> operation creates a new
/// file system under the specified account. If the file system with the
/// same name already exists, the operation fails.
///
/// For more information, see
/// <see href="https://docs.microsoft.com/rest/api/storageservices/create-container">
/// Create Container</see>.
/// </summary>
/// <param name="fileSystemName">
/// The name of the file system to create.
/// </param>
/// <param name="publicAccessType">
/// Optionally specifies whether data in the file system may be accessed
/// publicly and the level of access. <see cref="PublicAccessType.FileSystem"/>
/// specifies full public read access for file system and path data.
/// Clients can enumerate paths within the file system via anonymous
/// request, but cannot enumerate file systems within the storage
/// account. <see cref="PublicAccessType.Path"/> specifies public
/// read access for paths. Path data within this file system can be
/// read via anonymous request, but file system data is not available.
/// Clients cannot enumerate paths within the file system via anonymous
/// request. <see cref="PublicAccessType.None"/> specifies that the
/// file system data is private to the account owner.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this file system.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{FileSystemClient}"/> referencing the
/// newly created file system.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public virtual async Task <Response <DataLakeFileSystemClient> > CreateFileSystemAsync(
string fileSystemName,
PublicAccessType publicAccessType = PublicAccessType.None,
Metadata metadata = default,
CancellationToken cancellationToken = default)
{
DiagnosticScope scope = ClientDiagnostics.CreateScope($"{nameof(DataLakeServiceClient)}.{nameof(CreateFileSystem)}");
try
{
scope.Start();
DataLakeFileSystemClient fileSystem = GetFileSystemClient(fileSystemName);
Response <FileSystemInfo> response = await fileSystem.CreateAsync(publicAccessType, metadata, cancellationToken).ConfigureAwait(false);
return(Response.FromValue(fileSystem, response.GetRawResponse()));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
public async Task UndeleteFileSystemAsync()
{
// Arrange
DataLakeServiceClient service = GetServiceClient_SharedKey();
string fileSystemName = GetNewFileSystemName();
DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName));
await fileSystem.CreateAsync();
await fileSystem.DeleteAsync();
IList <FileSystemItem> fileSystems = await service.GetFileSystemsAsync(states : FileSystemStates.Deleted).ToListAsync();
FileSystemItem fileSystemItem = fileSystems.Where(c => c.Name == fileSystemName).FirstOrDefault();
// It takes some time for the FileSystem to be deleted.
await Delay(30000);
// Act
Response <DataLakeFileSystemClient> response = await service.UndeleteFileSystemAsync(
fileSystemItem.Name,
fileSystemItem.VersionId,
GetNewFileSystemName());
// Assert
await response.Value.GetPropertiesAsync();
// Cleanup
await response.Value.DeleteAsync();
}
public async Task Ctor_ConnectionString_GenerateSas()
{
// Arrage
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={TestConfigHierarchicalNamespace.AccountName};AccountKey={TestConfigHierarchicalNamespace.AccountKey};EndpointSuffix=core.windows.net";
DataLakeServiceClient serviceClient = InstrumentClient(new DataLakeServiceClient(connectionString, GetOptions()));
string fileSystemName = GetNewFileSystemName();
DataLakeFileSystemClient fileSystem = InstrumentClient(serviceClient.GetFileSystemClient(fileSystemName));
try
{
await fileSystem.CreateAsync();
Uri accountSasUri = serviceClient.GenerateAccountSasUri(
AccountSasPermissions.All,
Recording.UtcNow.AddDays(1),
AccountSasResourceTypes.All);
DataLakeServiceClient sasServiceClient = InstrumentClient(new DataLakeServiceClient(accountSasUri, GetOptions()));
DataLakeFileSystemClient sasFileSystem = InstrumentClient(sasServiceClient.GetFileSystemClient(fileSystemName));
// Act
await sasFileSystem.GetPropertiesAsync();
}
// Cleanup
finally
{
await fileSystem.DeleteAsync();
}
}
public async Task AppendAsync()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength));
string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength));
string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Get a reference to a FileSystemClient
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Create the file
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Append data to an existing DataLake File. Append is currently limited to 4000 MB per call.
// To upload a large file all at once, consider using UploadAsync() instead.
await file.AppendAsync(File.OpenRead(sampleFileContentPart1), 0);
await file.AppendAsync(File.OpenRead(sampleFileContentPart2), contentLength);
await file.AppendAsync(File.OpenRead(sampleFileContentPart3), contentLength * 2);
await file.FlushAsync(contentLength * 3);
// Verify the contents of the file
PathProperties properties = await file.GetPropertiesAsync();
Assert.AreEqual(contentLength * 3, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public virtual async Task <Response <DataLakeFileSystemClient> > CreateFileSystemAsync(
string fileSystemName,
PublicAccessType publicAccessType = PublicAccessType.None,
Metadata metadata = default,
CancellationToken cancellationToken = default)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClient(fileSystemName);
Response <FileSystemInfo> response = await fileSystem.CreateAsync(publicAccessType, metadata, cancellationToken).ConfigureAwait(false);
return(Response.FromValue(fileSystem, response.GetRawResponse()));
}
public async Task ReadAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-readasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
await file.CreateAsync();
await file.AppendAsync(File.OpenRead(originalPath), 0);
await file.FlushAsync(SampleFileContent.Length);
// Download the DataLake file's contents and save it to a file
Response <FileDownloadInfo> fileContents = await file.ReadAsync();
using (FileStream stream = File.OpenWrite(downloadPath))
{
fileContents.Value.Content.CopyTo(stream);
}
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task RenameAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-rename" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-rename"));
await filesystem.CreateAsync();
try
{
// Create a DataLake Directory to rename it later
DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
await directoryClient.CreateAsync();
// Rename directory with new path/name and verify by making a service call (e.g. GetProperties)
DataLakeDirectoryClient renamedDirectoryClient = await directoryClient.RenameAsync("sample-directory2");
PathProperties directoryPathProperties = await renamedDirectoryClient.GetPropertiesAsync();
// Delete the sample directory using the new path/name
await filesystem.DeleteDirectoryAsync("sample-directory2");
// Create a DataLake file.
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
await fileClient.CreateAsync();
// Rename file with new path/name and verify by making a service call (e.g. GetProperties)
DataLakeFileClient renamedFileClient = await fileClient.RenameAsync("sample-file2");
PathProperties pathProperties = await renamedFileClient.GetPropertiesAsync();
// Delete the sample directory using the new path/name
await filesystem.DeleteFileAsync("sample-file2");
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public static async Task <DisposingFileSystem> GetNewFileSystem(
this DataLakeClientBuilder clientBuilder,
DataLakeServiceClient service = default,
string fileSystemName = default,
IDictionary <string, string> metadata = default,
PublicAccessType?publicAccessType = default,
bool premium = default,
bool hnsEnabled = true)
{
fileSystemName ??= clientBuilder.GetNewFileSystemName();
service ??= hnsEnabled?clientBuilder.GetServiceClient_Hns() : clientBuilder.GetServiceClient_NonHns();
if (publicAccessType == default)
{
publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem;
}
DataLakeFileSystemClient fileSystem = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(service.GetFileSystemClient(fileSystemName));
// due to a service issue, if the initial container creation request times out, subsequent requests
// can return a ContainerAlreadyExists code even though the container doesn't really exist.
// we delay until after the service cache timeout and then attempt to create the container one more time.
// If this attempt still fails, we mark the test as inconclusive.
// TODO Remove this handling after the service bug is fixed https://github.com/Azure/azure-sdk-for-net/issues/9399
try
{
await StorageTestBase <DataLakeTestEnvironment> .RetryAsync(
clientBuilder.AzureCoreRecordedTestBase.Recording.Mode,
async() => await fileSystem.CreateAsync(metadata: metadata, publicAccessType: publicAccessType.Value),
ex => ex.ErrorCode == Blobs.Models.BlobErrorCode.ContainerAlreadyExists,
retryDelay : TestConstants.DataLakeRetryDelay,
retryAttempts : 1);
}
catch (RequestFailedException storageRequestFailedException)
when(storageRequestFailedException.ErrorCode == Blobs.Models.BlobErrorCode.ContainerAlreadyExists)
{
// if we still get this error after retrying, mark the test as inconclusive
TestContext.Out.WriteLine(
$"{TestContext.CurrentContext.Test.Name} is inconclusive due to hitting " +
$"the DataLake service bug described in https://github.com/Azure/azure-sdk-for-net/issues/9399");
Assert.Inconclusive(); // passing the message in Inconclusive call doesn't show up in Console output.
}
return(new DisposingFileSystem(fileSystem));
}
public async Task Ctor_ConnectionString_RoundTrip()
{
// Arrage
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={TestConfigHierarchicalNamespace.AccountName};AccountKey={TestConfigHierarchicalNamespace.AccountKey};EndpointSuffix=core.windows.net";
DataLakeServiceClient serviceClient = InstrumentClient(new DataLakeServiceClient(connectionString, GetOptions()));
DataLakeFileSystemClient fileSystem = InstrumentClient(serviceClient.GetFileSystemClient(GetNewFileSystemName()));
// Act
try
{
await fileSystem.CreateAsync();
}
// Cleanup
finally
{
await fileSystem.DeleteAsync();
}
}
public async Task ReadToAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-readasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
await file.UploadAsync(File.OpenRead(originalPath));
// Download the DataLake file's contents directly to a file.
// For larger files, ReadToAsync() will download the file in multiple parallel requests.
await file.ReadToAsync(downloadPath);
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task <DisposingFileSystem> GetNewFileSystem(
DataLakeServiceClient service = default,
string fileSystemName = default,
IDictionary <string, string> metadata = default,
PublicAccessType publicAccessType = PublicAccessType.None,
bool premium = default)
{
fileSystemName ??= GetNewFileSystemName();
service ??= GetServiceClient_SharedKey();
if (publicAccessType == PublicAccessType.None)
{
publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem;
}
DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName));
await fileSystem.CreateAsync(metadata : metadata, publicAccessType : publicAccessType);
return(new DisposingFileSystem(fileSystem));
}
public async Task AppendAsync_Simple()
{
// Create Sample File to read content from
string sampleFilePath = CreateTempFile(SampleFileContent);
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
await filesystem.CreateAsync();
try
{
// Create a file
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Append data to the DataLake File
await file.AppendAsync(File.OpenRead(sampleFilePath), 0);
await file.FlushAsync(SampleFileContent.Length);
// Verify the contents of the file
PathProperties properties = await file.GetPropertiesAsync();
Assert.AreEqual(SampleFileContent.Length, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task UploadAsync()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Get a reference to a FileSystemClient
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Upload content to the file. When using the Upload API, you don't need to create the file first.
// If the file already exists, it will be overwritten.
// For larger files, Upload() will upload the file in multiple parallel requests.
await file.UploadAsync(File.OpenRead(sampleFileContent));
// Verify the contents of the file
PathProperties properties = await file.GetPropertiesAsync();
Assert.AreEqual(contentLength, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task ListAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-listasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-list"));
await filesystem.CreateAsync();
try
{
// Upload a couple of directories so we have something to list
await filesystem.CreateDirectoryAsync("sample-directory1");
await filesystem.CreateDirectoryAsync("sample-directory2");
await filesystem.CreateDirectoryAsync("sample-directory3");
// List all the directories
List <string> names = new List <string>();
await foreach (PathItem pathItem in filesystem.GetPathsAsync())
{
names.Add(pathItem.Name);
}
Assert.AreEqual(3, names.Count);
Assert.Contains("sample-directory1", names);
Assert.Contains("sample-directory2", names);
Assert.Contains("sample-directory3", names);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task GetFileSystemsAsync_Deleted()
{
// Arrange
DataLakeServiceClient service = GetServiceClient_SharedKey();
string fileSystemName = GetNewFileSystemName();
DataLakeFileSystemClient fileSystemClient = InstrumentClient(service.GetFileSystemClient(fileSystemName));
await fileSystemClient.CreateAsync();
await fileSystemClient.DeleteAsync();
// Act
IList <FileSystemItem> fileSystems = await service.GetFileSystemsAsync(states : FileSystemStates.Deleted).ToListAsync();
FileSystemItem fileSystemItem = fileSystems.Where(c => c.Name == fileSystemName).FirstOrDefault();
// Assert
Assert.IsTrue(fileSystemItem.IsDeleted);
Assert.IsNotNull(fileSystemItem.VersionId);
Assert.IsNotNull(fileSystemItem.Properties.DeletedOn);
Assert.IsNotNull(fileSystemItem.Properties.RemainingRetentionDays);
}
public async Task <DisposingFileSystem> GetNewFileSystem(
DataLakeServiceClient service = default,
string fileSystemName = default,
IDictionary <string, string> metadata = default,
PublicAccessType publicAccessType = PublicAccessType.None,
bool premium = default)
{
fileSystemName ??= GetNewFileSystemName();
service ??= GetServiceClient_SharedKey();
if (publicAccessType == PublicAccessType.None)
{
publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem;
}
DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName));
// due to a service issue, if the initial container creation request times out, subsequent requests
// can return a ContainerAlreadyExists code even though the container doesn't really exist.
// we delay until after the service cache timeout and then attempt to create the container one more time.
// If this attempt fails, we let the exception propagate.
// TODO Note this issue will be fixed in the 72 rollout. After that time, this try/catch can be removed.
try
{
await RetryAsync(
async() => await fileSystem.CreateAsync(metadata: metadata, publicAccessType: publicAccessType),
ex => ex.ErrorCode == Constants.Blob.Container.AlreadyExists,
retryDelay : TestConstants.DataLakeRetryDelay,
retryAttempts : 1);
}
catch (RequestFailedException storageRequestFailedException)
when(storageRequestFailedException.ErrorCode == Constants.Blob.Container.AlreadyExists)
{
// if we get this error after retrying once, that means the container really does
// exist, since the retry attempt occurred after the cache timeout.
// so we just swallow this error and continue on in the test.
}
return(new DisposingFileSystem(fileSystem));
}
public async Task AnonymousAuthAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string accountName = StorageAccountName;
string accountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey);
// Get a reference to a service Client
DataLakeServiceClient service = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem"
DataLakeFileSystemClient filesystem = service.GetFileSystemClient(Randomize("sample-filesystem"));
try
{
// Create a file that can be accessed publicly
await filesystem.CreateAsync(PublicAccessType.FileSystem);
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Append data to the file
string fileContent = "File Content";
await file.AppendAsync(new MemoryStream(Encoding.UTF8.GetBytes(fileContent)), 0);
await file.FlushAsync(fileContent.Length);
// Anonymously access a blob given its URI
Uri endpoint = file.Uri;
DataLakeFileClient anonymous = new DataLakeFileClient(endpoint);
// Make a service request to verify we've succesfully authenticated
await anonymous.GetPropertiesAsync();
}
finally
{
await filesystem.DeleteAsync();
}
}
public async Task SetGetAclsAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = NamespaceStorageAccountName;
string storageAccountKey = NamespaceStorageAccountKey;
Uri serviceUri = NamespaceBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-aclasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-acl"));
await filesystem.CreateAsync();
try
{
// Create a DataLake file so we can set the Access Controls on the files
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
await fileClient.CreateAsync();
IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r--,mask::rwx,other::---");
// Set Access Control List
await fileClient.SetAccessControlListAsync(accessControlList);
// Get Access Control List
PathAccessControl accessControlResponse = await fileClient.GetAccessControlAsync();
// Check Access Control permissions
Assert.AreEqual(
PathAccessControlExtensions.ToAccessControlListString(accessControlList),
PathAccessControlExtensions.ToAccessControlListString(accessControlResponse.AccessControlList.ToList()));
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task SetPermissionsAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = NamespaceStorageAccountName;
string storageAccountKey = NamespaceStorageAccountKey;
Uri serviceUri = NamespaceBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-aclasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-perasync"));
await filesystem.CreateAsync();
try
{
// Create a DataLake file so we can set the Access Controls on the files
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
await fileClient.CreateAsync();
// Set the Permissions of the file
PathPermissions pathPermissions = PathPermissions.ParseSymbolicPermissions("rwxrwxrwx");
await fileClient.SetPermissionsAsync(permissions : pathPermissions);
// Get Access Control List
PathAccessControl accessControlResponse = await fileClient.GetAccessControlAsync();
// Check Access Control permissions
Assert.AreEqual(pathPermissions.ToSymbolicPermissions(), accessControlResponse.Permissions.ToSymbolicPermissions());
Assert.AreEqual(pathPermissions.ToOctalPermissions(), accessControlResponse.Permissions.ToOctalPermissions());
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task GetPropertiesAsync()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-rename" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
await filesystem.CreateAsync();
try
{
// Create a DataLake Directory to rename it later
DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
await directoryClient.CreateAsync();
// Get Properties on a Directory
PathProperties directoryPathProperties = await directoryClient.GetPropertiesAsync();
// Create a DataLake file
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
await fileClient.CreateAsync();
// Get Properties on a File
PathProperties filePathProperties = await fileClient.GetPropertiesAsync();
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public DisposingFileSystem(DataLakeFileSystemClient fileSystem, IDictionary <string, string> metadata, Models.PublicAccessType publicAccessType = default)
{
fileSystem.CreateAsync(metadata: metadata, publicAccessType: publicAccessType).Wait();
FileSystemClient = fileSystem;
}
public async Task TraverseAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-traverseasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-traverse"));
await filesystem.CreateAsync();
try
{
// Create a bunch of directories and files within the directories
DataLakeDirectoryClient first = await filesystem.CreateDirectoryAsync("first");
await first.CreateSubDirectoryAsync("a");
await first.CreateSubDirectoryAsync("b");
DataLakeDirectoryClient second = await filesystem.CreateDirectoryAsync("second");
await second.CreateSubDirectoryAsync("c");
await second.CreateSubDirectoryAsync("d");
await filesystem.CreateDirectoryAsync("third");
DataLakeDirectoryClient fourth = await filesystem.CreateDirectoryAsync("fourth");
DataLakeDirectoryClient deepest = await fourth.CreateSubDirectoryAsync("e");
// Upload a DataLake file named "file"
DataLakeFileClient file = deepest.GetFileClient("file");
await file.CreateAsync();
using (FileStream stream = File.OpenRead(originalPath))
{
await file.AppendAsync(stream, 0);
}
// Keep track of all the names we encounter
List <string> names = new List <string>();
await foreach (PathItem pathItem in filesystem.GetPathsAsync(recursive: true))
{
names.Add(pathItem.Name);
}
// Verify we've seen everything
Assert.AreEqual(10, names.Count);
Assert.Contains("first", names);
Assert.Contains("second", names);
Assert.Contains("third", names);
Assert.Contains("fourth", names);
Assert.Contains("first/a", names);
Assert.Contains("first/b", names);
Assert.Contains("second/c", names);
Assert.Contains("second/d", names);
Assert.Contains("fourth/e", names);
Assert.Contains("fourth/e/file", names);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
