public DataLakeSasQueryParameters GetNewDataLakeServiceSasCredentialsPath(string fileSystemName, string path, StorageSharedKeyCredential sharedKeyCredentials = default)
{
var builder = new DataLakeSasBuilder
{
FileSystemName = fileSystemName,
Path = path,
Protocol = SasProtocol.None,
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(+1),
IPRange = new SasIPRange(IPAddress.None, IPAddress.None)
};
builder.SetPermissions(
DataLakeSasPermissions.Read |
DataLakeSasPermissions.Add |
DataLakeSasPermissions.Create |
DataLakeSasPermissions.Delete |
DataLakeSasPermissions.Write);
return(builder.ToSasQueryParameters(sharedKeyCredentials ?? GetNewSharedKeyCredentials()));
}
public DataLakeSasQueryParameters GetNewDataLakeServiceIdentitySasCredentialsPath(string fileSystemName, string path, UserDelegationKey userDelegationKey, string accountName)
{
var builder = new DataLakeSasBuilder
{
FileSystemName = fileSystemName,
Path = path,
Protocol = SasProtocol.None,
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(+1),
IPRange = new SasIPRange(IPAddress.None, IPAddress.None)
};
builder.SetPermissions(
DataLakeSasPermissions.Read |
DataLakeSasPermissions.Add |
DataLakeSasPermissions.Create |
DataLakeSasPermissions.Delete |
DataLakeSasPermissions.Write);
return(builder.ToSasQueryParameters(userDelegationKey, accountName));
}
public async Task DataLakeSasBuilder_DirectoryDepth_SharedKey()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory = await directory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory2 = await subdirectory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory3 = await subdirectory2.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeFileClient file = await subdirectory3.CreateFileAsync(GetNewFileName());
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
Path = subdirectory3.Path,
IsDirectory = true
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(TestConfigHierarchicalNamespace.AccountName, TestConfigHierarchicalNamespace.AccountKey);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(subdirectory3.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(sharedKeyCredential)
};
DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await sasDirectoryClient.ExistsAsync();
}
public async Task DataLakeSasBuilder_CorrelationId()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
CorrelationId = Recording.Random.NewGuid().ToString()
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.List);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await foreach (PathItem pathItem in sasFileSystemClient.GetPathsAsync())
{
// Just make sure the call succeeds.
}
}
public async Task DataLakeSasBuilder_AgentObjectId_Error()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
AgentObjectId = Recording.Random.NewGuid().ToString()
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
sasFileSystemClient.ExistsAsync(),
e => Assert.IsNotNull(e.ErrorCode));
}
public async Task DataLakeSasBuilder_DirectoryRawPermissions_Exists()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
Path = directory.Path,
IsDirectory = true
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(directory.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await sasDirectoryClient.ExistsAsync();
}
public void GenerateSas_Builder()
{
var constants = TestConstants.Create(this);
string fileSystemName = GetNewFileSystemName();
string path = GetNewFileName();
DataLakeSasPermissions permissions = DataLakeSasPermissions.Read;
DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
DateTimeOffset startsOn = Recording.UtcNow.AddHours(-1);
var blobEndpoint = new Uri("http://127.0.0.1/" + constants.Sas.Account + "/" + fileSystemName + "/" + path);
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(
blobEndpoint,
constants.Sas.SharedKeyCredential,
GetOptions()));
DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder(permissions, expiresOn)
{
FileSystemName = fileSystemName,
Path = path,
StartsOn = startsOn
};
// Act
Uri sasUri = pathClient.GenerateSasUri(sasBuilder);
// Assert
DataLakeUriBuilder expectedUri = new DataLakeUriBuilder(blobEndpoint);
DataLakeSasBuilder sasBuilder2 = new DataLakeSasBuilder(permissions, expiresOn)
{
FileSystemName = fileSystemName,
Path = path,
StartsOn = startsOn
};
expectedUri.Sas = sasBuilder2.ToSasQueryParameters(constants.Sas.SharedKeyCredential);
Assert.AreEqual(expectedUri.ToUri().ToString(), sasUri.ToString());
}
// </Snippet_ListBlobsWithSasAsync>
#endregion
#region
// <Snippet_GetUserDelegationSasDirectory>
async static Task <Uri> GetUserDelegationSasDirectory(DataLakeDirectoryClient directoryClient)
{
try
{
// Get service endpoint from the directory URI.
DataLakeUriBuilder dataLakeServiceUri = new DataLakeUriBuilder(directoryClient.Uri)
{
FileSystemName = null,
DirectoryOrFilePath = null
};
// Get service client.
DataLakeServiceClient dataLakeServiceClient =
new DataLakeServiceClient(dataLakeServiceUri.ToUri(),
new DefaultAzureCredential());
// Get a user delegation key that's valid for seven days.
// You can use the key to generate any number of shared access signatures
// over the lifetime of the key.
Azure.Storage.Files.DataLake.Models.UserDelegationKey userDelegationKey =
await dataLakeServiceClient.GetUserDelegationKeyAsync(DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow.AddDays(7));
// Create a SAS token that's valid for seven days.
DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder()
{
// Specify the file system name and path, and indicate that
// the client object points to a directory.
FileSystemName = directoryClient.FileSystemName,
Resource = "d",
IsDirectory = true,
Path = directoryClient.Path,
ExpiresOn = DateTimeOffset.UtcNow.AddDays(7)
};
// Specify racwl permissions for the SAS.
sasBuilder.SetPermissions(
DataLakeSasPermissions.Read |
DataLakeSasPermissions.Add |
DataLakeSasPermissions.Create |
DataLakeSasPermissions.Write |
DataLakeSasPermissions.List
);
// Construct the full URI, including the SAS token.
DataLakeUriBuilder fullUri = new DataLakeUriBuilder(directoryClient.Uri)
{
Sas = sasBuilder.ToSasQueryParameters(userDelegationKey,
dataLakeServiceClient.AccountName)
};
Console.WriteLine("Directory user delegation SAS URI: {0}", fullUri);
Console.WriteLine();
return(fullUri.ToUri());
}
catch (Exception e)
{
Console.WriteLine(e.Message);
throw;
}
}
private async Task <bool> DoesOAuthWorkAsync()
{
TestContext.Error.WriteLine($"Datalake Probing OAuth {Process.GetCurrentProcess().Id}");
try
{
for (int i = 0; i < 10; i++)
{
// Check flat account. For some reason we observe failures if that one doesn't work before we start datalake run.
{
BlobServiceClient serviceClient = new BlobServiceClient(
new Uri(TestConfigurations.DefaultTargetOAuthTenant.BlobServiceEndpoint),
GetOAuthCredential(TestConfigurations.DefaultTargetOAuthTenant));
await serviceClient.GetPropertiesAsync();
var containerName = Guid.NewGuid().ToString();
var containerClient = serviceClient.GetBlobContainerClient(containerName);
await containerClient.CreateIfNotExistsAsync();
try
{
await containerClient.GetPropertiesAsync();
var blobName = Guid.NewGuid().ToString();
var blobClient = containerClient.GetAppendBlobClient(blobName);
await blobClient.CreateIfNotExistsAsync();
await blobClient.GetPropertiesAsync();
var userDelegationKey = await serviceClient.GetUserDelegationKeyAsync(startsOn : null, expiresOn : DateTimeOffset.UtcNow.AddHours(1));
var sasBuilder = new BlobSasBuilder(BlobSasPermissions.All, DateTimeOffset.UtcNow.AddHours(1))
{
BlobContainerName = containerName,
BlobName = blobName,
};
var sas = sasBuilder.ToSasQueryParameters(userDelegationKey.Value, serviceClient.AccountName).ToString();
await new BlobBaseClient(blobClient.Uri, new AzureSasCredential(sas)).GetPropertiesAsync();
}
finally
{
await containerClient.DeleteIfExistsAsync();
}
}
// Check hierarchical account.
{
DataLakeServiceClient serviceClient = new DataLakeServiceClient(
new Uri(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant.BlobServiceEndpoint),
GetOAuthCredential(TestConfigurations.DefaultTargetHierarchicalNamespaceTenant));
await serviceClient.GetPropertiesAsync();
var fileSystemName = Guid.NewGuid().ToString();
var fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName);
await fileSystemClient.CreateIfNotExistsAsync();
try
{
var directoryName = Guid.NewGuid().ToString();
var directoryClient = fileSystemClient.GetDirectoryClient(directoryName);
await directoryClient.CreateIfNotExistsAsync();
await directoryClient.GetPropertiesAsync();
var fileName = Guid.NewGuid().ToString();
var fileClient = directoryClient.GetFileClient(fileName);
await fileClient.CreateIfNotExistsAsync();
await fileClient.GetPropertiesAsync();
// call some APIs that talk to DFS endoint as well.
await fileClient.AppendAsync(new MemoryStream(new byte[] { 1 }), 0);
await fileClient.GetAccessControlAsync();
var userDelegationKey = await serviceClient.GetUserDelegationKeyAsync(startsOn : null, expiresOn : DateTimeOffset.UtcNow.AddHours(1));
var sasBuilder = new DataLakeSasBuilder(DataLakeSasPermissions.All, DateTimeOffset.UtcNow.AddHours(1))
{
FileSystemName = fileSystemName,
Path = fileClient.Path,
};
var sas = sasBuilder.ToSasQueryParameters(userDelegationKey.Value, serviceClient.AccountName).ToString();
await new DataLakeFileClient(fileClient.Uri, new AzureSasCredential(sas)).GetPropertiesAsync();
}
finally
{
await fileSystemClient.DeleteIfExistsAsync();
}
}
}
}
catch (RequestFailedException e) when(e.Status == 403 && e.ErrorCode == "AuthorizationPermissionMismatch")
{
TestContext.Error.WriteLine($"Datalake Probing OAuth - not ready {Process.GetCurrentProcess().Id}");
return(false);
}
TestContext.Error.WriteLine($"Datalake Probing OAuth - ready {Process.GetCurrentProcess().Id}");
return(true);
}
