public void GenerateSas_RequiredParameters()
{
// Arrange
var constants = TestConstants.Create(this);
string fileSystemName = GetNewFileSystemName();
string path = GetNewFileName();
DataLakeSasPermissions permissions = DataLakeSasPermissions.Read;
DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
var blobEndpoint = new Uri("http://127.0.0.1/" + constants.Sas.Account + "/" + fileSystemName + "/" + path);
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(
blobEndpoint,
constants.Sas.SharedKeyCredential,
GetOptions()));
// Act
Uri sasUri = pathClient.GenerateSasUri(permissions, expiresOn);
// Assert
DataLakeSasBuilder sasBuilder2 = new DataLakeSasBuilder(permissions, expiresOn)
{
FileSystemName = fileSystemName,
Path = path
};
DataLakeUriBuilder expectedUri = new DataLakeUriBuilder(blobEndpoint)
{
Sas = sasBuilder2.ToSasQueryParameters(constants.Sas.SharedKeyCredential)
};
Assert.AreEqual(expectedUri.ToUri().ToString(), sasUri.ToString());
}
/// <inheritdoc cref="IFileStorageService"/>
public bool DoesBlobExist(string hash)
{
Uri uri = BuildUriForBlob(DataLakeConfiguration.Uri, DataLakeConfiguration.EnrichmentDataContainerName, hash);
DataLakePathClient pathClient = new DataLakePathClient(uri, SharedKeyCredential);
return(pathClient.Exists());
}
public void GenerateSas_BuilderWrongFileSystemName()
{
// Arrange
var constants = new TestConstants(this);
var blobEndpoint = new Uri("http://127.0.0.1/");
UriBuilder blobUriBuilder = new UriBuilder(blobEndpoint);
string path = GetNewFileName();
DataLakeSasPermissions permissions = DataLakeSasPermissions.Read;
DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
blobUriBuilder.Path += constants.Sas.Account + "/" + GetNewFileSystemName() + "/" + path;
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(
blobUriBuilder.Uri,
constants.Sas.SharedKeyCredential,
GetOptions()));
DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder(permissions, expiresOn)
{
FileSystemName = GetNewFileSystemName(), // different filesystem name
Path = path,
};
// Act
try
{
pathClient.GenerateSasUri(sasBuilder);
Assert.Fail("DataLakePathClient.GenerateSasUri should have failed with an ArgumentException.");
}
catch (InvalidOperationException)
{
//the correct exception came back
}
}
private bool ApplyACL(DataLakePathClient itemClient)
{
PathAccessControl pac = itemClient.GetAccessControl(true).Value;
var fileACLList = pac.AccessControlList.ToList();
if (_cfg.LogVerbose)
{
Log?.Invoke($" - ACL before:");
fileACLList.ForEach((item) => Log?.Invoke($" - {item.ToString()}"));
}
bool changed = false;
// new/updated permissions
foreach (var newACLItem in _newACL)
{
var existingACLItem = fileACLList.SingleOrDefault(x => x.AccessControlType == newACLItem.AccessControlType &&
x.EntityId == newACLItem.EntityId &&
x.DefaultScope == newACLItem.DefaultScope);
if (existingACLItem == null)
{
if (!(newACLItem.DefaultScope && itemClient is DataLakeFileClient))
{
fileACLList.Add(newACLItem);
changed = true;
}
}
else
{
if (existingACLItem.Permissions != newACLItem.Permissions)
{
existingACLItem.Permissions = newACLItem.Permissions;
changed = true;
}
}
}
// remove users/groups
var fileACLList2 = fileACLList.Where(x => !_cfg.RemoveList.Contains(x.EntityId)).ToList();
if (fileACLList.Count() != fileACLList2.Count())
{
changed = true;
}
// apply
if (changed)
{
itemClient.SetAccessControlList(fileACLList2);
}
if (_cfg.LogVerbose)
{
Log?.Invoke($" - ACL after:");
fileACLList.ForEach((item) => Log?.Invoke($" - {item.ToString()}"));
}
return(changed);
}
public void CanGenerateSas_ClientConstructors()
{
// Arrange
var constants = TestConstants.Create(this);
var blobEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account);
var blobSecondaryEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account + "-secondary");
var storageConnectionString = new StorageConnectionString(constants.Sas.SharedKeyCredential, blobStorageUri: (blobEndpoint, blobSecondaryEndpoint));
string connectionString = storageConnectionString.ToString(true);
// Act - DataLakePathClient(Uri blobContainerUri, BlobClientOptions options = default)
DataLakePathClient blob3 = InstrumentClient(new DataLakePathClient(
blobEndpoint,
GetOptions()));
Assert.IsFalse(blob3.CanGenerateSasUri);
// Act - DataLakePathClient(Uri blobContainerUri, StorageSharedKeyCredential credential, BlobClientOptions options = default)
DataLakePathClient blob4 = InstrumentClient(new DataLakePathClient(
blobEndpoint,
constants.Sas.SharedKeyCredential,
GetOptions()));
Assert.IsTrue(blob4.CanGenerateSasUri);
// Act - DataLakePathClient(Uri blobContainerUri, TokenCredential credential, BlobClientOptions options = default)
var tokenCredentials = new DefaultAzureCredential();
DataLakePathClient blob5 = InstrumentClient(new DataLakePathClient(
blobEndpoint,
tokenCredentials,
GetOptions()));
Assert.IsFalse(blob5.CanGenerateSasUri);
}
/// <summary>
/// Azure DataLakeGen2 Item constructor
/// </summary>
/// <param name="item">datalake gen2 listout item</param>
public AzureDataLakeGen2Item(PathItem item, DataLakeFileSystemClient fileSystem, bool fetchProperties = false)
{
this.Name = item.Name;
this.Path = item.Name;
this.ListPathItem = item;
this.IsDirectory = item.IsDirectory is null ? false : item.IsDirectory.Value;
DataLakePathClient pathclient = null;
if (this.IsDirectory) // Directory
{
this.Directory = fileSystem.GetDirectoryClient(item.Name);
pathclient = this.Directory;
}
else //File
{
this.File = fileSystem.GetFileClient(item.Name);
pathclient = this.File;
}
this.Owner = item.Owner;
this.Group = item.Group;
this.Permissions = PathPermissions.ParseSymbolicPermissions(item.Permissions);
this.LastModified = item.LastModified;
this.Length = item.ContentLength is null ? 0 : item.ContentLength.Value;
if (fetchProperties)
{
this.Properties = pathclient.GetProperties();
this.AccessControl = pathclient.GetAccessControl();
this.ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(this.AccessControl.AccessControlList);
this.ContentType = Properties.ContentType;
}
}
public void GenerateSas_BuilderIsDirectoryError()
{
var constants = TestConstants.Create(this);
var blobEndpoint = new Uri("http://127.0.0.1/");
UriBuilder blobUriBuilder = new UriBuilder(blobEndpoint);
string fileSystemName = GetNewFileSystemName();
string fileName = GetNewFileName();
DataLakeSasPermissions permissions = DataLakeSasPermissions.Read;
DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
blobUriBuilder.Path += constants.Sas.Account + "/" + fileSystemName + "/" + fileName;
DataLakePathClient containerClient = InstrumentClient(new DataLakePathClient(
blobUriBuilder.Uri,
constants.Sas.SharedKeyCredential,
GetOptions()));
DataLakeSasBuilder sasBuilder = new DataLakeSasBuilder(permissions, expiresOn)
{
FileSystemName = fileSystemName,
Path = fileName,
IsDirectory = true,
};
// Act
try
{
containerClient.GenerateSasUri(sasBuilder);
Assert.Fail("DataLakeFileClient.GenerateSasUri should have failed with an ArgumentException.");
}
catch (InvalidOperationException)
{
//the correct exception came back
}
}
//TODO consider removing this.
public async Task <string> SetupPathLeaseCondition(DataLakePathClient path, string leaseId, string garbageLeaseId)
{
Models.DataLakeLease lease = null;
if (leaseId == ReceivedLeaseId || leaseId == garbageLeaseId)
{
lease = await InstrumentClient(path.GetDataLakeLeaseClient(Recording.Random.NewGuid().ToString())).AcquireAsync(DataLakeLeaseClient.InfiniteLeaseDuration);
}
return(leaseId == ReceivedLeaseId ? lease.LeaseId : leaseId);
}
//TODO consider removing this.
public async Task <string> SetupPathMatchCondition(DataLakePathClient path, string match)
{
if (match == ReceivedETag)
{
Response <PathProperties> headers = await path.GetPropertiesAsync();
return(headers.Value.ETag.ToString());
}
else
{
return(match);
}
}
/// <summary>
/// Get Item string without SAS for confirmation string.
/// </summary>
/// <param name="item"></param>
/// <returns></returns>
protected static string GetDataLakeItemUriWithoutSas(DataLakePathClient item)
{
string uriString = item.Uri.ToString();
int length = uriString.IndexOf("?");
if (length < 0) // Not container "?"
{
return(uriString);
}
else
{
return(uriString.Substring(0, uriString.IndexOf("?")));
}
}
public async Task Ctor_FileSystemAndPath()
{
// Arrange
await using DisposingFileSystem test = await GetNewFileSystem();
DataLakeFileClient fileClient = await test.FileSystem.CreateFileAsync(GetNewFileName());
// Act
DataLakePathClient pathClient = new DataLakePathClient(test.FileSystem, fileClient.Path);
// Assert
await pathClient.GetPropertiesAsync();
await pathClient.GetAccessControlAsync();
}
/// <summary>
/// Set Metadata to a datalake gen2 item
/// </summary>
/// <param name="file">datalake gen2 item</param>
/// <param name="Metadata">Metadata to set</param>
/// <param name="setToServer">True will set to server, false only set to the local Datalakegen2Item object</param>
protected static IDictionary <string, string> SetDatalakegen2ItemMetaData(DataLakePathClient item, Hashtable Metadata, bool setToServer = true)
{
if (Metadata != null)
{
IDictionary <string, string> metadata = GetUpdatedMetaData(Metadata, null);
if (setToServer && item != null)
{
item.SetMetadata(metadata);
}
return(metadata);
}
else
{
return(null);
}
}
public async Task Ctor_ConnectionString_RoundTrip()
{
// Arrange
string fileSystemName = GetNewFileSystemName();
string path = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(fileSystemName : fileSystemName);
DataLakeDirectoryClient directoryClient = InstrumentClient(test.FileSystem.GetDirectoryClient(path));
await directoryClient.CreateAsync();
// Act
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={TestConfigHierarchicalNamespace.AccountName};AccountKey={TestConfigHierarchicalNamespace.AccountKey};EndpointSuffix=core.windows.net";
DataLakePathClient connStringDirectory = InstrumentClient(new DataLakePathClient(connectionString, fileSystemName, path, GetOptions()));
// Assert
await connStringDirectory.GetPropertiesAsync();
await connStringDirectory.GetAccessControlAsync();
}
/// <summary>
/// execute command
/// </summary>
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
if (ShouldProcess(ParameterSetName == ManualParameterSet ? this.Path : InputObject.Path, "remove"))
{
if (ParameterSetName == ManualParameterSet)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
DataLakePathClient pathClient = new DataLakePathClient(fileSystem, this.Path);
if (force || ShouldContinue(string.Format("Remove DatalakeGen2 Item: {0}", GetDataLakeItemUriWithoutSas(pathClient)), ""))
{
pathClient.Delete(true, cancellationToken: this.CmdletCancellationToken);
}
}
else //ItemPipeline
{
if (!InputObject.IsDirectory)
{
DataLakeFileClient fileClient = InputObject.File;
if (force || ShouldContinue(string.Format("Remove File: {0}", GetDataLakeItemUriWithoutSas(fileClient)), ""))
{
fileClient.Delete(cancellationToken: this.CmdletCancellationToken);
}
}
else
{
DataLakeDirectoryClient dirClient = InputObject.Directory;
if (force || ShouldContinue(string.Format("Remove Directory: {0}", GetDataLakeItemUriWithoutSas(dirClient)), ""))
{
dirClient.Delete(true, cancellationToken: this.CmdletCancellationToken);
}
}
}
if (PassThru)
{
WriteObject(true);
}
}
}
public async Task Ctor_TokenCredential()
{
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(fileSystemName : fileSystemName);
// Arrange
string directoryName = GetNewDirectoryName();
await test.FileSystem.CreateDirectoryAsync(directoryName);
TokenCredential tokenCredential = GetOAuthCredential(TestConfigHierarchicalNamespace);
Uri uri = new Uri($"{TestConfigHierarchicalNamespace.BlobServiceEndpoint}/{fileSystemName}/{directoryName}").ToHttps();
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(uri, tokenCredential, GetOptions()));
// Act
await pathClient.GetPropertiesAsync();
// Assert
Assert.AreEqual(fileSystemName, pathClient.FileSystemName);
Assert.AreEqual(uri, pathClient.Uri);
}
public async Task Ctor_Uri()
{
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(fileSystemName : fileSystemName);
// Arrange
string directoryName = GetNewDirectoryName();
await test.FileSystem.CreateDirectoryAsync(directoryName);
SasQueryParameters sasQueryParameters = GetNewAccountSasCredentials();
Uri uri = new Uri($"{TestConfigHierarchicalNamespace.BlobServiceEndpoint}/{fileSystemName}/{directoryName}?{sasQueryParameters}");
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(uri, GetOptions()));
// Act
await pathClient.GetPropertiesAsync();
// Assert
Assert.AreEqual(fileSystemName, pathClient.FileSystemName);
Assert.AreEqual(uri, pathClient.Uri);
}
public async Task Ctor_ConnectionString_GenerateSas()
{
// Arrange
string fileSystemName = GetNewFileSystemName();
string path = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(fileSystemName : fileSystemName);
DataLakeDirectoryClient directoryClient = InstrumentClient(test.FileSystem.GetDirectoryClient(path));
await directoryClient.CreateAsync();
// Act
string connectionString = $"DefaultEndpointsProtocol=https;AccountName={TestConfigHierarchicalNamespace.AccountName};AccountKey={TestConfigHierarchicalNamespace.AccountKey};BlobEndpoint={TestConfigHierarchicalNamespace.BlobServiceEndpoint};FileEndpoint={TestConfigHierarchicalNamespace.FileServiceEndpoint};QueueEndpoint={TestConfigHierarchicalNamespace.QueueServiceEndpoint};TableEndpoint={TestConfigHierarchicalNamespace.TableServiceEndpoint}";
DataLakePathClient connStringDirectory = InstrumentClient(new DataLakePathClient(connectionString, fileSystemName, path, GetOptions()));
Uri sasUri = connStringDirectory.GenerateSasUri(DataLakeSasPermissions.All, Recording.UtcNow.AddDays(1));
DataLakePathClient sasPathClient = InstrumentClient(new DataLakePathClient(sasUri, GetOptions()));
// Assert
await sasPathClient.GetPropertiesAsync();
await sasPathClient.GetAccessControlAsync();
}
public async Task Ctor_SharedKey()
{
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(fileSystemName : fileSystemName);
// Arrange
string directoryName = GetNewDirectoryName();
await test.FileSystem.CreateDirectoryAsync(directoryName);
StorageSharedKeyCredential sharedKey = new StorageSharedKeyCredential(
TestConfigHierarchicalNamespace.AccountName,
TestConfigHierarchicalNamespace.AccountKey);
Uri uri = new Uri($"{TestConfigHierarchicalNamespace.BlobServiceEndpoint}/{fileSystemName}/{directoryName}");
DataLakePathClient pathClient = InstrumentClient(new DataLakePathClient(uri, sharedKey, GetOptions()));
// Act
await pathClient.GetPropertiesAsync();
// Assert
Assert.AreEqual(fileSystemName, pathClient.FileSystemName);
Assert.AreEqual(uri, pathClient.Uri);
}
/// <summary>
/// Set properties to a datalake gen2 Datalakegen2Item
/// </summary>
/// <param name="item">datalake gen2 Datalakegen2Item</param>
/// <param name="BlobProperties">properties to set</param>
/// <param name="setToServer">True will set to server, false only set to the local Datalakegen2Item object</param>
protected static PathHttpHeaders SetDatalakegen2ItemProperties(DataLakePathClient item, Hashtable BlobProperties, bool setToServer = true)
{
if (BlobProperties != null)
{
// Valid Blob Dir properties
foreach (DictionaryEntry entry in BlobProperties)
{
if (!validDatalakeGen2FileProperties.ContainsKey(entry.Key.ToString()))
{
throw new ArgumentException(String.Format("InvalidDataLakeFileProperties", entry.Key.ToString(), entry.Value.ToString()));
}
}
PathHttpHeaders headers = new PathHttpHeaders();
foreach (DictionaryEntry entry in BlobProperties)
{
string key = entry.Key.ToString();
string value = entry.Value.ToString();
Action <PathHttpHeaders, string> action = validDatalakeGen2FileProperties[key];
if (action != null)
{
action(headers, value);
}
}
if (setToServer && item != null)
{
item.SetHttpHeaders(headers);
}
return(headers);
}
else
{
return(null);
}
}
public void Run()
{
Log?.Invoke("ACL:");
_newACL.ForEach((item) => Log?.Invoke($" - {item.ToString()}"));
var serviceClient = new DataLakeServiceClient(
new Uri($"https://{_cfg.AccountName}.dfs.core.windows.net"),
new StorageSharedKeyCredential(_cfg.AccountName, _cfg.AccountKey));
var fileSystemClient = serviceClient.GetFileSystemClient(_cfg.FileSystem);
Log?.Invoke("ADLS listing items...");
var allItems = fileSystemClient.GetPaths(_cfg.StartingPath, true).ToList();
Log?.Invoke($"Items count: {allItems.Count}");
var startDT = DateTime.UtcNow;
_counterAll = 0;
_counterUpdated = 0;
Parallel.ForEach(
allItems,
new ParallelOptions()
{
MaxDegreeOfParallelism = _cfg.Parallelism
},
(pathItem, state) =>
{
try
{
var a = pathItem;
Interlocked.Increment(ref _counterAll);
if (_cfg.LogVerbose)
{
Log?.Invoke($"Processing {pathItem.Name} {(pathItem.IsDirectory.Value ? " [D]" : "")}");
}
DataLakePathClient itemClient =
a.IsDirectory.Value ?
(DataLakePathClient)fileSystemClient.GetDirectoryClient(pathItem.Name) :
(DataLakePathClient)fileSystemClient.GetFileClient(pathItem.Name);
bool updated = ApplyACL(itemClient); // <<<===
if (updated)
{
Interlocked.Increment(ref _counterUpdated);
}
if (_counterAll % 10 == 0)
{
Log?.Invoke($"Progress: {_counterAll}/{allItems.Count} - Updated: {_counterUpdated}");
}
if (_cfg.ExitAfter.HasValue && _counterAll > _cfg.ExitAfter)
{
state.Break();
}
}
catch (Exception ex)
{
Console.WriteLine($"******* ITEM: {pathItem.Name} - EXCEPTION: {ex.ToString()}");
throw;
}
});
Log?.Invoke($"Elapsed time: {DateTime.UtcNow.Subtract(startDT).TotalSeconds}");
Log?.Invoke($"counterAll={_counterAll} counterUpdated={_counterUpdated}");
}
/// <summary>
/// Initializes a new instance of the <see cref="DataLakeLeaseClient"/> class.
/// </summary>
/// <param name="client">
/// A <see cref="BlobClient"/> representing the blob being leased.
/// </param>
/// <param name="leaseId">
/// An optional lease ID. If no lease ID is provided, a random lease
/// ID will be created.
/// </param>
public DataLakeLeaseClient(DataLakePathClient client, string leaseId = null)
{
_blobLeaseClient = new BlobLeaseClient(client.BlobClient, leaseId);
}
/// <summary>
/// Initializes a new instance of the <see cref="DataLakeLeaseClient"/> class.
/// </summary>
/// <param name="client">
/// A <see cref="BlobClient"/> representing the blob being leased.
/// </param>
/// <param name="leaseId">
/// An optional lease ID. If no lease ID is provided, a random lease
/// ID will be created.
/// </param>
public DataLakeLeaseClient(DataLakePathClient client, string leaseId = null)
{
_blobLeaseClient = new BlobLeaseClient(client.BlobClient, leaseId);
_clientDiagnostics = client.ClientDiagnostics;
}
/// <summary>
/// Initializes a new instance of the <see cref="DataLakeLeaseClient"/> class.
/// </summary>
/// <param name="client">
/// A <see cref="DataLakePathClient"/> representing the path being leased.
/// </param>
/// <param name="leaseId">
/// An optional lease ID. If no lease ID is provided, a random lease
/// ID will be created.
/// </param>
public static DataLakeLeaseClient GetDataLakeLeaseClient(
this DataLakePathClient client,
string leaseId = null) =>
new DataLakeLeaseClient(client, leaseId);
