/// <summary>
/// Get an Exist DataLakeGen2Item, return true is the item is a folder, return false if it's File
/// </summary>
/// <param name="container">the blob container</param>
/// <param name="path">the path of the Items</param>
/// <returns>return true if the item is a folder, else false</returns>
public static bool GetExistDataLakeGen2Item(DataLakeFileSystemClient fileSystem, string path, out DataLakeFileClient fileClient, out DataLakeDirectoryClient dirClient)
{
try
{
if (string.IsNullOrEmpty(path))
{
dirClient = fileSystem.GetDirectoryClient("");
fileClient = null;
return(true);
}
fileClient = fileSystem.GetFileClient(path);
PathProperties properties = fileClient.GetProperties().Value;
if (isDirectory(properties))
{
dirClient = fileSystem.GetDirectoryClient(path);
fileClient = null;
return(true);
}
else
{
dirClient = null;
return(false);
}
}
catch (RequestFailedException e) when(e.Status == 404)
{
// TODO: through exception that the item not exist
throw new ArgumentException(string.Format("The Item in File System {0} on path {1} does not exist.", fileSystem.Name, path));
}
}
/// <summary>
/// Azure DataLakeGen2 Item constructor
/// </summary>
/// <param name="blob">CloudBlockBlob blob object</param>
public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
{
Name = fileClient.Name;
Path = fileClient.Path;
File = fileClient;
IsDirectory = false;
try
{
Properties = fileClient.GetProperties();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
{
// skip get file properties if don't have read permission
}
try
{
AccessControl = File.GetAccessControl();
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
{
// skip get file ACL if don't have read permission
}
}
private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/
public static void Main(string[] args)
{
// Create Client Secret Credential
var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret);
// Create data lake file service client object
DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds);
var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8);
// Create data lake file system client object
DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name);
filesystemclient.CreateIfNotExists();
try
{
long length;
string fileName = "/Test/testFilename1.txt";
DataLakeFileClient file = filesystemclient.GetFileClient(fileName);
// Upload a file - automatically creates any parent directories that don't exist
length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length;
file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true);
file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length);
file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length);
//Read file contents
Response <FileDownloadInfo> fileContents = file.Read();
Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString());
// Get the properties of the file
PathProperties pathProperties = file.GetProperties();
PrintDirectoryEntry(pathProperties);
// Rename a file
string destFilePath = "/Test/testRenameDest3.txt";
file.Rename(destFilePath);
file = filesystemclient.GetFileClient(destFilePath);
Console.WriteLine("The file URI is " + file.Uri);
// Enumerate directory
foreach (var pathItem in filesystemclient.GetPaths("/Test"))
{
PrintDirectoryEntry(pathItem);
}
// Delete a directory and all it's subdirectories and files
filesystemclient.DeleteDirectory("/Test");
}
finally
{
filesystemclient.Delete();
}
Console.WriteLine("Done. Press ENTER to continue ...");
Console.ReadLine();
}
/// <inheritdoc cref="IFileStorageService"/>
public string GetFileMD5Hash(Uri fileUri)
{
DataLakeFileClient fcli = new DataLakeFileClient(fileUri, SharedKeyCredential);
Response <PathProperties> props = fcli.GetProperties();
byte[] hashBytes = props.Value.ContentHash;
string hexHash = BitConverter.ToString(hashBytes);
return(hexHash);
}
public void Append()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength));
string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength));
string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Create the file
file.Create();
// Verify we created one file
Assert.AreEqual(1, filesystem.GetPaths().Count());
// Append data to an existing DataLake File. Append is currently limited to 4000 MB per call.
// To upload a large file all at once, consider using Upload() instead.
file.Append(File.OpenRead(sampleFileContentPart1), 0);
file.Append(File.OpenRead(sampleFileContentPart2), contentLength);
file.Append(File.OpenRead(sampleFileContentPart3), contentLength * 2);
file.Flush(contentLength * 3);
// Verify the contents of the file
PathProperties properties = file.GetProperties();
Assert.AreEqual(contentLength * 3, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
public void Rename()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-rename" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-rename"));
filesystem.Create();
try
{
// Create a DataLake Directory to rename it later
DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
directoryClient.Create();
// Rename directory with new path/name and verify by making a service call (e.g. GetProperties)
#region Snippet:SampleSnippetDataLakeFileClient_RenameDirectory
DataLakeDirectoryClient renamedDirectoryClient = directoryClient.Rename("sample-directory2");
#endregion Snippet:SampleSnippetDataLakeFileClient_RenameDirectory
PathProperties directoryPathProperties = renamedDirectoryClient.GetProperties();
// Delete the sample directory using the new path/name
filesystem.DeleteDirectory("sample-directory2");
// Create a DataLake file.
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
fileClient.Create();
// Rename file with new path/name and verify by making a service call (e.g. GetProperties)
#region Snippet:SampleSnippetDataLakeFileClient_RenameFile
DataLakeFileClient renamedFileClient = fileClient.Rename("sample-file2");
#endregion Snippet:SampleSnippetDataLakeFileClient_RenameFile
PathProperties filePathProperties = renamedFileClient.GetProperties();
// Delete the sample directory using the new path/name
filesystem.DeleteFile("sample-file2");
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
/// <summary>
/// Azure DataLakeGen2 Item constructor
/// </summary>
/// <param name="blob">CloudBlockBlob blob object</param>
public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
{
Name = fileClient.Name;
Path = fileClient.Path;
File = fileClient;
Properties = fileClient.GetProperties();
AccessControl = File.GetAccessControl();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
IsDirectory = false;
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}
public void Upload()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Create the file
file.Create();
// Verify we created one file
Assert.AreEqual(1, filesystem.GetPaths().Count());
// Upload content to the file. When using the Upload API, you don't need to create the file first.
// If the file already exists, it will be overwritten.
// For larger files, Upload() will upload the file in multiple sequential requests.
file.Upload(File.OpenRead(sampleFileContent), true);
// Verify the contents of the file
PathProperties properties = file.GetProperties();
Assert.AreEqual(contentLength, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
public void GetProperties()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-rename" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
filesystem.Create();
try
{
// Create a DataLake Directory to rename it later
DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
directoryClient.Create();
#region Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties
// Get Properties on a Directory
PathProperties directoryPathProperties = directoryClient.GetProperties();
#endregion Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties
// Create a DataLake file
DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
fileClient.Create();
#region Snippet:SampleSnippetDataLakeFileClient_GetProperties
// Get Properties on a File
PathProperties filePathProperties = fileClient.GetProperties();
#endregion Snippet:SampleSnippetDataLakeFileClient_GetProperties
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
public void Append_Simple()
{
// Create Sample File to read content from
string sampleFilePath = CreateTempFile(SampleFileContent);
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
filesystem.Create();
try
{
#region Snippet:SampleSnippetDataLakeFileClient_Append
// Create a file
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
file.Create();
// Append data to the DataLake File
file.Append(File.OpenRead(sampleFilePath), 0);
file.Flush(SampleFileContent.Length);
#endregion Snippet:SampleSnippetDataLakeFileClient_Append
// Verify the contents of the file
PathProperties properties = file.GetProperties();
Assert.AreEqual(SampleFileContent.Length, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
/// <summary>
/// execute command
/// </summary>
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
BlobRequestOptions requestOptions = RequestOptions;
bool foundAFolder = false;
DataLakeFileClient srcBlob = null;
DataLakeDirectoryClient srcBlobDir = null;
if (ParameterSetName == ManualParameterSet)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir);
}
else //BlobParameterSet
{
if (!InputObject.IsDirectory)
{
srcBlob = InputObject.File;
}
else
{
srcBlobDir = InputObject.Directory;
foundAFolder = true;
}
}
if (foundAFolder)
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: "))
{
// check dest exist
bool destExist = true;
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath);
try
{
destBlobDir.GetProperties();
}
catch (RequestFailedException e) when(e.Status == 404)
{
destExist = false;
}
if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), ""))
{
destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destBlobDir);
}
}
}
else
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: "))
{
// check dest exist
bool destExist = true;
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath);
try
{
destFile.GetProperties();
}
catch (RequestFailedException e) when(e.Status == 404)
{
destExist = false;
}
if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), ""))
{
destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destFile);
}
}
}
}