/// <summary> /// Get an Exist DataLakeGen2Item, return true is the item is a folder, return false if it's File /// </summary> /// <param name="container">the blob container</param> /// <param name="path">the path of the Items</param> /// <returns>return true if the item is a folder, else false</returns> public static bool GetExistDataLakeGen2Item(DataLakeFileSystemClient fileSystem, string path, out DataLakeFileClient fileClient, out DataLakeDirectoryClient dirClient) { try { if (string.IsNullOrEmpty(path)) { dirClient = fileSystem.GetDirectoryClient(""); fileClient = null; return(true); } fileClient = fileSystem.GetFileClient(path); PathProperties properties = fileClient.GetProperties().Value; if (isDirectory(properties)) { dirClient = fileSystem.GetDirectoryClient(path); fileClient = null; return(true); } else { dirClient = null; return(false); } } catch (RequestFailedException e) when(e.Status == 404) { // TODO: through exception that the item not exist throw new ArgumentException(string.Format("The Item in File System {0} on path {1} does not exist.", fileSystem.Name, path)); } }
/// <summary> /// Azure DataLakeGen2 Item constructor /// </summary> /// <param name="blob">CloudBlockBlob blob object</param> public AzureDataLakeGen2Item(DataLakeFileClient fileClient) { Name = fileClient.Name; Path = fileClient.Path; File = fileClient; IsDirectory = false; try { Properties = fileClient.GetProperties(); Length = Properties.ContentLength; ContentType = Properties.ContentType; LastModified = Properties.LastModified; } catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404) { // skip get file properties if don't have read permission } try { AccessControl = File.GetAccessControl(); Permissions = AccessControl.Permissions; ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList); Owner = AccessControl.Owner; Group = AccessControl.Group; } catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404) { // skip get file ACL if don't have read permission } }
private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/ public static void Main(string[] args) { // Create Client Secret Credential var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret); // Create data lake file service client object DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds); var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8); // Create data lake file system client object DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name); filesystemclient.CreateIfNotExists(); try { long length; string fileName = "/Test/testFilename1.txt"; DataLakeFileClient file = filesystemclient.GetFileClient(fileName); // Upload a file - automatically creates any parent directories that don't exist length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length; file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true); file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length); file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length); //Read file contents Response <FileDownloadInfo> fileContents = file.Read(); Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString()); // Get the properties of the file PathProperties pathProperties = file.GetProperties(); PrintDirectoryEntry(pathProperties); // Rename a file string destFilePath = "/Test/testRenameDest3.txt"; file.Rename(destFilePath); file = filesystemclient.GetFileClient(destFilePath); Console.WriteLine("The file URI is " + file.Uri); // Enumerate directory foreach (var pathItem in filesystemclient.GetPaths("/Test")) { PrintDirectoryEntry(pathItem); } // Delete a directory and all it's subdirectories and files filesystemclient.DeleteDirectory("/Test"); } finally { filesystemclient.Delete(); } Console.WriteLine("Done. Press ENTER to continue ..."); Console.ReadLine(); }
/// <inheritdoc cref="IFileStorageService"/> public string GetFileMD5Hash(Uri fileUri) { DataLakeFileClient fcli = new DataLakeFileClient(fileUri, SharedKeyCredential); Response <PathProperties> props = fcli.GetProperties(); byte[] hashBytes = props.Value.ContentHash; string hexHash = BitConverter.ToString(hashBytes); return(hexHash); }
public void Append() { // Create three temporary Lorem Ipsum files on disk that we can upload int contentLength = 10; string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength)); string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength)); string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength)); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-append" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); filesystem.Create(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // Create the file file.Create(); // Verify we created one file Assert.AreEqual(1, filesystem.GetPaths().Count()); // Append data to an existing DataLake File. Append is currently limited to 4000 MB per call. // To upload a large file all at once, consider using Upload() instead. file.Append(File.OpenRead(sampleFileContentPart1), 0); file.Append(File.OpenRead(sampleFileContentPart2), contentLength); file.Append(File.OpenRead(sampleFileContentPart3), contentLength * 2); file.Flush(contentLength * 3); // Verify the contents of the file PathProperties properties = file.GetProperties(); Assert.AreEqual(contentLength * 3, properties.ContentLength); } finally { // Clean up after the test when we're finished filesystem.Delete(); } }
public void Rename() { // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-rename" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-rename")); filesystem.Create(); try { // Create a DataLake Directory to rename it later DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory")); directoryClient.Create(); // Rename directory with new path/name and verify by making a service call (e.g. GetProperties) #region Snippet:SampleSnippetDataLakeFileClient_RenameDirectory DataLakeDirectoryClient renamedDirectoryClient = directoryClient.Rename("sample-directory2"); #endregion Snippet:SampleSnippetDataLakeFileClient_RenameDirectory PathProperties directoryPathProperties = renamedDirectoryClient.GetProperties(); // Delete the sample directory using the new path/name filesystem.DeleteDirectory("sample-directory2"); // Create a DataLake file. DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file")); fileClient.Create(); // Rename file with new path/name and verify by making a service call (e.g. GetProperties) #region Snippet:SampleSnippetDataLakeFileClient_RenameFile DataLakeFileClient renamedFileClient = fileClient.Rename("sample-file2"); #endregion Snippet:SampleSnippetDataLakeFileClient_RenameFile PathProperties filePathProperties = renamedFileClient.GetProperties(); // Delete the sample directory using the new path/name filesystem.DeleteFile("sample-file2"); } finally { // Clean up after the test when we're finished filesystem.Delete(); } }
/// <summary> /// Azure DataLakeGen2 Item constructor /// </summary> /// <param name="blob">CloudBlockBlob blob object</param> public AzureDataLakeGen2Item(DataLakeFileClient fileClient) { Name = fileClient.Name; Path = fileClient.Path; File = fileClient; Properties = fileClient.GetProperties(); AccessControl = File.GetAccessControl(); Length = Properties.ContentLength; ContentType = Properties.ContentType; LastModified = Properties.LastModified; IsDirectory = false; Permissions = AccessControl.Permissions; ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList); Owner = AccessControl.Owner; Group = AccessControl.Group; }
public void Upload() { // Create three temporary Lorem Ipsum files on disk that we can upload int contentLength = 10; string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength)); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-append" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); filesystem.Create(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // Create the file file.Create(); // Verify we created one file Assert.AreEqual(1, filesystem.GetPaths().Count()); // Upload content to the file. When using the Upload API, you don't need to create the file first. // If the file already exists, it will be overwritten. // For larger files, Upload() will upload the file in multiple sequential requests. file.Upload(File.OpenRead(sampleFileContent), true); // Verify the contents of the file PathProperties properties = file.GetProperties(); Assert.AreEqual(contentLength, properties.ContentLength); } finally { // Clean up after the test when we're finished filesystem.Delete(); } }
public void GetProperties() { // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-rename" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem")); filesystem.Create(); try { // Create a DataLake Directory to rename it later DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory")); directoryClient.Create(); #region Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties // Get Properties on a Directory PathProperties directoryPathProperties = directoryClient.GetProperties(); #endregion Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties // Create a DataLake file DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file")); fileClient.Create(); #region Snippet:SampleSnippetDataLakeFileClient_GetProperties // Get Properties on a File PathProperties filePathProperties = fileClient.GetProperties(); #endregion Snippet:SampleSnippetDataLakeFileClient_GetProperties } finally { // Clean up after the test when we're finished filesystem.Delete(); } }
public void Append_Simple() { // Create Sample File to read content from string sampleFilePath = CreateTempFile(SampleFileContent); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-append" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); filesystem.Create(); try { #region Snippet:SampleSnippetDataLakeFileClient_Append // Create a file DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); file.Create(); // Append data to the DataLake File file.Append(File.OpenRead(sampleFilePath), 0); file.Flush(SampleFileContent.Length); #endregion Snippet:SampleSnippetDataLakeFileClient_Append // Verify the contents of the file PathProperties properties = file.GetProperties(); Assert.AreEqual(SampleFileContent.Length, properties.ContentLength); } finally { // Clean up after the test when we're finished filesystem.Delete(); } }
/// <summary> /// execute command /// </summary> public override void ExecuteCmdlet() { IStorageBlobManagement localChannel = Channel; BlobRequestOptions requestOptions = RequestOptions; bool foundAFolder = false; DataLakeFileClient srcBlob = null; DataLakeDirectoryClient srcBlobDir = null; if (ParameterSetName == ManualParameterSet) { DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir); } else //BlobParameterSet { if (!InputObject.IsDirectory) { srcBlob = InputObject.File; } else { srcBlobDir = InputObject.Directory; foundAFolder = true; } } if (foundAFolder) { if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: ")) { // check dest exist bool destExist = true; DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath); try { destBlobDir.GetProperties(); } catch (RequestFailedException e) when(e.Status == 404) { destExist = false; } if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), "")) { destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value; WriteDataLakeGen2Item(localChannel, destBlobDir); } } } else { if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: ")) { // check dest exist bool destExist = true; DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath); try { destFile.GetProperties(); } catch (RequestFailedException e) when(e.Status == 404) { destExist = false; } if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), "")) { destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value; WriteDataLakeGen2Item(localChannel, destFile); } } } }