示例#1
0
        /// <summary>
        /// Get an Exist DataLakeGen2Item, return true is the item is a folder, return false if it's File
        /// </summary>
        /// <param name="container">the blob container</param>
        /// <param name="path">the path of the Items</param>
        /// <returns>return true if the item is a folder, else false</returns>
        public static bool GetExistDataLakeGen2Item(DataLakeFileSystemClient fileSystem, string path, out DataLakeFileClient fileClient, out DataLakeDirectoryClient dirClient)
        {
            try
            {
                if (string.IsNullOrEmpty(path))
                {
                    dirClient  = fileSystem.GetDirectoryClient("");
                    fileClient = null;
                    return(true);
                }

                fileClient = fileSystem.GetFileClient(path);
                PathProperties properties = fileClient.GetProperties().Value;
                if (isDirectory(properties))
                {
                    dirClient  = fileSystem.GetDirectoryClient(path);
                    fileClient = null;
                    return(true);
                }
                else
                {
                    dirClient = null;
                    return(false);
                }
            }
            catch (RequestFailedException e) when(e.Status == 404)
            {
                // TODO: through exception that the item not exist
                throw new ArgumentException(string.Format("The Item in File System {0} on path {1} does not exist.", fileSystem.Name, path));
            }
        }
示例#2
0
 /// <summary>
 /// Azure DataLakeGen2 Item constructor
 /// </summary>
 /// <param name="blob">CloudBlockBlob blob object</param>
 public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
 {
     Name        = fileClient.Name;
     Path        = fileClient.Path;
     File        = fileClient;
     IsDirectory = false;
     try
     {
         Properties   = fileClient.GetProperties();
         Length       = Properties.ContentLength;
         ContentType  = Properties.ContentType;
         LastModified = Properties.LastModified;
     }
     catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
     {
         // skip get file properties if don't have read permission
     }
     try
     {
         AccessControl = File.GetAccessControl();
         Permissions   = AccessControl.Permissions;
         ACL           = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
         Owner         = AccessControl.Owner;
         Group         = AccessControl.Group;
     }
     catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
     {
         // skip get file ACL if don't have read permission
     }
 }
        private static string serviceUri    = "FILL-IN-HERE";     // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/

        public static void Main(string[] args)
        {
            // Create Client Secret Credential
            var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret);

            // Create data lake file service client object
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds);
            var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8);
            // Create data lake file system client object
            DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name);

            filesystemclient.CreateIfNotExists();

            try
            {
                long               length;
                string             fileName = "/Test/testFilename1.txt";
                DataLakeFileClient file     = filesystemclient.GetFileClient(fileName);

                // Upload a file - automatically creates any parent directories that don't exist
                length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length;

                file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true);

                file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length);
                file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length);
                //Read file contents
                Response <FileDownloadInfo> fileContents = file.Read();

                Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString());

                // Get the properties of the file
                PathProperties pathProperties = file.GetProperties();
                PrintDirectoryEntry(pathProperties);

                // Rename a file
                string destFilePath = "/Test/testRenameDest3.txt";
                file.Rename(destFilePath);
                file = filesystemclient.GetFileClient(destFilePath);
                Console.WriteLine("The file URI is " + file.Uri);

                // Enumerate directory
                foreach (var pathItem in filesystemclient.GetPaths("/Test"))
                {
                    PrintDirectoryEntry(pathItem);
                }

                // Delete a directory and all it's subdirectories and files
                filesystemclient.DeleteDirectory("/Test");
            }
            finally
            {
                filesystemclient.Delete();
            }

            Console.WriteLine("Done. Press ENTER to continue ...");
            Console.ReadLine();
        }
        /// <inheritdoc cref="IFileStorageService"/>
        public string GetFileMD5Hash(Uri fileUri)
        {
            DataLakeFileClient        fcli  = new DataLakeFileClient(fileUri, SharedKeyCredential);
            Response <PathProperties> props = fcli.GetProperties();

            byte[] hashBytes = props.Value.ContentHash;
            string hexHash   = BitConverter.ToString(hashBytes);

            return(hexHash);
        }
        public void Append()
        {
            // Create three temporary Lorem Ipsum files on disk that we can upload
            int    contentLength          = 10;
            string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength));
            string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength));
            string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength));

            // Make StorageSharedKeyCredential to pass to the serviceClient
            string storageAccountName = StorageAccountName;
            string storageAccountKey  = StorageAccountKey;
            Uri    serviceUri         = StorageAccountBlobUri;

            StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

            // Create DataLakeServiceClient using StorageSharedKeyCredentials
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);

            // Get a reference to a filesystem named "sample-filesystem-append" and then create it
            DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));

            filesystem.Create();
            try
            {
                // Get a reference to a file named "sample-file" in a filesystem
                DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));

                // Create the file
                file.Create();

                // Verify we created one file
                Assert.AreEqual(1, filesystem.GetPaths().Count());

                // Append data to an existing DataLake File.  Append is currently limited to 4000 MB per call.
                // To upload a large file all at once, consider using Upload() instead.
                file.Append(File.OpenRead(sampleFileContentPart1), 0);
                file.Append(File.OpenRead(sampleFileContentPart2), contentLength);
                file.Append(File.OpenRead(sampleFileContentPart3), contentLength * 2);
                file.Flush(contentLength * 3);

                // Verify the contents of the file
                PathProperties properties = file.GetProperties();
                Assert.AreEqual(contentLength * 3, properties.ContentLength);
            }
            finally
            {
                // Clean up after the test when we're finished
                filesystem.Delete();
            }
        }
        public void Rename()
        {
            // Make StorageSharedKeyCredential to pass to the serviceClient
            string storageAccountName = StorageAccountName;
            string storageAccountKey  = StorageAccountKey;
            Uri    serviceUri         = StorageAccountBlobUri;
            StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

            // Create DataLakeServiceClient using StorageSharedKeyCredentials
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);

            // Get a reference to a filesystem named "sample-filesystem-rename" and then create it
            DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-rename"));

            filesystem.Create();
            try
            {
                // Create a DataLake Directory to rename it later
                DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
                directoryClient.Create();

                // Rename directory with new path/name and verify by making a service call (e.g. GetProperties)
                #region Snippet:SampleSnippetDataLakeFileClient_RenameDirectory
                DataLakeDirectoryClient renamedDirectoryClient = directoryClient.Rename("sample-directory2");
                #endregion Snippet:SampleSnippetDataLakeFileClient_RenameDirectory
                PathProperties directoryPathProperties = renamedDirectoryClient.GetProperties();

                // Delete the sample directory using the new path/name
                filesystem.DeleteDirectory("sample-directory2");

                // Create a DataLake file.
                DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
                fileClient.Create();

                // Rename file with new path/name and verify by making a service call (e.g. GetProperties)
                #region Snippet:SampleSnippetDataLakeFileClient_RenameFile
                DataLakeFileClient renamedFileClient = fileClient.Rename("sample-file2");
                #endregion Snippet:SampleSnippetDataLakeFileClient_RenameFile
                PathProperties filePathProperties = renamedFileClient.GetProperties();

                // Delete the sample directory using the new path/name
                filesystem.DeleteFile("sample-file2");
            }
            finally
            {
                // Clean up after the test when we're finished
                filesystem.Delete();
            }
        }
示例#7
0
 /// <summary>
 /// Azure DataLakeGen2 Item constructor
 /// </summary>
 /// <param name="blob">CloudBlockBlob blob object</param>
 public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
 {
     Name          = fileClient.Name;
     Path          = fileClient.Path;
     File          = fileClient;
     Properties    = fileClient.GetProperties();
     AccessControl = File.GetAccessControl();
     Length        = Properties.ContentLength;
     ContentType   = Properties.ContentType;
     LastModified  = Properties.LastModified;
     IsDirectory   = false;
     Permissions   = AccessControl.Permissions;
     ACL           = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
     Owner         = AccessControl.Owner;
     Group         = AccessControl.Group;
 }
        public void Upload()
        {
            // Create three temporary Lorem Ipsum files on disk that we can upload
            int    contentLength     = 10;
            string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength));

            // Make StorageSharedKeyCredential to pass to the serviceClient
            string storageAccountName = StorageAccountName;
            string storageAccountKey  = StorageAccountKey;
            Uri    serviceUri         = StorageAccountBlobUri;

            StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

            // Create DataLakeServiceClient using StorageSharedKeyCredentials
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);

            // Get a reference to a filesystem named "sample-filesystem-append" and then create it
            DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));

            filesystem.Create();
            try
            {
                // Get a reference to a file named "sample-file" in a filesystem
                DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));

                // Create the file
                file.Create();

                // Verify we created one file
                Assert.AreEqual(1, filesystem.GetPaths().Count());

                // Upload content to the file.  When using the Upload API, you don't need to create the file first.
                // If the file already exists, it will be overwritten.
                // For larger files, Upload() will upload the file in multiple sequential requests.
                file.Upload(File.OpenRead(sampleFileContent), true);

                // Verify the contents of the file
                PathProperties properties = file.GetProperties();
                Assert.AreEqual(contentLength, properties.ContentLength);
            }
            finally
            {
                // Clean up after the test when we're finished
                filesystem.Delete();
            }
        }
        public void GetProperties()
        {
            // Make StorageSharedKeyCredential to pass to the serviceClient
            string storageAccountName = StorageAccountName;
            string storageAccountKey  = StorageAccountKey;
            Uri    serviceUri         = StorageAccountBlobUri;
            StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

            // Create DataLakeServiceClient using StorageSharedKeyCredentials
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);

            // Get a reference to a filesystem named "sample-filesystem-rename" and then create it
            DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));

            filesystem.Create();
            try
            {
                // Create a DataLake Directory to rename it later
                DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(Randomize("sample-directory"));
                directoryClient.Create();

                #region Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties
                // Get Properties on a Directory
                PathProperties directoryPathProperties = directoryClient.GetProperties();
                #endregion Snippet:SampleSnippetDataLakeDirectoryClient_GetProperties

                // Create a DataLake file
                DataLakeFileClient fileClient = filesystem.GetFileClient(Randomize("sample-file"));
                fileClient.Create();

                #region Snippet:SampleSnippetDataLakeFileClient_GetProperties
                // Get Properties on a File
                PathProperties filePathProperties = fileClient.GetProperties();
                #endregion Snippet:SampleSnippetDataLakeFileClient_GetProperties
            }
            finally
            {
                // Clean up after the test when we're finished
                filesystem.Delete();
            }
        }
        public void Append_Simple()
        {
            // Create Sample File to read content from
            string sampleFilePath = CreateTempFile(SampleFileContent);

            // Make StorageSharedKeyCredential to pass to the serviceClient
            string storageAccountName = StorageAccountName;
            string storageAccountKey  = StorageAccountKey;
            Uri    serviceUri         = StorageAccountBlobUri;

            StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

            // Create DataLakeServiceClient using StorageSharedKeyCredentials
            DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);

            // Get a reference to a filesystem named "sample-filesystem-append" and then create it
            DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));

            filesystem.Create();
            try
            {
                #region Snippet:SampleSnippetDataLakeFileClient_Append
                // Create a file
                DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
                file.Create();

                // Append data to the DataLake File
                file.Append(File.OpenRead(sampleFilePath), 0);
                file.Flush(SampleFileContent.Length);
                #endregion Snippet:SampleSnippetDataLakeFileClient_Append

                // Verify the contents of the file
                PathProperties properties = file.GetProperties();
                Assert.AreEqual(SampleFileContent.Length, properties.ContentLength);
            }
            finally
            {
                // Clean up after the test when we're finished
                filesystem.Delete();
            }
        }
        /// <summary>
        /// execute command
        /// </summary>
        public override void ExecuteCmdlet()
        {
            IStorageBlobManagement localChannel   = Channel;
            BlobRequestOptions     requestOptions = RequestOptions;

            bool foundAFolder = false;
            DataLakeFileClient      srcBlob    = null;
            DataLakeDirectoryClient srcBlobDir = null;

            if (ParameterSetName == ManualParameterSet)
            {
                DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
                foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir);
            }
            else //BlobParameterSet
            {
                if (!InputObject.IsDirectory)
                {
                    srcBlob = InputObject.File;
                }
                else
                {
                    srcBlobDir   = InputObject.Directory;
                    foundAFolder = true;
                }
            }

            if (foundAFolder)
            {
                if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: "))
                {
                    // check dest exist
                    bool destExist = true;
                    DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
                    DataLakeDirectoryClient  destBlobDir    = destFileSystem.GetDirectoryClient(this.DestPath);
                    try
                    {
                        destBlobDir.GetProperties();
                    }
                    catch (RequestFailedException e) when(e.Status == 404)
                    {
                        destExist = false;
                    }

                    if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), ""))
                    {
                        destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value;
                        WriteDataLakeGen2Item(localChannel, destBlobDir);
                    }
                }
            }
            else
            {
                if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: "))
                {
                    // check dest exist
                    bool destExist = true;
                    DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
                    DataLakeFileClient       destFile       = destFileSystem.GetFileClient(this.DestPath);
                    try
                    {
                        destFile.GetProperties();
                    }
                    catch (RequestFailedException e) when(e.Status == 404)
                    {
                        destExist = false;
                    }

                    if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), ""))
                    {
                        destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value;
                        WriteDataLakeGen2Item(localChannel, destFile);
                    }
                }
            }
        }