public async Task PathClient_CanGetParentDirectoryClient_WithContainerSAS() { // Arrange var parentDirName = DataLakeClientBuilder.GetNewDirectoryName(); await using DisposingFileSystem test = await DataLakeClientBuilder.GetNewFileSystem(); var fileName = DataLakeClientBuilder.GetNewFileName(); DataLakeFileClient fileClient = InstrumentClient( GetServiceClient_DataLakeServiceSas_FileSystem(test.Container.Name) .GetFileSystemClient(test.FileSystem.Name) .GetRootDirectoryClient() .GetSubDirectoryClient(parentDirName) .GetFileClient(fileName)); await fileClient.CreateAsync(); // Act DataLakeDirectoryClient parentDirClient = fileClient.GetParentDirectoryClient(); // make sure that client is functional var pathItems = await parentDirClient.GetPathsAsync().ToListAsync(); // Assert Assert.AreEqual(fileClient.Path.GetParentPath(), parentDirClient.Path); Assert.AreEqual(fileClient.AccountName, parentDirClient.AccountName); Assert.IsNotNull(pathItems); }
// </Snippet_DownloadFromDirectory> #endregion #region Download a binary file from a directory // --------------------------------------------------------- // Download file from directory (binary) //---------------------------------------------------------- // <Snippet_DownloadBinaryFromDirectory> public async Task DownloadFile(DataLakeFileSystemClient fileSystemClient) { DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient("my-directory"); DataLakeFileClient fileClient = directoryClient.GetFileClient("my-image.png"); Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync(); BinaryReader reader = new BinaryReader(downloadResponse.Value.Content); FileStream fileStream = File.OpenWrite("C:\\Users\\contoso\\my-image-downloaded.png"); int bufferSize = 4096; byte[] buffer = new byte[bufferSize]; int count; while ((count = reader.Read(buffer, 0, buffer.Length)) != 0) { fileStream.Write(buffer, 0, count); } await fileStream.FlushAsync(); fileStream.Close(); }
/// <summary> /// Connect and upload the data as file to the Azure Data Lake. /// </summary> /// <param name="storageAccountName">Azure storage account name</param> /// <param name="storageAccountKey">Azure storage account key</param> /// <param name="dataLakeUri">Azure Data Lake URI</param> /// <param name="directoryName">Azure Data Lake directory name</param> /// <param name="content">Upload data content</param> public async Task <bool> UploadData(string storageAccountName, string storageAccountKey, string dataLakeUri, string directoryName, string content) { try { Uri serviceUri = new Uri(dataLakeUri); StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(directoryName); DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(directoryName); DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(string.Format("data-{0}.json", Guid.NewGuid().ToString())); using (MemoryStream memoryStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(content))) { await fileClient.AppendAsync(memoryStream, offset : 0); await fileClient.FlushAsync(position : memoryStream.Length); } return(true); } catch (Exception exception) { logger.Error(exception.StackTrace); return(false); } }
public async Task DataLakeSasBuilder_BothObjectId_Error() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); string directoryName = GetNewDirectoryName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); // Arrange DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName); DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName()); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, PreauthorizedAgentObjectId = Recording.Random.NewGuid().ToString(), AgentObjectId = Recording.Random.NewGuid().ToString() }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); TestHelper.AssertExpectedException <InvalidOperationException>( () => dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName), new InvalidOperationException("SAS cannot have the following parameters specified in conjunction: PreauthorizedAgentObjectId, AgentObjectId")); }
public async Task CreateFileClientAsync_Filesystem() { // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-append" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); filesystem.Create(); // Create DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); await file.CreateAsync(); // Verify we created one file AsyncPageable <PathItem> response = filesystem.GetPathsAsync(); IList <PathItem> paths = await response.ToListAsync(); Assert.AreEqual(1, paths.Count); // Cleanup await filesystem.DeleteAsync(); }
public async Task CreateFileClientAsync_Directory() { // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Create a DataLake Filesystem DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem")); await filesystem.CreateAsync(); //Create a DataLake Directory DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory")); await directory.CreateAsync(); // Create a DataLake File using a DataLake Directory DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file")); await file.CreateAsync(); // Verify we created one file AsyncPageable <PathItem> response = filesystem.GetPathsAsync(); IList <PathItem> paths = await response.ToListAsync(); Assert.AreEqual(1, paths.Count); // Cleanup await filesystem.DeleteAsync(); }
public async Task <MemoryStream> DownloadFiles(List <Models.File> Files) { var DirectoryClient = FileSystemClient.GetDirectoryClient(Files[0].UserId); DataLakeFileClient FileClient = null; MemoryStream returnStream = new MemoryStream(); using (MemoryStream ms = new MemoryStream()) { using (ZipOutputStream zipOutputStream = new ZipOutputStream(ms)) { foreach (Models.File file in Files) { FileClient = DirectoryClient.GetFileClient(file.Categories[0].CategoryName + "/" + file.FileName); using (Stream stream = await FileClient.OpenReadAsync()) { var entry = new ZipEntry(file.FileName); zipOutputStream.PutNextEntry(entry); byte[] bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); await zipOutputStream.WriteAsync(bytes, 0, bytes.Length); } } } returnStream = ms; } return(returnStream); }
/// <summary> /// Azure DataLakeGen2 Item constructor /// </summary> /// <param name="blob">CloudBlockBlob blob object</param> public AzureDataLakeGen2Item(DataLakeFileClient fileClient) { Name = fileClient.Name; Path = fileClient.Path; File = fileClient; IsDirectory = false; try { Properties = fileClient.GetProperties(); Length = Properties.ContentLength; ContentType = Properties.ContentType; LastModified = Properties.LastModified; } catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404) { // skip get file properties if don't have read permission } try { AccessControl = File.GetAccessControl(); Permissions = AccessControl.Permissions; ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList); Owner = AccessControl.Owner; Group = AccessControl.Group; } catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404) { // skip get file ACL if don't have read permission } }
public void CreateFileClient_Directory() { // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; #region Snippet:SampleSnippetDataLakeFileSystemClient_Create StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Create a DataLake Filesystem DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem")); filesystem.Create(); #endregion Snippet:SampleSnippetDataLakeFileSystemClient_Create #region Snippet:SampleSnippetDataLakeFileClient_Create_Directory // Create a DataLake Directory DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory")); directory.Create(); // Create a DataLake File using a DataLake Directory DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file")); file.Create(); #endregion Snippet:SampleSnippetDataLakeFileClient_Create_Directory // Verify we created one file Assert.AreEqual(1, filesystem.ListPaths().Count()); // Cleanup filesystem.Delete(); }
// Create a sample hierarchical directory tree using async operations private static async Task CreateDirRecursiveAsync(DataLakeFileSystemClient client, string path, int recursLevel, int noDirEntries, int noFileEntries) { await client.CreateDirectoryAsync(path); byte[] writeData = Encoding.UTF8.GetBytes("This is the first line.\n"); string[] str = path.Split('/'); char nextLevel = str[str.Length - 1][0]; nextLevel++; for (int i = 0; i < noFileEntries; i++) { DataLakeFileClient file = client.GetFileClient(path + "/" + nextLevel + i + "File.txt"); using (var stream = new MemoryStream(writeData)) { file.Upload(stream, true); } } if (recursLevel == 0) { return; } string newPath = path + "/"; for (int i = 0; i < noDirEntries; i++) { await CreateDirRecursiveAsync(client, newPath + nextLevel + i, recursLevel - 1, noDirEntries, noFileEntries); } }
private static void PerformWriteFlushReadSeek(DataLakeFileSystemClient client) { string fileName = "/Test/dir1/testFilename1.txt"; DataLakeFileClient file = client.GetFileClient(fileName); // Create the file Stream stream = BinaryData.FromString("This is the first line.\nThis is the second line.\n").ToStream(); long length = stream.Length; file.Upload(stream, true); // Append to the file stream = BinaryData.FromString("This is the third line.\nThis is the fourth line.\n").ToStream(); file.Append(stream, length); file.Flush(length + stream.Length); // Read the file using (var readStream = file.OpenRead()) { byte[] readData = new byte[1024]; // Read 40 bytes at this offset int readBytes = readStream.Read(readData, 25, 40); Console.WriteLine("Read output of 40 bytes from offset 25: " + Encoding.UTF8.GetString(readData, 0, readBytes)); } }
/// <summary> /// Initializes a new instance of the <see cref="Read"/> class. /// </summary> /// /// <param name="options">The set of options to consider for configuring the scenario.</param> /// public Read(SizeOptions options) : base(options) { var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); FileClient = FileSystemClient.GetFileClient(Filename); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger log) { //connect to account string accountName = Common.GetEnvironmentVariable("DataLakeAccountName"); string accountKey = Common.GetEnvironmentVariable("DataLakeAccountKey"); string fileSystemName = Common.GetEnvironmentVariable("DataLakeFileSystemName"); StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey); string dfsUri = "https://" + accountName + ".dfs.core.windows.net"; DataLakeServiceClient dataLakeServiceClient = new DataLakeServiceClient(new Uri(dfsUri), sharedKeyCredential); //upload file DataLakeFileSystemClient fileSystemClient = dataLakeServiceClient.GetFileSystemClient(fileSystemName); string fileName = Guid.NewGuid().ToString() + ".json"; DataLakeFileClient fileClient = await fileSystemClient.CreateFileAsync(fileName); long fileSize = req.Body.Length; await fileClient.AppendAsync(req.Body, offset : 0); await fileClient.FlushAsync(position : fileSize); return((ActionResult) new OkResult()); }
public async Task Append(string path, byte[] data, CancellationToken token = default) { path = WithBasePath(path); _logger.LogTrace($"Appending to {path}, data.Length={data.Length}"); data .VerifyNotNull(nameof(data)) .VerifyAssert(x => x.Length > 0, $"{nameof(data)} length must be greater then 0"); using var memoryBuffer = new MemoryStream(data.ToArray()); try { DatalakePathProperties properties = await GetPathProperties(path, token); DataLakeFileClient file = _fileSystem.GetFileClient(path); await file.AppendAsync(memoryBuffer, properties.ContentLength, cancellationToken : token); await file.FlushAsync(properties.ContentLength + data.Length); } catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound" || ex.ErrorCode == "BlobNotFound") { await Write(path, data, true, token : token); } catch (TaskCanceledException) { } }
public List <Project> GetProjects( ProjectReader reader) { List <string> fileNames = GetFileNames(MIPPEN_FILE_SEARCH_TERM); List <Project> projects = new List <Project>(); DataLakeFileSystemClient fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName); foreach (var fileName in fileNames) { // Get file contents DataLakeFileClient fileClient = fileSystemClient.GetFileClient(fileName); // Try to get a project Project project; using (var stream = fileClient.OpenRead()) { project = reader.Read(stream); } if (project is not null) { projects.Add(project); } } return(projects); }
/// <summary> /// Upload File with Datalake API /// </summary> internal virtual async Task UploadDataLakeFile(long taskId, DataLakeFileClient fileClient, string filePath) { if (this.Force.IsPresent || !fileClient.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, GetDataLakeItemUriWithoutSas(fileClient)), null)) { // Set Item Properties and MetaData PathHttpHeaders pathHttpHeaders = SetDatalakegen2ItemProperties(fileClient, BlobProperties, setToServer: false); IDictionary <string, string> metadata = SetDatalakegen2ItemMetaData(fileClient, BlobMetadata, setToServer: false); fileClient.Create(pathHttpHeaders, metadata, this.Permission, this.Umask != null ? DataLakeModels.PathPermissions.ParseSymbolicPermissions(this.Umask).ToOctalPermissions() : null); long fileSize = new FileInfo(ResolvedFileName).Length; string activity = String.Format(Resources.SendAzureBlobActivity, this.Source, this.Path, this.FileSystem); string status = Resources.PrepareUploadingBlob; ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); IProgress <long> progressHandler = new Progress <long>((finishedBytes) => { if (pr != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize); pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete); this.OutputStream.WriteProgress(pr); } }); using (FileStream stream = File.OpenRead(ResolvedFileName)) { await fileClient.AppendAsync(stream, 0, progressHandler : progressHandler, cancellationToken : CmdletCancellationToken).ConfigureAwait(false); } WriteDataLakeGen2Item(Channel, fileClient, taskId: taskId); } }
private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/ public static void Main(string[] args) { // Create Client Secret Credential var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret); // Create data lake file service client object DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds); var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8); // Create data lake file system client object DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name); filesystemclient.CreateIfNotExists(); try { long length; string fileName = "/Test/testFilename1.txt"; DataLakeFileClient file = filesystemclient.GetFileClient(fileName); // Upload a file - automatically creates any parent directories that don't exist length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length; file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true); file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length); file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length); //Read file contents Response <FileDownloadInfo> fileContents = file.Read(); Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString()); // Get the properties of the file PathProperties pathProperties = file.GetProperties(); PrintDirectoryEntry(pathProperties); // Rename a file string destFilePath = "/Test/testRenameDest3.txt"; file.Rename(destFilePath); file = filesystemclient.GetFileClient(destFilePath); Console.WriteLine("The file URI is " + file.Uri); // Enumerate directory foreach (var pathItem in filesystemclient.GetPaths("/Test")) { PrintDirectoryEntry(pathItem); } // Delete a directory and all it's subdirectories and files filesystemclient.DeleteDirectory("/Test"); } finally { filesystemclient.Delete(); } Console.WriteLine("Done. Press ENTER to continue ..."); Console.ReadLine(); }
public async Task <bool> Exist(string path, CancellationToken token) { path.VerifyNotEmpty(nameof(path)); DataLakeFileClient file = _fileSystem.GetFileClient(path); Response <bool> response = await file.ExistsAsync(token); return(response.Value); }
public async Task Read(string path, Stream toStream, CancellationToken token = default) { path = WithBasePath(path); toStream.VerifyNotNull(nameof(toStream)); _logger.LogTrace($"Reading {path} to stream"); DataLakeFileClient file = _fileSystem.GetFileClient(path); await file.ReadToAsync(toStream, cancellationToken : token); }
public async Task AppendAsync() { // Create three temporary Lorem Ipsum files on disk that we can upload int contentLength = 10; string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength)); string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength)); string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength)); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Get a reference to a FileSystemClient DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); await filesystem.CreateAsync(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // Create the file await file.CreateAsync(); // Verify we created one file AsyncPageable <PathItem> response = filesystem.GetPathsAsync(); IList <PathItem> paths = await response.ToListAsync(); Assert.AreEqual(1, paths.Count); // Append data to an existing DataLake File. Append is currently limited to 4000 MB per call. // To upload a large file all at once, consider using UploadAsync() instead. await file.AppendAsync(File.OpenRead(sampleFileContentPart1), 0); await file.AppendAsync(File.OpenRead(sampleFileContentPart2), contentLength); await file.AppendAsync(File.OpenRead(sampleFileContentPart3), contentLength * 2); await file.FlushAsync(contentLength * 3); // Verify the contents of the file PathProperties properties = await file.GetPropertiesAsync(); Assert.AreEqual(contentLength * 3, properties.ContentLength); } finally { // Clean up after the test when we're finished await filesystem.DeleteAsync(); } }
static async Task RemoveACLsForFile(DataLakeFileClient fileClient, AppSettings settings) { PathAccessControl fileAccessControl = await fileClient.GetAccessControlAsync(); List <PathAccessControlItem> accessControlList = RemoveACLs(fileAccessControl.AccessControlList, settings); await fileClient.SetAccessControlListAsync(accessControlList); }
static async Task ApplyACLsForFile(DataLakeFileClient fileClient, RolePermissions newACLs, AppSettings settings) { PathAccessControl fileAccessControl = await fileClient.GetAccessControlAsync(); List <PathAccessControlItem> accessControlList = UpdateACLs(fileAccessControl.AccessControlList, newACLs, settings); await fileClient.SetAccessControlListAsync(accessControlList); }
public async Task Upload(Stream fromStream, string toPath, bool force, CancellationToken token) { fromStream.VerifyNotNull(nameof(fromStream)); toPath.VerifyNotEmpty(nameof(toPath)); _logger.LogTrace($"{nameof(Upload)} from stream to {toPath}"); DataLakeFileClient file = _fileSystem.GetFileClient(toPath); await file.UploadAsync(fromStream, force, token); }
public async Task <bool> Delete(string path, ETag?eTag = null, CancellationToken token = default) { path = WithBasePath(path); _logger.LogTrace($"Deleting to {path}, ETag={eTag}"); DataLakeFileClient file = _fileSystem.GetFileClient(path); Response <bool> response = await file.DeleteIfExistsAsync(cancellationToken : token); return(response.Value); }
public async Task <bool> Exist(string path, CancellationToken token = default) { path = WithBasePath(path); _logger.LogTrace($"Is path {path} exist"); DataLakeFileClient file = _fileSystem.GetFileClient(path); Response <bool> response = await file.ExistsAsync(token); return(response.Value); }
/// <summary> /// Create file /// </summary> /// <param name="file"></param> /// <param name="leaseId"></param> /// <param name="logger"></param> private FileSystemFile(DataLakeFileClient file, string leaseId, ILogger logger) { _file = file; _logger = logger.ForContext <FileSystemFile>(); _conditions = leaseId == null ? null : new DataLakeRequestConditions { LeaseId = leaseId }; }
public async Task <DatalakePathProperties> GetPathProperties(string path, CancellationToken token) { path.VerifyNotEmpty(nameof(path)); DataLakeFileClient file = _fileSystem.GetFileClient(path); return((await file.GetPropertiesAsync(cancellationToken: token)) .Value .ConvertTo()); }
public async Task Download(string path, Stream toStream, CancellationToken token) { path.VerifyNotEmpty(nameof(path)); toStream.VerifyNotNull(nameof(toStream)); _logger.LogTrace($"{nameof(Download)} downloading {path} to stream"); DataLakeFileClient file = _fileSystem.GetFileClient(path); await file.ReadToAsync(toStream, cancellationToken : token); }
/// <inheritdoc cref="IFileStorageService"/> public string GetFileMD5Hash(Uri fileUri) { DataLakeFileClient fcli = new DataLakeFileClient(fileUri, SharedKeyCredential); Response <PathProperties> props = fcli.GetProperties(); byte[] hashBytes = props.Value.ContentHash; string hexHash = BitConverter.ToString(hashBytes); return(hexHash); }
/// <summary> /// execute command /// </summary> public override void ExecuteCmdlet() { IStorageBlobManagement localChannel = Channel; bool foundAFolder = false; DataLakeFileClient srcBlob = null; DataLakeDirectoryClient srcBlobDir = null; if (ParameterSetName == ManualParameterSet) { DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem); foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir); } else //BlobParameterSet { if (!InputObject.IsDirectory) { srcBlob = InputObject.File; } else { srcBlobDir = InputObject.Directory; foundAFolder = true; } } if (foundAFolder) { if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: ")) { DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath); if (this.Force || !destBlobDir.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), "")) { destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value; WriteDataLakeGen2Item(localChannel, destBlobDir); } } } else { if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: ")) { DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem); DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath); if (this.Force || !destFile.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), "")) { destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value; WriteDataLakeGen2Item(localChannel, destFile); } } } }