public async Task PathClient_CanGetParentDirectoryClient_WithContainerSAS()
{
// Arrange
var parentDirName = DataLakeClientBuilder.GetNewDirectoryName();
await using DisposingFileSystem test = await DataLakeClientBuilder.GetNewFileSystem();
var fileName = DataLakeClientBuilder.GetNewFileName();
DataLakeFileClient fileClient = InstrumentClient(
GetServiceClient_DataLakeServiceSas_FileSystem(test.Container.Name)
.GetFileSystemClient(test.FileSystem.Name)
.GetRootDirectoryClient()
.GetSubDirectoryClient(parentDirName)
.GetFileClient(fileName));
await fileClient.CreateAsync();
// Act
DataLakeDirectoryClient parentDirClient = fileClient.GetParentDirectoryClient();
// make sure that client is functional
var pathItems = await parentDirClient.GetPathsAsync().ToListAsync();
// Assert
Assert.AreEqual(fileClient.Path.GetParentPath(), parentDirClient.Path);
Assert.AreEqual(fileClient.AccountName, parentDirClient.AccountName);
Assert.IsNotNull(pathItems);
}
// </Snippet_DownloadFromDirectory>
#endregion
#region Download a binary file from a directory
// ---------------------------------------------------------
// Download file from directory (binary)
//----------------------------------------------------------
// <Snippet_DownloadBinaryFromDirectory>
public async Task DownloadFile(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient =
directoryClient.GetFileClient("my-image.png");
Response <FileDownloadInfo> downloadResponse = await fileClient.ReadAsync();
BinaryReader reader = new BinaryReader(downloadResponse.Value.Content);
FileStream fileStream =
File.OpenWrite("C:\\Users\\contoso\\my-image-downloaded.png");
int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
fileStream.Write(buffer, 0, count);
}
await fileStream.FlushAsync();
fileStream.Close();
}
/// <summary>
/// Connect and upload the data as file to the Azure Data Lake.
/// </summary>
/// <param name="storageAccountName">Azure storage account name</param>
/// <param name="storageAccountKey">Azure storage account key</param>
/// <param name="dataLakeUri">Azure Data Lake URI</param>
/// <param name="directoryName">Azure Data Lake directory name</param>
/// <param name="content">Upload data content</param>
public async Task <bool> UploadData(string storageAccountName, string storageAccountKey, string dataLakeUri, string directoryName, string content)
{
try
{
Uri serviceUri = new Uri(dataLakeUri);
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(directoryName);
DataLakeDirectoryClient directoryClient =
filesystem.GetDirectoryClient(directoryName);
DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(string.Format("data-{0}.json", Guid.NewGuid().ToString()));
using (MemoryStream memoryStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(content)))
{
await fileClient.AppendAsync(memoryStream, offset : 0);
await fileClient.FlushAsync(position : memoryStream.Length);
}
return(true);
}
catch (Exception exception)
{
logger.Error(exception.StackTrace);
return(false);
}
}
public async Task DataLakeSasBuilder_BothObjectId_Error()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
PreauthorizedAgentObjectId = Recording.Random.NewGuid().ToString(),
AgentObjectId = Recording.Random.NewGuid().ToString()
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
TestHelper.AssertExpectedException <InvalidOperationException>(
() => dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName),
new InvalidOperationException("SAS cannot have the following parameters specified in conjunction: PreauthorizedAgentObjectId, AgentObjectId"));
}
public async Task CreateFileClientAsync_Filesystem()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
filesystem.Create();
// Create
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Cleanup
await filesystem.DeleteAsync();
}
public async Task CreateFileClientAsync_Directory()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Create a DataLake Filesystem
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
await filesystem.CreateAsync();
//Create a DataLake Directory
DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory"));
await directory.CreateAsync();
// Create a DataLake File using a DataLake Directory
DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file"));
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Cleanup
await filesystem.DeleteAsync();
}
public async Task <MemoryStream> DownloadFiles(List <Models.File> Files)
{
var DirectoryClient = FileSystemClient.GetDirectoryClient(Files[0].UserId);
DataLakeFileClient FileClient = null;
MemoryStream returnStream = new MemoryStream();
using (MemoryStream ms = new MemoryStream())
{
using (ZipOutputStream zipOutputStream = new ZipOutputStream(ms))
{
foreach (Models.File file in Files)
{
FileClient = DirectoryClient.GetFileClient(file.Categories[0].CategoryName + "/" + file.FileName);
using (Stream stream = await FileClient.OpenReadAsync())
{
var entry = new ZipEntry(file.FileName);
zipOutputStream.PutNextEntry(entry);
byte[] bytes = new byte[stream.Length];
stream.Read(bytes, 0, (int)stream.Length);
await zipOutputStream.WriteAsync(bytes, 0, bytes.Length);
}
}
}
returnStream = ms;
}
return(returnStream);
}
/// <summary>
/// Azure DataLakeGen2 Item constructor
/// </summary>
/// <param name="blob">CloudBlockBlob blob object</param>
public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
{
Name = fileClient.Name;
Path = fileClient.Path;
File = fileClient;
IsDirectory = false;
try
{
Properties = fileClient.GetProperties();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
{
// skip get file properties if don't have read permission
}
try
{
AccessControl = File.GetAccessControl();
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404)
{
// skip get file ACL if don't have read permission
}
}
public void CreateFileClient_Directory()
{
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
#region Snippet:SampleSnippetDataLakeFileSystemClient_Create
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Create a DataLake Filesystem
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem"));
filesystem.Create();
#endregion Snippet:SampleSnippetDataLakeFileSystemClient_Create
#region Snippet:SampleSnippetDataLakeFileClient_Create_Directory
// Create a DataLake Directory
DataLakeDirectoryClient directory = filesystem.CreateDirectory(Randomize("sample-directory"));
directory.Create();
// Create a DataLake File using a DataLake Directory
DataLakeFileClient file = directory.GetFileClient(Randomize("sample-file"));
file.Create();
#endregion Snippet:SampleSnippetDataLakeFileClient_Create_Directory
// Verify we created one file
Assert.AreEqual(1, filesystem.ListPaths().Count());
// Cleanup
filesystem.Delete();
}
// Create a sample hierarchical directory tree using async operations
private static async Task CreateDirRecursiveAsync(DataLakeFileSystemClient client, string path, int recursLevel, int noDirEntries, int noFileEntries)
{
await client.CreateDirectoryAsync(path);
byte[] writeData = Encoding.UTF8.GetBytes("This is the first line.\n");
string[] str = path.Split('/');
char nextLevel = str[str.Length - 1][0];
nextLevel++;
for (int i = 0; i < noFileEntries; i++)
{
DataLakeFileClient file = client.GetFileClient(path + "/" + nextLevel + i + "File.txt");
using (var stream = new MemoryStream(writeData))
{
file.Upload(stream, true);
}
}
if (recursLevel == 0)
{
return;
}
string newPath = path + "/";
for (int i = 0; i < noDirEntries; i++)
{
await CreateDirRecursiveAsync(client, newPath + nextLevel + i, recursLevel - 1, noDirEntries, noFileEntries);
}
}
private static void PerformWriteFlushReadSeek(DataLakeFileSystemClient client)
{
string fileName = "/Test/dir1/testFilename1.txt";
DataLakeFileClient file = client.GetFileClient(fileName);
// Create the file
Stream stream = BinaryData.FromString("This is the first line.\nThis is the second line.\n").ToStream();
long length = stream.Length;
file.Upload(stream, true);
// Append to the file
stream = BinaryData.FromString("This is the third line.\nThis is the fourth line.\n").ToStream();
file.Append(stream, length);
file.Flush(length + stream.Length);
// Read the file
using (var readStream = file.OpenRead())
{
byte[] readData = new byte[1024];
// Read 40 bytes at this offset
int readBytes = readStream.Read(readData, 25, 40);
Console.WriteLine("Read output of 40 bytes from offset 25: " + Encoding.UTF8.GetString(readData, 0, readBytes));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Read"/> class.
/// </summary>
///
/// <param name="options">The set of options to consider for configuring the scenario.</param>
///
public Read(SizeOptions options) : base(options)
{
var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential);
FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName);
FileClient = FileSystemClient.GetFileClient(Filename);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
//connect to account
string accountName = Common.GetEnvironmentVariable("DataLakeAccountName");
string accountKey = Common.GetEnvironmentVariable("DataLakeAccountKey");
string fileSystemName = Common.GetEnvironmentVariable("DataLakeFileSystemName");
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey);
string dfsUri = "https://" + accountName + ".dfs.core.windows.net";
DataLakeServiceClient dataLakeServiceClient = new DataLakeServiceClient(new Uri(dfsUri), sharedKeyCredential);
//upload file
DataLakeFileSystemClient fileSystemClient = dataLakeServiceClient.GetFileSystemClient(fileSystemName);
string fileName = Guid.NewGuid().ToString() + ".json";
DataLakeFileClient fileClient = await fileSystemClient.CreateFileAsync(fileName);
long fileSize = req.Body.Length;
await fileClient.AppendAsync(req.Body, offset : 0);
await fileClient.FlushAsync(position : fileSize);
return((ActionResult) new OkResult());
}
public async Task Append(string path, byte[] data, CancellationToken token = default)
{
path = WithBasePath(path);
_logger.LogTrace($"Appending to {path}, data.Length={data.Length}");
data
.VerifyNotNull(nameof(data))
.VerifyAssert(x => x.Length > 0, $"{nameof(data)} length must be greater then 0");
using var memoryBuffer = new MemoryStream(data.ToArray());
try
{
DatalakePathProperties properties = await GetPathProperties(path, token);
DataLakeFileClient file = _fileSystem.GetFileClient(path);
await file.AppendAsync(memoryBuffer, properties.ContentLength, cancellationToken : token);
await file.FlushAsync(properties.ContentLength + data.Length);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound" || ex.ErrorCode == "BlobNotFound")
{
await Write(path, data, true, token : token);
}
catch (TaskCanceledException) { }
}
public List <Project> GetProjects(
ProjectReader reader)
{
List <string> fileNames = GetFileNames(MIPPEN_FILE_SEARCH_TERM);
List <Project> projects = new List <Project>();
DataLakeFileSystemClient fileSystemClient =
serviceClient.GetFileSystemClient(fileSystemName);
foreach (var fileName in fileNames)
{
// Get file contents
DataLakeFileClient fileClient =
fileSystemClient.GetFileClient(fileName);
// Try to get a project
Project project;
using (var stream = fileClient.OpenRead())
{
project = reader.Read(stream);
}
if (project is not null)
{
projects.Add(project);
}
}
return(projects);
}
/// <summary>
/// Upload File with Datalake API
/// </summary>
internal virtual async Task UploadDataLakeFile(long taskId, DataLakeFileClient fileClient, string filePath)
{
if (this.Force.IsPresent || !fileClient.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, GetDataLakeItemUriWithoutSas(fileClient)), null))
{
// Set Item Properties and MetaData
PathHttpHeaders pathHttpHeaders = SetDatalakegen2ItemProperties(fileClient, BlobProperties, setToServer: false);
IDictionary <string, string> metadata = SetDatalakegen2ItemMetaData(fileClient, BlobMetadata, setToServer: false);
fileClient.Create(pathHttpHeaders,
metadata,
this.Permission,
this.Umask != null ? DataLakeModels.PathPermissions.ParseSymbolicPermissions(this.Umask).ToOctalPermissions() : null);
long fileSize = new FileInfo(ResolvedFileName).Length;
string activity = String.Format(Resources.SendAzureBlobActivity, this.Source, this.Path, this.FileSystem);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (pr != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize);
pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete);
this.OutputStream.WriteProgress(pr);
}
});
using (FileStream stream = File.OpenRead(ResolvedFileName))
{
await fileClient.AppendAsync(stream, 0, progressHandler : progressHandler, cancellationToken : CmdletCancellationToken).ConfigureAwait(false);
}
WriteDataLakeGen2Item(Channel, fileClient, taskId: taskId);
}
}
private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/
public static void Main(string[] args)
{
// Create Client Secret Credential
var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret);
// Create data lake file service client object
DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds);
var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8);
// Create data lake file system client object
DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name);
filesystemclient.CreateIfNotExists();
try
{
long length;
string fileName = "/Test/testFilename1.txt";
DataLakeFileClient file = filesystemclient.GetFileClient(fileName);
// Upload a file - automatically creates any parent directories that don't exist
length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length;
file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true);
file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length);
file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length);
//Read file contents
Response <FileDownloadInfo> fileContents = file.Read();
Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString());
// Get the properties of the file
PathProperties pathProperties = file.GetProperties();
PrintDirectoryEntry(pathProperties);
// Rename a file
string destFilePath = "/Test/testRenameDest3.txt";
file.Rename(destFilePath);
file = filesystemclient.GetFileClient(destFilePath);
Console.WriteLine("The file URI is " + file.Uri);
// Enumerate directory
foreach (var pathItem in filesystemclient.GetPaths("/Test"))
{
PrintDirectoryEntry(pathItem);
}
// Delete a directory and all it's subdirectories and files
filesystemclient.DeleteDirectory("/Test");
}
finally
{
filesystemclient.Delete();
}
Console.WriteLine("Done. Press ENTER to continue ...");
Console.ReadLine();
}
public async Task <bool> Exist(string path, CancellationToken token)
{
path.VerifyNotEmpty(nameof(path));
DataLakeFileClient file = _fileSystem.GetFileClient(path);
Response <bool> response = await file.ExistsAsync(token);
return(response.Value);
}
public async Task Read(string path, Stream toStream, CancellationToken token = default)
{
path = WithBasePath(path);
toStream.VerifyNotNull(nameof(toStream));
_logger.LogTrace($"Reading {path} to stream");
DataLakeFileClient file = _fileSystem.GetFileClient(path);
await file.ReadToAsync(toStream, cancellationToken : token);
}
public async Task AppendAsync()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContentPart1 = CreateTempFile(SampleFileContent.Substring(0, contentLength));
string sampleFileContentPart2 = CreateTempFile(SampleFileContent.Substring(contentLength, contentLength));
string sampleFileContentPart3 = CreateTempFile(SampleFileContent.Substring(contentLength * 2, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Get a reference to a FileSystemClient
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Create the file
await file.CreateAsync();
// Verify we created one file
AsyncPageable <PathItem> response = filesystem.GetPathsAsync();
IList <PathItem> paths = await response.ToListAsync();
Assert.AreEqual(1, paths.Count);
// Append data to an existing DataLake File. Append is currently limited to 4000 MB per call.
// To upload a large file all at once, consider using UploadAsync() instead.
await file.AppendAsync(File.OpenRead(sampleFileContentPart1), 0);
await file.AppendAsync(File.OpenRead(sampleFileContentPart2), contentLength);
await file.AppendAsync(File.OpenRead(sampleFileContentPart3), contentLength * 2);
await file.FlushAsync(contentLength * 3);
// Verify the contents of the file
PathProperties properties = await file.GetPropertiesAsync();
Assert.AreEqual(contentLength * 3, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
static async Task RemoveACLsForFile(DataLakeFileClient fileClient, AppSettings settings)
{
PathAccessControl fileAccessControl =
await fileClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlList = RemoveACLs(fileAccessControl.AccessControlList, settings);
await fileClient.SetAccessControlListAsync(accessControlList);
}
static async Task ApplyACLsForFile(DataLakeFileClient fileClient, RolePermissions newACLs, AppSettings settings)
{
PathAccessControl fileAccessControl =
await fileClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlList = UpdateACLs(fileAccessControl.AccessControlList, newACLs, settings);
await fileClient.SetAccessControlListAsync(accessControlList);
}
public async Task Upload(Stream fromStream, string toPath, bool force, CancellationToken token)
{
fromStream.VerifyNotNull(nameof(fromStream));
toPath.VerifyNotEmpty(nameof(toPath));
_logger.LogTrace($"{nameof(Upload)} from stream to {toPath}");
DataLakeFileClient file = _fileSystem.GetFileClient(toPath);
await file.UploadAsync(fromStream, force, token);
}
public async Task <bool> Delete(string path, ETag?eTag = null, CancellationToken token = default)
{
path = WithBasePath(path);
_logger.LogTrace($"Deleting to {path}, ETag={eTag}");
DataLakeFileClient file = _fileSystem.GetFileClient(path);
Response <bool> response = await file.DeleteIfExistsAsync(cancellationToken : token);
return(response.Value);
}
public async Task <bool> Exist(string path, CancellationToken token = default)
{
path = WithBasePath(path);
_logger.LogTrace($"Is path {path} exist");
DataLakeFileClient file = _fileSystem.GetFileClient(path);
Response <bool> response = await file.ExistsAsync(token);
return(response.Value);
}
/// <summary>
/// Create file
/// </summary>
/// <param name="file"></param>
/// <param name="leaseId"></param>
/// <param name="logger"></param>
private FileSystemFile(DataLakeFileClient file, string leaseId,
ILogger logger)
{
_file = file;
_logger = logger.ForContext <FileSystemFile>();
_conditions = leaseId == null ? null :
new DataLakeRequestConditions {
LeaseId = leaseId
};
}
public async Task <DatalakePathProperties> GetPathProperties(string path, CancellationToken token)
{
path.VerifyNotEmpty(nameof(path));
DataLakeFileClient file = _fileSystem.GetFileClient(path);
return((await file.GetPropertiesAsync(cancellationToken: token))
.Value
.ConvertTo());
}
public async Task Download(string path, Stream toStream, CancellationToken token)
{
path.VerifyNotEmpty(nameof(path));
toStream.VerifyNotNull(nameof(toStream));
_logger.LogTrace($"{nameof(Download)} downloading {path} to stream");
DataLakeFileClient file = _fileSystem.GetFileClient(path);
await file.ReadToAsync(toStream, cancellationToken : token);
}
/// <inheritdoc cref="IFileStorageService"/>
public string GetFileMD5Hash(Uri fileUri)
{
DataLakeFileClient fcli = new DataLakeFileClient(fileUri, SharedKeyCredential);
Response <PathProperties> props = fcli.GetProperties();
byte[] hashBytes = props.Value.ContentHash;
string hexHash = BitConverter.ToString(hashBytes);
return(hexHash);
}
/// <summary>
/// execute command
/// </summary>
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
bool foundAFolder = false;
DataLakeFileClient srcBlob = null;
DataLakeDirectoryClient srcBlobDir = null;
if (ParameterSetName == ManualParameterSet)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out srcBlob, out srcBlobDir);
}
else //BlobParameterSet
{
if (!InputObject.IsDirectory)
{
srcBlob = InputObject.File;
}
else
{
srcBlobDir = InputObject.Directory;
foundAFolder = true;
}
}
if (foundAFolder)
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: "))
{
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath);
if (this.Force || !destBlobDir.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), ""))
{
destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destBlobDir);
}
}
}
else
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: "))
{
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath);
if (this.Force || !destFile.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), ""))
{
destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destFile);
}
}
}
}
