private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/
public static void Main(string[] args)
{
// Create Client Secret Credential
var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret);
// Create data lake file service client object
DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds);
var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8);
// Create data lake file system client object
DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name);
filesystemclient.CreateIfNotExists();
try
{
long length;
string fileName = "/Test/testFilename1.txt";
DataLakeFileClient file = filesystemclient.GetFileClient(fileName);
// Upload a file - automatically creates any parent directories that don't exist
length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length;
file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true);
file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length);
file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length);
//Read file contents
Response <FileDownloadInfo> fileContents = file.Read();
Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString());
// Get the properties of the file
PathProperties pathProperties = file.GetProperties();
PrintDirectoryEntry(pathProperties);
// Rename a file
string destFilePath = "/Test/testRenameDest3.txt";
file.Rename(destFilePath);
file = filesystemclient.GetFileClient(destFilePath);
Console.WriteLine("The file URI is " + file.Uri);
// Enumerate directory
foreach (var pathItem in filesystemclient.GetPaths("/Test"))
{
PrintDirectoryEntry(pathItem);
}
// Delete a directory and all it's subdirectories and files
filesystemclient.DeleteDirectory("/Test");
}
finally
{
filesystemclient.Delete();
}
Console.WriteLine("Done. Press ENTER to continue ...");
Console.ReadLine();
}
public void Read()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-read" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
file.Create();
file.Append(File.OpenRead(originalPath), 0);
file.Flush(SampleFileContent.Length);
// Download the DataLake file's contents and save it to a file
#region Snippet:SampleSnippetDataLakeFileClient_Read
Response <FileDownloadInfo> fileContents = file.Read();
#endregion Snippet:SampleSnippetDataLakeFileClient_Read
using (FileStream stream = File.OpenWrite(downloadPath))
{
fileContents.Value.Content.CopyTo(stream);
}
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
// Bulk upload and download
private static void RunFileTransfer(DataLakeFileSystemClient client)
{
Directory.CreateDirectory(localFileTransferPath);
var fileName = localFileTransferPath + @"\testUploadFile.txt";
Console.WriteLine("Creating the test file to upload:");
using (var stream = new StreamWriter(new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite)))
{
stream.WriteLine("Hello I am the first line of upload.");
stream.WriteLine("Hello I am the second line of upload.");
}
var destFile = remoteFileTransferPath + "/testremoteUploadFile.txt";
DataLakeFileClient file = client.GetFileClient(destFile);
Console.WriteLine("Upload of the file:");
file.Upload(fileName); // Source and destination could also be directories
Response <FileDownloadInfo> fileContents = file.Read();
MemoryStream fileContentDown = new MemoryStream();
fileContents.Value.Content.CopyTo(fileContentDown);
using (var readStream = new StreamReader(fileContents.Value.Content))
{
string line;
while ((line = readStream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
var localDestFile = localFileTransferPath + @"\testlocalDownloadFile.txt";
Console.WriteLine("Download of the uploaded file:");
using (FileStream stream = File.OpenWrite(localDestFile))
{
fileContentDown.CopyTo(stream);
}
fileContentDown.Close();
using (var stream = new StreamReader(File.OpenRead(localDestFile)))
{
string line;
while ((line = stream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Directory.Delete(localFileTransferPath, true);
client.DeleteDirectory(remoteFileTransferPath);
}
public List <Metadata> GetMetadatas()
{
List <string> files = GetFileNames();
List <Metadata> metadatas = new List <Metadata>();
DataLakeFileSystemClient fileSystemClient =
serviceClient.GetFileSystemClient(fileSystemName);
MetadataParser parser =
new MetadataParser(
new MetadataConverter());
foreach (var file in files)
{
// Get file contents as json string
DataLakeFileClient fileClient =
fileSystemClient.GetFileClient(file);
Response <FileDownloadInfo> fileContents = fileClient.Read();
string json;
using (MemoryStream ms = new MemoryStream())
{
fileContents.Value.Content.CopyTo(ms);
json = Encoding.UTF8.GetString(ms.ToArray());
}
// Parse json string and add relative path to Dataset
Metadata metadata = parser.Parse(json);
string filePath = fileClient.Path.Replace(FILE_EXTENSION, "");
metadata.Dataset.DatasetPath = filePath;
metadatas.Add(metadata);
}
return(metadatas);
}
/// <inheritdoc cref="IFileStorageService"/>
public Stream GetFileContent(Uri fileUri)
{
DataLakeFileClient fcli = new DataLakeFileClient(fileUri, SharedKeyCredential);
return(fcli.Read().Value.Content);
}