private static void PerformWriteFlushReadSeek(DataLakeFileSystemClient client)
{
string fileName = "/Test/dir1/testFilename1.txt";
DataLakeFileClient file = client.GetFileClient(fileName);
// Create the file
Stream stream = BinaryData.FromString("This is the first line.\nThis is the second line.\n").ToStream();
long length = stream.Length;
file.Upload(stream, true);
// Append to the file
stream = BinaryData.FromString("This is the third line.\nThis is the fourth line.\n").ToStream();
file.Append(stream, length);
file.Flush(length + stream.Length);
// Read the file
using (var readStream = file.OpenRead())
{
byte[] readData = new byte[1024];
// Read 40 bytes at this offset
int readBytes = readStream.Read(readData, 25, 40);
Console.WriteLine("Read output of 40 bytes from offset 25: " + Encoding.UTF8.GetString(readData, 0, readBytes));
}
}
// Create a sample hierarchical directory tree using async operations
private static async Task CreateDirRecursiveAsync(DataLakeFileSystemClient client, string path, int recursLevel, int noDirEntries, int noFileEntries)
{
await client.CreateDirectoryAsync(path);
byte[] writeData = Encoding.UTF8.GetBytes("This is the first line.\n");
string[] str = path.Split('/');
char nextLevel = str[str.Length - 1][0];
nextLevel++;
for (int i = 0; i < noFileEntries; i++)
{
DataLakeFileClient file = client.GetFileClient(path + "/" + nextLevel + i + "File.txt");
using (var stream = new MemoryStream(writeData))
{
file.Upload(stream, true);
}
}
if (recursLevel == 0)
{
return;
}
string newPath = path + "/";
for (int i = 0; i < noDirEntries; i++)
{
await CreateDirRecursiveAsync(client, newPath + nextLevel + i, recursLevel - 1, noDirEntries, noFileEntries);
}
}
private static string serviceUri = "FILL-IN-HERE"; // full account FQDN, not just the account name - it should look like https://{ACCOUNTNAME}.dfs.core.windows.net/
public static void Main(string[] args)
{
// Create Client Secret Credential
var creds = new ClientSecretCredential(tenantId, applicationId, clientSecret);
// Create data lake file service client object
DataLakeServiceClient serviceClient = new DataLakeServiceClient(new Uri(serviceUri), creds);
var name = "sample-filesystem" + Guid.NewGuid().ToString("n").Substring(0, 8);
// Create data lake file system client object
DataLakeFileSystemClient filesystemclient = serviceClient.GetFileSystemClient(name);
filesystemclient.CreateIfNotExists();
try
{
long length;
string fileName = "/Test/testFilename1.txt";
DataLakeFileClient file = filesystemclient.GetFileClient(fileName);
// Upload a file - automatically creates any parent directories that don't exist
length = BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream().Length;
file.Upload(BinaryData.FromString("This is test data to write.\r\nThis is the second line.\r\n").ToStream(), true);
file.Append(BinaryData.FromString("This is the added line.\r\n").ToStream(), length);
file.Flush(length + BinaryData.FromString("This is the added line.\r\n").ToStream().Length);
//Read file contents
Response <FileDownloadInfo> fileContents = file.Read();
Console.WriteLine(BinaryData.FromStream(fileContents.Value.Content).ToString());
// Get the properties of the file
PathProperties pathProperties = file.GetProperties();
PrintDirectoryEntry(pathProperties);
// Rename a file
string destFilePath = "/Test/testRenameDest3.txt";
file.Rename(destFilePath);
file = filesystemclient.GetFileClient(destFilePath);
Console.WriteLine("The file URI is " + file.Uri);
// Enumerate directory
foreach (var pathItem in filesystemclient.GetPaths("/Test"))
{
PrintDirectoryEntry(pathItem);
}
// Delete a directory and all it's subdirectories and files
filesystemclient.DeleteDirectory("/Test");
}
finally
{
filesystemclient.Delete();
}
Console.WriteLine("Done. Press ENTER to continue ...");
Console.ReadLine();
}
public void Read()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-read" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
file.Upload(File.OpenRead(originalPath));
// Download the DataLake file's contents and save it to a file
// The ReadAsync() API downloads a file in a single requests.
// For large files, it may be faster to call ReadTo()
#region Snippet:SampleSnippetDataLakeFileClient_Read
Response <FileDownloadInfo> fileContents = file.Read();
#endregion Snippet:SampleSnippetDataLakeFileClient_Read
using (FileStream stream = File.OpenWrite(downloadPath))
{
fileContents.Value.Content.CopyTo(stream);
}
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
// Bulk upload and download
private static void RunFileTransfer(DataLakeFileSystemClient client)
{
Directory.CreateDirectory(localFileTransferPath);
var fileName = localFileTransferPath + @"\testUploadFile.txt";
Console.WriteLine("Creating the test file to upload:");
using (var stream = new StreamWriter(new FileStream(fileName, FileMode.Create, FileAccess.ReadWrite)))
{
stream.WriteLine("Hello I am the first line of upload.");
stream.WriteLine("Hello I am the second line of upload.");
}
var destFile = remoteFileTransferPath + "/testremoteUploadFile.txt";
DataLakeFileClient file = client.GetFileClient(destFile);
Console.WriteLine("Upload of the file:");
file.Upload(fileName); // Source and destination could also be directories
Response <FileDownloadInfo> fileContents = file.Read();
MemoryStream fileContentDown = new MemoryStream();
fileContents.Value.Content.CopyTo(fileContentDown);
using (var readStream = new StreamReader(fileContents.Value.Content))
{
string line;
while ((line = readStream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
var localDestFile = localFileTransferPath + @"\testlocalDownloadFile.txt";
Console.WriteLine("Download of the uploaded file:");
using (FileStream stream = File.OpenWrite(localDestFile))
{
fileContentDown.CopyTo(stream);
}
fileContentDown.Close();
using (var stream = new StreamReader(File.OpenRead(localDestFile)))
{
string line;
while ((line = stream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
Directory.Delete(localFileTransferPath, true);
client.DeleteDirectory(remoteFileTransferPath);
}
public void Upload()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-append" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Create the file
file.Create();
// Verify we created one file
Assert.AreEqual(1, filesystem.GetPaths().Count());
// Upload content to the file. When using the Upload API, you don't need to create the file first.
// If the file already exists, it will be overwritten.
// For larger files, Upload() will upload the file in multiple sequential requests.
file.Upload(File.OpenRead(sampleFileContent), true);
// Verify the contents of the file
PathProperties properties = file.GetProperties();
Assert.AreEqual(contentLength, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
public void ReadTo()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-read" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
filesystem.Create();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
file.Upload(File.OpenRead(originalPath));
// Download the DataLake file's directly to a file.
// For larger files, ReadTo() will download the file in multiple sequential requests.
#region Snippet:SampleSnippetDataLakeFileClient_ReadTo
file.ReadTo(downloadPath);
#endregion Snippet:SampleSnippetDataLakeFileClient_ReadTo
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
filesystem.Delete();
}
}
private static void TestTokenRefresh(DataLakeFileSystemClient client)
{
string path = "/Test/TokenRefresh.txt";
DataLakeFileClient file = client.GetFileClient(path);
// Create file
file.Upload(
BinaryData.FromString("This is the first file.").ToStream(), overwrite: true);
// Wait 1 mins
Console.WriteLine("The token is refreshing...");
Thread.Sleep(60 * 1000);
// Read file - this still works, because token is internally refreshed
using (var readStream = new StreamReader(file.OpenRead()))
{
string line;
while ((line = readStream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
}