public async Task Upload(Stream fromStream, string toPath, bool force, CancellationToken token)
{
fromStream.VerifyNotNull(nameof(fromStream));
toPath.VerifyNotEmpty(nameof(toPath));
_logger.LogTrace($"{nameof(Upload)} from stream to {toPath}");
DataLakeFileClient file = _fileSystem.GetFileClient(toPath);
await file.UploadAsync(fromStream, force, token);
}
// </Snippet_UploadFile>
#endregion
#region Upload files to a directory in bulk
// ---------------------------------------------------------
// Upload files to the directory - bulk uploads
//----------------------------------------------------------
// <Snippet_UploadFileBulk>
public async Task UploadFileBulk(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient = directoryClient.GetFileClient("uploaded-file.txt");
FileStream fileStream =
File.OpenRead("C:\\Users\\contoso\\file-to-upload.txt");
await fileClient.UploadAsync(fileStream);
}
public async Task Write(string path, byte[] data, bool force, CancellationToken token)
{
path.VerifyNotEmpty(nameof(path));
data
.VerifyNotNull(nameof(data))
.VerifyAssert(x => x.Length > 0, $"{nameof(data)} length must be greater then 0");
_logger.LogTrace($"{nameof(Write)} to {path}");
using var memoryBuffer = new MemoryStream(data.ToArray());
DataLakeFileClient file = _fileSystem.GetFileClient(path);
await file.UploadAsync(memoryBuffer, force, token);
}
public async Task ReadAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-readasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
await file.UploadAsync(File.OpenRead(originalPath));
// Download the DataLake file's contents and save it to a file
// The ReadAsync() API downloads a file in a single requests.
// For large files, it may be faster to call ReadToAsync()
Response <FileDownloadInfo> fileContents = await file.ReadAsync();
using (FileStream stream = File.OpenWrite(downloadPath))
{
fileContents.Value.Content.CopyTo(stream);
}
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task ReadToAsync()
{
// Create a temporary Lorem Ipsum file on disk that we can upload
string originalPath = CreateTempFile(SampleFileContent);
// Get a temporary path on disk where we can download the file
string downloadPath = CreateTempPath();
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-readasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// First upload something the DataLake file so we have something to download
await file.UploadAsync(File.OpenRead(originalPath));
// Download the DataLake file's contents directly to a file.
// For larger files, ReadToAsync() will download the file in multiple parallel requests.
await file.ReadToAsync(downloadPath);
// Verify the contents
Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath));
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
public async Task UploadAsync()
{
// Create three temporary Lorem Ipsum files on disk that we can upload
int contentLength = 10;
string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength));
// Make StorageSharedKeyCredential to pass to the serviceClient
string storageAccountName = StorageAccountName;
string storageAccountKey = StorageAccountKey;
Uri serviceUri = StorageAccountBlobUri;
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Get a reference to a FileSystemClient
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
// Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append"));
await filesystem.CreateAsync();
try
{
// Get a reference to a file named "sample-file" in a filesystem
DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file"));
// Upload content to the file. When using the Upload API, you don't need to create the file first.
// If the file already exists, it will be overwritten.
// For larger files, Upload() will upload the file in multiple parallel requests.
await file.UploadAsync(File.OpenRead(sampleFileContent));
// Verify the contents of the file
PathProperties properties = await file.GetPropertiesAsync();
Assert.AreEqual(contentLength, properties.ContentLength);
}
finally
{
// Clean up after the test when we're finished
await filesystem.DeleteAsync();
}
}
static async Task UploadSampleDataIfNotExistsAsync(string localDirectory, DataLakeDirectoryClient directoryClient)
{
// Upload all sample data files in this directory
foreach (string filePath in Directory.GetFiles(localDirectory))
{
string fileName = Path.GetFileName(filePath);
DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName);
if (!await fileClient.ExistsAsync())
{
await fileClient.UploadAsync(filePath);
}
}
// Recursively create subdirectories, and upload all sample data files in those subdirectories
foreach (string directory in Directory.GetDirectories(localDirectory))
{
string directoryName = Path.GetFileNameWithoutExtension(directory);
DataLakeDirectoryClient subDirectoryClient = directoryClient.GetSubDirectoryClient(directoryName);
await subDirectoryClient.CreateIfNotExistsAsync();
await UploadSampleDataIfNotExistsAsync(directory, subDirectoryClient);
}
}
public static async Task OnFileCreated(
[EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.url}", FileAccess.ReadWrite, Connection = "AzureWebJobsStorageInput")] Stream input,
ILogger log)
{
try
{
if (input != null)
{
var createdEvent = ((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
var blobName = GetBlobPathFromUrl(createdEvent.Url);
DataLakeFileClient dataLakeFileClient = new DataLakeFileClient(BLOB_OUTPUT_CONNECTION_STRING, "backup", blobName);
await dataLakeFileClient.UploadAsync(input, overwrite : true);
}
}
catch (Exception ex)
{
log.LogInformation(ex.Message);
throw;
}
}
public async Task Run([ServiceBusTrigger("datalake-create", Connection = "servicebusconnection")] Message message, ILogger log)
{
//telemetryClient.TrackTrace($"CreateDirectory: Message {message.MessageId} dequeued. Attempt {message.SystemProperties.DeliveryCount}");
// amended in Github.com
// parse incoming CreateRequest message
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
CreateRequest request = JsonConvert.DeserializeObject <CreateRequest>(payload);
log.LogInformation($"CreateDirectory: path:{request.Path} depth:{request.CurrentDepth} numDir:{request.NumberOfDirectories} numFiles:{request.NumberOfFiles} createDir:{request.CreateDirectories} CreateFiles:{request.CreateFiles}");
telemetryClient.TrackEvent($"CreateDirectory called {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
// using path, set where we are in terms of the current directory
var directoryClient = createFileSystemClient.GetDirectoryClient(request.Path);
// we are in a directory, so create the file for this directory
if (request.CreateFiles == true)
{
for (int i = 0; i < request.NumberOfFiles; i++)
{
var newFile = string.Format("{0}{1}.txt", request.FilePattern, i);
DataLakeFileClient fileClient = directoryClient.GetFileClient(newFile);
// test if this is another run and file exists already
if (await fileClient.ExistsAsync() == false)
{
var binDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); // get path of test file
var rootDirectory = Path.GetFullPath(Path.Combine(binDirectory, ".."));
FileStream fileStream = File.OpenRead(rootDirectory + "/file-to-upload.txt");
var response = await fileClient.UploadAsync(fileStream, true);
if (response.GetRawResponse().Status == 200)
{
telemetryClient.TrackEvent($"File created {request.FilePattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() },
{ "response", response.ToString() }
});
}
if (request.CreateAcls == true)
{
// set ACLs for the file
IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-");
await fileClient.SetAccessControlListAsync(accessControlList);
telemetryClient.TrackEvent($"File ACL created {request.FilePattern}");
}
}
else
{
telemetryClient.TrackEvent($"File exists {request.FilePattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
}
}
// increment the depth, or it will never stop
request.CurrentDepth++;
// if we are at the max path or I see the terminate flag, do nothing - your work here is complete
if (request.CurrentDepth >= request.MaxDepth || TerminateCreate == "1")
{
log.LogInformation($"CreateLakeFolder: Message {message.MessageId} dequeued. No more to do");
telemetryClient.TrackEvent("Nothing to do");
return;
}
if (request.CreateDirectories == true)
{
// create a number of directories
for (int i = 0; i < request.NumberOfDirectories; i++)
{
// create directory
var newDir = string.Format("{0}{1}", request.DirectoryPattern, i);
var response = await directoryClient.CreateSubDirectoryAsync(newDir);
telemetryClient.TrackEvent($"Directory created {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() },
{ "response", response.ToString() }
});
if (request.CreateAcls == true)
{
// set ACL for the directory
IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-");
await directoryClient.SetAccessControlListAsync(accessControlList);
telemetryClient.TrackEvent($"Dir ACL created {request.FilePattern}");
}
}
// create service bus messages for sub directories
var requestPath = request.Path;
for (int i = 0; i < request.NumberOfDirectories; i++)
{
var newDir = string.Format("{0}{1}", request.DirectoryPattern, i);
// send a service bus message with this new path
var childRequest = request;
childRequest.Path = requestPath + "/" + newDir; // new path
string data = JsonConvert.SerializeObject(childRequest);
Message newMessage = new Message(Encoding.UTF8.GetBytes(data));
await _queueClient.SendAsync(newMessage);
telemetryClient.TrackEvent($"Message sent {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
}
telemetryClient.TrackEvent($"CreateDirectory ended {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
