public async Task <bool> Exist(string path, CancellationToken token)
{
path.VerifyNotEmpty(nameof(path));
DataLakeFileClient file = _fileSystem.GetFileClient(path);
Response <bool> response = await file.ExistsAsync(token);
return(response.Value);
}
public async Task <bool> Exist(string path, CancellationToken token = default)
{
path = WithBasePath(path);
_logger.LogTrace($"Is path {path} exist");
DataLakeFileClient file = _fileSystem.GetFileClient(path);
Response <bool> response = await file.ExistsAsync(token);
return(response.Value);
}
static async Task UploadSampleDataIfNotExistsAsync(string localDirectory, DataLakeDirectoryClient directoryClient)
{
// Upload all sample data files in this directory
foreach (string filePath in Directory.GetFiles(localDirectory))
{
string fileName = Path.GetFileName(filePath);
DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName);
if (!await fileClient.ExistsAsync())
{
await fileClient.UploadAsync(filePath);
}
}
// Recursively create subdirectories, and upload all sample data files in those subdirectories
foreach (string directory in Directory.GetDirectories(localDirectory))
{
string directoryName = Path.GetFileNameWithoutExtension(directory);
DataLakeDirectoryClient subDirectoryClient = directoryClient.GetSubDirectoryClient(directoryName);
await subDirectoryClient.CreateIfNotExistsAsync();
await UploadSampleDataIfNotExistsAsync(directory, subDirectoryClient);
}
}
public async Task Run([ServiceBusTrigger("datalake-create", Connection = "servicebusconnection")] Message message, ILogger log)
{
//telemetryClient.TrackTrace($"CreateDirectory: Message {message.MessageId} dequeued. Attempt {message.SystemProperties.DeliveryCount}");
// amended in Github.com
// parse incoming CreateRequest message
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
CreateRequest request = JsonConvert.DeserializeObject <CreateRequest>(payload);
log.LogInformation($"CreateDirectory: path:{request.Path} depth:{request.CurrentDepth} numDir:{request.NumberOfDirectories} numFiles:{request.NumberOfFiles} createDir:{request.CreateDirectories} CreateFiles:{request.CreateFiles}");
telemetryClient.TrackEvent($"CreateDirectory called {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
// using path, set where we are in terms of the current directory
var directoryClient = createFileSystemClient.GetDirectoryClient(request.Path);
// we are in a directory, so create the file for this directory
if (request.CreateFiles == true)
{
for (int i = 0; i < request.NumberOfFiles; i++)
{
var newFile = string.Format("{0}{1}.txt", request.FilePattern, i);
DataLakeFileClient fileClient = directoryClient.GetFileClient(newFile);
// test if this is another run and file exists already
if (await fileClient.ExistsAsync() == false)
{
var binDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); // get path of test file
var rootDirectory = Path.GetFullPath(Path.Combine(binDirectory, ".."));
FileStream fileStream = File.OpenRead(rootDirectory + "/file-to-upload.txt");
var response = await fileClient.UploadAsync(fileStream, true);
if (response.GetRawResponse().Status == 200)
{
telemetryClient.TrackEvent($"File created {request.FilePattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() },
{ "response", response.ToString() }
});
}
if (request.CreateAcls == true)
{
// set ACLs for the file
IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-");
await fileClient.SetAccessControlListAsync(accessControlList);
telemetryClient.TrackEvent($"File ACL created {request.FilePattern}");
}
}
else
{
telemetryClient.TrackEvent($"File exists {request.FilePattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
}
}
// increment the depth, or it will never stop
request.CurrentDepth++;
// if we are at the max path or I see the terminate flag, do nothing - your work here is complete
if (request.CurrentDepth >= request.MaxDepth || TerminateCreate == "1")
{
log.LogInformation($"CreateLakeFolder: Message {message.MessageId} dequeued. No more to do");
telemetryClient.TrackEvent("Nothing to do");
return;
}
if (request.CreateDirectories == true)
{
// create a number of directories
for (int i = 0; i < request.NumberOfDirectories; i++)
{
// create directory
var newDir = string.Format("{0}{1}", request.DirectoryPattern, i);
var response = await directoryClient.CreateSubDirectoryAsync(newDir);
telemetryClient.TrackEvent($"Directory created {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path },
{ "deliveryCount", message.SystemProperties.DeliveryCount.ToString() },
{ "response", response.ToString() }
});
if (request.CreateAcls == true)
{
// set ACL for the directory
IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-");
await directoryClient.SetAccessControlListAsync(accessControlList);
telemetryClient.TrackEvent($"Dir ACL created {request.FilePattern}");
}
}
// create service bus messages for sub directories
var requestPath = request.Path;
for (int i = 0; i < request.NumberOfDirectories; i++)
{
var newDir = string.Format("{0}{1}", request.DirectoryPattern, i);
// send a service bus message with this new path
var childRequest = request;
childRequest.Path = requestPath + "/" + newDir; // new path
string data = JsonConvert.SerializeObject(childRequest);
Message newMessage = new Message(Encoding.UTF8.GetBytes(data));
await _queueClient.SendAsync(newMessage);
telemetryClient.TrackEvent($"Message sent {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
}
telemetryClient.TrackEvent($"CreateDirectory ended {request.DirectoryPattern}",
new Dictionary <string, string> {
{ "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }
});
}
