/// <summary>
/// Connect and upload the data as file to the Azure Data Lake.
/// </summary>
/// <param name="storageAccountName">Azure storage account name</param>
/// <param name="storageAccountKey">Azure storage account key</param>
/// <param name="dataLakeUri">Azure Data Lake URI</param>
/// <param name="directoryName">Azure Data Lake directory name</param>
/// <param name="content">Upload data content</param>
public async Task <bool> UploadData(string storageAccountName, string storageAccountKey, string dataLakeUri, string directoryName, string content)
{
try
{
Uri serviceUri = new Uri(dataLakeUri);
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Create DataLakeServiceClient using StorageSharedKeyCredentials
DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential);
DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(directoryName);
DataLakeDirectoryClient directoryClient =
filesystem.GetDirectoryClient(directoryName);
DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(string.Format("data-{0}.json", Guid.NewGuid().ToString()));
using (MemoryStream memoryStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(content)))
{
await fileClient.AppendAsync(memoryStream, offset : 0);
await fileClient.FlushAsync(position : memoryStream.Length);
}
return(true);
}
catch (Exception exception)
{
logger.Error(exception.StackTrace);
return(false);
}
}
public async Task DataLakeSasBuilder_BothObjectId_Error()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
PreauthorizedAgentObjectId = Recording.Random.NewGuid().ToString(),
AgentObjectId = Recording.Random.NewGuid().ToString()
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
TestHelper.AssertExpectedException <InvalidOperationException>(
() => dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName),
new InvalidOperationException("SAS cannot have the following parameters specified in conjunction: PreauthorizedAgentObjectId, AgentObjectId"));
}
/// <summary>
/// A Task() which we will use for uploading a million blobs in parallel
/// </summary>
public async Task UploadFile(DataLakeDirectoryClient directoryClient, int numTasks, int taskId, int numFilesPerTask)
{
for (int i = 0; i < numFilesPerTask; i++)
{
// Giving unique name to each path
string fileName = "Dir_" + i.ToString("D3") + "/" + ((taskId * numTasks) + i).ToString("D6");
// Send request to create the file
Console.WriteLine("Creating file = {0}.", fileName);
DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(fileName);
}
}
public async Task DataLakeSasBuilder_AgentObjectId()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
string unknownGuid = Recording.Random.NewGuid().ToString();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = test.FileSystem.GetRootDirectoryClient();
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
// Give UnknownGuid rights
IList <PathAccessControlItem> accessControlList = new List <PathAccessControlItem>()
{
new PathAccessControlItem(
AccessControlType.User,
RolePermissions.Read | RolePermissions.Write | RolePermissions.Execute,
false,
unknownGuid)
};
await directory.SetAccessControlListAsync(accessControlList);
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
AgentObjectId = unknownGuid
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeDirectoryClient sasDirectoryClient = new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions());
// Act
DataLakeFileClient file = await sasDirectoryClient.CreateFileAsync(GetNewFileName());
}
// </Snippet_ListFilesInDirectory>
#endregion
#region Upload files to a directory
// ---------------------------------------------------------
// Upload files to the directory.
//----------------------------------------------------------
// <Snippet_UploadFile>
public async Task UploadFile(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("my-directory");
DataLakeFileClient fileClient = await directoryClient.CreateFileAsync("uploaded-file.txt");
FileStream fileStream =
File.OpenRead("C:\\Users\\contoso\\Temp\\file-to-upload.txt");
long fileSize = fileStream.Length;
await fileClient.AppendAsync(fileStream, offset : 0);
await fileClient.FlushAsync(position : fileSize);
}
public async Task UploadJsonData(DataLakeFileSystemClient fileSystemClient, string directory, string jsonData, string dataId)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient(directory);
DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(dataId + ".json");
byte[] byteArray = Encoding.ASCII.GetBytes(jsonData);
MemoryStream stream = new MemoryStream(byteArray);
long fileSize = stream.Length;
await fileClient.AppendAsync(stream, offset : 0);
await fileClient.FlushAsync(position : fileSize);
}
public async Task DataLakeSasBuilder_DirectoryDepth_Exists()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory = await directory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory2 = await subdirectory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory3 = await subdirectory2.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeFileClient file = await subdirectory3.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
Path = subdirectory3.Path,
IsDirectory = true
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(subdirectory3.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await sasDirectoryClient.ExistsAsync();
}
public async Task DataLakeSasBuilderRawPermissions_2020_02_10(string permissionsString)
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name
};
dataLakeSasBuilder.SetPermissions(
rawPermissions: permissionsString,
normalize: true);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await foreach (PathItem pathItem in sasFileSystemClient.GetPathsAsync())
{
// Just make sure the call succeeds.
}
}
public async Task DataLakeSasBuilder_DirectoryDepth_SharedKey()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory = await directory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory2 = await subdirectory.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeDirectoryClient subdirectory3 = await subdirectory2.CreateSubDirectoryAsync(GetNewDirectoryName());
DataLakeFileClient file = await subdirectory3.CreateFileAsync(GetNewFileName());
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
Path = subdirectory3.Path,
IsDirectory = true
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(TestConfigHierarchicalNamespace.AccountName, TestConfigHierarchicalNamespace.AccountKey);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(subdirectory3.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(sharedKeyCredential)
};
DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await sasDirectoryClient.ExistsAsync();
}
public async Task DataLakeSasBuilder_AgentObjectId_Error()
{
// Arrange
DataLakeServiceClient oauthService = GetServiceClient_OAuth();
string fileSystemName = GetNewFileSystemName();
string directoryName = GetNewDirectoryName();
await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName);
// Arrange
DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName);
DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName());
Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync(
startsOn : null,
expiresOn : Recording.UtcNow.AddHours(1));
DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder
{
StartsOn = Recording.UtcNow.AddHours(-1),
ExpiresOn = Recording.UtcNow.AddHours(1),
FileSystemName = test.FileSystem.Name,
AgentObjectId = Recording.Random.NewGuid().ToString()
};
dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All);
DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri)
{
Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName)
};
DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions()));
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
sasFileSystemClient.ExistsAsync(),
e => Assert.IsNotNull(e.ErrorCode));
}
public async Task UploadFile(string fullPath)
{
var dataLakeServiceClient = GetDataLakeServiceClient();
var dataLakeFileSystemClient = dataLakeServiceClient.GetFileSystemClient(ContainerName);
var directoryDate = dataLakeFileSystemClient.GetDirectoryClient(DateTime.Now.ToString("yyyy-MM-dd"));
#region Create and Check Folder
DataLakeDirectoryClient directoryDistributorCode = null;
if (!directoryDate.Exists())
{
dataLakeFileSystemClient.CreateDirectory(DateTime.Now.ToString("yyyy-MM-dd"));
directoryDate = dataLakeFileSystemClient.GetDirectoryClient(DateTime.Now.ToString("yyyy-MM-dd"));
}
if (FormFileUpload.Category.ToUpper() == "STOCK")
{
var directoryStock = directoryDate.GetSubDirectoryClient("Stock");
if (!directoryStock.Exists())
{
directoryDate.GetSubDirectoryClient("Stock");
directoryStock = directoryDate.GetSubDirectoryClient("Stock");
}
directoryDistributorCode = directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode);
if (!directoryDistributorCode.Exists())
{
directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode);
directoryDistributorCode = directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode);
}
}
if (FormFileUpload.Category.ToUpper() == "SALE")
{
var directorySales = directoryDate.GetSubDirectoryClient("Sales");
if (!directorySales.Exists())
{
directoryDate.GetSubDirectoryClient("Sales");
directorySales = directoryDate.GetSubDirectoryClient("Sales");
}
directoryDistributorCode = directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode);
if (!directoryDistributorCode.Exists())
{
directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode);
directoryDistributorCode = directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode);
}
}
#endregion
string fileName = Path.GetFileName(fullPath);
DataLakeFileClient fileClient = await directoryDistributorCode.CreateFileAsync(fileName);
using var fileStream = System.IO.File.OpenRead(fullPath);
long fileSize = fileStream.Length;
await fileClient.AppendAsync(fileStream, offset : 0);
await fileClient.FlushAsync(position : fileSize);
}
