/// <summary> /// Connect and upload the data as file to the Azure Data Lake. /// </summary> /// <param name="storageAccountName">Azure storage account name</param> /// <param name="storageAccountKey">Azure storage account key</param> /// <param name="dataLakeUri">Azure Data Lake URI</param> /// <param name="directoryName">Azure Data Lake directory name</param> /// <param name="content">Upload data content</param> public async Task <bool> UploadData(string storageAccountName, string storageAccountKey, string dataLakeUri, string directoryName, string content) { try { Uri serviceUri = new Uri(dataLakeUri); StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(directoryName); DataLakeDirectoryClient directoryClient = filesystem.GetDirectoryClient(directoryName); DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(string.Format("data-{0}.json", Guid.NewGuid().ToString())); using (MemoryStream memoryStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(content))) { await fileClient.AppendAsync(memoryStream, offset : 0); await fileClient.FlushAsync(position : memoryStream.Length); } return(true); } catch (Exception exception) { logger.Error(exception.StackTrace); return(false); } }
public async Task DataLakeSasBuilder_BothObjectId_Error() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); string directoryName = GetNewDirectoryName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); // Arrange DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName); DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName()); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, PreauthorizedAgentObjectId = Recording.Random.NewGuid().ToString(), AgentObjectId = Recording.Random.NewGuid().ToString() }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); TestHelper.AssertExpectedException <InvalidOperationException>( () => dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName), new InvalidOperationException("SAS cannot have the following parameters specified in conjunction: PreauthorizedAgentObjectId, AgentObjectId")); }
/// <summary> /// A Task() which we will use for uploading a million blobs in parallel /// </summary> public async Task UploadFile(DataLakeDirectoryClient directoryClient, int numTasks, int taskId, int numFilesPerTask) { for (int i = 0; i < numFilesPerTask; i++) { // Giving unique name to each path string fileName = "Dir_" + i.ToString("D3") + "/" + ((taskId * numTasks) + i).ToString("D6"); // Send request to create the file Console.WriteLine("Creating file = {0}.", fileName); DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(fileName); } }
public async Task DataLakeSasBuilder_AgentObjectId() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); string directoryName = GetNewDirectoryName(); string unknownGuid = Recording.Random.NewGuid().ToString(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); // Arrange DataLakeDirectoryClient directory = test.FileSystem.GetRootDirectoryClient(); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); // Give UnknownGuid rights IList <PathAccessControlItem> accessControlList = new List <PathAccessControlItem>() { new PathAccessControlItem( AccessControlType.User, RolePermissions.Read | RolePermissions.Write | RolePermissions.Execute, false, unknownGuid) }; await directory.SetAccessControlListAsync(accessControlList); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, AgentObjectId = unknownGuid }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri) { Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName) }; DataLakeDirectoryClient sasDirectoryClient = new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions()); // Act DataLakeFileClient file = await sasDirectoryClient.CreateFileAsync(GetNewFileName()); }
// </Snippet_ListFilesInDirectory> #endregion #region Upload files to a directory // --------------------------------------------------------- // Upload files to the directory. //---------------------------------------------------------- // <Snippet_UploadFile> public async Task UploadFile(DataLakeFileSystemClient fileSystemClient) { DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient("my-directory"); DataLakeFileClient fileClient = await directoryClient.CreateFileAsync("uploaded-file.txt"); FileStream fileStream = File.OpenRead("C:\\Users\\contoso\\Temp\\file-to-upload.txt"); long fileSize = fileStream.Length; await fileClient.AppendAsync(fileStream, offset : 0); await fileClient.FlushAsync(position : fileSize); }
public async Task UploadJsonData(DataLakeFileSystemClient fileSystemClient, string directory, string jsonData, string dataId) { DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient(directory); DataLakeFileClient fileClient = await directoryClient.CreateFileAsync(dataId + ".json"); byte[] byteArray = Encoding.ASCII.GetBytes(jsonData); MemoryStream stream = new MemoryStream(byteArray); long fileSize = stream.Length; await fileClient.AppendAsync(stream, offset : 0); await fileClient.FlushAsync(position : fileSize); }
public async Task DataLakeSasBuilder_DirectoryDepth_Exists() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory = await directory.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory2 = await subdirectory.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory3 = await subdirectory2.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeFileClient file = await subdirectory3.CreateFileAsync(GetNewFileName()); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, Path = subdirectory3.Path, IsDirectory = true }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(subdirectory3.Uri) { Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName) }; DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions())); // Act await sasDirectoryClient.ExistsAsync(); }
public async Task DataLakeSasBuilderRawPermissions_2020_02_10(string permissionsString) { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); string directoryName = GetNewDirectoryName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); // Arrange DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName); DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName()); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name }; dataLakeSasBuilder.SetPermissions( rawPermissions: permissionsString, normalize: true); DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri) { Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName) }; DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions())); // Act await foreach (PathItem pathItem in sasFileSystemClient.GetPathsAsync()) { // Just make sure the call succeeds. } }
public async Task DataLakeSasBuilder_DirectoryDepth_SharedKey() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory = await directory.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory2 = await subdirectory.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeDirectoryClient subdirectory3 = await subdirectory2.CreateSubDirectoryAsync(GetNewDirectoryName()); DataLakeFileClient file = await subdirectory3.CreateFileAsync(GetNewFileName()); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, Path = subdirectory3.Path, IsDirectory = true }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(TestConfigHierarchicalNamespace.AccountName, TestConfigHierarchicalNamespace.AccountKey); DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(subdirectory3.Uri) { Sas = dataLakeSasBuilder.ToSasQueryParameters(sharedKeyCredential) }; DataLakeDirectoryClient sasDirectoryClient = InstrumentClient(new DataLakeDirectoryClient(dataLakeUriBuilder.ToUri(), GetOptions())); // Act await sasDirectoryClient.ExistsAsync(); }
public async Task DataLakeSasBuilder_AgentObjectId_Error() { // Arrange DataLakeServiceClient oauthService = GetServiceClient_OAuth(); string fileSystemName = GetNewFileSystemName(); string directoryName = GetNewDirectoryName(); await using DisposingFileSystem test = await GetNewFileSystem(service : oauthService, fileSystemName : fileSystemName); // Arrange DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(directoryName); DataLakeFileClient file = await directory.CreateFileAsync(GetNewFileName()); Response <UserDelegationKey> userDelegationKey = await oauthService.GetUserDelegationKeyAsync( startsOn : null, expiresOn : Recording.UtcNow.AddHours(1)); DataLakeSasBuilder dataLakeSasBuilder = new DataLakeSasBuilder { StartsOn = Recording.UtcNow.AddHours(-1), ExpiresOn = Recording.UtcNow.AddHours(1), FileSystemName = test.FileSystem.Name, AgentObjectId = Recording.Random.NewGuid().ToString() }; dataLakeSasBuilder.SetPermissions(DataLakeSasPermissions.All); DataLakeUriBuilder dataLakeUriBuilder = new DataLakeUriBuilder(test.FileSystem.Uri) { Sas = dataLakeSasBuilder.ToSasQueryParameters(userDelegationKey, test.FileSystem.AccountName) }; DataLakeFileSystemClient sasFileSystemClient = InstrumentClient(new DataLakeFileSystemClient(dataLakeUriBuilder.ToUri(), GetOptions())); // Act await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>( sasFileSystemClient.ExistsAsync(), e => Assert.IsNotNull(e.ErrorCode)); }
public async Task UploadFile(string fullPath) { var dataLakeServiceClient = GetDataLakeServiceClient(); var dataLakeFileSystemClient = dataLakeServiceClient.GetFileSystemClient(ContainerName); var directoryDate = dataLakeFileSystemClient.GetDirectoryClient(DateTime.Now.ToString("yyyy-MM-dd")); #region Create and Check Folder DataLakeDirectoryClient directoryDistributorCode = null; if (!directoryDate.Exists()) { dataLakeFileSystemClient.CreateDirectory(DateTime.Now.ToString("yyyy-MM-dd")); directoryDate = dataLakeFileSystemClient.GetDirectoryClient(DateTime.Now.ToString("yyyy-MM-dd")); } if (FormFileUpload.Category.ToUpper() == "STOCK") { var directoryStock = directoryDate.GetSubDirectoryClient("Stock"); if (!directoryStock.Exists()) { directoryDate.GetSubDirectoryClient("Stock"); directoryStock = directoryDate.GetSubDirectoryClient("Stock"); } directoryDistributorCode = directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode); if (!directoryDistributorCode.Exists()) { directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode); directoryDistributorCode = directoryStock.GetSubDirectoryClient(FormFileUpload.DistributorCode); } } if (FormFileUpload.Category.ToUpper() == "SALE") { var directorySales = directoryDate.GetSubDirectoryClient("Sales"); if (!directorySales.Exists()) { directoryDate.GetSubDirectoryClient("Sales"); directorySales = directoryDate.GetSubDirectoryClient("Sales"); } directoryDistributorCode = directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode); if (!directoryDistributorCode.Exists()) { directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode); directoryDistributorCode = directorySales.GetSubDirectoryClient(FormFileUpload.DistributorCode); } } #endregion string fileName = Path.GetFileName(fullPath); DataLakeFileClient fileClient = await directoryDistributorCode.CreateFileAsync(fileName); using var fileStream = System.IO.File.OpenRead(fullPath); long fileSize = fileStream.Length; await fileClient.AppendAsync(fileStream, offset : 0); await fileClient.FlushAsync(position : fileSize); }