public async Task Upload(Stream fromStream, string toPath, bool force, CancellationToken token) { fromStream.VerifyNotNull(nameof(fromStream)); toPath.VerifyNotEmpty(nameof(toPath)); _logger.LogTrace($"{nameof(Upload)} from stream to {toPath}"); DataLakeFileClient file = _fileSystem.GetFileClient(toPath); await file.UploadAsync(fromStream, force, token); }
// </Snippet_UploadFile> #endregion #region Upload files to a directory in bulk // --------------------------------------------------------- // Upload files to the directory - bulk uploads //---------------------------------------------------------- // <Snippet_UploadFileBulk> public async Task UploadFileBulk(DataLakeFileSystemClient fileSystemClient) { DataLakeDirectoryClient directoryClient = fileSystemClient.GetDirectoryClient("my-directory"); DataLakeFileClient fileClient = directoryClient.GetFileClient("uploaded-file.txt"); FileStream fileStream = File.OpenRead("C:\\Users\\contoso\\file-to-upload.txt"); await fileClient.UploadAsync(fileStream); }
public async Task Write(string path, byte[] data, bool force, CancellationToken token) { path.VerifyNotEmpty(nameof(path)); data .VerifyNotNull(nameof(data)) .VerifyAssert(x => x.Length > 0, $"{nameof(data)} length must be greater then 0"); _logger.LogTrace($"{nameof(Write)} to {path}"); using var memoryBuffer = new MemoryStream(data.ToArray()); DataLakeFileClient file = _fileSystem.GetFileClient(path); await file.UploadAsync(memoryBuffer, force, token); }
public async Task ReadAsync() { // Create a temporary Lorem Ipsum file on disk that we can upload string originalPath = CreateTempFile(SampleFileContent); // Get a temporary path on disk where we can download the file string downloadPath = CreateTempPath(); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-readasync" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read")); await filesystem.CreateAsync(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // First upload something the DataLake file so we have something to download await file.UploadAsync(File.OpenRead(originalPath)); // Download the DataLake file's contents and save it to a file // The ReadAsync() API downloads a file in a single requests. // For large files, it may be faster to call ReadToAsync() Response <FileDownloadInfo> fileContents = await file.ReadAsync(); using (FileStream stream = File.OpenWrite(downloadPath)) { fileContents.Value.Content.CopyTo(stream); } // Verify the contents Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); } finally { // Clean up after the test when we're finished await filesystem.DeleteAsync(); } }
public async Task ReadToAsync() { // Create a temporary Lorem Ipsum file on disk that we can upload string originalPath = CreateTempFile(SampleFileContent); // Get a temporary path on disk where we can download the file string downloadPath = CreateTempPath(); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Create DataLakeServiceClient using StorageSharedKeyCredentials DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-readasync" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read")); await filesystem.CreateAsync(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // First upload something the DataLake file so we have something to download await file.UploadAsync(File.OpenRead(originalPath)); // Download the DataLake file's contents directly to a file. // For larger files, ReadToAsync() will download the file in multiple parallel requests. await file.ReadToAsync(downloadPath); // Verify the contents Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); } finally { // Clean up after the test when we're finished await filesystem.DeleteAsync(); } }
public async Task UploadAsync() { // Create three temporary Lorem Ipsum files on disk that we can upload int contentLength = 10; string sampleFileContent = CreateTempFile(SampleFileContent.Substring(0, contentLength)); // Make StorageSharedKeyCredential to pass to the serviceClient string storageAccountName = StorageAccountName; string storageAccountKey = StorageAccountKey; Uri serviceUri = StorageAccountBlobUri; StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); // Get a reference to a FileSystemClient DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); // Get a reference to a filesystem named "sample-filesystem-appendasync" and then create it DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-append")); await filesystem.CreateAsync(); try { // Get a reference to a file named "sample-file" in a filesystem DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); // Upload content to the file. When using the Upload API, you don't need to create the file first. // If the file already exists, it will be overwritten. // For larger files, Upload() will upload the file in multiple parallel requests. await file.UploadAsync(File.OpenRead(sampleFileContent)); // Verify the contents of the file PathProperties properties = await file.GetPropertiesAsync(); Assert.AreEqual(contentLength, properties.ContentLength); } finally { // Clean up after the test when we're finished await filesystem.DeleteAsync(); } }
static async Task UploadSampleDataIfNotExistsAsync(string localDirectory, DataLakeDirectoryClient directoryClient) { // Upload all sample data files in this directory foreach (string filePath in Directory.GetFiles(localDirectory)) { string fileName = Path.GetFileName(filePath); DataLakeFileClient fileClient = directoryClient.GetFileClient(fileName); if (!await fileClient.ExistsAsync()) { await fileClient.UploadAsync(filePath); } } // Recursively create subdirectories, and upload all sample data files in those subdirectories foreach (string directory in Directory.GetDirectories(localDirectory)) { string directoryName = Path.GetFileNameWithoutExtension(directory); DataLakeDirectoryClient subDirectoryClient = directoryClient.GetSubDirectoryClient(directoryName); await subDirectoryClient.CreateIfNotExistsAsync(); await UploadSampleDataIfNotExistsAsync(directory, subDirectoryClient); } }
public static async Task OnFileCreated( [EventGridTrigger] EventGridEvent eventGridEvent, [Blob("{data.url}", FileAccess.ReadWrite, Connection = "AzureWebJobsStorageInput")] Stream input, ILogger log) { try { if (input != null) { var createdEvent = ((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>(); var blobName = GetBlobPathFromUrl(createdEvent.Url); DataLakeFileClient dataLakeFileClient = new DataLakeFileClient(BLOB_OUTPUT_CONNECTION_STRING, "backup", blobName); await dataLakeFileClient.UploadAsync(input, overwrite : true); } } catch (Exception ex) { log.LogInformation(ex.Message); throw; } }
public async Task Run([ServiceBusTrigger("datalake-create", Connection = "servicebusconnection")] Message message, ILogger log) { //telemetryClient.TrackTrace($"CreateDirectory: Message {message.MessageId} dequeued. Attempt {message.SystemProperties.DeliveryCount}"); // amended in Github.com // parse incoming CreateRequest message string payload = System.Text.Encoding.UTF8.GetString(message.Body); CreateRequest request = JsonConvert.DeserializeObject <CreateRequest>(payload); log.LogInformation($"CreateDirectory: path:{request.Path} depth:{request.CurrentDepth} numDir:{request.NumberOfDirectories} numFiles:{request.NumberOfFiles} createDir:{request.CreateDirectories} CreateFiles:{request.CreateFiles}"); telemetryClient.TrackEvent($"CreateDirectory called {request.DirectoryPattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() } }); // using path, set where we are in terms of the current directory var directoryClient = createFileSystemClient.GetDirectoryClient(request.Path); // we are in a directory, so create the file for this directory if (request.CreateFiles == true) { for (int i = 0; i < request.NumberOfFiles; i++) { var newFile = string.Format("{0}{1}.txt", request.FilePattern, i); DataLakeFileClient fileClient = directoryClient.GetFileClient(newFile); // test if this is another run and file exists already if (await fileClient.ExistsAsync() == false) { var binDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); // get path of test file var rootDirectory = Path.GetFullPath(Path.Combine(binDirectory, "..")); FileStream fileStream = File.OpenRead(rootDirectory + "/file-to-upload.txt"); var response = await fileClient.UploadAsync(fileStream, true); if (response.GetRawResponse().Status == 200) { telemetryClient.TrackEvent($"File created {request.FilePattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }, { "response", response.ToString() } }); } if (request.CreateAcls == true) { // set ACLs for the file IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-"); await fileClient.SetAccessControlListAsync(accessControlList); telemetryClient.TrackEvent($"File ACL created {request.FilePattern}"); } } else { telemetryClient.TrackEvent($"File exists {request.FilePattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() } }); } } } // increment the depth, or it will never stop request.CurrentDepth++; // if we are at the max path or I see the terminate flag, do nothing - your work here is complete if (request.CurrentDepth >= request.MaxDepth || TerminateCreate == "1") { log.LogInformation($"CreateLakeFolder: Message {message.MessageId} dequeued. No more to do"); telemetryClient.TrackEvent("Nothing to do"); return; } if (request.CreateDirectories == true) { // create a number of directories for (int i = 0; i < request.NumberOfDirectories; i++) { // create directory var newDir = string.Format("{0}{1}", request.DirectoryPattern, i); var response = await directoryClient.CreateSubDirectoryAsync(newDir); telemetryClient.TrackEvent($"Directory created {request.DirectoryPattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() }, { "response", response.ToString() } }); if (request.CreateAcls == true) { // set ACL for the directory IList <PathAccessControlItem> accessControlList = PathAccessControlExtensions.ParseAccessControlList("user::rwx,group::r-x,other::rw-"); await directoryClient.SetAccessControlListAsync(accessControlList); telemetryClient.TrackEvent($"Dir ACL created {request.FilePattern}"); } } // create service bus messages for sub directories var requestPath = request.Path; for (int i = 0; i < request.NumberOfDirectories; i++) { var newDir = string.Format("{0}{1}", request.DirectoryPattern, i); // send a service bus message with this new path var childRequest = request; childRequest.Path = requestPath + "/" + newDir; // new path string data = JsonConvert.SerializeObject(childRequest); Message newMessage = new Message(Encoding.UTF8.GetBytes(data)); await _queueClient.SendAsync(newMessage); telemetryClient.TrackEvent($"Message sent {request.DirectoryPattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() } }); } } telemetryClient.TrackEvent($"CreateDirectory ended {request.DirectoryPattern}", new Dictionary <string, string> { { "path", request.Path }, { "deliveryCount", message.SystemProperties.DeliveryCount.ToString() } }); }