/// <summary> /// Downloads the "NBytesPerBlob" of the given blob into the given stream and returns the amount of time taken. /// </summary> /// <param name="ms">Memory stream to populate with he </param> /// <param name="cbb"></param> /// <returns></returns> private async Task <DownloadTestDataEntry> DownloadBlobAsync(Stream stream, CloudBlockBlob cbb) { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); await cbb.DownloadRangeToStreamAsync(stream, 0, NBytesPerBlob); stopwatch.Stop(); return(new DownloadTestDataEntry(stopwatch.Elapsed, cbb.Container.Name, cbb.Name)); }
} // End of the DownloadToStream method public async Task DownloadRangeToStream(CloudBlockBlob blob, Stream stream, long?offset, long?length) { // Download the blob to a stream try { await blob.DownloadRangeToStreamAsync(stream, offset, length); } catch (Exception ex) { // Log the exception logger.LogError(ex, $"DownloadRangeToStream", null); } } // End of the DownloadRangeToStream method
/// <summary> /// Download a remote blob on the Azure Cloud Storage by blocks to a local file. The downloading progress is tracked during the downloading process. /// </summary> /// <param name="containerName">The name of the remote container containing the blob</param> /// <param name="blobName">The name of the remote blob</param> /// <param name="filePath">The full path of the local file</param> /// <param name="blockSizeInBytes">The maximum amount of bytes of each downloaded block</param> /// <param name="onProgressChange">The callback function to call when the downloading progress changes</param> /// <param name="onComplete">The callback function to call when the function gets completed</param> public static async Task DownloadToFileProgressively(string containerName, string blobName, string filePath, int blockSizeInBytes, Action <double> onProgressChange = null, Action onComplete = null) { Debug.Log(containerName); Debug.Log(blobName); CloudBlobContainer container = blobClient.GetContainerReference(containerName); CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); await blockBlob.FetchAttributesAsync(); var bytesToDownloadTotal = blockBlob.Properties.Length; var bytesToDownloadRemaining = bytesToDownloadTotal; long startPosition = 0; do { long bytesToWrite = Math.Min(blockSizeInBytes, bytesToDownloadRemaining); byte[] blobContents = new byte[bytesToWrite]; using (MemoryStream ms = new MemoryStream()) { //blockBlob.DownloadRangeToStream(ms, startPosition, blockSize); await blockBlob.DownloadRangeToStreamAsync(ms, startPosition, bytesToWrite); ms.Position = 0; ms.Read(blobContents, 0, blobContents.Length); using (FileStream fs = new FileStream(filePath, FileMode.OpenOrCreate)) { fs.Position = startPosition; fs.Write(blobContents, 0, blobContents.Length); } } startPosition += bytesToWrite; bytesToDownloadRemaining -= bytesToWrite; double percentCompleted = startPosition / (double)bytesToDownloadTotal; if (onProgressChange != null) { onProgressChange(percentCompleted); } }while (bytesToDownloadRemaining > 0); onComplete?.Invoke(); }
/// <summary> /// Attempts to open a stream to download a range, and if it fails with <see cref="StorageException"/> /// then the message is compared to a string representation of the expected message if the MD5 /// property does not match the property sent. /// </summary> /// <param name="instance">The instance of <see cref="CloudBlockBlob"/></param> /// <returns>Returns a false if the calculated MD5 does not match the existing property.</returns> /// <exception cref="ArgumentNullException">If <paramref name="instance"/> is null.</exception> /// <remarks>This is a hack, and if the message from storage API changes, then this will fail.</remarks> public static async Task <bool> IsValidContentMD5(this CloudBlockBlob instance) { if (instance == null) { throw new ArgumentNullException(nameof(instance)); } try { using (var ms = new MemoryStream()) { await instance.DownloadRangeToStreamAsync(ms, null, null); } } catch (StorageException ex) { return(!ex.Message.Equals("Calculated MD5 does not match existing property", StringComparison.Ordinal)); } return(true); }
/// <summary> /// Obtains a part of the file. /// </summary> /// <param name="fileURI">The file URI.</param> /// <param name="from">The cursor init.</param> /// <param name="length">The length of the data.</param> /// <returns>The content of the block of information.</returns> public async Task <string> GetFileBlock(string fileURI, int from, int length) { string result = string.Empty; try { CloudBlob blob = new CloudBlockBlob(new Uri(fileURI)); MemoryStream ms = null; try { ms = new MemoryStream(); var accessCondition = new AccessCondition(); var blobRequestOptions = new BlobRequestOptions(); //// configurar. var operationContext = new OperationContext(); await blob.DownloadRangeToStreamAsync(ms, from, length, accessCondition, blobRequestOptions, operationContext).ConfigureAwait(false); ms.Seek(0, SeekOrigin.Begin); using (StreamReader sr = new StreamReader(ms, true)) { ms = null; result = sr.ReadToEnd(); } } finally { if (ms != null) { ms.Dispose(); } } } catch (Exception ex) { this.logger.LogError(ex, $"Error al obtener el trozo del fichero ({fileURI},{from},{length})"); throw; } return(result); }
// Overload: Override the default file overwrite setting at the class level for this specific file public async Task <string> DownloadStorageBlockBlobSegmentedOperationAsync(string MediaFile, bool overwrite) { try { // Create a blob client for interacting with the blob service. CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient(); // Create a container for organizing blobs within the storage account. WriteLine("Opening Blob Container in Azure Storage."); CloudBlobContainer container = blobClient.GetContainerReference(BlobContainerName); try { await container.CreateIfNotExistsAsync(); } catch (StorageException) { WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample."); throw; } // Access a specific blob in the container WriteLine("Get Specific Blob in Container and its size"); CloudBlockBlob blockBlob = container.GetBlockBlobReference(MediaFile); int segmentSize = SegmentSizeKB * 1024; // SegmentSizeKB is set in the inspector chunk if (blockBlob != null) { // Obtain the size of the blob await blockBlob.FetchAttributesAsync(); long blobSize = blockBlob.Properties.Length; long blobLengthRemaining = blobSize; float completion = 0f; long startPosition = 0; WriteLine("3. Blob size (bytes):" + blobLengthRemaining.ToString()); // Download a blob to your file system string path = ""; WriteLine(string.Format("Downloading Blob from {0}, please wait...", blockBlob.Uri.AbsoluteUri)); string fileName = MediaFile; // string.Format("CopyOf{0}", MediaFile); bool fileExists = false; #if WINDOWS_UWP StorageFolder storageFolder = ApplicationData.Current.TemporaryFolder; StorageFile sf; try { CreationCollisionOption collisionoption = (overwrite ? CreationCollisionOption.ReplaceExisting : CreationCollisionOption.FailIfExists); sf = await storageFolder.CreateFileAsync(fileName, collisionoption); fileExists = false; // if the file existed but we were allowed to overwrite it, let's treat it as if it didn't exist } catch (Exception) { // The file already exists and we're not supposed to overwrite it fileExists = true; sf = await storageFolder.GetFileAsync(fileName); // Necessary to avoid a compilation error below } path = sf.Path; #else path = Path.Combine(Application.temporaryCachePath, fileName); fileExists = File.Exists(path); #endif if (fileExists) { if (overwrite) { WriteLine(string.Format("Already exists. Deleting file {0}", fileName)); #if WINDOWS_UWP // Nothing to do here in UWP, we already Replaced it when we created the StorageFile #else File.Delete(path); #endif } else { WriteLine(string.Format("File {0} already exists and overwriting is disabled. Download operation cancelled.", fileName)); return(path); } } ProgressBar.AddDownload(); #if WINDOWS_UWP var fs = await sf.OpenAsync(FileAccessMode.ReadWrite); #else FileStream fs = new FileStream(path, FileMode.OpenOrCreate); #endif // Start the timer to measure performance var sw = Stopwatch.StartNew(); do { long blockSize = Math.Min(segmentSize, blobLengthRemaining); byte[] blobContents = new byte[blockSize]; using (MemoryStream ms = new MemoryStream()) { await blockBlob.DownloadRangeToStreamAsync(ms, (long)startPosition, blockSize); ms.Position = 0; ms.Read(blobContents, 0, blobContents.Length); #if WINDOWS_UWP fs.Seek((ulong)startPosition); await fs.WriteAsync(blobContents.AsBuffer()); #else fs.Position = startPosition; fs.Write(blobContents, 0, blobContents.Length); #endif } completion = (float)startPosition / (float)blobSize; WriteLine("Completed: " + (completion).ToString("P")); ProgressBar.Value = (completion * 100); startPosition += blockSize; blobLengthRemaining -= blockSize; }while (blobLengthRemaining > 0); WriteLine("Completed: 100.00%"); ProgressBar.Value = 100; ProgressBar.RemoveDownload(); #if !WINDOWS_UWP // Required for Mono & .NET or we'll get a file IO access violation the next time we try to access it fs.Close(); #else fs.Dispose(); #endif fs = null; // Stop the timer and report back on completion + performance sw.Stop(); TimeSpan time = sw.Elapsed; WriteLine(string.Format("5. Blob file downloaded to {0} in {1}s", path, time.TotalSeconds.ToString())); return(path); } else { WriteLine(string.Format("3. File {0} not found in blob {1}", MediaFile, blockBlob.Uri.AbsoluteUri)); return(string.Empty); } } catch (Exception ex) { // Woops! WriteLine(string.Format("Error while downloading file {0}", MediaFile)); WriteLine("Error: " + ex.ToString()); WriteLine("Error: " + ex.InnerException.ToString()); return(string.Empty); } }
public static async Task <TimeSpan> GetBlocksAsync (CloudBlobClient blobClient, CloudBlobContainer container, string blobName, long startingIndex, long downloadSizeBytes, long chunkSizeBytes, uint levelOfConcurrency) { // Create a buffer representing a single chunk. // NOTE: This example aims to illustrate total transfer speed and thus seeks to fully utilize available network bandwidth. // To avoid being bound by disk write speed, we write only to memory. // To avoid reserving enough memory to contain the entire portion of the blob for which this instance is responsible, // we only reserve enough memory to contain a single chunk - and overwrite it. This effectively throws the downloaded data away. byte[] buffer = new byte[chunkSizeBytes]; try { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); List <Task> tasks = new List <Task>(); SemaphoreSlim semaphore = new SemaphoreSlim((int)levelOfConcurrency, (int)levelOfConcurrency); Random rng = new Random(); for (long index = startingIndex; index < startingIndex + downloadSizeBytes; index += chunkSizeBytes) { MemoryStream ms = new MemoryStream(buffer); await semaphore.WaitAsync(); BlobRequestOptions options = new BlobRequestOptions { // Tell the server to give up on the operation if it takes an exceptionally long period of time. // Note: This timeout should be fine-tuned based on a number of factors including: // 1) Performance Tier (Standard vs Premium) // 2) Size of the Operation // 3) Network Latency between Client and Server ServerTimeout = TimeSpan.FromSeconds(rng.Next(10, 11)) }; tasks.Add(blockBlob.DownloadRangeToStreamAsync(ms, index, chunkSizeBytes).ContinueWith((t) => { semaphore.Release(); })); } Console.WriteLine("All DownloadRange requests issued. Waiting for the remainder to complete."); Task completionTask = Task.WhenAll(tasks); await completionTask; // If any failures occurred, communicate them and propagate an aggregate exception to indicate a failed test. if (completionTask.IsFaulted) { Console.WriteLine("An error occurred while executing one or more of the DownloadRange operations. Details:"); foreach (Exception ex in completionTask.Exception.InnerExceptions) { Console.WriteLine($"\t{ex.Message}"); } throw completionTask.Exception; } stopwatch.Stop(); return(stopwatch.Elapsed); } catch (Exception ex) { Console.WriteLine($"An error occurred while executing GetBlocksAsync. Details: {ex.Message}"); throw; } }
/// <summary> /// Copies large file as chunks from Azure BLOB to Amazon S3. /// </summary> /// <returns></returns> public async Task CopyLargeFileFromAzureBlobToAwsS3() { AmazonS3Client s3Client = new AmazonS3Client(AwsAccessKeyId, AwsSecretKey, Amazon.RegionEndpoint.APSouth1); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageAccount); //Create Storage account reference. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Create the blob client. CloudBlobContainer container = blobClient.GetContainerReference(ContainerName); // Retrieve reference to a container. container.CreateIfNotExists(); CloudBlockBlob blob = container.GetBlockBlobReference(BlobFileName); // Create Blob reference. blob.FetchAttributes(); // Prepare blob instance To get the file length. var remainingBytes = blob.Properties.Length; long readPosition = 0; // To be used offset / position from where to start reading from BLOB. InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest { BucketName = AwsS3BucketName, Key = TargetFileName }; // Will use UploadId from this response. InitiateMultipartUploadResponse initiateMultipartUploadResponse = s3Client.InitiateMultipartUpload(initiateMultipartUploadRequest); List <UploadPartResponse> uploadPartResponses = new List <UploadPartResponse>(); Stopwatch stopwatch = Stopwatch.StartNew(); try { int partCounter = 0; // To increment on each read of parts and use it as part number. while (remainingBytes > 0) { // Determine the size when final block reached as it might be less than Part size. // Will be PartSize except final block. long bytesToCopy = Math.Min(PartSize, remainingBytes); using (MemoryStream memoryStream = new MemoryStream()) { // To download part from BLOB. await blob.DownloadRangeToStreamAsync(memoryStream, readPosition, bytesToCopy).ConfigureAwait(false); memoryStream.Position = 0; partCounter++; UploadPartRequest uploadRequest = new UploadPartRequest { BucketName = AwsS3BucketName, Key = TargetFileName, UploadId = initiateMultipartUploadResponse.UploadId, PartNumber = partCounter, PartSize = bytesToCopy, InputStream = memoryStream }; UploadPartResponse uploadPartResponse = s3Client.UploadPart(uploadRequest); uploadPartResponses.Add(uploadPartResponse); remainingBytes -= bytesToCopy; readPosition += bytesToCopy; this.logger.WriteLine($"Uploaded part with part number {partCounter}, size {bytesToCopy}bytes and remaining {remainingBytes}bytes to read."); } } CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest { BucketName = AwsS3BucketName, Key = TargetFileName, UploadId = initiateMultipartUploadResponse.UploadId }; completeMultipartUploadRequest.AddPartETags(uploadPartResponses); CompleteMultipartUploadResponse completeMultipartUploadResponse = await s3Client.CompleteMultipartUploadAsync(completeMultipartUploadRequest).ConfigureAwait(false); } catch (Exception exception) { this.logger.WriteLine($"Exception : {exception.Message}"); AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest { BucketName = AwsS3BucketName, Key = TargetFileName, UploadId = initiateMultipartUploadResponse.UploadId }; await s3Client.AbortMultipartUploadAsync(abortMultipartUploadRequest).ConfigureAwait(false); } finally { stopwatch.Stop(); this.logger.WriteLine($"Execution time in mins: {stopwatch.Elapsed.TotalMinutes}"); } }
public async Task <string> DownloadPublicFile(Uri fileUrl, string localPath, string localFile) { if (!Directory.Exists(localPath)) { throw new DirectoryNotFoundException("Directory is not found"); } //1 MB chunk; var blobSize = genericHttpClient.GetSizeOFile(fileUrl); long blockSize = (this.blobStorageSettings.ChunkBlockSize * 1024 * 1024); var fileToCreate = Path.Combine(localPath, localFile); CloudBlockBlob blob = new CloudBlockBlob(fileUrl); using (FileStream fileStream = new FileStream(fileToCreate, FileMode.Create)) { fileStream.SetLength(blobSize); } long offset = 0; long bytesRemaining = blobSize; var blobRequestOptions = this.GetBlobRequestsOptions(); var operationContext = new OperationContext(); var accessCondition = new AccessCondition(); do { if (operationContext.LastResult == null) { Console.WriteLine("Downloading file from blobStorage..."); } else { Console.WriteLine("Last partial chunk was {0}", operationContext.LastResult.RequestDate); } var bytesToFetch = Math.Min(blockSize, bytesRemaining); using (MemoryStream memoryStream = new MemoryStream()) { await blob.DownloadRangeToStreamAsync(target : memoryStream, offset : offset, length : bytesToFetch, accessCondition : accessCondition, options : blobRequestOptions, operationContext : operationContext); memoryStream.Position = 0; var contents = memoryStream.ToArray(); using (var fileStream = new FileStream(fileToCreate, FileMode.Open)) { fileStream.Position = offset; fileStream.Write(contents, 0, contents.Length); } offset += contents.Length; bytesRemaining -= contents.Length; } }while (bytesRemaining > 0); return(fileToCreate); }
private static async Task MainAsync(string[] args) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(@"DefaultEndpointsProtocol=https;AccountName=demosaehandsb;AccountKey=M/E07gdsv/wRTGokKnmPAmrS+M9F3hPBYqR+46kCsHwG4YlVwYQSf8b1DIwIsRSTCM0oiazbNbDFeBDg9latQw==;EndpointSuffix=core.windows.net"); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference("democapture"); Console.WriteLine("Iterating through blobs:"); BlobContinuationToken continuationToken = null; BlobResultSegment resultSegment = null; do { //This overload allows control of the page size. You can return all remaining results by passing null for the maxResults parameter, //or by calling a different overload. resultSegment = await container.ListBlobsSegmentedAsync("", true, BlobListingDetails.All, 5000, continuationToken, null, null); foreach (var blobItem in resultSegment.Results) { //Console.WriteLine("\t{0}", blobItem.StorageUri.PrimaryUri); string[] theURI = blobItem.Uri.Segments; string blobReference = ""; for (var i = 0; i < theURI.Length; i++) { if (i >= 2) { blobReference += theURI[i]; } } CloudBlockBlob myBlockBlob = container.GetBlockBlobReference(blobReference); MemoryStream memoryStream = new MemoryStream(); await myBlockBlob.DownloadRangeToStreamAsync(memoryStream, null, null); if (myBlockBlob.Properties.Length > 0) { //string pathSource = @"C:\temp\37.avro"; using (var reader = Avro.File.DataFileReader <GenericRecord> .OpenReader(memoryStream)) { while (reader.HasNext()) { GenericRecord record = reader.Next(); long sn = (long)record["SequenceNumber"]; string offset = (string)record["Offset"]; byte[] body = (byte[])record["Body"]; string bodyText = Encoding.UTF8.GetString(body); Console.WriteLine($"{sn}: {offset}, {bodyText}"); // process other property according to the schema: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview } } } } //Get the continuation token. continuationToken = resultSegment.ContinuationToken; }while (continuationToken != null); Console.WriteLine("Press any key to exit"); Console.ReadLine(); }
/// <summary> /// Appends a stream's contents to the end of a blob (creating the blob and appending a header if the blob doesn't exist). /// Before calling this method, set the blob's ContentType Property (ex: "text/csv; charset=utf-8") /// </summary> /// <param name="blob">The blob to have the byte array appended to.</param> /// <param name="data">The stream with contents to append.</param> /// <param name="header">The header byte array to put in the blob if the blob is being created.</param> /// <param name="maxBlockSize">The maximum block sized in bytes (0=Azure default of 4MB)</param> /// <param name="options">A Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions object that specifies additional options for the request.</param> /// <param name="operationContext">An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation.</param> /// <param name="cancellationToken">A System.Threading.CancellationToken to observe while waiting for a task to complete.</param> /// <returns>A System.Threading.Tasks.Task object that represents the current operation.</returns> public static async Task AppendAsync(this CloudBlockBlob blob, Stream data, Byte[] header, Int32 maxBlockSize = 0, BlobRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { if (maxBlockSize == 0) { maxBlockSize = 4 * 1024 * 1024; } if (data.Length > maxBlockSize) { throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + maxBlockSize.ToString() + " bytes."); } if (header != null && header.Length > maxBlockSize) { throw new ArgumentOutOfRangeException("header", "The header cannot be larger than " + maxBlockSize.ToString() + " bytes."); } while (true) { using (var ms = new MemoryStream()) { AccessCondition accessCondition; IEnumerable <ListBlockItem> blockList = Enumerable.Empty <ListBlockItem>(); try { blockList = await blob.DownloadBlockListAsync(BlockListingFilter.Committed, null, options, operationContext, cancellationToken).ConfigureAwait(false); // 404 if blob not found accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read } catch (StorageException se) { if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound)) { throw; } accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist } try { List <String> blockIds = blockList.Select(lbi => lbi.Name).ToList(); ListBlockItem lastBlock = blockList.LastOrDefault(); if (lastBlock == null) { if (header != null) { // No blocks exist, add header (if specified) ms.Write(header, 0, header.Length); } } else { // A block exists, can it hold the new data? if (lastBlock.Length + data.Length < maxBlockSize) { // Yes, download the last block's current data as long as the blob's etag hasn't changed Int64 lastBlockOffset = blockList.Sum(lbi => lbi.Length) - lastBlock.Length; await blob.DownloadRangeToStreamAsync(ms, lastBlockOffset, lastBlock.Length, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 412 if blob modified behind our back blockIds.Remove(lastBlock.Name); // Remove the block we're appending to } } await data.CopyToAsync(ms).ConfigureAwait(false); // Append new data to end of stream ms.Seek(0, SeekOrigin.Begin); // Upload new block and append it to the blob String newBlockId = Guid.NewGuid().ToString("N").Encode().ToBase64String(); blockIds.Add(newBlockId); // Append new block to end of blob await blob.PutBlockAsync(newBlockId, ms, null, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // PutBlock ignores access condition so this always succeeds await blob.PutBlockListAsync(blockIds, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block ID doesn't exist (happens in another PC calls PutBlockList after our PutBlock) break; // If successful, we're done; don't retry } catch (StorageException se) { // Blob got created behind our back, retry if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists)) { continue; } // Blob got created or modified behind our back, retry if (se.Matches(HttpStatusCode.PreconditionFailed)) { continue; } // Another PC called PutBlockList between our PutBlock & PutBlockList, // our block got destroyed, retry if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList)) { continue; } throw; } } } }