public async Task <Response <BlobContentInfo> > UploadFromStream(string blobContainer, string fileName, Stream content, BlobHttpHeaders header = null, bool createContainerIfNotExists = true) { var container = GetContainer(blobContainer, createContainerIfNotExists); var blob = container.GetBlobClient(fileName); return(await blob.UploadAsync(content, header)); }
private async Task <Response <BlobContentInfo> > UploadInParallelAsync( Stream content, int blockSize, BlobHttpHeaders blobHttpHeaders, IDictionary <string, string> metadata, BlobRequestConditions conditions, IProgress <long> progressHandler, AccessTier?accessTier, CancellationToken cancellationToken) { // Wrap the staging and commit calls in an Upload span for // distributed tracing DiagnosticScope scope = _client.ClientDiagnostics.CreateScope( _operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}"); try { scope.Start(); // Wrap progressHandler in a AggregatingProgressIncrementer to prevent // progress from being reset with each stage blob operation. if (progressHandler != null) { progressHandler = new AggregatingProgressIncrementer(progressHandler); } // The list tracking blocks IDs we're going to commit List <string> blockIds = new List <string>(); // A list of tasks that are currently executing which will // always be smaller than _maxWorkerCount List <Task> runningTasks = new List <Task>(); // Partition the stream into individual blocks await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync( content, blockSize, async: true, _arrayPool, cancellationToken).ConfigureAwait(false)) { // Start staging the next block (but don't await the Task!) string blockId = GenerateBlockId(block.AbsolutePosition); Task task = StageBlockAsync( block, blockId, conditions, progressHandler, cancellationToken); // Add the block to our task and commit lists runningTasks.Add(task); blockIds.Add(blockId); // If we run out of workers if (runningTasks.Count >= _maxWorkerCount) { // Wait for at least one of them to finish await Task.WhenAny(runningTasks).ConfigureAwait(false); // Clear any completed blocks from the task list for (int i = 0; i < runningTasks.Count; i++) { Task runningTask = runningTasks[i]; if (!runningTask.IsCompleted) { continue; } await runningTask.ConfigureAwait(false); runningTasks.RemoveAt(i); i--; } } } // Wait for all the remaining blocks to finish staging and then // commit the block list to complete the upload await Task.WhenAll(runningTasks).ConfigureAwait(false); return(await _client.CommitBlockListAsync( blockIds, blobHttpHeaders, metadata, conditions, accessTier, cancellationToken) .ConfigureAwait(false)); } catch (Exception ex) { scope.Failed(ex); throw; } finally { scope.Dispose(); } }
/// <summary> /// Upload File to blob with storage Client library API /// </summary> internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob) { BlobClientOptions options = this.ClientOptions; if (!string.IsNullOrEmpty(this.EncryptionScope)) { options = SetClientOptionsWithEncryptionScope(this.EncryptionScope); } if (this.Force.IsPresent || !blob.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null)) { // Prepare blob Properties, MetaData, accessTier BlobHttpHeaders blobHttpHeaders = CreateBlobHttpHeaders(BlobProperties); IDictionary <string, string> metadata = new Dictionary <string, string>(); SetBlobMeta_Track2(metadata, this.Metadata); AccessTier?accesstier = GetAccessTier_Track2(this.standardBlobTier, this.pageBlobTier); //Prepare progress handler long fileSize = new FileInfo(ResolvedFileName).Length; string activity = String.Format(Resources.SendAzureBlobActivity, this.File, blob.Name, blob.Container.Name); string status = Resources.PrepareUploadingBlob; ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status); IProgress <long> progressHandler = new Progress <long>((finishedBytes) => { if (pr != null) { // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize % 101); pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete); this.OutputStream.WriteProgress(pr); } }); BlobBaseClient outputBlobClient = null; using (FileStream stream = System.IO.File.OpenRead(ResolvedFileName)) { //block blob if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase)) { BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options); outputBlobClient = blobClient; StorageTransferOptions trasnferOption = new StorageTransferOptions() { MaximumConcurrency = this.GetCmdletConcurrency() }; BlobUploadOptions uploadOptions = new BlobUploadOptions(); if (this.BlobTag != null) { uploadOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value); } uploadOptions.Metadata = metadata; uploadOptions.HttpHeaders = blobHttpHeaders; uploadOptions.Conditions = this.BlobRequestConditions; uploadOptions.AccessTier = accesstier; uploadOptions.ProgressHandler = progressHandler; uploadOptions.TransferOptions = trasnferOption; await blobClient.UploadAsync(stream, uploadOptions, CmdletCancellationToken).ConfigureAwait(false); } //Page or append blob else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) || string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase)) { PageBlobClient pageblobClient = null; AppendBlobClient appendblobClient = null; //Create Blob if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page { if (fileSize % 512 != 0) { throw new ArgumentException(String.Format("File size {0} Bytes is invalid for PageBlob, must be a multiple of 512 bytes.", fileSize.ToString())); } pageblobClient = GetTrack2PageBlobClient(blob, localChannel.StorageContext, options); outputBlobClient = pageblobClient; PageBlobCreateOptions createOptions = new PageBlobCreateOptions(); if (this.BlobTag != null) { createOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value); } createOptions.Metadata = metadata; createOptions.HttpHeaders = blobHttpHeaders; createOptions.Conditions = this.PageBlobRequestConditions; Response <BlobContentInfo> blobInfo = await pageblobClient.CreateAsync(fileSize, createOptions, CmdletCancellationToken).ConfigureAwait(false); } else //append { appendblobClient = GetTrack2AppendBlobClient(blob, localChannel.StorageContext, options); outputBlobClient = appendblobClient; AppendBlobCreateOptions createOptions = new AppendBlobCreateOptions(); if (this.BlobTag != null) { createOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value); } createOptions.Metadata = metadata; createOptions.HttpHeaders = blobHttpHeaders; createOptions.Conditions = this.AppendBlobRequestConditions; Response <BlobContentInfo> blobInfo = await appendblobClient.CreateAsync(createOptions, CmdletCancellationToken).ConfigureAwait(false); } // Upload blob content byte[] uploadcache4MB = null; byte[] uploadcache = null; progressHandler.Report(0); long offset = 0; while (offset < fileSize) { // Get chunk size and prepare cache int chunksize = size4MB; if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB { if (uploadcache4MB == null) { uploadcache4MB = new byte[size4MB]; } uploadcache = uploadcache4MB; } else // last chunk can < 4MB { chunksize = (int)(fileSize - offset); if (uploadcache4MB == null) { uploadcache = new byte[chunksize]; } else { uploadcache = uploadcache4MB; } } //Get content to upload for the chunk int readoutcount = await stream.ReadAsync(uploadcache, 0, (int)chunksize).ConfigureAwait(false); MemoryStream chunkContent = new MemoryStream(uploadcache, 0, readoutcount); //Upload content if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page { Response <PageInfo> pageInfo = await pageblobClient.UploadPagesAsync(chunkContent, offset, null, null, null, CmdletCancellationToken).ConfigureAwait(false); } else //append { Response <BlobAppendInfo> pageInfo = await appendblobClient.AppendBlockAsync(chunkContent, null, null, null, CmdletCancellationToken).ConfigureAwait(false); } // Update progress offset += readoutcount; progressHandler.Report(offset); } if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) && accesstier != null) { await pageblobClient.SetAccessTierAsync(accesstier.Value, cancellationToken : CmdletCancellationToken).ConfigureAwait(false); } } else { throw new InvalidOperationException(string.Format( CultureInfo.CurrentCulture, Resources.InvalidBlobType, blobType, BlobName)); } } AzureStorageBlob outputBlob = new AzureStorageBlob(outputBlobClient, localChannel.StorageContext, null, ClientOptions); OutputStream.WriteObject(taskId, outputBlob); } }
public static async void ProcessFileExports( [QueueTrigger(QueueNames.ProcessFileExports)] string processFileExportsQueue, [Table(TableNames.RedirectSessions)] CloudTable redirectTable, ILogger log, ExecutionContext context) { FileExportEntity queueFileExport = JsonConvert.DeserializeObject <FileExportEntity>(processFileExportsQueue); var config = new ConfigurationBuilder() .SetBasePath(context.FunctionAppDirectory) .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() .Build(); Stream stream = new MemoryStream(); if (queueFileExport.SourceUrl.StartsWith("https://") || queueFileExport.SourceUrl.StartsWith("http://")) { var client = new HttpClient(); stream = await client.GetStreamAsync(queueFileExport.SourceUrl); } else { if (queueFileExport.SourceUrl == FileExportTypes.AllVideos) { dynamic[] data = await GetVideo.All(redirectTable); stream = new MemoryStream(); var writer = new StreamWriter(stream); writer.Write(JsonConvert.SerializeObject(data)); writer.Flush(); stream.Position = 0; } } // Create a BlobServiceClient object which will be used to create a container client BlobServiceClient blobServiceClient = new BlobServiceClient(config.GetValue <string>(queueFileExport.DestinationStorage)); // Create the container and return a container client object BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(queueFileExport.DestinationContainer); BlobClient blobClient = containerClient.GetBlobClient(queueFileExport.DestinationLocation); await blobClient.DeleteIfExistsAsync(); await blobClient.UploadAsync(stream); BlobHttpHeaders headers = new BlobHttpHeaders(); bool addHeaders = false; if (queueFileExport.DestinationLocation.ToLower().EndsWith(".json")) { headers.ContentType = "application/json"; addHeaders = true; } if (config.GetValue <string>("FILE_EXPORT_CACHE_CONTROL") != null) { headers.CacheControl = config.GetValue <string>("FILE_EXPORT_CACHE_CONTROL"); addHeaders = true; } if (addHeaders) { await blobClient.SetHttpHeadersAsync(headers); } // log.LogError($"URL lookup {queueFileExport.SourceUrl} failed with status code {getResponse.StatusCode}"); // throw new System.Exception($"URL lookup {queueFileExport.SourceUrl} failed with status code {getResponse.StatusCode}"); }
private Response <BlobContentInfo> UploadInSequence( Stream content, int blockSize, BlobHttpHeaders blobHttpHeaders, IDictionary <string, string> metadata, BlobRequestConditions conditions, IProgress <long> progressHandler, AccessTier?accessTier, CancellationToken cancellationToken) { // Wrap the staging and commit calls in an Upload span for // distributed tracing DiagnosticScope scope = _client.ClientDiagnostics.CreateScope( _operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}"); try { scope.Start(); // Wrap progressHandler in a AggregatingProgressIncrementer to prevent // progress from being reset with each stage blob operation. if (progressHandler != null) { progressHandler = new AggregatingProgressIncrementer(progressHandler); } // The list tracking blocks IDs we're going to commit List <string> blockIds = new List <string>(); // Partition the stream into individual blocks and stage them IAsyncEnumerator <ChunkedStream> enumerator = PartitionedUploadExtensions.GetBlocksAsync(content, blockSize, async: false, _arrayPool, cancellationToken) .GetAsyncEnumerator(cancellationToken); #pragma warning disable AZC0107 while (enumerator.MoveNextAsync().EnsureCompleted()) #pragma warning restore AZC0107 { // Dispose the block after the loop iterates and return its // memory to our ArrayPool using ChunkedStream block = enumerator.Current; // Stage the next block string blockId = GenerateBlockId(block.AbsolutePosition); _client.StageBlock( blockId, new MemoryStream(block.Bytes, 0, block.Length, writable: false), conditions: conditions, progressHandler: progressHandler, cancellationToken: cancellationToken); blockIds.Add(blockId); } // Commit the block list after everything has been staged to // complete the upload return(_client.CommitBlockList( blockIds, blobHttpHeaders, metadata, conditions, accessTier, cancellationToken)); } catch (Exception ex) { scope.Failed(ex); throw; } finally { scope.Dispose(); } }
public async Task <IActionResult> Upload() { var formCollection = await Request.ReadFormAsync(); var file = formCollection.Files.First(); var requestedContainer = formCollection["container"].ToString(); var fileExtension = Path.GetExtension(file.FileName.ToLowerInvariant()); // check the file type is allowed if (!permittedFileExtensions.Contains(fileExtension) && fileExtension != "") { Console.WriteLine("The file type '" + fileExtension + "' is not allowed"); return(BadRequest(new ApiResponse(400))); } // check the file size (in bytes) if (file.Length > fileSizeLimit) { Console.WriteLine("The file size was too large"); return(BadRequest(new ApiResponse(400))); } // check the file name length isn't excessive if (file.FileName.Length > 75) { Console.WriteLine("The file name is too long: " + file.FileName.Length + " characters"); return(BadRequest(new ApiResponse(400))); } // check container name isn't empty if (requestedContainer.Length == 0) { Console.WriteLine("Azure container name is empty"); return(BadRequest(new ApiResponse(400))); } // check requested container name is in an allowed set of names if (!azureContainersAllowed.Contains(requestedContainer.ToLowerInvariant())) { Console.WriteLine("Invalid azure container name supplied: " + requestedContainer); return(BadRequest(new ApiResponse(400))); } if (file.Length > 0) { var azureContainer = new BlobContainerClient(azureConnectionString, requestedContainer); var createResponse = await azureContainer.CreateIfNotExistsAsync(); // in case the container doesn't exist if (createResponse != null && createResponse.GetRawResponse().Status == 201) { await azureContainer.SetAccessPolicyAsync(Azure.Storage.Blobs.Models.PublicAccessType.Blob); } // generate a unique upload file name // [original_filename_without_extension]_[8_random_chars].[original_filename_extension] // eg. filename_xgh38tye.jpg var fileName = HttpUtility.HtmlEncode(Path.GetFileNameWithoutExtension(file.FileName)) + "_" + Path.GetRandomFileName().Substring(0, 8) + Path.GetExtension(file.FileName); var blob = azureContainer.GetBlobClient(fileName); //await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots); // set the content type (which may or may not have been provided by the client) var blobHttpHeader = new BlobHttpHeaders(); if (file.ContentType != null) { blobHttpHeader.ContentType = file.ContentType; } else { blobHttpHeader.ContentType = fileExtension switch { ".jpg" => "image/jpeg", ".jpeg" => "image/jpeg", ".png" => "image/png", _ => null }; } using (var fileStream = file.OpenReadStream()) { await blob.UploadAsync(fileStream, blobHttpHeader); } return(Ok(new { filename = blob.Name, container = blob.BlobContainerName, uri = blob.Uri })); } Console.WriteLine("The file could not be uploaded"); return(BadRequest(new ApiResponse(400))); } }
public static async Task Run( [EventGridTrigger] EventGridEvent eventGridEvent, [Blob("{data.url}", FileAccess.Read)] Stream input, ILogger log) { try { if (input != null) { var createdEvent = ((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>(); var extension = Path.GetExtension(createdEvent.Url); var encoder = GetEncoder(extension); log.LogInformation($"encoder: {encoder}"); if (encoder != null) { var thumbnailWidth = 100; //Convert.ToInt32(Environment.GetEnvironmentVariable("THUMBNAIL_WIDTH")); var blobContainerName = Environment.GetEnvironmentVariable("THUMBNAIL_CONTAINER_NAME"); var connectionString = BLOB_STORAGE_CONNECTION_STRING; //var blobServiceClient = new BlobServiceClient(BLOB_STORAGE_CONNECTION_STRING); //var blobContainerClient = blobServiceClient.GetBlobContainerClient(thumbContainerName); var blobName = GetBlobNameFromUrl(createdEvent.Url); var blobClient = new BlobClient(connectionString, blobContainerName, blobName); var contentType = GetContentType(extension); var blobHttpHeaders = new BlobHttpHeaders { ContentType = contentType }; using (var output = new MemoryStream()) using (Image <Rgba32> image = Image.Load(input)) { log.LogInformation($"image.Width: {image.Width}"); log.LogInformation($"image.Height: {image.Height}"); log.LogInformation($"thumbnailWidth: {thumbnailWidth}"); var divisor = image.Width / thumbnailWidth; log.LogInformation($"divisor: {divisor}"); if (divisor < 1) { divisor = 1; } var height = Convert.ToInt32(Math.Round((decimal)(image.Height / divisor))); if (divisor >= 1) { image.Mutate(x => x.Resize(thumbnailWidth, height)); } image.Save(output, encoder); output.Position = 0; //await blobContainerClient.UploadBlobAsync(blobName, output); await blobClient.UploadAsync(output, blobHttpHeaders); } //log.LogInformation($"blobName: {blobName}"); //log.LogInformation($"createdEvent: {createdEvent.Url}"); log.LogInformation($"contentType: {contentType}"); log.LogInformation($"blobContainerClient: {blobName}"); } else { log.LogInformation($"No encoder support for: {createdEvent.Url}"); } } } catch (Exception ex) { log.LogInformation(ex.Message); throw; } }
/// <summary> /// This operation will create a new /// block blob of arbitrary size by uploading it as indiviually staged /// blocks if it's larger than the /// <paramref name="singleBlockThreshold"/>. /// </summary> /// <param name="file"> /// A <see cref="FileInfo"/> of the file to upload. /// </param> /// <param name="blobHttpHeaders"> /// Optional standard HTTP header properties that can be set for the /// block blob. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this block blob. /// </param> /// <param name="blobAccessConditions"> /// Optional <see cref="BlobAccessConditions"/> to add conditions on /// the creation of this new block blob. /// </param> /// <param name="progressHandler"> /// Optional <see cref="IProgress{StorageProgress}"/> to provide /// progress updates about data transfers. /// </param> /// <param name="accessTier"> /// Optional <see cref="AccessTier"/> /// Indicates the tier to be set on the blob. /// </param> /// <param name="singleBlockThreshold"> /// The maximum size stream that we'll upload as a single block. The /// default value is 256MB. /// </param> /// <param name="transferOptions"> /// Optional <see cref="StorageTransferOptions"/> to configure /// parallel transfer behavior. /// </param> /// <param name="async"> /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Response{BlobContentInfo}"/> describing the /// state of the updated block blob. /// </returns> /// <remarks> /// A <see cref="StorageRequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> internal async Task <Response <BlobContentInfo> > StagedUploadAsync( FileInfo file, BlobHttpHeaders blobHttpHeaders, Metadata metadata, BlobAccessConditions?blobAccessConditions, IProgress <StorageProgress> progressHandler, AccessTier?accessTier = default, long?singleBlockThreshold = default, StorageTransferOptions transferOptions = default, bool async = true, CancellationToken cancellationToken = default) { var client = new BlockBlobClient(Uri, Pipeline); singleBlockThreshold ??= client.BlockBlobMaxUploadBlobBytes; Debug.Assert(singleBlockThreshold <= client.BlockBlobMaxUploadBlobBytes); var blockMap = new ConcurrentDictionary <long, string>(); var blockName = 0; Task <Response <BlobContentInfo> > uploadTask = PartitionedUploader.UploadAsync( UploadStreamAsync, StageBlockAsync, CommitBlockListAsync, threshold => file.Length < threshold, memoryPool => new StreamPartitioner(file, memoryPool), singleBlockThreshold.Value, transferOptions, async, cancellationToken); return(async ? await uploadTask.ConfigureAwait(false) : uploadTask.EnsureCompleted()); string GetNewBase64BlockId(long blockOrdinal) { // Create and record a new block ID, storing the order information // (nominally the block's start position in the original stream) var newBlockName = Interlocked.Increment(ref blockName); var blockId = Constants.BlockNameFormat; blockId = String.Format(CultureInfo.InvariantCulture, blockId, newBlockName); blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId)); var success = blockMap.TryAdd(blockOrdinal, blockId); Debug.Assert(success); return(blockId); } // Upload the entire stream async Task <Response <BlobContentInfo> > UploadStreamAsync() { using (FileStream stream = file.OpenRead()) { Stream transformedStream; // cannot reassign to a "using" variable (transformedStream, metadata) = TransformContent(stream, metadata); return (await client.UploadInternal( transformedStream, blobHttpHeaders, metadata, blobAccessConditions, accessTier, progressHandler, async, cancellationToken) .ConfigureAwait(false)); } } // Upload a single partition of the stream Task <Response <BlockInfo> > StageBlockAsync( Stream partition, long blockOrdinal, bool async, CancellationToken cancellation) { var base64BlockId = GetNewBase64BlockId(blockOrdinal); //var bytes = new byte[10]; //partition.Read(bytes, 0, 10); partition.Position = 0; //Console.WriteLine($"Commiting partition {blockOrdinal} => {base64BlockId}, {String.Join(" ", bytes)}"); // Upload the block return(client.StageBlockInternal( base64BlockId, partition, null, blobAccessConditions?.LeaseAccessConditions, progressHandler, async, cancellationToken)); } // Commit a series of partitions Task <Response <BlobContentInfo> > CommitBlockListAsync( bool async, CancellationToken cancellation) { var base64BlockIds = blockMap.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray(); //Console.WriteLine($"Commiting block list:\n{String.Join("\n", base64BlockIds)}"); return (client.CommitBlockListInternal( base64BlockIds, blobHttpHeaders, metadata, blobAccessConditions, accessTier, async, cancellationToken)); } }
public void UploadFileinBlocks(string filePath, string fileName) { string _filename = DateTime.Now.ToString("dd.MM.yyyy_hh.mm.ss") + Path.GetFileName(filePath); BlockBlobClient blobClient = blobContainer.GetBlockBlobClient(fileName);// Path.GetFileName(filePath)); BlobHttpHeaders blobHttpHeaders = new BlobHttpHeaders(); blobHttpHeaders.ContentType = "application/octet-stream"; blobClient.SetHttpHeadersAsync(blobHttpHeaders); int blockSize = 256 * 1024; //256 kb using (FileStream fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { long fileSize = fileStream.Length; //block count is the number of blocks + 1 for the last one int blockCount = (int)((float)fileSize / (float)blockSize) + 1; //List of block ids; the blocks will be committed in the order of this list List <string> blockIDs = new List <string>(); //starting block number - 1 int blockNumber = 0; try { int bytesRead = 0; //number of bytes read so far long bytesLeft = fileSize; //number of bytes left to read and upload //do until all of the bytes are uploaded while (bytesLeft > 0) { blockNumber++; int bytesToRead; if (bytesLeft >= blockSize) { //more than one block left, so put up another whole block bytesToRead = blockSize; } else { //less than one block left, read the rest of it bytesToRead = (int)bytesLeft; } //create a blockID from the block number, add it to the block ID list //the block ID is a base64 string string blockId = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", blockNumber.ToString("0000000")))); blockIDs.Add(blockId); //set up new buffer with the right size, and read that many bytes into it byte[] bytes = new byte[bytesToRead]; fileStream.Read(bytes, 0, bytesToRead); //calculate the MD5 hash of the byte array byte [] blockHash = GetMD5HashByte(bytes); //upload the block, provide the hash so Azure can verify it blobClient.StageBlock(blockId, new MemoryStream(bytes), blockHash); //increment/decrement counters bytesRead += bytesToRead; bytesLeft -= bytesToRead; } //commit the blocks blobClient.CommitBlockList(blockIDs); } catch (Exception ex) { throw new Exception(ex.Message); } } }
public override Task <Response <BlobContentInfo> > UploadAsync(Stream content, BlobHttpHeaders httpHeaders = null, IDictionary <string, string> metadata = null, BlobRequestConditions conditions = null, IProgress <long> progressHandler = null, AccessTier?accessTier = null, StorageTransferOptions transferOptions = default, CancellationToken cancellationToken = default) { if (UploadBlobException != null) { throw UploadBlobException; } if (BlobInfo != null) { throw new RequestFailedException(409, BlobErrorCode.BlobAlreadyExists.ToString(), BlobErrorCode.BlobAlreadyExists.ToString(), default); } return(Task.FromResult( Response.FromValue( BlobsModelFactory.BlobContentInfo(new ETag("etag"), DateTime.UtcNow, new byte[] { }, string.Empty, 0L), Mock.Of <Response>()))); }
private async void CaptureProcess(object state) { var configuration = ((TimerState)state).configuration; try { // Need to get all of our processes this._logger.LogDebug($"Searching for process '{configuration.processName}' with window title '{configuration.windowTitle}' for configuration '{configuration.id}'"); // Find the pointer(s) for this window var ptrs = WindowFilter.FindWindowsWithText(configuration.windowTitle, true).ToList(); // We will attempt to auto start if we have been provided the right information if (ptrs.Count == 0 && configuration.autoStart && this._canLaunch) { // Attempt to launch the application and wait 10 seconds to continue // If we have already launched this application, do we have a problem? if (configuration._process != null) { if (configuration._process.HasExited) { if (configuration._attempts >= this._maxProcessAttempts) { this._logger.LogError($"Auto started process for configuration '{configuration.id}' has exited prematurely and failed too many times. Disabling auto start."); configuration.autoStart = false; return; } this._logger.LogError($"Auto started process for configuration '{configuration.id}' has exited prematurely. Will re-attempt."); } else { // So we have launched a process but we don't have a window, this is a failure configuration._process.Kill(); if (configuration._attempts >= this._maxProcessAttempts) { this._logger.LogError($"Auto started process for configuration '{configuration.id}' was launched but no corresponding window found. Process '{configuration._process.Id}' killed. Disabling auto start"); configuration.autoStart = false; return; } this._logger.LogError($"Auto started process for configuration '{configuration.id}' was launched but no corresponding window found. Process '{configuration._process.Id}' killed. Will re-attempt."); } configuration._process.Dispose(); } // The process expects just the profile file name that exists in its own directory try { // Increment our counter configuration._attempts++; this._logger.LogDebug($"Launching process '{this._ATCSFullPath}' with profile '{configuration.profile}' for configuration '{configuration.id}' (Attempt {configuration._attempts} of {this._maxProcessAttempts})"); configuration._process = Process.Start(this._ATCSFullPath, new List <string>() { configuration.profile }); // If we have a null response, it failed to start if (configuration._process == null) { throw new Exception("Error auto starting the process but no exception provided."); } await Task.Delay(this._processLaunchTime); // Attempt to find the windows again ptrs = WindowFilter.FindWindowsWithText(configuration.windowTitle, true).ToList(); // We don't check agian because we will do it below. We will also give this one more timer interval before failing it out with the above code } catch (Exception e) { this._logger.LogError(e, $"Unable to auto start the process for configuration '{configuration.id}', auto start disabled."); configuration.autoStart = false; } } // If we have more than one, we need to fail out here if (ptrs.Count > 1) { this._logger.LogWarning($"Found multiple windows for '{configuration.id}', unable to proceed."); } else if (ptrs.Count == 0) { this._logger.LogWarning($"Found no windows for '{configuration.id}', unable to proceed."); } else { try { // Capture this and save it this._logger.LogDebug($"Capturing window with handle '{ptrs[0].ToString()}' for configuration '{configuration.id}'."); var img = CaptureWindow(ptrs[0]); if (img == null) { throw new Exception("Received zero bytes for screenshot."); } // Determine the filename // Keep these separate for now, blob may change down the road var blobPath = $"{configuration.blobName}{this._ImageExt}".Trim(); var filePath = $"{configuration.blobName}{this._ImageExt}".Trim(); // Note that we have something recent this._lastUpdate[configuration.id] = DateTimeOffset.UtcNow; // Upload it to Azure if (this._enableUpload) { using (var ms = new MemoryStream(img)) { using (var ctx = new CancellationTokenSource()) { // Set our maximum upload time ctx.CancelAfter(this._maxUploadTime); try { var blobClient = this._blobContainerClient.GetBlobClient(blobPath); await blobClient.UploadAsync(ms, true, ctx.Token); // Set information about this screenshot value await blobClient.SetMetadataAsync(new Dictionary <string, string> { { "id", configuration.id }, { "updateTime", DateTimeOffset.UtcNow.ToString("O") }, { "name", configuration.name } }); var headers = new BlobHttpHeaders(); headers.ContentDisposition = "inline"; headers.ContentType = this._ImageMime; blobClient.SetHttpHeaders(headers); this._logger.LogInformation($"Blob '{blobPath}' saved for configuration '{configuration.id}'."); } catch (TaskCanceledException e) { this._logger.LogError(e, $"Blob '{blobPath}' took too long to upload, cancelled."); } } } } else { this._logger.LogDebug("Upload skipped due to configuration."); } // Save it if (configuration.saveFile) { SaveImage(img, filePath, this._ImageFormat); this._logger.LogDebug($"File '{filePath}' saved for configuration '{configuration.id}'."); } else { this._logger.LogDebug("Save file skipped due to configuration."); } } catch (Exception e) { this._logger.LogError(e, $"Exception thrown while capturing the window for '{configuration.windowTitle}': {e.Message}"); } } } catch (Exception e) { this._logger.LogError(e, $"Uncaught exception in {nameof(CaptureProcess)}."); } finally { // Re-initiate the timer this._timers.First(a => a.Key == configuration.id).Value.Change(this._frequency, Timeout.Infinite); } }
/// <summary> /// Upload to file to blob container /// </summary> /// <param name="file">file from the form</param> /// <param name="fileName">Name of file to the new uploaded file</param> /// <param name="blobHttpHeader">Set Http properties</param> /// <returns>BlobClient where the file with the same name is located</returns> public void UploadFileToStorage(IFormFile file, string fileName, BlobHttpHeaders blobHttpHeader) { GetBlobByFileName(fileName) .Upload(file.OpenReadStream(), blobHttpHeader); }
static void Main(string[] args) { string CONNECTIONSTRING = "<YOURCONNECTIONSTRING>"; string CONTAINERNAME = "<YOURCONTAINERNAME>"; var blobServiceClient = new BlobServiceClient(CONNECTIONSTRING); var containerClient = blobServiceClient.GetBlobContainerClient(CONTAINERNAME); var FOLDERNAMES = new List <string>() { "video", "image", "thumbnail" }; var messages = new List <string>(); var unsupportedFiles = new List <string>(); var errorFiles = new List <string>(); foreach (var folderName in FOLDERNAMES) { var blobs = containerClient.GetBlobs(prefix: folderName).ToList(); Console.WriteLine("connected to folder => " + folderName); int i = 0; for (; i < blobs.Count; i++) { try { var blob = containerClient.GetBlobClient(blobs[i].Name); Console.WriteLine($"Updating {i + 1}=> {blobs[i].Name}"); string mimeType = GetContentType(blobs[i].Name); if (!string.IsNullOrEmpty(mimeType)) { var blobHttpHeader = new BlobHttpHeaders { ContentType = mimeType }; var blobStream = blob.OpenRead(); blob.UploadAsync(blobStream, blobHttpHeader).Wait(); } else { unsupportedFiles.Add($"{CONTAINERNAME}/{blobs[i].Name}"); } } catch (Exception ex) { Console.WriteLine("============> Error in updating"); Console.WriteLine(ex.Message); errorFiles.Add($"{CONTAINERNAME}/{blobs[i].Name} => {ex.Message}"); Console.WriteLine(new string('=', 20)); } } messages.Add($"{CONTAINERNAME}/{folderName} => {i}"); } var lineSperator = new string('=', 40); Console.WriteLine(lineSperator); messages.ForEach(x => Console.WriteLine(x)); Console.WriteLine(lineSperator); unsupportedFiles.ForEach(x => Console.WriteLine(x)); Console.WriteLine(lineSperator); errorFiles.ForEach(x => Console.WriteLine(x)); }
public static async Task <string> UploadToBlob(TaskModuleSubmitDataDeserializer taskModuleOutput, ITurnContext turnContext) { List <string> imageURL = new List <string>(); string audioURL = taskModuleOutput.data.RecorderOutput; string requestID = taskModuleOutput.data.RequestId; byte[] streamConverter; string assignedToUserImageUrl; string submittedByUserImageUrl; List <Images> imgURL = new List <Images> { }; var blobHttpHeader = new BlobHttpHeaders(); blobHttpHeader.ContentType = "image/jpeg"; BlobContainerClient container = new BlobContainerClient(Constants.BlobConnectionString, Constants.BlobContainerName); BlobClient blob; var assignedToUserImage = GetImage(taskModuleOutput.data.AssignedTo.objectId).Result; if (assignedToUserImage != null) { blob = container.GetBlobClient(requestID + "-" + taskModuleOutput.data.AssignedTo.objectId + ".jpg"); blob.UploadAsync(assignedToUserImage, blobHttpHeader); assignedToUserImageUrl = blob.Uri.ToString(); } else { assignedToUserImageUrl = Constants.MemberGenericImageUrl; } var submittedByUserImage = GetImage(turnContext.Activity.From.AadObjectId).Result; if (submittedByUserImage != null) { blob = container.GetBlobClient(requestID + "-" + turnContext.Activity.From.AadObjectId + ".jpg"); blob.UploadAsync(submittedByUserImage, blobHttpHeader); submittedByUserImageUrl = blob.Uri.ToString(); } else { submittedByUserImageUrl = Constants.MemberGenericImageUrl; } if (taskModuleOutput.data.ImagesOutput.Length != 0) { foreach (string item in taskModuleOutput.data.ImagesOutput) { imageURL.Add(item); } } else { imageURL = null; } if (imageURL != null) { foreach (var item in imageURL) { var imgs = new Images { url = item }; imgURL.Add(imgs); } } string[] blobStorage = { ".", "Resources", "BlobStorage.json" }; var blobStorageText = File.ReadAllText(Path.Combine(blobStorage)); AdaptiveCardTemplate template = new AdaptiveCardTemplate(blobStorageText); var payloadData = new { requestID = requestID, status = Status.Pending, itemName = taskModuleOutput.data.ItemName, itemCode = taskModuleOutput.data.ItemCode, assignedToName = taskModuleOutput.data.AssignedTo.displayName, assignedToId = taskModuleOutput.data.AssignedTo.objectId, assignedToMail = taskModuleOutput.data.AssignedTo.email, submittedByName = turnContext.Activity.From.Name, submittedById = turnContext.Activity.From.AadObjectId, submittedByMail = taskModuleOutput.data.SubmittedByMail, assignedToUserImage = assignedToUserImageUrl, submittedByUserImage = submittedByUserImageUrl, imageURL = imgURL, audioURL = audioURL, comments = taskModuleOutput.data.Comments, conversationId = turnContext.Activity.Conversation.Id, messageId = "", }; var blobTextString = template.Expand(payloadData); streamConverter = Encoding.ASCII.GetBytes(blobTextString); blob = container.GetBlobClient(requestID + ".txt"); await blob.UploadAsync(new MemoryStream(streamConverter)); return(requestID); }
public static async Task Run([TimerTrigger("%ScheduleTriggerTime%")] TimerInfo timer, [Blob("%RSSPath%", FileAccess.ReadWrite, Connection = "StorageConnection")] BlockBlobClient feedBlob, ILogger log) { log.LogInformation($"AzureStack Hun MarketPlace RSS Generator launched at: {DateTime.Now}"); List <MarketPlaceItem> marketPlaceItems = new List <MarketPlaceItem>(); try { var stream = await httpClient.GetStreamAsync(marketPlaceUpdatesURL); using (StreamReader reader = new StreamReader(stream)) { string line; string section = ""; while (null != (line = await reader.ReadLineAsync())) { if (line.Contains("ms.date:")) { lastUpdated = DateTime.Parse(line.Split(":") [1].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-US").DateTimeFormat, System.Globalization.DateTimeStyles.AssumeUniversal); log.LogInformation($"GitHub page was last updated on: {lastUpdated.ToShortDateString()}"); if (feedBlob.Exists()) { var blobProperties = await feedBlob.GetPropertiesAsync(); if (blobProperties.Value.Metadata.ContainsKey(lastUpdatedMetadataKey) && blobProperties.Value.Metadata[lastUpdatedMetadataKey].Equals(lastUpdated.ToShortDateString())) { log.LogInformation("RSS Feed is already up to date, will quit"); return; } } } var matchSection = sectionRegex.Match(line); if (matchSection.Success) { section = matchSection.Groups[1].Value; continue; } var matchItem = itemRegex.Match(line); if (matchItem.Success) { marketPlaceItems.Add( new MarketPlaceItem { Name = matchItem.Groups["product"].Value, Change = section, ReleaseDate = DateTime.Parse(matchItem.Groups["date"].Value, System.Globalization.CultureInfo.GetCultureInfo("en-US").DateTimeFormat, System.Globalization.DateTimeStyles.AssumeUniversal) } ); } } } } catch (Exception e) { log.LogError($"Error while fetch Marketplace Items page: {e.Message}"); return; } if (marketPlaceItems.Count > 0) { log.LogInformation($"Got {marketPlaceItems.Count} items"); var feed = FormatRssFeed(marketPlaceItems); var settings = new XmlWriterSettings { Encoding = System.Text.Encoding.UTF8, NewLineHandling = NewLineHandling.Entitize, NewLineOnAttributes = true, Indent = true, Async = true }; using (var stream = new MemoryStream()) { using (var xmlWriter = XmlWriter.Create(stream, settings)) { var rssFormatter = new Rss20FeedFormatter(feed, false); rssFormatter.WriteTo(xmlWriter); await xmlWriter.FlushAsync(); } stream.Seek(0, SeekOrigin.Begin); var properties = new BlobHttpHeaders() { ContentType = "application/rss+xml; charset=utf-8" }; var metadata = new Dictionary <string, string> { { lastUpdatedMetadataKey, lastUpdated.ToShortDateString() } }; await feedBlob.UploadAsync(stream, new BlobUploadOptions { HttpHeaders = properties, Metadata = metadata }); } } }
static async Task Main(string[] args) { MemoryStream ms = new MemoryStream(); var input = File.OpenWrite(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\file2.jpg"); const int chunkSize = 506; using (var file = File.OpenRead(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\25thnov2020TestImg.txt")) { int bytesRead; var buffer = new byte[chunkSize]; while ((bytesRead = file.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, bytesRead); } } BlobContainerClient container = new BlobContainerClient("DefaultEndpointsProtocol=https;AccountName=fatihteststorage;AccountKey=N01DqDtMWIjzyGiSf2e3mBNJcqvGqBnM7lD6SSKjIT6/WbqT8ecMng5IYyH9WsST3iLbuoEGjHHACR9zoO51Uw==;EndpointSuffix=core.windows.net", "main"); var blockBlobClient = container.GetBlockBlobClient("myImage.png"); int blockSize = 50 * 1024; int offset = 0; int counter = 0; List <string> blockIds = new List <string>(); using (var fs = File.OpenRead(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\test-image.png")) { var bytesRemaining = fs.Length; do { var dataToRead = Math.Min(bytesRemaining, blockSize); byte[] data = new byte[dataToRead]; var dataRead = fs.Read(data, offset, (int)dataToRead); bytesRemaining -= dataRead; if (dataRead > 0) { var blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(counter.ToString("d6"))); blockBlobClient.StageBlock(blockId, new MemoryStream(data)); Console.WriteLine(string.Format("Block {0} uploaded successfully.", counter.ToString("d6"))); blockIds.Add(blockId); counter++; } } while (bytesRemaining > 0); var headers = new BlobHttpHeaders() { ContentType = "image/png" }; blockBlobClient.CommitBlockList(blockIds, headers); } try { // Get a reference to a blob BlobClient blob = container.GetBlobClient("test.png"); ms.Position = 0; // Open the file and upload its data await blob.UploadAsync(ms); // Verify we uploaded some content BlobProperties properties = await blob.GetPropertiesAsync(); } catch (Exception e) { } finally { // Clean up after the test when we're finished } }