public async Task <Response <BlobContentInfo> > UploadFromStream(string blobContainer, string fileName, Stream content, BlobHttpHeaders header = null, bool createContainerIfNotExists = true)
{
var container = GetContainer(blobContainer, createContainerIfNotExists);
var blob = container.GetBlobClient(fileName);
return(await blob.UploadAsync(content, header));
}
private async Task <Response <BlobContentInfo> > UploadInParallelAsync(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// A list of tasks that are currently executing which will
// always be smaller than _maxWorkerCount
List <Task> runningTasks = new List <Task>();
// Partition the stream into individual blocks
await foreach (ChunkedStream block in PartitionedUploadExtensions.GetBlocksAsync(
content,
blockSize,
async: true,
_arrayPool,
cancellationToken).ConfigureAwait(false))
{
// Start staging the next block (but don't await the Task!)
string blockId = GenerateBlockId(block.AbsolutePosition);
Task task = StageBlockAsync(
block,
blockId,
conditions,
progressHandler,
cancellationToken);
// Add the block to our task and commit lists
runningTasks.Add(task);
blockIds.Add(blockId);
// If we run out of workers
if (runningTasks.Count >= _maxWorkerCount)
{
// Wait for at least one of them to finish
await Task.WhenAny(runningTasks).ConfigureAwait(false);
// Clear any completed blocks from the task list
for (int i = 0; i < runningTasks.Count; i++)
{
Task runningTask = runningTasks[i];
if (!runningTask.IsCompleted)
{
continue;
}
await runningTask.ConfigureAwait(false);
runningTasks.RemoveAt(i);
i--;
}
}
}
// Wait for all the remaining blocks to finish staging and then
// commit the block list to complete the upload
await Task.WhenAll(runningTasks).ConfigureAwait(false);
return(await _client.CommitBlockListAsync(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken)
.ConfigureAwait(false));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
/// <summary>
/// Upload File to blob with storage Client library API
/// </summary>
internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
BlobClientOptions options = this.ClientOptions;
if (!string.IsNullOrEmpty(this.EncryptionScope))
{
options = SetClientOptionsWithEncryptionScope(this.EncryptionScope);
}
if (this.Force.IsPresent ||
!blob.Exists() ||
ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null))
{
// Prepare blob Properties, MetaData, accessTier
BlobHttpHeaders blobHttpHeaders = CreateBlobHttpHeaders(BlobProperties);
IDictionary <string, string> metadata = new Dictionary <string, string>();
SetBlobMeta_Track2(metadata, this.Metadata);
AccessTier?accesstier = GetAccessTier_Track2(this.standardBlobTier, this.pageBlobTier);
//Prepare progress handler
long fileSize = new FileInfo(ResolvedFileName).Length;
string activity = String.Format(Resources.SendAzureBlobActivity, this.File, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (pr != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
pr.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize % 101);
pr.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, pr.PercentComplete);
this.OutputStream.WriteProgress(pr);
}
});
BlobBaseClient outputBlobClient = null;
using (FileStream stream = System.IO.File.OpenRead(ResolvedFileName))
{
//block blob
if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase))
{
BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options);
outputBlobClient = blobClient;
StorageTransferOptions trasnferOption = new StorageTransferOptions()
{
MaximumConcurrency = this.GetCmdletConcurrency()
};
BlobUploadOptions uploadOptions = new BlobUploadOptions();
if (this.BlobTag != null)
{
uploadOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value);
}
uploadOptions.Metadata = metadata;
uploadOptions.HttpHeaders = blobHttpHeaders;
uploadOptions.Conditions = this.BlobRequestConditions;
uploadOptions.AccessTier = accesstier;
uploadOptions.ProgressHandler = progressHandler;
uploadOptions.TransferOptions = trasnferOption;
await blobClient.UploadAsync(stream, uploadOptions, CmdletCancellationToken).ConfigureAwait(false);
}
//Page or append blob
else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) ||
string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))
{
PageBlobClient pageblobClient = null;
AppendBlobClient appendblobClient = null;
//Create Blob
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
if (fileSize % 512 != 0)
{
throw new ArgumentException(String.Format("File size {0} Bytes is invalid for PageBlob, must be a multiple of 512 bytes.", fileSize.ToString()));
}
pageblobClient = GetTrack2PageBlobClient(blob, localChannel.StorageContext, options);
outputBlobClient = pageblobClient;
PageBlobCreateOptions createOptions = new PageBlobCreateOptions();
if (this.BlobTag != null)
{
createOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value);
}
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.PageBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await pageblobClient.CreateAsync(fileSize, createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
appendblobClient = GetTrack2AppendBlobClient(blob, localChannel.StorageContext, options);
outputBlobClient = appendblobClient;
AppendBlobCreateOptions createOptions = new AppendBlobCreateOptions();
if (this.BlobTag != null)
{
createOptions.Tags = this.BlobTag.Cast <DictionaryEntry>().ToDictionary(d => (string)d.Key, d => (string)d.Value);
}
createOptions.Metadata = metadata;
createOptions.HttpHeaders = blobHttpHeaders;
createOptions.Conditions = this.AppendBlobRequestConditions;
Response <BlobContentInfo> blobInfo = await appendblobClient.CreateAsync(createOptions, CmdletCancellationToken).ConfigureAwait(false);
}
// Upload blob content
byte[] uploadcache4MB = null;
byte[] uploadcache = null;
progressHandler.Report(0);
long offset = 0;
while (offset < fileSize)
{
// Get chunk size and prepare cache
int chunksize = size4MB;
if (chunksize <= (fileSize - offset)) // Chunk size will be 4MB
{
if (uploadcache4MB == null)
{
uploadcache4MB = new byte[size4MB];
}
uploadcache = uploadcache4MB;
}
else // last chunk can < 4MB
{
chunksize = (int)(fileSize - offset);
if (uploadcache4MB == null)
{
uploadcache = new byte[chunksize];
}
else
{
uploadcache = uploadcache4MB;
}
}
//Get content to upload for the chunk
int readoutcount = await stream.ReadAsync(uploadcache, 0, (int)chunksize).ConfigureAwait(false);
MemoryStream chunkContent = new MemoryStream(uploadcache, 0, readoutcount);
//Upload content
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) //page
{
Response <PageInfo> pageInfo = await pageblobClient.UploadPagesAsync(chunkContent, offset, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
else //append
{
Response <BlobAppendInfo> pageInfo = await appendblobClient.AppendBlockAsync(chunkContent, null, null, null, CmdletCancellationToken).ConfigureAwait(false);
}
// Update progress
offset += readoutcount;
progressHandler.Report(offset);
}
if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase) && accesstier != null)
{
await pageblobClient.SetAccessTierAsync(accesstier.Value, cancellationToken : CmdletCancellationToken).ConfigureAwait(false);
}
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.CurrentCulture,
Resources.InvalidBlobType,
blobType,
BlobName));
}
}
AzureStorageBlob outputBlob = new AzureStorageBlob(outputBlobClient, localChannel.StorageContext, null, ClientOptions);
OutputStream.WriteObject(taskId, outputBlob);
}
}
public static async void ProcessFileExports(
[QueueTrigger(QueueNames.ProcessFileExports)] string processFileExportsQueue,
[Table(TableNames.RedirectSessions)] CloudTable redirectTable,
ILogger log,
ExecutionContext context)
{
FileExportEntity queueFileExport = JsonConvert.DeserializeObject <FileExportEntity>(processFileExportsQueue);
var config = new ConfigurationBuilder()
.SetBasePath(context.FunctionAppDirectory)
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
Stream stream = new MemoryStream();
if (queueFileExport.SourceUrl.StartsWith("https://") || queueFileExport.SourceUrl.StartsWith("http://"))
{
var client = new HttpClient();
stream = await client.GetStreamAsync(queueFileExport.SourceUrl);
}
else
{
if (queueFileExport.SourceUrl == FileExportTypes.AllVideos)
{
dynamic[] data = await GetVideo.All(redirectTable);
stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(JsonConvert.SerializeObject(data));
writer.Flush();
stream.Position = 0;
}
}
// Create a BlobServiceClient object which will be used to create a container client
BlobServiceClient blobServiceClient = new BlobServiceClient(config.GetValue <string>(queueFileExport.DestinationStorage));
// Create the container and return a container client object
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(queueFileExport.DestinationContainer);
BlobClient blobClient = containerClient.GetBlobClient(queueFileExport.DestinationLocation);
await blobClient.DeleteIfExistsAsync();
await blobClient.UploadAsync(stream);
BlobHttpHeaders headers = new BlobHttpHeaders();
bool addHeaders = false;
if (queueFileExport.DestinationLocation.ToLower().EndsWith(".json"))
{
headers.ContentType = "application/json";
addHeaders = true;
}
if (config.GetValue <string>("FILE_EXPORT_CACHE_CONTROL") != null)
{
headers.CacheControl = config.GetValue <string>("FILE_EXPORT_CACHE_CONTROL");
addHeaders = true;
}
if (addHeaders)
{
await blobClient.SetHttpHeadersAsync(headers);
}
// log.LogError($"URL lookup {queueFileExport.SourceUrl} failed with status code {getResponse.StatusCode}");
// throw new System.Exception($"URL lookup {queueFileExport.SourceUrl} failed with status code {getResponse.StatusCode}");
}
private Response <BlobContentInfo> UploadInSequence(
Stream content,
int blockSize,
BlobHttpHeaders blobHttpHeaders,
IDictionary <string, string> metadata,
BlobRequestConditions conditions,
IProgress <long> progressHandler,
AccessTier?accessTier,
CancellationToken cancellationToken)
{
// Wrap the staging and commit calls in an Upload span for
// distributed tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope(
_operationName ?? $"{nameof(Azure)}.{nameof(Storage)}.{nameof(Blobs)}.{nameof(BlobClient)}.{nameof(BlobClient.Upload)}");
try
{
scope.Start();
// Wrap progressHandler in a AggregatingProgressIncrementer to prevent
// progress from being reset with each stage blob operation.
if (progressHandler != null)
{
progressHandler = new AggregatingProgressIncrementer(progressHandler);
}
// The list tracking blocks IDs we're going to commit
List <string> blockIds = new List <string>();
// Partition the stream into individual blocks and stage them
IAsyncEnumerator <ChunkedStream> enumerator =
PartitionedUploadExtensions.GetBlocksAsync(content, blockSize, async: false, _arrayPool, cancellationToken)
.GetAsyncEnumerator(cancellationToken);
#pragma warning disable AZC0107
while (enumerator.MoveNextAsync().EnsureCompleted())
#pragma warning restore AZC0107
{
// Dispose the block after the loop iterates and return its
// memory to our ArrayPool
using ChunkedStream block = enumerator.Current;
// Stage the next block
string blockId = GenerateBlockId(block.AbsolutePosition);
_client.StageBlock(
blockId,
new MemoryStream(block.Bytes, 0, block.Length, writable: false),
conditions: conditions,
progressHandler: progressHandler,
cancellationToken: cancellationToken);
blockIds.Add(blockId);
}
// Commit the block list after everything has been staged to
// complete the upload
return(_client.CommitBlockList(
blockIds,
blobHttpHeaders,
metadata,
conditions,
accessTier,
cancellationToken));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
public async Task <IActionResult> Upload()
{
var formCollection = await Request.ReadFormAsync();
var file = formCollection.Files.First();
var requestedContainer = formCollection["container"].ToString();
var fileExtension = Path.GetExtension(file.FileName.ToLowerInvariant());
// check the file type is allowed
if (!permittedFileExtensions.Contains(fileExtension) && fileExtension != "")
{
Console.WriteLine("The file type '" + fileExtension + "' is not allowed");
return(BadRequest(new ApiResponse(400)));
}
// check the file size (in bytes)
if (file.Length > fileSizeLimit)
{
Console.WriteLine("The file size was too large");
return(BadRequest(new ApiResponse(400)));
}
// check the file name length isn't excessive
if (file.FileName.Length > 75)
{
Console.WriteLine("The file name is too long: " + file.FileName.Length + " characters");
return(BadRequest(new ApiResponse(400)));
}
// check container name isn't empty
if (requestedContainer.Length == 0)
{
Console.WriteLine("Azure container name is empty");
return(BadRequest(new ApiResponse(400)));
}
// check requested container name is in an allowed set of names
if (!azureContainersAllowed.Contains(requestedContainer.ToLowerInvariant()))
{
Console.WriteLine("Invalid azure container name supplied: " + requestedContainer);
return(BadRequest(new ApiResponse(400)));
}
if (file.Length > 0)
{
var azureContainer = new BlobContainerClient(azureConnectionString, requestedContainer);
var createResponse = await azureContainer.CreateIfNotExistsAsync();
// in case the container doesn't exist
if (createResponse != null && createResponse.GetRawResponse().Status == 201)
{
await azureContainer.SetAccessPolicyAsync(Azure.Storage.Blobs.Models.PublicAccessType.Blob);
}
// generate a unique upload file name
// [original_filename_without_extension]_[8_random_chars].[original_filename_extension]
// eg. filename_xgh38tye.jpg
var fileName = HttpUtility.HtmlEncode(Path.GetFileNameWithoutExtension(file.FileName)) +
"_" + Path.GetRandomFileName().Substring(0, 8) + Path.GetExtension(file.FileName);
var blob = azureContainer.GetBlobClient(fileName);
//await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots);
// set the content type (which may or may not have been provided by the client)
var blobHttpHeader = new BlobHttpHeaders();
if (file.ContentType != null)
{
blobHttpHeader.ContentType = file.ContentType;
}
else
{
blobHttpHeader.ContentType = fileExtension switch
{
".jpg" => "image/jpeg",
".jpeg" => "image/jpeg",
".png" => "image/png",
_ => null
};
}
using (var fileStream = file.OpenReadStream())
{
await blob.UploadAsync(fileStream, blobHttpHeader);
}
return(Ok(new {
filename = blob.Name,
container = blob.BlobContainerName,
uri = blob.Uri
}));
}
Console.WriteLine("The file could not be uploaded");
return(BadRequest(new ApiResponse(400)));
}
}
public static async Task Run(
[EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.url}", FileAccess.Read)] Stream input,
ILogger log)
{
try
{
if (input != null)
{
var createdEvent = ((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
var extension = Path.GetExtension(createdEvent.Url);
var encoder = GetEncoder(extension);
log.LogInformation($"encoder: {encoder}");
if (encoder != null)
{
var thumbnailWidth = 100; //Convert.ToInt32(Environment.GetEnvironmentVariable("THUMBNAIL_WIDTH"));
var blobContainerName = Environment.GetEnvironmentVariable("THUMBNAIL_CONTAINER_NAME");
var connectionString = BLOB_STORAGE_CONNECTION_STRING;
//var blobServiceClient = new BlobServiceClient(BLOB_STORAGE_CONNECTION_STRING);
//var blobContainerClient = blobServiceClient.GetBlobContainerClient(thumbContainerName);
var blobName = GetBlobNameFromUrl(createdEvent.Url);
var blobClient = new BlobClient(connectionString, blobContainerName, blobName);
var contentType = GetContentType(extension);
var blobHttpHeaders = new BlobHttpHeaders {
ContentType = contentType
};
using (var output = new MemoryStream())
using (Image <Rgba32> image = Image.Load(input))
{
log.LogInformation($"image.Width: {image.Width}");
log.LogInformation($"image.Height: {image.Height}");
log.LogInformation($"thumbnailWidth: {thumbnailWidth}");
var divisor = image.Width / thumbnailWidth;
log.LogInformation($"divisor: {divisor}");
if (divisor < 1)
{
divisor = 1;
}
var height = Convert.ToInt32(Math.Round((decimal)(image.Height / divisor)));
if (divisor >= 1)
{
image.Mutate(x => x.Resize(thumbnailWidth, height));
}
image.Save(output, encoder);
output.Position = 0;
//await blobContainerClient.UploadBlobAsync(blobName, output);
await blobClient.UploadAsync(output, blobHttpHeaders);
}
//log.LogInformation($"blobName: {blobName}");
//log.LogInformation($"createdEvent: {createdEvent.Url}");
log.LogInformation($"contentType: {contentType}");
log.LogInformation($"blobContainerClient: {blobName}");
}
else
{
log.LogInformation($"No encoder support for: {createdEvent.Url}");
}
}
}
catch (Exception ex)
{
log.LogInformation(ex.Message);
throw;
}
}
/// <summary>
/// This operation will create a new
/// block blob of arbitrary size by uploading it as indiviually staged
/// blocks if it's larger than the
/// <paramref name="singleBlockThreshold"/>.
/// </summary>
/// <param name="file">
/// A <see cref="FileInfo"/> of the file to upload.
/// </param>
/// <param name="blobHttpHeaders">
/// Optional standard HTTP header properties that can be set for the
/// block blob.
/// </param>
/// <param name="metadata">
/// Optional custom metadata to set for this block blob.
/// </param>
/// <param name="blobAccessConditions">
/// Optional <see cref="BlobAccessConditions"/> to add conditions on
/// the creation of this new block blob.
/// </param>
/// <param name="progressHandler">
/// Optional <see cref="IProgress{StorageProgress}"/> to provide
/// progress updates about data transfers.
/// </param>
/// <param name="accessTier">
/// Optional <see cref="AccessTier"/>
/// Indicates the tier to be set on the blob.
/// </param>
/// <param name="singleBlockThreshold">
/// The maximum size stream that we'll upload as a single block. The
/// default value is 256MB.
/// </param>
/// <param name="transferOptions">
/// Optional <see cref="StorageTransferOptions"/> to configure
/// parallel transfer behavior.
/// </param>
/// <param name="async">
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="StorageRequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
internal async Task <Response <BlobContentInfo> > StagedUploadAsync(
FileInfo file,
BlobHttpHeaders blobHttpHeaders,
Metadata metadata,
BlobAccessConditions?blobAccessConditions,
IProgress <StorageProgress> progressHandler,
AccessTier?accessTier = default,
long?singleBlockThreshold = default,
StorageTransferOptions transferOptions = default,
bool async = true,
CancellationToken cancellationToken = default)
{
var client = new BlockBlobClient(Uri, Pipeline);
singleBlockThreshold ??= client.BlockBlobMaxUploadBlobBytes;
Debug.Assert(singleBlockThreshold <= client.BlockBlobMaxUploadBlobBytes);
var blockMap = new ConcurrentDictionary <long, string>();
var blockName = 0;
Task <Response <BlobContentInfo> > uploadTask = PartitionedUploader.UploadAsync(
UploadStreamAsync,
StageBlockAsync,
CommitBlockListAsync,
threshold => file.Length < threshold,
memoryPool => new StreamPartitioner(file, memoryPool),
singleBlockThreshold.Value,
transferOptions,
async,
cancellationToken);
return(async ?
await uploadTask.ConfigureAwait(false) :
uploadTask.EnsureCompleted());
string GetNewBase64BlockId(long blockOrdinal)
{
// Create and record a new block ID, storing the order information
// (nominally the block's start position in the original stream)
var newBlockName = Interlocked.Increment(ref blockName);
var blockId = Constants.BlockNameFormat;
blockId = String.Format(CultureInfo.InvariantCulture, blockId, newBlockName);
blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));
var success = blockMap.TryAdd(blockOrdinal, blockId);
Debug.Assert(success);
return(blockId);
}
// Upload the entire stream
async Task <Response <BlobContentInfo> > UploadStreamAsync()
{
using (FileStream stream = file.OpenRead())
{
Stream transformedStream; // cannot reassign to a "using" variable
(transformedStream, metadata) = TransformContent(stream, metadata);
return
(await client.UploadInternal(
transformedStream,
blobHttpHeaders,
metadata,
blobAccessConditions,
accessTier,
progressHandler,
async,
cancellationToken)
.ConfigureAwait(false));
}
}
// Upload a single partition of the stream
Task <Response <BlockInfo> > StageBlockAsync(
Stream partition,
long blockOrdinal,
bool async,
CancellationToken cancellation)
{
var base64BlockId = GetNewBase64BlockId(blockOrdinal);
//var bytes = new byte[10];
//partition.Read(bytes, 0, 10);
partition.Position = 0;
//Console.WriteLine($"Commiting partition {blockOrdinal} => {base64BlockId}, {String.Join(" ", bytes)}");
// Upload the block
return(client.StageBlockInternal(
base64BlockId,
partition,
null,
blobAccessConditions?.LeaseAccessConditions,
progressHandler,
async,
cancellationToken));
}
// Commit a series of partitions
Task <Response <BlobContentInfo> > CommitBlockListAsync(
bool async,
CancellationToken cancellation)
{
var base64BlockIds = blockMap.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray();
//Console.WriteLine($"Commiting block list:\n{String.Join("\n", base64BlockIds)}");
return
(client.CommitBlockListInternal(
base64BlockIds,
blobHttpHeaders,
metadata,
blobAccessConditions,
accessTier,
async,
cancellationToken));
}
}
public void UploadFileinBlocks(string filePath, string fileName)
{
string _filename = DateTime.Now.ToString("dd.MM.yyyy_hh.mm.ss") + Path.GetFileName(filePath);
BlockBlobClient blobClient = blobContainer.GetBlockBlobClient(fileName);// Path.GetFileName(filePath));
BlobHttpHeaders blobHttpHeaders = new BlobHttpHeaders();
blobHttpHeaders.ContentType = "application/octet-stream";
blobClient.SetHttpHeadersAsync(blobHttpHeaders);
int blockSize = 256 * 1024; //256 kb
using (FileStream fileStream =
new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
long fileSize = fileStream.Length;
//block count is the number of blocks + 1 for the last one
int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
//List of block ids; the blocks will be committed in the order of this list
List <string> blockIDs = new List <string>();
//starting block number - 1
int blockNumber = 0;
try
{
int bytesRead = 0; //number of bytes read so far
long bytesLeft = fileSize; //number of bytes left to read and upload
//do until all of the bytes are uploaded
while (bytesLeft > 0)
{
blockNumber++;
int bytesToRead;
if (bytesLeft >= blockSize)
{
//more than one block left, so put up another whole block
bytesToRead = blockSize;
}
else
{
//less than one block left, read the rest of it
bytesToRead = (int)bytesLeft;
}
//create a blockID from the block number, add it to the block ID list
//the block ID is a base64 string
string blockId =
Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}",
blockNumber.ToString("0000000"))));
blockIDs.Add(blockId);
//set up new buffer with the right size, and read that many bytes into it
byte[] bytes = new byte[bytesToRead];
fileStream.Read(bytes, 0, bytesToRead);
//calculate the MD5 hash of the byte array
byte [] blockHash = GetMD5HashByte(bytes);
//upload the block, provide the hash so Azure can verify it
blobClient.StageBlock(blockId, new MemoryStream(bytes), blockHash);
//increment/decrement counters
bytesRead += bytesToRead;
bytesLeft -= bytesToRead;
}
//commit the blocks
blobClient.CommitBlockList(blockIDs);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
}
public override Task <Response <BlobContentInfo> > UploadAsync(Stream content, BlobHttpHeaders httpHeaders = null, IDictionary <string, string> metadata = null, BlobRequestConditions conditions = null, IProgress <long> progressHandler = null, AccessTier?accessTier = null, StorageTransferOptions transferOptions = default, CancellationToken cancellationToken = default)
{
if (UploadBlobException != null)
{
throw UploadBlobException;
}
if (BlobInfo != null)
{
throw new RequestFailedException(409, BlobErrorCode.BlobAlreadyExists.ToString(), BlobErrorCode.BlobAlreadyExists.ToString(), default);
}
return(Task.FromResult(
Response.FromValue(
BlobsModelFactory.BlobContentInfo(new ETag("etag"), DateTime.UtcNow, new byte[] { }, string.Empty, 0L),
Mock.Of <Response>())));
}
private async void CaptureProcess(object state)
{
var configuration = ((TimerState)state).configuration;
try {
// Need to get all of our processes
this._logger.LogDebug($"Searching for process '{configuration.processName}' with window title '{configuration.windowTitle}' for configuration '{configuration.id}'");
// Find the pointer(s) for this window
var ptrs = WindowFilter.FindWindowsWithText(configuration.windowTitle, true).ToList();
// We will attempt to auto start if we have been provided the right information
if (ptrs.Count == 0 && configuration.autoStart && this._canLaunch)
{
// Attempt to launch the application and wait 10 seconds to continue
// If we have already launched this application, do we have a problem?
if (configuration._process != null)
{
if (configuration._process.HasExited)
{
if (configuration._attempts >= this._maxProcessAttempts)
{
this._logger.LogError($"Auto started process for configuration '{configuration.id}' has exited prematurely and failed too many times. Disabling auto start.");
configuration.autoStart = false;
return;
}
this._logger.LogError($"Auto started process for configuration '{configuration.id}' has exited prematurely. Will re-attempt.");
}
else
{
// So we have launched a process but we don't have a window, this is a failure
configuration._process.Kill();
if (configuration._attempts >= this._maxProcessAttempts)
{
this._logger.LogError($"Auto started process for configuration '{configuration.id}' was launched but no corresponding window found. Process '{configuration._process.Id}' killed. Disabling auto start");
configuration.autoStart = false;
return;
}
this._logger.LogError($"Auto started process for configuration '{configuration.id}' was launched but no corresponding window found. Process '{configuration._process.Id}' killed. Will re-attempt.");
}
configuration._process.Dispose();
}
// The process expects just the profile file name that exists in its own directory
try {
// Increment our counter
configuration._attempts++;
this._logger.LogDebug($"Launching process '{this._ATCSFullPath}' with profile '{configuration.profile}' for configuration '{configuration.id}' (Attempt {configuration._attempts} of {this._maxProcessAttempts})");
configuration._process = Process.Start(this._ATCSFullPath, new List <string>()
{
configuration.profile
});
// If we have a null response, it failed to start
if (configuration._process == null)
{
throw new Exception("Error auto starting the process but no exception provided.");
}
await Task.Delay(this._processLaunchTime);
// Attempt to find the windows again
ptrs = WindowFilter.FindWindowsWithText(configuration.windowTitle, true).ToList();
// We don't check agian because we will do it below. We will also give this one more timer interval before failing it out with the above code
} catch (Exception e) {
this._logger.LogError(e, $"Unable to auto start the process for configuration '{configuration.id}', auto start disabled.");
configuration.autoStart = false;
}
}
// If we have more than one, we need to fail out here
if (ptrs.Count > 1)
{
this._logger.LogWarning($"Found multiple windows for '{configuration.id}', unable to proceed.");
}
else if (ptrs.Count == 0)
{
this._logger.LogWarning($"Found no windows for '{configuration.id}', unable to proceed.");
}
else
{
try {
// Capture this and save it
this._logger.LogDebug($"Capturing window with handle '{ptrs[0].ToString()}' for configuration '{configuration.id}'.");
var img = CaptureWindow(ptrs[0]);
if (img == null)
{
throw new Exception("Received zero bytes for screenshot.");
}
// Determine the filename
// Keep these separate for now, blob may change down the road
var blobPath = $"{configuration.blobName}{this._ImageExt}".Trim();
var filePath = $"{configuration.blobName}{this._ImageExt}".Trim();
// Note that we have something recent
this._lastUpdate[configuration.id] = DateTimeOffset.UtcNow;
// Upload it to Azure
if (this._enableUpload)
{
using (var ms = new MemoryStream(img)) {
using (var ctx = new CancellationTokenSource()) {
// Set our maximum upload time
ctx.CancelAfter(this._maxUploadTime);
try {
var blobClient = this._blobContainerClient.GetBlobClient(blobPath);
await blobClient.UploadAsync(ms, true, ctx.Token);
// Set information about this screenshot value
await blobClient.SetMetadataAsync(new Dictionary <string, string>
{
{ "id", configuration.id },
{ "updateTime", DateTimeOffset.UtcNow.ToString("O") },
{ "name", configuration.name }
});
var headers = new BlobHttpHeaders();
headers.ContentDisposition = "inline";
headers.ContentType = this._ImageMime;
blobClient.SetHttpHeaders(headers);
this._logger.LogInformation($"Blob '{blobPath}' saved for configuration '{configuration.id}'.");
} catch (TaskCanceledException e) {
this._logger.LogError(e, $"Blob '{blobPath}' took too long to upload, cancelled.");
}
}
}
}
else
{
this._logger.LogDebug("Upload skipped due to configuration.");
}
// Save it
if (configuration.saveFile)
{
SaveImage(img, filePath, this._ImageFormat);
this._logger.LogDebug($"File '{filePath}' saved for configuration '{configuration.id}'.");
}
else
{
this._logger.LogDebug("Save file skipped due to configuration.");
}
} catch (Exception e) {
this._logger.LogError(e, $"Exception thrown while capturing the window for '{configuration.windowTitle}': {e.Message}");
}
}
} catch (Exception e) {
this._logger.LogError(e, $"Uncaught exception in {nameof(CaptureProcess)}.");
} finally {
// Re-initiate the timer
this._timers.First(a => a.Key == configuration.id).Value.Change(this._frequency, Timeout.Infinite);
}
}
/// <summary>
/// Upload to file to blob container
/// </summary>
/// <param name="file">file from the form</param>
/// <param name="fileName">Name of file to the new uploaded file</param>
/// <param name="blobHttpHeader">Set Http properties</param>
/// <returns>BlobClient where the file with the same name is located</returns>
public void UploadFileToStorage(IFormFile file, string fileName, BlobHttpHeaders blobHttpHeader)
{
GetBlobByFileName(fileName)
.Upload(file.OpenReadStream(), blobHttpHeader);
}
static void Main(string[] args)
{
string CONNECTIONSTRING = "<YOURCONNECTIONSTRING>";
string CONTAINERNAME = "<YOURCONTAINERNAME>";
var blobServiceClient = new BlobServiceClient(CONNECTIONSTRING);
var containerClient = blobServiceClient.GetBlobContainerClient(CONTAINERNAME);
var FOLDERNAMES = new List <string>()
{
"video", "image", "thumbnail"
};
var messages = new List <string>();
var unsupportedFiles = new List <string>();
var errorFiles = new List <string>();
foreach (var folderName in FOLDERNAMES)
{
var blobs = containerClient.GetBlobs(prefix: folderName).ToList();
Console.WriteLine("connected to folder => " + folderName);
int i = 0;
for (; i < blobs.Count; i++)
{
try
{
var blob = containerClient.GetBlobClient(blobs[i].Name);
Console.WriteLine($"Updating {i + 1}=> {blobs[i].Name}");
string mimeType = GetContentType(blobs[i].Name);
if (!string.IsNullOrEmpty(mimeType))
{
var blobHttpHeader = new BlobHttpHeaders
{
ContentType = mimeType
};
var blobStream = blob.OpenRead();
blob.UploadAsync(blobStream, blobHttpHeader).Wait();
}
else
{
unsupportedFiles.Add($"{CONTAINERNAME}/{blobs[i].Name}");
}
}
catch (Exception ex)
{
Console.WriteLine("============> Error in updating");
Console.WriteLine(ex.Message);
errorFiles.Add($"{CONTAINERNAME}/{blobs[i].Name} => {ex.Message}");
Console.WriteLine(new string('=', 20));
}
}
messages.Add($"{CONTAINERNAME}/{folderName} => {i}");
}
var lineSperator = new string('=', 40);
Console.WriteLine(lineSperator);
messages.ForEach(x => Console.WriteLine(x));
Console.WriteLine(lineSperator);
unsupportedFiles.ForEach(x => Console.WriteLine(x));
Console.WriteLine(lineSperator);
errorFiles.ForEach(x => Console.WriteLine(x));
}
public static async Task <string> UploadToBlob(TaskModuleSubmitDataDeserializer taskModuleOutput, ITurnContext turnContext)
{
List <string> imageURL = new List <string>();
string audioURL = taskModuleOutput.data.RecorderOutput;
string requestID = taskModuleOutput.data.RequestId;
byte[] streamConverter;
string assignedToUserImageUrl;
string submittedByUserImageUrl;
List <Images> imgURL = new List <Images> {
};
var blobHttpHeader = new BlobHttpHeaders();
blobHttpHeader.ContentType = "image/jpeg";
BlobContainerClient container = new BlobContainerClient(Constants.BlobConnectionString, Constants.BlobContainerName);
BlobClient blob;
var assignedToUserImage = GetImage(taskModuleOutput.data.AssignedTo.objectId).Result;
if (assignedToUserImage != null)
{
blob = container.GetBlobClient(requestID + "-" + taskModuleOutput.data.AssignedTo.objectId + ".jpg");
blob.UploadAsync(assignedToUserImage, blobHttpHeader);
assignedToUserImageUrl = blob.Uri.ToString();
}
else
{
assignedToUserImageUrl = Constants.MemberGenericImageUrl;
}
var submittedByUserImage = GetImage(turnContext.Activity.From.AadObjectId).Result;
if (submittedByUserImage != null)
{
blob = container.GetBlobClient(requestID + "-" + turnContext.Activity.From.AadObjectId + ".jpg");
blob.UploadAsync(submittedByUserImage, blobHttpHeader);
submittedByUserImageUrl = blob.Uri.ToString();
}
else
{
submittedByUserImageUrl = Constants.MemberGenericImageUrl;
}
if (taskModuleOutput.data.ImagesOutput.Length != 0)
{
foreach (string item in taskModuleOutput.data.ImagesOutput)
{
imageURL.Add(item);
}
}
else
{
imageURL = null;
}
if (imageURL != null)
{
foreach (var item in imageURL)
{
var imgs = new Images {
url = item
};
imgURL.Add(imgs);
}
}
string[] blobStorage = { ".", "Resources", "BlobStorage.json" };
var blobStorageText = File.ReadAllText(Path.Combine(blobStorage));
AdaptiveCardTemplate template = new AdaptiveCardTemplate(blobStorageText);
var payloadData = new
{
requestID = requestID,
status = Status.Pending,
itemName = taskModuleOutput.data.ItemName,
itemCode = taskModuleOutput.data.ItemCode,
assignedToName = taskModuleOutput.data.AssignedTo.displayName,
assignedToId = taskModuleOutput.data.AssignedTo.objectId,
assignedToMail = taskModuleOutput.data.AssignedTo.email,
submittedByName = turnContext.Activity.From.Name,
submittedById = turnContext.Activity.From.AadObjectId,
submittedByMail = taskModuleOutput.data.SubmittedByMail,
assignedToUserImage = assignedToUserImageUrl,
submittedByUserImage = submittedByUserImageUrl,
imageURL = imgURL,
audioURL = audioURL,
comments = taskModuleOutput.data.Comments,
conversationId = turnContext.Activity.Conversation.Id,
messageId = "",
};
var blobTextString = template.Expand(payloadData);
streamConverter = Encoding.ASCII.GetBytes(blobTextString);
blob = container.GetBlobClient(requestID + ".txt");
await blob.UploadAsync(new MemoryStream(streamConverter));
return(requestID);
}
public static async Task Run([TimerTrigger("%ScheduleTriggerTime%")] TimerInfo timer, [Blob("%RSSPath%", FileAccess.ReadWrite, Connection = "StorageConnection")] BlockBlobClient feedBlob,
ILogger log)
{
log.LogInformation($"AzureStack Hun MarketPlace RSS Generator launched at: {DateTime.Now}");
List <MarketPlaceItem> marketPlaceItems = new List <MarketPlaceItem>();
try {
var stream = await httpClient.GetStreamAsync(marketPlaceUpdatesURL);
using (StreamReader reader = new StreamReader(stream)) {
string line;
string section = "";
while (null != (line = await reader.ReadLineAsync()))
{
if (line.Contains("ms.date:"))
{
lastUpdated = DateTime.Parse(line.Split(":") [1].Trim(), System.Globalization.CultureInfo.GetCultureInfo("en-US").DateTimeFormat, System.Globalization.DateTimeStyles.AssumeUniversal);
log.LogInformation($"GitHub page was last updated on: {lastUpdated.ToShortDateString()}");
if (feedBlob.Exists())
{
var blobProperties = await feedBlob.GetPropertiesAsync();
if (blobProperties.Value.Metadata.ContainsKey(lastUpdatedMetadataKey) &&
blobProperties.Value.Metadata[lastUpdatedMetadataKey].Equals(lastUpdated.ToShortDateString()))
{
log.LogInformation("RSS Feed is already up to date, will quit");
return;
}
}
}
var matchSection = sectionRegex.Match(line);
if (matchSection.Success)
{
section = matchSection.Groups[1].Value;
continue;
}
var matchItem = itemRegex.Match(line);
if (matchItem.Success)
{
marketPlaceItems.Add(
new MarketPlaceItem {
Name = matchItem.Groups["product"].Value,
Change = section,
ReleaseDate = DateTime.Parse(matchItem.Groups["date"].Value, System.Globalization.CultureInfo.GetCultureInfo("en-US").DateTimeFormat, System.Globalization.DateTimeStyles.AssumeUniversal)
}
);
}
}
}
} catch (Exception e) {
log.LogError($"Error while fetch Marketplace Items page: {e.Message}");
return;
}
if (marketPlaceItems.Count > 0)
{
log.LogInformation($"Got {marketPlaceItems.Count} items");
var feed = FormatRssFeed(marketPlaceItems);
var settings = new XmlWriterSettings {
Encoding = System.Text.Encoding.UTF8,
NewLineHandling = NewLineHandling.Entitize,
NewLineOnAttributes = true,
Indent = true,
Async = true
};
using (var stream = new MemoryStream()) {
using (var xmlWriter = XmlWriter.Create(stream, settings)) {
var rssFormatter = new Rss20FeedFormatter(feed, false);
rssFormatter.WriteTo(xmlWriter);
await xmlWriter.FlushAsync();
}
stream.Seek(0, SeekOrigin.Begin);
var properties = new BlobHttpHeaders()
{
ContentType = "application/rss+xml; charset=utf-8"
};
var metadata = new Dictionary <string, string> {
{ lastUpdatedMetadataKey, lastUpdated.ToShortDateString() }
};
await feedBlob.UploadAsync(stream, new BlobUploadOptions { HttpHeaders = properties, Metadata = metadata });
}
}
}
static async Task Main(string[] args)
{
MemoryStream ms = new MemoryStream();
var input = File.OpenWrite(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\file2.jpg");
const int chunkSize = 506;
using (var file = File.OpenRead(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\25thnov2020TestImg.txt"))
{
int bytesRead;
var buffer = new byte[chunkSize];
while ((bytesRead = file.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, bytesRead);
}
}
BlobContainerClient container = new BlobContainerClient("DefaultEndpointsProtocol=https;AccountName=fatihteststorage;AccountKey=N01DqDtMWIjzyGiSf2e3mBNJcqvGqBnM7lD6SSKjIT6/WbqT8ecMng5IYyH9WsST3iLbuoEGjHHACR9zoO51Uw==;EndpointSuffix=core.windows.net", "main");
var blockBlobClient = container.GetBlockBlobClient("myImage.png");
int blockSize = 50 * 1024;
int offset = 0;
int counter = 0;
List <string> blockIds = new List <string>();
using (var fs = File.OpenRead(@"C:\\Users\\Fatih\\source\\repos\\image-serializer\\test-image.png"))
{
var bytesRemaining = fs.Length;
do
{
var dataToRead = Math.Min(bytesRemaining, blockSize);
byte[] data = new byte[dataToRead];
var dataRead = fs.Read(data, offset, (int)dataToRead);
bytesRemaining -= dataRead;
if (dataRead > 0)
{
var blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(counter.ToString("d6")));
blockBlobClient.StageBlock(blockId, new MemoryStream(data));
Console.WriteLine(string.Format("Block {0} uploaded successfully.", counter.ToString("d6")));
blockIds.Add(blockId);
counter++;
}
} while (bytesRemaining > 0);
var headers = new BlobHttpHeaders()
{
ContentType = "image/png"
};
blockBlobClient.CommitBlockList(blockIds, headers);
}
try
{
// Get a reference to a blob
BlobClient blob = container.GetBlobClient("test.png");
ms.Position = 0;
// Open the file and upload its data
await blob.UploadAsync(ms);
// Verify we uploaded some content
BlobProperties properties = await blob.GetPropertiesAsync();
}
catch (Exception e)
{
}
finally
{
// Clean up after the test when we're finished
}
}
