public async Task <IActionResult> Get(string token, string filename) { if (!long.TryParse(token, NumberStyles.HexNumber, null, out long actualToken)) { throw new ArgumentException("Invalid Token.", nameof(token)); } string contentType = this.fileRepository.GetContentTypeForExtension(Path.GetExtension(filename)); BlobContainerClient azureContainer = await this.fileRepository.GetFile(actualToken, filename); BlobClient blob = azureContainer.GetBlobClient(filename); Azure.Response <Azure.Storage.Blobs.Models.BlobDownloadInfo> info = await blob.DownloadAsync(); return(this.File(info.Value.Content, contentType)); }
public async Task <ChangeFeed> BuildChangeFeed( DateTimeOffset?startTime, DateTimeOffset?endTime, string continuation, bool async, CancellationToken cancellationToken) { DateTimeOffset lastConsumable; Queue <string> years = new Queue <string>(); Queue <string> segments = new Queue <string>(); ChangeFeedCursor cursor = null; // Create cursor if (continuation != null) { cursor = JsonSerializer.Deserialize <ChangeFeedCursor>(continuation); ValidateCursor(_containerClient, cursor); startTime = BlobChangeFeedExtensions.ToDateTimeOffset(cursor.CurrentSegmentCursor.SegmentPath).Value; endTime = cursor.EndTime; } // Round start and end time if we are not using the cursor. else { startTime = startTime.RoundDownToNearestHour(); endTime = endTime.RoundUpToNearestHour(); } // Check if Change Feed has been abled for this account. bool changeFeedContainerExists; if (async) { changeFeedContainerExists = await _containerClient.ExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } else { changeFeedContainerExists = _containerClient.Exists(cancellationToken: cancellationToken); } if (!changeFeedContainerExists) { throw new ArgumentException("Change Feed hasn't been enabled on this account, or is currently being enabled."); } // Get last consumable BlobClient blobClient = _containerClient.GetBlobClient(Constants.ChangeFeed.MetaSegmentsPath); BlobDownloadInfo blobDownloadInfo; if (async) { blobDownloadInfo = await blobClient.DownloadAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } else { blobDownloadInfo = blobClient.Download(cancellationToken: cancellationToken); } JsonDocument jsonMetaSegment; if (async) { jsonMetaSegment = await JsonDocument.ParseAsync( blobDownloadInfo.Content, cancellationToken : cancellationToken ).ConfigureAwait(false); } else { jsonMetaSegment = JsonDocument.Parse(blobDownloadInfo.Content); } lastConsumable = jsonMetaSegment.RootElement.GetProperty("lastConsumable").GetDateTimeOffset(); // Get year paths years = await GetYearPathsInternal( async, cancellationToken).ConfigureAwait(false); // Dequeue any years that occur before start time if (startTime.HasValue) { while (years.Count > 0 && BlobChangeFeedExtensions.ToDateTimeOffset(years.Peek()) < startTime.RoundDownToNearestYear()) { years.Dequeue(); } } // There are no years. if (years.Count == 0) { return(ChangeFeed.Empty()); } while (segments.Count == 0 && years.Count > 0) { // Get Segments for year segments = await BlobChangeFeedExtensions.GetSegmentsInYearInternal( containerClient : _containerClient, yearPath : years.Dequeue(), startTime : startTime, endTime : BlobChangeFeedExtensions.MinDateTime(lastConsumable, endTime), async : async, cancellationToken : cancellationToken) .ConfigureAwait(false); } // We were on the last year, and there were no more segments. if (segments.Count == 0) { return(ChangeFeed.Empty()); } Segment currentSegment = await _segmentFactory.BuildSegment( async, segments.Dequeue(), cursor?.CurrentSegmentCursor) .ConfigureAwait(false); return(new ChangeFeed( _containerClient, _segmentFactory, years, segments, currentSegment, lastConsumable, startTime, endTime)); }
/// <summary> /// Converts and stores the image. /// </summary> /// <param name="log"></param> /// <param name="uploadedImage"></param> /// <param name="convertedImagesContainer"></param> /// <param name="blobName"></param> /// <param name="failedImagesContainer"></param> /// <param name="jobId"></param> /// <param name="imageSource"></param> public static async Task ConvertAndStoreImage(CloudStorageAccount storageAccount, string blobName, int imageConversionMode, string jobId) { // Set the name of the converted blob to the same as the uploaded blob string convertedBlobName = blobName; try { // Update Job Status - about to convert image await UpdateJobEntityStatus(storageAccount, jobId, 2, "Job is running."); // Get the Blob Container Client BlobServiceClient blobServiceClient = new BlobServiceClient(ConfigurationManager.ConnectionStrings["AzureWebJobsStorage"].ConnectionString); BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(ConfigSettings.UPLOADEDIMAGES_CONTAINERNAME); // Get the Blob Client for the specified blob BlobClient blobClient = blobContainerClient.GetBlobClient(blobName); BlobDownloadInfo blobDownloadInfo = await blobClient.DownloadAsync(); // Set up blob stream using (MemoryStream inStream = new MemoryStream()) { await blobDownloadInfo.Content.CopyToAsync(inStream); inStream.Position = 0; // Convert the image based on the imageConversionMode using (MemoryStream outStream = new MemoryStream()) { using (ImageFactory imageFactory = new ImageFactory(preserveExifData: true)) { switch (imageConversionMode) { case 1: { imageFactory.Load(inStream) .Filter(ImageProcessor.Imaging.Filters.Photo.MatrixFilters.GreyScale) .Save(outStream); break; } case 2: { imageFactory.Load(inStream) .Filter(ImageProcessor.Imaging.Filters.Photo.MatrixFilters.Sepia) .Save(outStream); break; } case 3: { imageFactory.Load(inStream) .Filter(ImageProcessor.Imaging.Filters.Photo.MatrixFilters.Comic) .Save(outStream); break; } } outStream.Seek(0, SeekOrigin.Begin); // Create a blob client CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); // Create or retrieve a reference to the converted images container CloudBlobContainer convertedImagesContainer = cloudBlobClient.GetContainerReference(ConfigSettings.CONVERTED_IMAGES_CONTAINERNAME); bool created = await convertedImagesContainer.CreateIfNotExistsAsync(); CloudBlockBlob convertedBlockBlob = convertedImagesContainer.GetBlockBlobReference(convertedBlobName); // Upload the converted blob to the converted images container convertedBlockBlob.Properties.ContentType = System.Net.Mime.MediaTypeNames.Image.Jpeg; await convertedBlockBlob.UploadFromStreamAsync(outStream); // Update Job Status - image converted successfully await UpdateJobEntityStatus(storageAccount, jobId, 3, "Job completed with success."); } } } } catch (Exception ex) { // Update Job Status - image conversion failed await UpdateJobEntityStatus(storageAccount, jobId, 4, ex.Message); } }
static async Task Run(string connectionString, Options options) { #if DEBUG if (!options.Debug) { throw new InvalidOperationException("Requires release configuration"); } #endif var httpClientHandler = new HttpClientHandler(); httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true; var httpClient = new HttpClient(httpClientHandler); var blobClientOptions = new BlobClientOptions(); blobClientOptions.Transport = new HttpClientTransport(httpClient); var blockClient = new BlockBlobClient(connectionString, _containerName, _blobName, blobClientOptions); var blockList = (await blockClient.GetBlockListAsync()).Value; PrintBlockList(blockList); var client = new BlobClient(connectionString, _containerName, _blobName, blobClientOptions); var parallelTransferOptions = new ParallelTransferOptions() { MaximumThreadCount = options.MaximumThreadCount, MaximumTransferLength = options.MaximumTransferLength }; var payload = new byte[options.Size]; // Initialize payload with stable random data since all-zeros may be compressed or optimized (new Random(0)).NextBytes(payload); var payloadStream = new MemoryStream(payload, writable: false); Console.WriteLine($"Uploading and downloading blob of size {options.Size} with {options.MaximumThreadCount} threads..."); Console.WriteLine(); var sw = new Stopwatch(); for (var i = 0; i < options.Count; i++) { payloadStream.Seek(0, SeekOrigin.Begin); sw.Restart(); await client.UploadAsync(payloadStream, parallelTransferOptions : parallelTransferOptions); sw.Stop(); var elapsedSeconds = sw.Elapsed.TotalSeconds; var megabytesPerSecond = (options.Size / (1024 * 1024)) / elapsedSeconds; Console.WriteLine($"Uploaded {options.Size} bytes in {elapsedSeconds:N2} seconds ({megabytesPerSecond:N2} MB/s)"); sw.Restart(); await client.DownloadAsync(Stream.Null, parallelTransferOptions : parallelTransferOptions); sw.Stop(); elapsedSeconds = sw.Elapsed.TotalSeconds; megabytesPerSecond = (options.Size / (1024 * 1024)) / elapsedSeconds; Console.WriteLine($"Downloaded {options.Size} bytes in {elapsedSeconds:N2} seconds ({megabytesPerSecond:N2} MB/s)"); Console.WriteLine(); } }
private static async void ImportCsvFileIntoTable(BlobItem blobItem, ILogger log, DateTime lastProcessedDate, BlobContainerClient icontainerClient, CloudTable icloudTable) { // string lastProcessedDateString = lastProcessedDate.Date.ToString("dd/MM/yyyy"); DateTime resourceUsageDateTime; log.LogInformation("lastproccessdatestring= " + lastProcessedDateString); log.LogInformation("**** CSV Import ****"); TableOperation itableOperation; int DayDiff = 0; //string storageConnectionString = SourceConnection; //CloudStorageAccount icloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString); //CloudTableClient itableClient = icloudStorageAccount.CreateCloudTableClient(); //string tableName = "Costs"; //CloudTable icloudTable = itableClient.GetTableReference(tableName); BlobClient blobClient = icontainerClient.GetBlobClient(blobItem.Name); log.LogInformation("blob is " + blobItem.Name); if (await blobClient.ExistsAsync()) { var response = await blobClient.DownloadAsync(); using (var streamReader = new StreamReader(response.Value.Content)) { bool Continue = true; int count = 0; while (!streamReader.EndOfStream && Continue) { var csvLine = await streamReader.ReadLineAsync(); // log.LogInformation(csvLine); string[] values = csvLine.Split(','); // data usage record date must be higher than last pprocessed date if (values[0] != "DepartmentName") { resourceUsageDateTime = DateTime.Parse(values[7]); TimeSpan t = resourceUsageDateTime.Date.Subtract(lastProcessedDate.Date); DayDiff = t.Days; // DayDiff = 1 ; if (DayDiff >= 0 || DayDiff < 0) { // Skip first line as contains header string rowKey = Guid.NewGuid().ToString("N"); ResourceUsageEntity resourceUsageEntity = new ResourceUsageEntity(values[7], rowKey); resourceUsageEntity.DepartmentName = values[0]; resourceUsageEntity.AccountName = values[1]; resourceUsageEntity.AccountOwnerId = values[2]; resourceUsageEntity.SubscriptionGuid = values[3]; resourceUsageEntity.SubscriptionName = values[4]; resourceUsageEntity.ResourceGroup = values[5]; resourceUsageEntity.ResourceLocation = values[6]; resourceUsageEntity.UsageDateTime = values[7]; resourceUsageEntity.ProductName = values[8]; resourceUsageEntity.MeterCategory = values[9]; resourceUsageEntity.MeterSubcategory = values[10]; resourceUsageEntity.MeterId = values[11]; resourceUsageEntity.MeterName = values[12]; resourceUsageEntity.MeterRegion = values[13]; resourceUsageEntity.UnitOfMeasure = values[14]; resourceUsageEntity.UsageQuantity = values[15]; resourceUsageEntity.ResourceRate = values[16]; resourceUsageEntity.PreTaxCost = values[17]; resourceUsageEntity.CostCenter = values[18]; resourceUsageEntity.ConsumedService = values[19]; resourceUsageEntity.ResourceType = values[20]; resourceUsageEntity.InstanceId = values[21]; resourceUsageEntity.Tags = values[22]; resourceUsageEntity.OfferId = values[23]; resourceUsageEntity.AdditionalInfo = values[24]; // resourceUsageEntity.ETag = "*"; // log.LogInformation("About to write to table"); itableOperation = TableOperation.Insert(resourceUsageEntity); var tableresult = icloudTable.ExecuteAsync(itableOperation); count++; } } // } log.LogInformation("Completed write of csv, count= " + count.ToString()); } } }
static async Task Main() { Console.WriteLine("Azure Blob storage v12 - .NET quickstart sample\n"); // Retrieve the connection string for use with the application. The storage // connection string is stored in an environment variable on the machine // running the application called AZURE_STORAGE_CONNECTION_STRING. If the // environment variable is created after the application is launched in a // console or with Visual Studio, the shell or application needs to be closed // and reloaded to take the environment variable into account. string connectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING"); // Create a BlobServiceClient object which will be used to create a container client BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString); //Create a unique name for the container string containerName = "quickstartblobs" + Guid.NewGuid().ToString(); // Create the container and return a container client object BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName); // Create a local file in the ./data/ directory for uploading and downloading string localPath = "./data/"; string fileName = "quickstart" + Guid.NewGuid().ToString() + ".txt"; string localFilePath = Path.Combine(localPath, fileName); // Write text to the file await File.WriteAllTextAsync(localFilePath, "Hello, World!"); // Get a reference to a blob BlobClient blobClient = containerClient.GetBlobClient(fileName); Console.WriteLine("Uploading to Blob storage as blob:\n\t {0}\n", blobClient.Uri); using (var FileStream = File.OpenRead(localFilePath)) { await blobClient.UploadAsync(FileStream, true); FileStream.Close(); } //Console.WriteLine("Listing the blobs......."); ////Listing all the blobs //await foreach (BlobItem blobItem in blobContainer.GetBlobsAsync()) //{ // Console.WriteLine("\t" + blobItem.Name); //} // Download the blob to a local file // Append the string "DOWNLOAD" before the .txt extension // so you can compare the files in the data directory string downloadFilePath = localFilePath.Replace(".txt", "DOWNLOAD.txt"); Console.WriteLine("\nDownloading blob to\n\t{0}\n", downloadFilePath); // Download the blob's contents and save it to a file BlobDownloadInfo download = await blobClient.DownloadAsync(); using (FileStream downloadFileStream = File.OpenWrite(downloadFilePath)) { await download.Content.CopyToAsync(downloadFileStream); downloadFileStream.Close(); } // Clean up Console.Write("Press any key to begin clean up"); Console.ReadLine(); Console.WriteLine("Deleting blob container..."); await containerClient.DeleteAsync(); Console.WriteLine("Deleting the local source and downloaded files..."); File.Delete(localFilePath); File.Delete(downloadFilePath); Console.WriteLine("Done"); }
#pragma warning disable CA1822 // Does not acces instance data can be marked static. public virtual async Task <Segment> BuildSegment( #pragma warning restore CA1822 // Can't mock static methods in MOQ. bool async, string manifestPath, SegmentCursor cursor = default) { // Models we need for later List <Shard> shards = new List <Shard>(); DateTimeOffset dateTime = manifestPath.ToDateTimeOffset().Value; int shardIndex = cursor?.ShardIndex ?? 0; // Download segment manifest BlobClient blobClient = _containerClient.GetBlobClient(manifestPath); BlobDownloadInfo blobDownloadInfo; if (async) { blobDownloadInfo = await blobClient.DownloadAsync().ConfigureAwait(false); } else { blobDownloadInfo = blobClient.Download(); } // Parse segment manifest JsonDocument jsonManifest; if (async) { jsonManifest = await JsonDocument.ParseAsync(blobDownloadInfo.Content).ConfigureAwait(false); } else { jsonManifest = JsonDocument.Parse(blobDownloadInfo.Content); } // Initalized Finalized field string statusString = jsonManifest.RootElement.GetProperty("status").GetString(); bool finalized = statusString == "Finalized"; int i = 0; foreach (JsonElement shardJsonElement in jsonManifest.RootElement.GetProperty("chunkFilePaths").EnumerateArray()) { string shardPath = shardJsonElement.ToString().Substring("$blobchangefeed/".Length); Shard shard = await _shardFactory.BuildShard( async, shardPath, cursor?.ShardCursors?[i]) .ConfigureAwait(false); shards.Add(shard); i++; } return(new Segment( shards, shardIndex, dateTime, finalized)); }
private static async Task OperateBlobAsync() { string tempDirectory = null; string destinationPath = null; string sourcePath = null; BlobContainerClient blobContainerClient = null; // Retrieve the connection string for use with the application. The storage connection string is stored // in an environment variable on the machine running the application called AZURE_STORAGE_CONNECTIONSTRING. // If the environment variable is created after the application is launched in a console or with Visual // Studio, the shell needs to be closed and reloaded to take the environment variable into account. string storageConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTIONSTRING"); if (storageConnectionString == null) { Console.WriteLine("A connection string has not been defined in the system environment variables. " + "Add a environment variable named 'AZURE_STORAGE_CONNECTIONSTRING' with your storage " + "connection string as a value."); return; } try { // Create a container called 'quickstartblob' and append a GUID value to it to make the name unique. string containerName = "quickstartblob" + Guid.NewGuid().ToString(); blobContainerClient = new BlobContainerClient(storageConnectionString, containerName); await blobContainerClient.CreateAsync(); Console.WriteLine($"Created container '{blobContainerClient.Uri}'"); Console.WriteLine(); // Set the permissions so the blobs are public. await blobContainerClient.SetAccessPolicyAsync(PublicAccessType.Blob); Console.WriteLine("Setting the Blob access policy to public."); Console.WriteLine(); // Create a file in a temp directory folder to upload to a blob. tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); Directory.CreateDirectory(tempDirectory); string blobFileName = $"QuickStart_{Path.GetRandomFileName()}.txt"; sourcePath = Path.Combine(tempDirectory, blobFileName); // Write text to this file. File.WriteAllText(sourcePath, "Storage Blob Quickstart."); Console.WriteLine($"Created Temp file = {sourcePath}"); Console.WriteLine(); // Get a reference to the blob named "sample-blob", then upload the file to the blob. Console.WriteLine($"Uploading file to Blob storage as blob '{blobFileName}'"); string blobName = "sample-blob"; BlobClient blob = blobContainerClient.GetBlobClient(blobName); // Open this file and upload it to blob using (FileStream fileStream = File.OpenRead(sourcePath)) { await blob.UploadAsync(fileStream); } Console.WriteLine("Uploaded successfully."); Console.WriteLine(); // List the blobs in the container. Console.WriteLine("Listing blobs in container."); await foreach (BlobItem item in blobContainerClient.GetBlobsAsync()) { Console.WriteLine($"The blob name is '{item.Name}'"); } Console.WriteLine("Listed successfully."); Console.WriteLine(); // Append the string "_DOWNLOADED" before the .txt extension so that you can see both files in the temp directory. destinationPath = sourcePath.Replace(".txt", "_DOWNLOADED.txt"); // Download the blob to a file in same directory, using the reference created earlier. Console.WriteLine($"Downloading blob to file in the temp directory {destinationPath}"); BlobDownloadInfo blobDownload = await blob.DownloadAsync(); using (FileStream fileStream = File.OpenWrite(destinationPath)) { await blobDownload.Content.CopyToAsync(fileStream); } Console.WriteLine("Downloaded successfully."); Console.WriteLine(); } catch (RequestFailedException ex) { Console.WriteLine($"Error returned from the service: {ex.Message}"); } finally { Console.WriteLine("Press enter to delete the sample files and example container."); Console.ReadLine(); // Clean up resources. This includes the container and the two temp files. Console.WriteLine("Deleting the container and any blobs it contains."); if (blobContainerClient != null) { await blobContainerClient.DeleteAsync(); } Console.WriteLine("Deleting the local source file and local downloaded files."); Console.WriteLine(); if (File.Exists(sourcePath)) { File.Delete(sourcePath); } if (File.Exists(destinationPath)) { File.Delete(destinationPath); } if (Directory.Exists(tempDirectory)) { Directory.Delete(tempDirectory); } } }
/// <summary>Fetch raw text from storage.</summary> /// <param name="id">The storage ID returned by <see cref="SaveAsync"/>.</param> public async Task <StoredFileInfo> GetAsync(string id) { // fetch from blob storage if (Guid.TryParseExact(id, "N", out Guid _)) { // Azure Blob storage if (this.HasAzure) { try { BlobClient blob = this.GetAzureBlobClient(id); Response <BlobDownloadInfo> response = await blob.DownloadAsync(); using BlobDownloadInfo result = response.Value; using StreamReader reader = new StreamReader(result.Content); DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return(new StoredFileInfo { Success = true, Content = content, Expiry = expiry.UtcDateTime }); } catch (RequestFailedException ex) { return(new StoredFileInfo { Error = ex.ErrorCode == "BlobNotFound" ? "There's no file with that ID." : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }); } } // local filesystem for testing else { FileInfo file = new FileInfo(this.GetDevFilePath(id)); if (file.Exists) { if (file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) { file.Delete(); } else { return(new StoredFileInfo { Success = true, Content = File.ReadAllText(file.FullName), Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays), Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment." }); } } return(new StoredFileInfo { Error = "There's no file with that ID." }); } } // get from Pastebin else { PasteInfo response = await this.Pastebin.GetAsync(id); response.Content = this.GzipHelper.DecompressString(response.Content); return(new StoredFileInfo { Success = response.Success, Content = response.Content, Error = response.Error }); } }
/// <inheritdoc /> public async Task <StoredFileInfo> GetAsync(string id, bool renew) { // fetch from blob storage if (Guid.TryParseExact(id, "N", out Guid _)) { // Azure Blob storage if (this.HasAzure) { try { // get client BlobClient blob = this.GetAzureBlobClient(id); // extend expiry if (renew) { await blob.SetMetadataAsync(new Dictionary <string, string> { ["expiryRenewed"] = DateTime.UtcNow.ToString("O") }); // change the blob's last-modified date (the specific property set doesn't matter) } // fetch file Response <BlobDownloadInfo> response = await blob.DownloadAsync(); using BlobDownloadInfo result = response.Value; using StreamReader reader = new StreamReader(result.Content); DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); // build model return(new StoredFileInfo { Success = true, Content = content, Expiry = expiry.UtcDateTime }); } catch (RequestFailedException ex) { return(new StoredFileInfo { Error = ex.ErrorCode == "BlobNotFound" ? "There's no file with that ID." : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }); } } // local filesystem for testing else { // get file FileInfo file = new FileInfo(this.GetDevFilePath(id)); if (file.Exists && file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) // expired { file.Delete(); } if (!file.Exists) { return(new StoredFileInfo { Error = "There's no file with that ID." }); } // renew if (renew) { File.SetLastWriteTimeUtc(file.FullName, DateTime.UtcNow); file.Refresh(); } // build model return(new StoredFileInfo { Success = true, Content = File.ReadAllText(file.FullName), Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays), Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment." }); } } // get from Pastebin else { PasteInfo response = await this.Pastebin.GetAsync(id); response.Content = this.GzipHelper.DecompressString(response.Content); return(new StoredFileInfo { Success = response.Success, Content = response.Content, Error = response.Error }); } }