Ejemplo n.º 1
0
        public async Task <bool> ExecuteAsync(CancellationToken ct)
        {
            if (Items.Length == 0)
            {
                Log.LogError("No items were provided for upload.");
                return(false);
            }

            Log.LogMessage("Begin uploading blobs to Azure account {0} in container {1}.",
                           AccountName,
                           ContainerName);

            try
            {
                AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName);

                List <Task> uploadTasks = new List <Task>();

                foreach (var item in Items)
                {
                    uploadTasks.Add(Task.Run(async() =>
                    {
                        string relativeBlobPath = item.GetMetadata("RelativeBlobPath");

                        if (string.IsNullOrEmpty(relativeBlobPath))
                        {
                            throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec));
                        }

                        if (!File.Exists(item.ItemSpec))
                        {
                            throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec));
                        }

                        BlobClient blobReference = blobUtils.GetBlob(relativeBlobPath);

                        if (!Overwrite && await blobReference.ExistsAsync())
                        {
                            if (PassIfExistingItemIdentical)
                            {
                                if (await blobUtils.IsFileIdenticalToBlobAsync(item.ItemSpec, blobReference))
                                {
                                    return;
                                }
                            }

                            throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath));
                        }

                        CancellationTokenSource timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(UploadTimeoutInMinutes));

                        using (Stream localFileStream = File.OpenRead(item.ItemSpec))
                        {
                            await blobReference.UploadAsync(localFileStream, timeoutTokenSource.Token);
                        }
                    }));
                }

                await Task.WhenAll(uploadTasks);

                Log.LogMessage("Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
            }
            catch (Exception e)
            {
                Log.LogErrorFromException(e, true);
            }

            return(!Log.HasLoggedErrors);
        }
Ejemplo n.º 2
0
 public async Task UploadFileToStorageContainer(Stream stream, string path)
 {
     BlobContainerClient containerClient = GetFileShareContainer();
     BlobClient          blobClient      = containerClient.GetBlobClient(path);
     await blobClient.UploadAsync(stream);
 }
        public static async Task <IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("C# HTTP trigger function processed a request.");

            string name = req.Query["name"];

            string  requestBody = await new StreamReader(req.Body).ReadToEndAsync();
            dynamic data        = JsonConvert.DeserializeObject(requestBody);

            name = name ?? data?.name;


            var connection    = Environment.GetEnvironmentVariable("azureconnection");
            var containerName = "commondataservice-kapish2020-ca7639d241a143b695207c2fb17d54";

            BlobContainerClient container = new BlobContainerClient(connection, containerName);

            var model = container.GetBlobs().Where(blb => blb.Name == "model.json").FirstOrDefault();
            var blobC = new BlobClient(connection, containerName, model.Name);
            var resp  = await blobC.DownloadAsync();

            StringBuilder sb = new StringBuilder();

            using (var streamReader = new StreamReader(resp.Value.Content))
            {
                while (!streamReader.EndOfStream)
                {
                    var line = await streamReader.ReadLineAsync();

                    sb.Append(line);
                }
            }

            dynamic       modeldata      = JsonConvert.DeserializeObject(sb.ToString());
            JObject       jobject        = modeldata;
            JObject       entity         = (JObject)jobject.GetValue("entities").FirstOrDefault();
            var           attributes     = entity.GetValue("attributes");
            List <string> attributesList = new List <string>();

            foreach (var att in attributes)
            {
                attributesList.Add(((JObject)att).GetValue("name").ToString());
            }


            var blobs = container.GetBlobs().Where(blb => blb.Name == "opportunity/Snapshot/2020_1608038345.csv");
            //kan jobba med filtret här för att få rätt blobs, så att jag får en snapshot i månaden osv.

            List <OpportunityHistory> opportunities = new List <OpportunityHistory>();

            foreach (var blob in blobs)
            {
                log.LogInformation(blob.Name);
                var blockBlob = new BlockBlobClient(connection, containerName, blob.Name);
                opportunities.AddRange(await QueryFile(blockBlob, log, attributesList));
            }

            return(new OkObjectResult(opportunities));
        }
Ejemplo n.º 4
0
 public override void Run(CancellationToken cancellationToken)
 {
     using var stream = RandomStream.Create(Options.Size);
     BlobClient.Upload(stream, transferOptions: Options.StorageTransferOptions, cancellationToken: cancellationToken);
 }
Ejemplo n.º 5
0
        /// <summary>
        ///   Attempts to claim ownership of partitions for processing.
        /// </summary>
        ///
        /// <param name="partitionOwnership">An enumerable containing all the ownership to claim.</param>
        ///
        /// <returns>An enumerable containing the successfully claimed ownership.</returns>
        ///
        public override async Task <IEnumerable <PartitionOwnership> > ClaimOwnershipAsync(IEnumerable <PartitionOwnership> partitionOwnership)
        {
            var claimedOwnership = new List <PartitionOwnership>();
            var metadata         = new Dictionary <string, string>();

            Response <BlobContentInfo> contentInfoResponse;
            Response <BlobInfo>        infoResponse;

            foreach (PartitionOwnership ownership in partitionOwnership)
            {
                metadata[BlobMetadataKey.OwnerIdentifier] = ownership.OwnerIdentifier;
                metadata[BlobMetadataKey.Offset]          = ownership.Offset?.ToString() ?? string.Empty;
                metadata[BlobMetadataKey.SequenceNumber]  = ownership.SequenceNumber?.ToString() ?? string.Empty;

                var blobAccessConditions = new BlobAccessConditions();

                var        blobName   = $"{ ownership.FullyQualifiedNamespace }/{ ownership.EventHubName }/{ ownership.ConsumerGroup }/{ ownership.PartitionId }";
                BlobClient blobClient = _containerClient.GetBlobClient(blobName);

                try
                {
                    // Even though documentation states otherwise, we cannot use UploadAsync when the blob already exists in
                    // the current storage SDK.  For this reason, we are using the specified ETag as an indication of what
                    // method to use.

                    if (ownership.ETag == null)
                    {
                        blobAccessConditions.HttpAccessConditions = new HttpAccessConditions {
                            IfNoneMatch = new ETag("*")
                        };

                        MemoryStream blobContent = null;

                        try
                        {
                            blobContent         = new MemoryStream(new byte[0]);
                            contentInfoResponse = await blobClient.UploadAsync(blobContent, metadata : metadata, accessConditions : blobAccessConditions).ConfigureAwait(false);
                        }
                        catch (StorageRequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.BlobAlreadyExists)
                        {
                            // A blob could have just been created by another Event Processor that claimed ownership of this
                            // partition.  In this case, there's no point in retrying because we don't have the correct ETag.

                            Log($"Ownership with partition id = '{ ownership.PartitionId }' is not claimable.");
                            continue;
                        }
                        finally
                        {
                            blobContent?.Dispose();
                        }

                        ownership.LastModifiedTime = contentInfoResponse.Value.LastModified;
                        ownership.ETag             = contentInfoResponse.Value.ETag.ToString();
                    }
                    else
                    {
                        blobAccessConditions.HttpAccessConditions = new HttpAccessConditions {
                            IfMatch = new ETag(ownership.ETag)
                        };

                        try
                        {
                            infoResponse = await blobClient.SetMetadataAsync(metadata, blobAccessConditions).ConfigureAwait(false);
                        }
                        catch (StorageRequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.BlobNotFound)
                        {
                            // No ownership was found, which means the ETag should have been set to null in order to
                            // claim this ownership.  For this reason, we consider it a failure and don't try again.

                            Log($"Ownership with partition id = '{ ownership.PartitionId }' is not claimable.");
                            continue;
                        }

                        ownership.LastModifiedTime = infoResponse.Value.LastModified;
                        ownership.ETag             = infoResponse.Value.ETag.ToString();
                    }

                    // Small workaround to retrieve the eTag.  The current storage SDK returns it enclosed in
                    // double quotes ('"ETAG_VALUE"' instead of 'ETAG_VALUE').

                    Match match = s_doubleQuotesExpression.Match(ownership.ETag);

                    if (match.Success)
                    {
                        ownership.ETag = match.Groups[1].ToString();
                    }

                    claimedOwnership.Add(ownership);

                    Log($"Ownership with partition id = '{ ownership.PartitionId }' claimed.");
                }
                catch (StorageRequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ConditionNotMet)
                {
                    Log($"Ownership with partition id = '{ ownership.PartitionId }' is not claimable.");
                }
            }

            return(claimedOwnership);
        }
        public async Task ClaimCheck()
        {
            await using (var scope = await ServiceBusScope.CreateWithQueue(enablePartitioning: false, enableSession: false))
            {
                #region Snippet:CreateBlobContainer
#if SNIPPET
                var containerClient = new BlobContainerClient("<storage connection string>", "claim-checks");
#else
                var containerClient = new BlobContainerClient(TestEnvironment.StorageClaimCheckConnectionString, "claim-checks");
#endif
                await containerClient.CreateIfNotExistsAsync();

                #endregion

                try
                {
                    #region Snippet:UploadMessage

                    byte[] body     = ServiceBusTestUtilities.GetRandomBuffer(1000000);
                    string blobName = Guid.NewGuid().ToString();
                    await containerClient.UploadBlobAsync(blobName, new BinaryData(body));

                    var message = new ServiceBusMessage
                    {
                        ApplicationProperties =
                        {
                            ["blob-name"] = blobName
                        }
                    };

                    #endregion

                    #region Snippet:ClaimCheckSendMessage

#if SNIPPET
                    var client = new ServiceBusClient("<service bus connection string>");
#else
                    var client = new ServiceBusClient(TestEnvironment.ServiceBusConnectionString);
#endif
                    ServiceBusSender sender = client.CreateSender(scope.QueueName);
                    await sender.SendMessageAsync(message);

                    #endregion

                    #region Snippet:ReceiveClaimCheck

                    ServiceBusReceiver        receiver        = client.CreateReceiver(scope.QueueName);
                    ServiceBusReceivedMessage receivedMessage = await receiver.ReceiveMessageAsync();

                    if (receivedMessage.ApplicationProperties.TryGetValue("blob-name", out object blobNameReceived))
                    {
#if SNIPPET
                        var blobClient = new BlobClient("<storage connection string>", "claim-checks", (string)blobNameReceived);
#else
                        var blobClient = new BlobClient(
                            TestEnvironment.StorageClaimCheckConnectionString,
                            "claim-checks",
                            (string)blobNameReceived);
#endif
                        BlobDownloadResult downloadResult = await blobClient.DownloadContentAsync();

                        BinaryData messageBody = downloadResult.Content;

                        // Once we determine that we are done with the message, we complete it and delete the corresponding blob.
                        await receiver.CompleteMessageAsync(receivedMessage);

                        await blobClient.DeleteAsync();

#if !SNIPPET
                        Assert.AreEqual(body, messageBody.ToArray());
#endif
                    }
                    #endregion
                }
                finally
                {
                    await containerClient.DeleteAsync();
                }
            }
        }
        public async Task StartTranslationWithAzureBlob()
        {
            /**
             * FILE: SampleTranslationWithAzureBlob.cs
             * DESCRIPTION:
             *  This sample demonstrates how to use Azure Blob Storage to set up the necessary resources to create a translation
             *  operation. Run the sample to create containers, upload documents, and generate SAS tokens for the source/target
             *  containers. Once the operation is completed, use the storage library to download your documents locally.
             *
             * PREREQUISITE:
             *  This sample requires you install Azure.Storage.Blobs nuget package:
             *  https://www.nuget.org/packages/Azure.Storage.Blobs
             *
             * USAGE:
             *  Set the environment variables with your own values before running the sample:
             *  1) DOCUMENT_TRANSLATION_ENDPOINT - the endpoint to your Document Translation resource.
             *  2) DOCUMENT_TRANSLATION_API_KEY - your Document Translation API key.
             *  3) DOCUMENT_TRANSLATION_CONNECTION_STRING - the connection string to your Storage account
             *  4) AZURE_DOCUMENT_PATH - (optional) the path and file extension of your document in this directory
             *      e.g. "path/mydocument.txt"
             *  Optionally, you can also set the following variables in code:
             *  5) sourceContainerName - the name of your source container
             *  6) targetContainerName - the name of your target container
             **/
#if SNIPPET
            string endpoint = "<Document Translator Resource Endpoint>";
            string apiKey   = "<Document Translator Resource API Key>";
#else
            string endpoint = TestEnvironment.Endpoint;
            string apiKey   = TestEnvironment.ApiKey;
#endif

            var client = new DocumentTranslationClient(new Uri(endpoint), new AzureKeyCredential(apiKey));

            var storageConnectionString = Environment.GetEnvironmentVariable("DOCUMENT_TRANSLATION_CONNECTION_STRING");
#if SNIPPET
            string sourceContainerName = "<Source Container Name>";
            string targetContainerName = "<Target Container Name>";
#else
            string sourceContainerName = GenerateRandomName("source");
            string targetContainerName = GenerateRandomName("target");
#endif
            string documentPath = Environment.GetEnvironmentVariable("AZURE_DOCUMENT_PATH");

            // Create source and target storage containers
            BlobServiceClient   blobServiceClient     = new BlobServiceClient(storageConnectionString);
            BlobContainerClient sourceContainerClient = await blobServiceClient.CreateBlobContainerAsync(sourceContainerName ?? "translation-source-container", PublicAccessType.BlobContainer).ConfigureAwait(false);

            BlobContainerClient targetContainerClient = await blobServiceClient.CreateBlobContainerAsync(targetContainerName ?? "translation-target-container", PublicAccessType.BlobContainer).ConfigureAwait(false);

            // Upload blob (file) to the source container
            BlobClient srcBlobClient = sourceContainerClient.GetBlobClient(!string.IsNullOrWhiteSpace(documentPath) ? Path.GetFileName(documentPath) : "example_source_document.txt");

            if (!string.IsNullOrWhiteSpace(documentPath))
            {
                using (FileStream uploadFileStream = File.OpenRead(documentPath))
                {
                    await srcBlobClient.UploadAsync(uploadFileStream, true).ConfigureAwait(false);
                }
            }
            else
            {
                await srcBlobClient.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Hello.\nThis is a testing text.")), true).ConfigureAwait(false);
            }

            Console.WriteLine($"Uploaded document {srcBlobClient.Uri} to source storage container");

            // Generate SAS tokens for source & target
            Uri srcSasUri = sourceContainerClient.GenerateSasUri(BlobContainerSasPermissions.List | BlobContainerSasPermissions.Read, DateTime.UtcNow.AddMinutes(30));
            Uri tgtSasUri = targetContainerClient.GenerateSasUri(BlobContainerSasPermissions.List | BlobContainerSasPermissions.Write | BlobContainerSasPermissions.Delete, DateTime.UtcNow.AddMinutes(30));

            // Submit the translation operation and wait for it to finish
            var operationRequest = new DocumentTranslationInput(srcSasUri, tgtSasUri, "es");
            DocumentTranslationOperation operationResult = await client.StartTranslationAsync(operationRequest);

            await operationResult.WaitForCompletionAsync();

            Console.WriteLine($"Operation status: {operationResult.Status}");
            Console.WriteLine($"Operation created on: {operationResult.CreatedOn}");
            Console.WriteLine($"Operation last updated on: {operationResult.LastModified}");
            Console.WriteLine($"Total number of translations on documents: {operationResult.DocumentsTotal}");
            Console.WriteLine("\nOf total documents...");
            Console.WriteLine($"{operationResult.DocumentsFailed} failed");
            Console.WriteLine($"{operationResult.DocumentsSucceeded} succeeded");

            await foreach (DocumentStatusResult document in operationResult.GetDocumentStatusesAsync())
            {
                if (document.Status == DocumentTranslationStatus.Succeeded)
                {
                    Console.WriteLine($"Document at {document.SourceDocumentUri} was translated to {document.TranslatedToLanguageCode} language.You can find translated document at {document.TranslatedDocumentUri}");
                }
                else
                {
                    Console.WriteLine($"Document ID: {document.Id}, Error Code: {document.Error.Code}, Message: {document.Error.Message}");
                }
            }
        }
Ejemplo n.º 8
0
 /// <summary>
 /// Initializes a new instance of the <see cref="AzureBlobStorageImageResolver"/> class.
 /// </summary>
 /// <param name="blob">The Azure blob.</param>
 public AzureBlobStorageImageResolver(BlobClient blob) => this.blob = blob;
 public DefaultBlobReaderFtpResponseAction(BlobClient blobClient, IMapper mapper = null) : base(blobClient, mapper)
 {
 }
Ejemplo n.º 10
0
 public AzureBlobFileHandle(BlobClient client)
 {
     this.Client = client;
 }
Ejemplo n.º 11
0
        public async Task <ChangeFeed> BuildChangeFeed(
            DateTimeOffset?startTime,
            DateTimeOffset?endTime,
            string continuation,
            bool async,
            CancellationToken cancellationToken)
        {
            DateTimeOffset   lastConsumable;
            Queue <string>   years    = new Queue <string>();
            Queue <string>   segments = new Queue <string>();
            ChangeFeedCursor cursor   = null;

            // Create cursor
            if (continuation != null)
            {
                cursor = JsonSerializer.Deserialize <ChangeFeedCursor>(continuation);
                ValidateCursor(_containerClient, cursor);
                startTime = BlobChangeFeedExtensions.ToDateTimeOffset(cursor.CurrentSegmentCursor.SegmentPath).Value;
                endTime   = cursor.EndTime;
            }
            // Round start and end time if we are not using the cursor.
            else
            {
                startTime = startTime.RoundDownToNearestHour();
                endTime   = endTime.RoundUpToNearestHour();
            }

            // Check if Change Feed has been abled for this account.
            bool changeFeedContainerExists;

            if (async)
            {
                changeFeedContainerExists = await _containerClient.ExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
            }
            else
            {
                changeFeedContainerExists = _containerClient.Exists(cancellationToken: cancellationToken);
            }

            if (!changeFeedContainerExists)
            {
                throw new ArgumentException("Change Feed hasn't been enabled on this account, or is currently being enabled.");
            }

            // Get last consumable
            BlobClient blobClient = _containerClient.GetBlobClient(Constants.ChangeFeed.MetaSegmentsPath);
            BlobDownloadStreamingResult blobDownloadInfo;

            if (async)
            {
                blobDownloadInfo = await blobClient.DownloadStreamingAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
            }
            else
            {
                blobDownloadInfo = blobClient.DownloadStreaming(cancellationToken: cancellationToken);
            }

            JsonDocument jsonMetaSegment;

            if (async)
            {
                jsonMetaSegment = await JsonDocument.ParseAsync(
                    blobDownloadInfo.Content,
                    cancellationToken : cancellationToken
                    ).ConfigureAwait(false);
            }
            else
            {
                jsonMetaSegment = JsonDocument.Parse(blobDownloadInfo.Content);
            }

            lastConsumable = jsonMetaSegment.RootElement.GetProperty("lastConsumable").GetDateTimeOffset();

            // Get year paths
            years = await GetYearPathsInternal(
                async,
                cancellationToken).ConfigureAwait(false);

            // Dequeue any years that occur before start time
            if (startTime.HasValue)
            {
                while (years.Count > 0 &&
                       BlobChangeFeedExtensions.ToDateTimeOffset(years.Peek()) < startTime.RoundDownToNearestYear())
                {
                    years.Dequeue();
                }
            }

            // There are no years.
            if (years.Count == 0)
            {
                return(ChangeFeed.Empty());
            }

            while (segments.Count == 0 && years.Count > 0)
            {
                // Get Segments for year
                segments = await BlobChangeFeedExtensions.GetSegmentsInYearInternal(
                    containerClient : _containerClient,
                    yearPath : years.Dequeue(),
                    startTime : startTime,
                    endTime : BlobChangeFeedExtensions.MinDateTime(lastConsumable, endTime),
                    async : async,
                    cancellationToken : cancellationToken)
                           .ConfigureAwait(false);
            }

            // We were on the last year, and there were no more segments.
            if (segments.Count == 0)
            {
                return(ChangeFeed.Empty());
            }

            Segment currentSegment = await _segmentFactory.BuildSegment(
                async,
                segments.Dequeue(),
                cursor?.CurrentSegmentCursor)
                                     .ConfigureAwait(false);

            return(new ChangeFeed(
                       _containerClient,
                       _segmentFactory,
                       years,
                       segments,
                       currentSegment,
                       lastConsumable,
                       startTime,
                       endTime));
        }
Ejemplo n.º 12
0
 private static async Task DeleteBlobAsync(string blobName)
 {
     BlobClient blobClient = new BlobClient(_connectionString, _blobContainerName, blobName);
     await blobClient.DeleteIfExistsAsync();
 }
Ejemplo n.º 13
0
        /// <summary>
        /// Tests a blob SAS to determine which operations it allows.
        /// </summary>
        /// <param name="sasUri">A string containing a URI with a SAS appended.</param>
        /// <param name="blobContent">A string content content to write to the blob.</param>
        static void TestBlobSAS(Uri sasUri, string blobContent)
        {
            //Try performing blob operations using the SAS provided.

            //Return a reference to the blob using the SAS URI.
            BlobClient blob = new BlobClient(sasUri);

            //Create operation: Upload a blob with the specified name to the container.
            //If the blob does not exist, it will be created. If it does exist, it will be overwritten.
            try
            {
                //string blobContent = "This blob was created with a shared access signature granting write permissions to the blob. ";
                blob.Upload(BinaryData.FromString(blobContent));
                Console.WriteLine("Create operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Create operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }

            // Write operation: Add metadata to the blob
            try
            {
                IDictionary <string, string> metadata = new Dictionary <string, string>();
                metadata.Add("name", "value");
                blob.SetMetadata(metadata);
                Console.WriteLine("Write operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Write operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }

            //Read operation: Read the contents of the blob.
            try
            {
                BlobDownloadResult download = blob.DownloadContent();
                string             content  = download.Content.ToString();
                Console.WriteLine(content);
                Console.WriteLine();

                Console.WriteLine("Read operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Read operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }

            //Delete operation: Delete the blob.
            try
            {
                blob.Delete();
                Console.WriteLine("Delete operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Delete operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }
        }
Ejemplo n.º 14
0
        /// <summary>
        /// Tests a container SAS to determine which operations it allows.
        /// </summary>
        /// <param name="sasUri">A string containing a URI with a SAS appended.</param>
        /// <param name="blobName">A string containing the name of the blob.</param>
        /// <param name="blobContent">A string content content to write to the blob.</param>
        static void TestContainerSAS(Uri sasUri, string blobName, string blobContent)
        {
            //Try performing container operations with the SAS provided.
            //Note that the storage account credentials are not required here; the SAS provides the necessary
            //authentication information on the URI.

            //Return a reference to the container using the SAS URI.
            BlobContainerClient container = new BlobContainerClient(sasUri);

            //Return a reference to a blob to be created in the container.
            BlobClient blob = container.GetBlobClient(blobName);

            //Write operation: Upload a new blob to the container.
            try
            {
                blob.Upload(BinaryData.FromString(blobContent));
                Console.WriteLine("Write operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Write operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }

            //List operation: List the blobs in the container.
            try
            {
                foreach (BlobItem blobItem in container.GetBlobs())
                {
                    Console.WriteLine(blobItem.Name);
                }
                Console.WriteLine("List operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("List operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }

            //Read operation: Read the contents of the blob we created above.
            try
            {
                BlobDownloadInfo download = blob.Download();
                Console.WriteLine(download.ContentLength);
                Console.WriteLine();
                Console.WriteLine("Read operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Read operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }
            Console.WriteLine();

            //Delete operation: Delete the blob we created above.
            try
            {
                blob.Delete();
                Console.WriteLine("Delete operation succeeded for SAS " + sasUri);
                Console.WriteLine();
            }
            catch (RequestFailedException e)
            {
                Console.WriteLine("Delete operation failed for SAS " + sasUri);
                Console.WriteLine("Additional error information: " + e.Message);
                Console.WriteLine();
            }
        }
Ejemplo n.º 15
0
 public AzureStoredBlob(AzureBlobStorage storage, BlobClient blob, BlobProperties properties)
     : base(blob)
 {
     Storage    = storage ?? throw new ArgumentNullException(nameof(storage));
     Properties = properties ?? throw new ArgumentNullException(nameof(properties));
 }
        public BlobTriggerBinding(ParameterInfo parameter,
                                  IArgumentBinding <IStorageBlob> argumentBinding,
                                  IStorageAccount hostAccount,
                                  IStorageAccount dataAccount,
                                  IBlobPathSource path,
                                  IHostIdProvider hostIdProvider,
                                  IQueueConfiguration queueConfiguration,
                                  JobHostBlobsConfiguration blobsConfiguration,
                                  IWebJobsExceptionHandler exceptionHandler,
                                  IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
                                  IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
                                  ISharedContextProvider sharedContextProvider,
                                  SingletonManager singletonManager,
                                  TraceWriter trace,
                                  ILoggerFactory loggerFactory)
        {
            if (parameter == null)
            {
                throw new ArgumentNullException("parameter");
            }

            if (argumentBinding == null)
            {
                throw new ArgumentNullException("argumentBinding");
            }

            if (hostAccount == null)
            {
                throw new ArgumentNullException("hostAccount");
            }

            if (dataAccount == null)
            {
                throw new ArgumentNullException("dataAccount");
            }

            if (path == null)
            {
                throw new ArgumentNullException("path");
            }

            if (hostIdProvider == null)
            {
                throw new ArgumentNullException("hostIdProvider");
            }

            if (queueConfiguration == null)
            {
                throw new ArgumentNullException("queueConfiguration");
            }

            if (blobsConfiguration == null)
            {
                throw new ArgumentNullException("blobsConfiguration");
            }

            if (exceptionHandler == null)
            {
                throw new ArgumentNullException("exceptionHandler");
            }

            if (blobWrittenWatcherSetter == null)
            {
                throw new ArgumentNullException("blobWrittenWatcherSetter");
            }

            if (messageEnqueuedWatcherSetter == null)
            {
                throw new ArgumentNullException("messageEnqueuedWatcherSetter");
            }

            if (sharedContextProvider == null)
            {
                throw new ArgumentNullException("sharedContextProvider");
            }

            if (singletonManager == null)
            {
                throw new ArgumentNullException("singletonManager");
            }

            if (trace == null)
            {
                throw new ArgumentNullException("trace");
            }

            _parameter       = parameter;
            _argumentBinding = argumentBinding;
            _hostAccount     = hostAccount;
            _dataAccount     = dataAccount;
            StorageClientFactoryContext context = new StorageClientFactoryContext
            {
                Parameter = parameter
            };

            _blobClient                   = dataAccount.CreateBlobClient(context);
            _accountName                  = BlobClient.GetAccountName(_blobClient);
            _path                         = path;
            _hostIdProvider               = hostIdProvider;
            _queueConfiguration           = queueConfiguration;
            _blobsConfiguration           = blobsConfiguration;
            _exceptionHandler             = exceptionHandler;
            _blobWrittenWatcherSetter     = blobWrittenWatcherSetter;
            _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
            _sharedContextProvider        = sharedContextProvider;
            _singletonManager             = singletonManager;
            _trace                        = trace;
            _loggerFactory                = loggerFactory;
            _converter                    = CreateConverter(_blobClient);
            _bindingDataContract          = CreateBindingDataContract(path);
        }
        /// <summary>
        /// Test some of the file storage operations.
        /// </summary>
        public async Task RunFileStorageOperationsAsync()
        {
            // These are used in the finally block to clean up the objects created during the demo.
            ShareClient          shareClient     = null;
            ShareFileClient      shareFileClient = null;
            ShareDirectoryClient fileDirectory   = null;

            BlobClient          targetBlob    = null;
            BlobContainerClient blobContainer = null;

            string destFile       = null;
            string downloadFolder = null;
            // Name to be used for the file when downloading it so you can inspect it locally
            string downloadFile = null;

            try
            {
                //***** Setup *****//
                Console.WriteLine("Getting reference to the storage account.");

                // How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
                string storageConnectionString = ConfigurationManager.AppSettings.Get("StorageConnectionString");
                string storageAccountName      = ConfigurationManager.AppSettings.Get("StorageAccountName");
                string storageAccountKey       = ConfigurationManager.AppSettings.Get("StorageAccountKey");

                Console.WriteLine("Instantiating file client.");

                // Create a share client for interacting with the file service.
                var shareServiceClient = new ShareServiceClient(storageConnectionString);

                // Create the share name -- use a guid in the name so it's unique.
                // This will also be used as the container name for blob storage when copying the file to blob storage.
                string shareName = "demotest-" + System.Guid.NewGuid().ToString();

                // Name of folder to put the files in
                string sourceFolder = "testfolder";

                // Name of file to upload and download
                string testFile = "HelloWorld.png";

                // Folder where the HelloWorld.png file resides
                string localFolder = @".\";

                // It won't let you download in the same folder as the exe file,
                //   so use a temporary folder with the same name as the share.
                downloadFolder = Path.Combine(Path.GetTempPath(), shareName);

                //***** Create a file share *****//

                // Create the share if it doesn't already exist.
                Console.WriteLine("Creating share with name {0}", shareName);
                shareClient = shareServiceClient.GetShareClient(shareName);
                try
                {
                    await shareClient.CreateIfNotExistsAsync();

                    Console.WriteLine("    Share created successfully.");
                }
                catch (RequestFailedException exRequest)
                {
                    Common.WriteException(exRequest);
                    Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
                    Console.WriteLine("Press any key to exit");
                    Console.ReadLine();
                    throw;
                }
                catch (Exception ex)
                {
                    Console.WriteLine("    Exception thrown creating share.");
                    Common.WriteException(ex);
                    throw;
                }

                //***** Create a directory on the file share *****//

                // Create a directory on the share.
                Console.WriteLine("Creating directory named {0}", sourceFolder);

                ShareDirectoryClient rootDirectory = shareClient.GetRootDirectoryClient();

                // If the source folder is null, then use the root folder.
                // If the source folder is specified, then get a reference to it.
                if (string.IsNullOrWhiteSpace(sourceFolder))
                {
                    // There is no folder specified, so return a reference to the root directory.
                    fileDirectory = rootDirectory;
                    Console.WriteLine("    Using root directory.");
                }
                else
                {
                    // There was a folder specified, so return a reference to that folder.
                    fileDirectory = rootDirectory.GetSubdirectoryClient(sourceFolder);

                    await fileDirectory.CreateIfNotExistsAsync();

                    Console.WriteLine("    Directory created successfully.");
                }

                //***** Upload a file to the file share *****//

                // Get a file client.
                shareFileClient = fileDirectory.GetFileClient(testFile);

                // Upload a file to the share.
                Console.WriteLine("Uploading file {0} to share", testFile);

                // Set up the name and path of the local file.
                string sourceFile = Path.Combine(localFolder, testFile);
                if (File.Exists(sourceFile))
                {
                    using (FileStream stream = File.OpenRead(sourceFile))
                    {
                        // Upload from the local file to the file share in azure.
                        await shareFileClient.CreateAsync(stream.Length);

                        await shareFileClient.UploadAsync(stream);
                    }
                    Console.WriteLine("    Successfully uploaded file to share.");
                }
                else
                {
                    Console.WriteLine("File not found, so not uploaded.");
                }

                //***** Get list of all files/directories on the file share*****//

                // List all files/directories under the root directory.
                Console.WriteLine("Getting list of all files/directories under the root directory of the share.");

                var fileList = rootDirectory.GetFilesAndDirectoriesAsync();

                // Print all files/directories listed above.
                await foreach (ShareFileItem listItem in fileList)
                {
                    // listItem type will be ShareClient or ShareDirectoryClient.
                    Console.WriteLine("    - {0} (type: {1})", listItem.Name, listItem.GetType());
                }

                Console.WriteLine("Getting list of all files/directories in the file directory on the share.");

                // Now get the list of all files/directories in your directory.
                // Ordinarily, you'd write something recursive to do this for all directories and subdirectories.

                fileList = fileDirectory.GetFilesAndDirectoriesAsync();

                // Print all files/directories in the folder.
                await foreach (ShareFileItem listItem in fileList)
                {
                    // listItem type will be a file or directory
                    Console.WriteLine("    - {0} (IsDirectory: {1})", listItem.Name, listItem.IsDirectory);
                }

                //***** Download a file from the file share *****//

                // Download the file to the downloadFolder in the temp directory.
                // Check and if the directory doesn't exist (which it shouldn't), create it.
                Console.WriteLine("Downloading file from share to local temp folder {0}.", downloadFolder);
                if (!Directory.Exists(downloadFolder))
                {
                    Directory.CreateDirectory(downloadFolder);
                }

                // Download the file.
                ShareFileDownloadInfo download = await shareFileClient.DownloadAsync();

                downloadFile = Path.Combine(downloadFolder, testFile);
                using (FileStream stream = File.OpenWrite(downloadFile))
                {
                    await download.Content.CopyToAsync(stream);
                }

                Console.WriteLine("    Successfully downloaded file from share to local temp folder.");

                //***** Copy a file from the file share to blob storage, then abort the copy *****//

                // Copies can sometimes complete before there's a chance to abort.
                // If that happens with the file you're testing with, try copying the file
                //   to a storage account in a different region. If it still finishes too fast,
                //   try using a bigger file and copying it to a different region. That will almost always
                //   take long enough to give you time to abort the copy.
                // If you want to change the file you're testing the Copy with without changing the value for the
                //   rest of the sample code, upload the file to the share, then assign the name of the file
                //   to the testFile variable right here before calling GetFileClient.
                //   Then it will use the new file for the copy and abort but the rest of the code
                //   will still use the original file.
                ShareFileClient shareFileCopy = fileDirectory.GetFileClient(testFile);

                // Upload a file to the share.
                Console.WriteLine("Uploading file {0} to share", testFile);

                // Set up the name and path of the local file.
                string sourceFileCopy = Path.Combine(localFolder, testFile);
                using (FileStream stream = File.OpenRead(sourceFile))
                {
                    // Upload from the local file to the file share in azure.
                    await shareFileCopy.CreateAsync(stream.Length);

                    await shareFileCopy.UploadAsync(stream);
                }
                Console.WriteLine("    Successfully uploaded file to share.");

                // Copy the file to blob storage.
                Console.WriteLine("Copying file to blob storage. Container name = {0}", shareName);

                // First get a blob service client.
                var blobServiceClient = new BlobServiceClient(storageConnectionString);

                // Get a blob container client and create it if it doesn't already exist.
                blobContainer = blobServiceClient.GetBlobContainerClient(shareName);
                await blobContainer.CreateIfNotExistsAsync();

                // Get a blob client to the target blob.
                targetBlob = blobContainer.GetBlobClient(testFile);

                string copyId = string.Empty;

                // Get a share file client to be copied.
                shareFileClient = fileDirectory.GetFileClient(testFile);

                // Create a SAS for the file that's valid for 24 hours.
                // Note that when you are copying a file to a blob, or a blob to a file, you must use a SAS
                // to authenticate access to the source object, even if you are copying within the same
                // storage account.
                var sas = new AccountSasBuilder
                {
                    // Allow access to Files
                    Services = AccountSasServices.Files,

                    // Allow access to the service level APIs
                    ResourceTypes = AccountSasResourceTypes.All,

                    // Access expires in 1 day!
                    ExpiresOn = DateTime.UtcNow.AddDays(1)
                };
                sas.SetPermissions(AccountSasPermissions.Read);

                var credential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);

                // Build a SAS URI
                var sasUri = new UriBuilder(shareFileClient.Uri)
                {
                    Query = sas.ToSasQueryParameters(credential).ToString()
                };
                // Start the copy of the file to the blob.
                CopyFromUriOperation operation = await targetBlob.StartCopyFromUriAsync(sasUri.Uri);

                copyId = operation.Id;

                Console.WriteLine("   File copy started successfully. copyID = {0}", copyId);

                // Now clean up after yourself.
                Console.WriteLine("Deleting the files from the file share.");

                // Delete the files because cloudFile is a different file in the range sample.
                shareFileClient = fileDirectory.GetFileClient(testFile);
                await shareFileClient.DeleteIfExistsAsync();

                Console.WriteLine("Setting up files to test WriteRange and ListRanges.");

                //***** Write 2 ranges to a file, then list the ranges *****//

                // This is the code for trying out writing data to a range in a file,
                //   and then listing those ranges.
                // Get a reference to a file and write a range of data to it      .
                // Then write another range to it.
                // Then list the ranges.

                // Start at the very beginning of the file.
                long startOffset = 0;

                // Set the destination file name -- this is the file on the file share that you're writing to.
                destFile        = "rangeops.txt";
                shareFileClient = fileDirectory.GetFileClient(destFile);

                // Create a string with 512 a's in it. This will be used to write the range.
                int    testStreamLen = 512;
                string textToStream  = string.Empty;
                textToStream = textToStream.PadRight(testStreamLen, 'a');

                using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
                {
                    // Max size of the output file; have to specify this when you create the file
                    // I picked this number arbitrarily.
                    long maxFileSize = 65536;

                    Console.WriteLine("Write first range.");

                    // Set the stream back to the beginning, in case it's been read at all.
                    ms.Position = 0;

                    // If the file doesn't exist, create it.
                    // The maximum file size is passed in. It has to be big enough to hold
                    //   all the data you're going to write, so don't set it to 256k and try to write two 256-k blocks to it.
                    if (!shareFileClient.Exists())
                    {
                        Console.WriteLine("File doesn't exist, create empty file to write ranges to.");

                        // Create a file with a maximum file size of 64k.
                        await shareFileClient.CreateAsync(maxFileSize);

                        Console.WriteLine("    Empty file created successfully.");
                    }

                    // Write the stream to the file starting at startOffset for the length of the stream.
                    Console.WriteLine("Writing range to file.");
                    var range = new HttpRange(startOffset, textToStream.Length);
                    await shareFileClient.UploadRangeAsync(range, ms);

                    // Download the file to your temp directory so you can inspect it locally.
                    downloadFile = Path.Combine(downloadFolder, "__testrange.txt");
                    Console.WriteLine("Downloading file to examine.");
                    download = await shareFileClient.DownloadAsync();

                    using (FileStream stream = File.OpenWrite(downloadFile))
                    {
                        await download.Content.CopyToAsync(stream);
                    }

                    Console.WriteLine("    Successfully downloaded file with ranges in it to examine.");
                }

                // Now add the second range, but don't make it adjacent to the first one, or it will show only
                //   one range, with the two combined. Put it like 1000 spaces away. When you get the range back, it will
                //   start at the position at the 512-multiple border prior or equal to the beginning of the data written,
                //   and it will end at the 512-multliple border after the actual end of the data.
                //For example, if you write to 2000-3000, the range will be the 512-multiple prior to 2000, which is
                //   position 1536, or offset 1535 (because it's 0-based).
                //   And the right offset of the range will be the 512-multiple after 3000, which is position 3072,
                //   or offset 3071 (because it's 0-based).
                Console.WriteLine("Getting ready to write second range to file.");

                startOffset += testStreamLen + 1000; //randomly selected number

                // Create a string with 512 b's in it. This will be used to write the range.
                textToStream = string.Empty;
                textToStream = textToStream.PadRight(testStreamLen, 'b');

                using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
                {
                    ms.Position = 0;

                    // Write the stream to the file starting at startOffset for the length of the stream.
                    Console.WriteLine("Write second range to file.");
                    var range = new HttpRange(startOffset, textToStream.Length);
                    await shareFileClient.UploadRangeAsync(range, ms);

                    Console.WriteLine("    Successful writing second range to file.");

                    // Download the file to your temp directory so you can examine it.
                    downloadFile = Path.Combine(downloadFolder, "__testrange2.txt");
                    Console.WriteLine("Downloading file with two ranges in it to examine.");

                    download = await shareFileClient.DownloadAsync();

                    using (FileStream stream = File.OpenWrite(downloadFile))
                    {
                        await download.Content.CopyToAsync(stream);
                    }
                    Console.WriteLine("    Successfully downloaded file to examine.");
                }

                // Query and view the list of ranges.
                Console.WriteLine("Call to get the list of ranges.");
                var listOfRanges = await shareFileClient.GetRangeListAsync(new HttpRange());


                Console.WriteLine("    Successfully retrieved list of ranges.");
                foreach (HttpRange range in listOfRanges.Value.Ranges)
                {
                    Console.WriteLine("    --> filerange startOffset = {0}, endOffset = {1}", range.Offset, range.Offset + range.Length);
                }

                //***** Clean up *****//
            }
            catch (Exception ex)
            {
                Console.WriteLine("    Exception thrown. Message = {0}{1}    Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace);
            }
            finally
            {
                //Clean up after you're done.

                Console.WriteLine("Removing all files, folders, shares, blobs, and containers created in this demo.");

                // ****NOTE: You can just delete the file share, and everything will be removed.
                // This samples deletes everything off of the file share first for the purpose of
                //   showing you how to delete specific files and directories.


                // Delete the file with the ranges in it.
                destFile        = "rangeops.txt";
                shareFileClient = fileDirectory.GetFileClient(destFile);
                await shareFileClient.DeleteIfExistsAsync();

                Console.WriteLine("Deleting the directory on the file share.");

                // Delete the directory.
                bool success = await fileDirectory.DeleteIfExistsAsync();

                if (success)
                {
                    Console.WriteLine("    Directory on the file share deleted successfully.");
                }
                else
                {
                    Console.WriteLine("    Directory on the file share NOT deleted successfully; may not exist.");
                }

                Console.WriteLine("Deleting the file share.");

                // Delete the share.
                await shareClient.DeleteAsync();

                Console.WriteLine("    Deleted the file share successfully.");

                Console.WriteLine("Deleting the temporary download directory and the file in it.");

                // Delete the download folder and its contents.
                Directory.Delete(downloadFolder, true);
                Console.WriteLine("    Successfully deleted the temporary download directory.");

                Console.WriteLine("Deleting the container and blob used in the Copy/Abort test.");
                await targetBlob.DeleteIfExistsAsync();

                await blobContainer.DeleteIfExistsAsync();

                Console.WriteLine("    Successfully deleted the blob and its container.");
            }
        }
Ejemplo n.º 18
0
 public static void Call([BlobTrigger(BlobPath)] BlobClient blob)
 {
     TaskSource.TrySetResult(blob);
 }
Ejemplo n.º 19
0
 /// <summary>
 /// Deletes the image from azure storage
 /// </summary>
 /// <param name="fileName"> image file name </param>
 /// <returns> no return </returns>
 public async Task DeleteBlobImage(string fileName)
 {
     BlobContainerClient container = new BlobContainerClient(Configuration["ImageBlob"], "images");
     BlobClient          blob      = container.GetBlobClient(fileName);
     await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, default);
 }
Ejemplo n.º 20
0
 public static void BindToBlobClient([Blob(BlobPath)] BlobClient blob)
 {
     Assert.NotNull(blob);
     Assert.AreEqual(BlobName, blob.Name);
 }
Ejemplo n.º 21
0
 public void Run(
     [Blob(BlobPath)] BlobClient blob)
 {
     this.Result = blob;
 }
 /// <summary>
 /// Creates a new instance of the <see cref="AzureBlobXmlRepository"/>.
 /// </summary>
 /// <param name="blobClient">A <see cref="BlobClient"/> that is connected to the blob we are reading from and writing to.</param>
 public AzureBlobXmlRepository(BlobClient blobClient)
 {
     _random     = new Random();
     _blobClient = blobClient;
 }
Ejemplo n.º 23
0
 public override async Task RunAsync(CancellationToken cancellationToken)
 {
     using var stream = RandomStream.Create(Options.Size);
     await BlobClient.UploadAsync(stream, transferOptions : Options.StorageTransferOptions, cancellationToken : cancellationToken);
 }
        protected virtual void CreateTestStorageEntities()
        {
            TestQueue = QueueClient.GetQueueReference(string.Format("test-input-{0}", FixtureId));
            TestQueue.CreateIfNotExists();
            TestQueue.Clear();

            // This queue name should really be suffixed by -fsharp, -csharp, -node etc.
            MobileTablesQueue = QueueClient.GetQueueReference("mobiletables-input");
            MobileTablesQueue.CreateIfNotExists(); // do not clear this queue since it is currently shared between fixtures

            TestInputContainer = BlobClient.GetContainerReference(string.Format("test-input-{0}", FixtureId));
            TestInputContainer.CreateIfNotExists();

            // Processing a large number of blobs on startup can take a while,
            // so let's start with an empty container.
            TestHelpers.ClearContainer(TestInputContainer);

            TestOutputContainer = BlobClient.GetContainerReference(string.Format("test-output-{0}", FixtureId));
            TestOutputContainer.CreateIfNotExists();
            TestHelpers.ClearContainer(TestOutputContainer);

            TestTable = TableClient.GetTableReference("test");
            TestTable.CreateIfNotExists();

            DeleteEntities(TestTable, "AAA");
            DeleteEntities(TestTable, "BBB");

            var batch = new TableBatchOperation();

            batch.Insert(new TestEntity {
                PartitionKey = "AAA", RowKey = "001", Region = "West", Name = "Test Entity 1", Status = 0
            });
            batch.Insert(new TestEntity {
                PartitionKey = "AAA", RowKey = "002", Region = "East", Name = "Test Entity 2", Status = 1
            });
            batch.Insert(new TestEntity {
                PartitionKey = "AAA", RowKey = "003", Region = "West", Name = "Test Entity 3", Status = 1
            });
            batch.Insert(new TestEntity {
                PartitionKey = "AAA", RowKey = "004", Region = "West", Name = "Test Entity 4", Status = 1
            });
            batch.Insert(new TestEntity {
                PartitionKey = "AAA", RowKey = "005", Region = "East", Name = "Test Entity 5", Status = 0
            });
            TestTable.ExecuteBatch(batch);

            batch = new TableBatchOperation();
            batch.Insert(new TestEntity {
                PartitionKey = "BBB", RowKey = "001", Region = "South", Name = "Test Entity 1", Status = 0
            });
            batch.Insert(new TestEntity {
                PartitionKey = "BBB", RowKey = "002", Region = "West", Name = "Test Entity 2", Status = 1
            });
            batch.Insert(new TestEntity {
                PartitionKey = "BBB", RowKey = "003", Region = "West", Name = "Test Entity 3", Status = 0
            });
            TestTable.ExecuteBatch(batch);

            string serviceBusQueueName = string.Format("test-input-{0}", FixtureId);
            string connectionString    = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.ServiceBus);
            var    namespaceManager    = NamespaceManager.CreateFromConnectionString(connectionString);

            namespaceManager.DeleteQueue(serviceBusQueueName);
            namespaceManager.CreateQueue(serviceBusQueueName);

            ServiceBusQueueClient = Microsoft.ServiceBus.Messaging.QueueClient.CreateFromConnectionString(connectionString, serviceBusQueueName);
        }
Ejemplo n.º 25
0
        public BlobTriggerBinding(string parameterName,
                                  IArgumentBinding <IStorageBlob> argumentBinding,
                                  IStorageAccount account,
                                  IBlobPathSource path,
                                  IHostIdProvider hostIdProvider,
                                  IQueueConfiguration queueConfiguration,
                                  IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
                                  IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
                                  IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
                                  ISharedContextProvider sharedContextProvider,
                                  TextWriter log)
        {
            if (argumentBinding == null)
            {
                throw new ArgumentNullException("argumentBinding");
            }

            if (account == null)
            {
                throw new ArgumentNullException("account");
            }

            if (path == null)
            {
                throw new ArgumentNullException("path");
            }

            if (hostIdProvider == null)
            {
                throw new ArgumentNullException("hostIdProvider");
            }

            if (queueConfiguration == null)
            {
                throw new ArgumentNullException("queueConfiguration");
            }

            if (backgroundExceptionDispatcher == null)
            {
                throw new ArgumentNullException("backgroundExceptionDispatcher");
            }

            if (blobWrittenWatcherSetter == null)
            {
                throw new ArgumentNullException("blobWrittenWatcherSetter");
            }

            if (messageEnqueuedWatcherSetter == null)
            {
                throw new ArgumentNullException("messageEnqueuedWatcherSetter");
            }

            if (sharedContextProvider == null)
            {
                throw new ArgumentNullException("sharedContextProvider");
            }

            if (log == null)
            {
                throw new ArgumentNullException("log");
            }

            _parameterName                 = parameterName;
            _argumentBinding               = argumentBinding;
            _account                       = account;
            _client                        = account.CreateBlobClient();
            _accountName                   = BlobClient.GetAccountName(_client);
            _path                          = path;
            _hostIdProvider                = hostIdProvider;
            _queueConfiguration            = queueConfiguration;
            _backgroundExceptionDispatcher = backgroundExceptionDispatcher;
            _blobWrittenWatcherSetter      = blobWrittenWatcherSetter;
            _messageEnqueuedWatcherSetter  = messageEnqueuedWatcherSetter;
            _sharedContextProvider         = sharedContextProvider;
            _log                 = log;
            _converter           = CreateConverter(_client);
            _bindingDataContract = CreateBindingDataContract(path);
        }
Ejemplo n.º 26
0
 public BlobTests(string account, string key)
 {
     this.client = new BlobClient(new CloudStorageAccount(account, key));
 }
Ejemplo n.º 27
0
        public async Task <IActionResult> Post([FromForm] IFormFileCollection files, string group, string groupidentifier)
        {
            Regex FileNameRegex         = new Regex(@"([a-zA-Z0-9\s_\.-:])+\.+.*$");
            Regex PictureRegexExtension = new Regex(@"^.*\.(jpg|JPG|gif|GIF|png|PNG|jpeg|JPEG)$");
            Regex AudioRegexExtension   = new Regex(@"^.*\.(mp3|wav|WAV|MP3|flac|FLAC)$");
            Regex VideoRegexExtension   = new Regex(@"^.*\.(mp4|MP4|MOV|mov|WMV|wmv|AVI|avi|WEBM|webm)$");
            Regex Extensions            = new Regex(@"\.(mp3|wav|WAV|MP3|jpg|JPG|gif|GIF|png|PNG|flac|FLAC|jpeg|JPEG|mp4|MP4|MOV|mov|WMV|wmv|AVI|avi|WEBM|webm)$");

            if (!files.Any())
            {
                return(BadRequest("No files given"));
            }

            foreach (var file in files)
            {
                if (!FileNameRegex.IsMatch(file.FileName))
                {
                    return(BadRequest("Invalid file name"));
                }
                if ((!PictureRegexExtension.IsMatch(file.FileName)) && (!AudioRegexExtension.IsMatch(file.FileName)) && (!VideoRegexExtension.IsMatch(file.FileName)))
                {
                    return(BadRequest("Invalid file extention"));
                }
                if (PictureRegexExtension.IsMatch(file.FileName))
                {
                    if (file.Length > (5 * 1024 * 1024))
                    {
                        return(BadRequest("File too large (5mb Max)"));
                    }
                }
                if (AudioRegexExtension.IsMatch(file.FileName))
                {
                    if (file.Length > (12 * 1024 * 1024))
                    {
                        return(BadRequest("File too large (12mb Max)"));
                    }
                }
                if (VideoRegexExtension.IsMatch(file.FileName))
                {
                    if (file.Length > (100 * 1024 * 1024))
                    {
                        return(BadRequest("File too large (100mb Max)"));
                    }
                }
            }
            BlobServiceClient blobServiceClient = new BlobServiceClient(_configuration.GetConnectionString("storage"));

            foreach (var file in files)
            {
                Match      extensionmatch = Extensions.Match(file.FileName);
                string     extension      = extensionmatch.ToString();
                MediaModel model          = new MediaModel();
                model.Group           = group;
                model.GroupIdentifier = groupidentifier;

                switch (group)
                {
                case "audio":
                case "video":
                case "profiles":
                case "campgrounds":
                case "campsites":
                {
                    _logger.LogDebug("uploading media");

                    BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(model.Group);
                    BlobClient          blobClient      = containerClient.GetBlobClient(model.GroupIdentifier + System.Guid.NewGuid().ToString() + extension);

                    await blobClient.UploadAsync(file.OpenReadStream());

                    _logger.LogDebug("uploaded media");

                    model.Uri = blobClient.Uri.ToString();
                    if (PictureRegexExtension.IsMatch(file.FileName))
                    {
                        model.AltText = "Picture of " + model.GroupIdentifier;
                    }
                    else if (AudioRegexExtension.IsMatch(file.FileName))
                    {
                        model.AltText = "Audio of " + model.GroupIdentifier;
                    }
                    else if (VideoRegexExtension.IsMatch(file.FileName))
                    {
                        model.AltText = "Video of " + model.GroupIdentifier;
                    }
                    _logger.LogDebug("adding media model");

                    await _unitOfWork.Media.InsertAsync(model);

                    await _unitOfWork.CommitAsync();

                    _logger.LogInformation($"added media model");

                    break;
                }

                default:
                {
                    return(BadRequest("Invalid group entered"));
                }
                }
            }

            return(Accepted());
        }
Ejemplo n.º 28
0
#pragma warning disable CA1806 // Do not ignore method results
        public override void Run(CancellationToken cancellationToken)
        {
            // traverse hierarchy down
            BlobServiceClient.GetBlobContainerClient(ContainerName);
            BlobContainerClient.GetBlobClient(BlobName);
            BlobContainerClient.GetBlobBaseClient(BlobName);
            BlobContainerClient.GetBlockBlobClient(BlobName);
            BlobContainerClient.GetPageBlobClient(BlobName);
            BlobContainerClient.GetAppendBlobClient(BlobName);

            // traverse hierarchy up
            BlobClient.GetParentBlobContainerClient();
            BlobContainerClient.GetParentBlobServiceClient();

            // BlobServiceClient ctors
            new BlobServiceClient(s_connectionString);
            new BlobServiceClient(BlobServiceClient.Uri);
            new BlobServiceClient(BlobServiceClient.Uri, s_azureSasCredential);
            new BlobServiceClient(BlobServiceClient.Uri, s_tokenCredential);
            new BlobServiceClient(BlobServiceClient.Uri, StorageSharedKeyCredential);

            // BlobContainerClient ctors
            new BlobContainerClient(s_connectionString, ContainerName);
            new BlobContainerClient(BlobContainerClient.Uri);
            new BlobContainerClient(BlobContainerClient.Uri, s_azureSasCredential);
            new BlobContainerClient(BlobContainerClient.Uri, s_tokenCredential);
            new BlobContainerClient(BlobContainerClient.Uri, StorageSharedKeyCredential);

            // BlobClient ctors
            new BlobClient(s_connectionString, ContainerName, BlobName);
            new BlobClient(BlobContainerClient.Uri);
            new BlobClient(BlobContainerClient.Uri, s_azureSasCredential);
            new BlobClient(BlobContainerClient.Uri, s_tokenCredential);
            new BlobClient(BlobContainerClient.Uri, StorageSharedKeyCredential);

            // BlobBaseClient ctors
            new BlobBaseClient(s_connectionString, ContainerName, BlobName);
            new BlobBaseClient(BlobContainerClient.Uri);
            new BlobBaseClient(BlobContainerClient.Uri, s_azureSasCredential);
            new BlobBaseClient(BlobContainerClient.Uri, s_tokenCredential);
            new BlobBaseClient(BlobContainerClient.Uri, StorageSharedKeyCredential);

            // AppendBlobClient ctors
            new AppendBlobClient(s_connectionString, ContainerName, BlobName);
            new AppendBlobClient(BlobContainerClient.Uri);
            new AppendBlobClient(BlobContainerClient.Uri, s_azureSasCredential);
            new AppendBlobClient(BlobContainerClient.Uri, s_tokenCredential);
            new AppendBlobClient(BlobContainerClient.Uri, StorageSharedKeyCredential);

            // BlockBlobClient ctors
            new BlockBlobClient(s_connectionString, ContainerName, BlobName);
            new BlockBlobClient(BlobContainerClient.Uri);
            new BlockBlobClient(BlobContainerClient.Uri, s_azureSasCredential);
            new BlockBlobClient(BlobContainerClient.Uri, s_tokenCredential);
            new BlockBlobClient(BlobContainerClient.Uri, StorageSharedKeyCredential);

            // PageBlobClient ctors
            new PageBlobClient(s_connectionString, ContainerName, BlobName);
            new PageBlobClient(BlobContainerClient.Uri);
            new PageBlobClient(BlobContainerClient.Uri, s_azureSasCredential);
            new PageBlobClient(BlobContainerClient.Uri, s_tokenCredential);
            new PageBlobClient(BlobContainerClient.Uri, StorageSharedKeyCredential);
        }
Ejemplo n.º 29
0
        private static async Task <bool> TryCreateAsync(BlobClient blob, CancellationToken cancellationToken)
        {
            try
            {
                var bytes = Encoding.UTF8.GetBytes(string.Empty);
                using (var stream = new MemoryStream(bytes))
                {
                    await blob.UploadAsync(stream, cancellationToken).ConfigureAwait(false);
                }

                return(true);
            }
            catch (RequestFailedException exception)
            {
                switch (exception.Status)
                {
                case 404:
                    break;

                case 409:
                case 412:
                    // The blob already exists, or is leased by someone else
                    return(false);

                default:
                    throw;
                }
            }

            var container = blob.GetParentBlobContainerClient();

            try
            {
                await container.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
            }
            catch (RequestFailedException exc)
                when(exc.Status == 409 && exc.ErrorCode == BlobErrorCode.ContainerBeingDeleted)
                {
                    throw;
                }

            try
            {
                var bytes = Encoding.UTF8.GetBytes(string.Empty);
                using (var stream = new MemoryStream(bytes))
                {
                    await blob.UploadAsync(stream, cancellationToken).ConfigureAwait(false);
                }
                return(true);
            }
            catch (RequestFailedException exception)
            {
                if (exception.Status == 409 || exception.Status == 412)
                {
                    // The blob already exists, or is leased by someone else
                    return(false);
                }

                throw;
            }
        }
        private async Task CleanupDevices(int deviceCount)
        {
            Console.WriteLine($"Using storage container {_blobContainerClient.Name} for importing device delete requests.");

            // Step 1: Collect the devices that need to be deleted.
            IReadOnlyList <ExportImportDevice> devicesToBeDeleted = await GetDeviceIdsToDeleteAsync(deviceCount);

            Console.WriteLine($"Discovered {devicesToBeDeleted.Count} devices for deletion.");

            string currentJobId = null;

            if (devicesToBeDeleted.Any())
            {
                try
                {
                    // Step 2: Write the new import data back to the blob.
                    using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);

                    // Retrieve the SAS Uri that will be used to grant access to the storage containers.
                    BlobClient blobClient   = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
                    var        uploadResult = await blobClient.UploadAsync(devicesFile, overwrite : true);

                    string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();

                    // Step 3: Call import using the same blob to delete all devices.
                    JobProperties importDevicesToBeDeletedProperties = JobProperties
                                                                       .CreateForImportJob(
                        inputBlobContainerUri: storageAccountSasUri,
                        outputBlobContainerUri: storageAccountSasUri,
                        inputBlobName: ImportExportDevicesFileName,
                        storageAuthenticationType: StorageAuthenticationType.KeyBased);

                    JobProperties importDevicesToBeDeletedJob = null;

                    Stopwatch jobTimer = Stopwatch.StartNew();
                    do
                    {
                        try
                        {
                            importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

                            currentJobId = importDevicesToBeDeletedJob.JobId;
                            break;
                        }
                        // Wait for pending jobs to finish.
                        catch (JobQuotaExceededException)
                        {
                            Console.WriteLine($"JobQuotaExceededException... waiting.");
                            await Task.Delay(s_waitDuration);
                        }
                    } while (jobTimer.Elapsed < s_maxJobDuration);

                    // Wait until job is finished.
                    jobTimer.Restart();
                    while (importDevicesToBeDeletedJob != null &&
                           jobTimer.Elapsed < s_maxJobDuration)
                    {
                        importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);

                        if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
                        {
                            // Job has finished executing.
                            Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
                            currentJobId = null;
                            break;
                        }

                        Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} after {jobTimer.Elapsed}.");
                        await Task.Delay(s_waitDuration);
                    }

                    if (importDevicesToBeDeletedJob?.Status != JobStatus.Completed)
                    {
                        throw new Exception("Importing devices job failed; exiting.");
                    }
                }
                finally
                {
                    if (!String.IsNullOrWhiteSpace(currentJobId))
                    {
                        Console.WriteLine($"Cancelling job {currentJobId}");
                        await _registryManager.CancelJobAsync(currentJobId);
                    }
                }
            }

            // Step 4: Delete the storage container created.
            await _blobContainerClient.DeleteAsync();

            Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
        }