/// <summary> /// Get blog info by url /// </summary> /// <param name="url"></param> /// <returns></returns> public virtual async Task <BlobInfo> GetBlobInfoAsync(string url) { if (string.IsNullOrEmpty(url)) { throw new ArgumentNullException(nameof(url)); } var uri = url.IsAbsoluteUrl() ? new Uri(url) : new Uri(_cloudBlobClient.BaseUri, url.TrimStart('/')); BlobInfo retVal = null; try { var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri); retVal = AbstractTypeFactory <BlobInfo> .TryCreateInstance(); retVal.Url = Uri.EscapeUriString(cloudBlob.Uri.ToString()); retVal.Name = Path.GetFileName(Uri.UnescapeDataString(cloudBlob.Uri.ToString())); retVal.ContentType = cloudBlob.Properties.ContentType; retVal.Size = cloudBlob.Properties.Length; retVal.ModifiedDate = cloudBlob.Properties.LastModified?.DateTime; retVal.RelativeUrl = cloudBlob.Uri.LocalPath; } catch (Exception) { //Azure blob storage client does not provide method to check blob url exist without throwing exception } return(retVal); }
public async Task <DocumentProcessingJob> StartPreprocessor([ActivityTrigger] DocumentProcessingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec) { var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -"; var orchestrationContainer = orchestrationBlobClient.GetContainerReference($"{job.OrchestrationId}"); job.OrchestrationContainerName = orchestrationContainer.Name; await orchestrationContainer.CreateIfNotExistsAsync(); log.LogTrace($"orchestrationContainerName={job.OrchestrationContainerName}"); var stagingBlobUri = new Uri(job.StagingBlobUrl); var stagingContainerName = stagingBlobUri.Segments[1].Split('/')[0]; job.DocumentFormat = stagingContainerName; var stagingContainer = stagingBlobClient.GetContainerReference(stagingContainerName); string orchestrationBlobName = $"{stagingContainerName}-{Uri.UnescapeDataString(stagingBlobUri.Segments.Last())}"; job.OrchestrationBlobName = orchestrationBlobName; log.LogTrace($"orchestrationBlobName={job.OrchestrationBlobName}"); var orchestrationBlob = orchestrationContainer.GetBlockBlobReference(orchestrationBlobName); job.OrchestrationBlobUrl = orchestrationBlob.Uri.ToString(); var stagingBlob = await stagingBlobClient.GetBlobReferenceFromServerAsync(stagingBlobUri); await orchestrationBlob.StartCopyAsync(new Uri(GetSharedAccessUri(stagingBlob.Name, stagingContainer))); log.LogInformation($"{snip} - Completed successfully"); return(job); }
/// <summary> /// Delete the file from storage container if exists /// </summary> /// <param name="uri">The uri</param> /// <returns></returns> public async Task DeleteFileAsync(Uri uri) { _logger.LogInformation($"Delete File at {uri}"); var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri); await blob.DeleteIfExistsAsync(); }
public async Task UploadBlob(StorageFile targetFile, string containerName, string prefix) { var container = blobClient.GetContainerReference(containerName); var fullUrl = container.Uri.AbsolutePath + prefix + targetFile.Name; var blobUri = new Uri(fullUrl); var blobRef = await blobClient.GetBlobReferenceFromServerAsync(blobUri); blobRef.UploadFromFileAsync(targetFile); }
public async Task <byte[]> GetBlob(Uri key) { var blob = await _client.GetBlobReferenceFromServerAsync(key); using (var stream = new MemoryStream()) { await blob.DownloadToStreamAsync(stream); return(stream.ToArray()); } }
public Task DeleteAsync(CancellationToken cancellationToken = default) { return(Task.Run(async() => { foreach (IListBlobItem listItem in _directory.ListBlobs()) { ICloudBlob blob = await _client.GetBlobReferenceFromServerAsync( listItem.Uri, cancellationToken) .ConfigureAwait(false); await blob.DeleteIfExistsAsync(cancellationToken) .ConfigureAwait(false); } }, cancellationToken)); }
public ICloudBlob GetBlobReferenceFromServer(Uri blobUri) { var task = _client.GetBlobReferenceFromServerAsync(blobUri); task.Wait(); return(task.Result); }
public async Task <Dictionary <string, object> > GetPropertiesAsync(Uri resourceUri, CancellationToken cancellationToken) { EnsureArg.IsNotNull(resourceUri, nameof(resourceUri)); try { return(await _integrationStoreRetryExceptionPolicyFactory .RetryPolicy .ExecuteAsync(async() => { CloudBlobClient cloudBlobClient = await _integrationDataStoreClientInitializer.GetAuthorizedClientAsync(cancellationToken); ICloudBlob blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(resourceUri); Dictionary <string, object> result = new Dictionary <string, object>(); result[IntegrationDataStoreClientConstants.BlobPropertyETag] = blob.Properties.ETag; result[IntegrationDataStoreClientConstants.BlobPropertyLength] = blob.Properties.Length; return result; })); } catch (StorageException storageEx) { _logger.LogInformation(storageEx, "Failed to get properties of blob {0}", resourceUri); HttpStatusCode statusCode = StorageExceptionParser.ParseStorageException(storageEx); throw new IntegrationDataStoreException(storageEx.Message, statusCode); } }
public static async Task <string> Run([ActivityTrigger] DurableActivityContextBase myBlob, TraceWriter log) { //log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes"); log.Info("in final step......................"); string storagePath = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storagePath); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer destinationContainer = blobClient.GetContainerReference(Environment.GetEnvironmentVariable("AcceptDestinationPath")); //await cloudBlobContainer.CreateAsync(); await destinationContainer.CreateIfNotExistsAsync(); // Set the permissions so the blobs are public. BlobContainerPermissions permissions = new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }; await destinationContainer.SetPermissionsAsync(permissions); RequestApplication requestMetadata = myBlob.GetInput <RequestApplication>(); var sourceBlob = blobClient.GetBlobReferenceFromServerAsync(new Uri(requestMetadata.LocationUrl)).Result; // var destinationContainer = blobClient.GetContainerReference(destinationContainer); var destinationBlob = destinationContainer.GetBlobReference(sourceBlob.Name); await destinationBlob.StartCopyAsync(sourceBlob.Uri); Task.Delay(TimeSpan.FromSeconds(15)).Wait(); await sourceBlob.DeleteAsync(); return("Copied"); }
public async override Task <DocumentProcessingJob> Save(DocumentProcessingJob job, ILogger log, string snip) { var documentBlob = await orchestrationBlobClient.GetBlobReferenceFromServerAsync(new Uri(job.DocumentBlobUrl)); string documentBlobContents; using (var memoryStream = new MemoryStream()) { await documentBlob.DownloadToStreamAsync(memoryStream); documentBlobContents = Encoding.UTF8.GetString(memoryStream.ToArray()); } var document = JsonConvert.DeserializeObject <Document>(documentBlobContents); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId); ContainerProperties containerProperties = new ContainerProperties(cosmosContainerId, partitionKeyPath: "/Account"); Container container = await database.CreateContainerIfNotExistsAsync( containerProperties, throughput : 400); _ = await container.CreateItemAsync(document, new PartitionKey(document.Account), new ItemRequestOptions() { EnableContentResponseOnWrite = false }); log.LogDebug($"{snip} document {document.DocumentNumber} was saved to Cosmos - database={cosmosDatabaseId}, container={cosmosContainerId})"); return(job); }
public async Task DeleteBlob([ActivityTrigger] DataBusBlobData blobData, ILogger log) { var blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(blobData.Path)); log.LogInformation($"Deleting blob at {blobData.Path}"); await blob.DeleteIfExistsAsync(); }
/// <summary> /// 스토리지 파일 제거 /// </summary> /// <param name="sourceUri"></param> public async Task DeleteAsync(Uri sourceUri) { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri); await blob.DeleteAsync(); }
public async virtual Task <bool> PathExistsAsync(string path) { path = NormalizePath(path); var cacheKey = CacheKey.With(GetType(), "PathExistsAsync", path); return(await _memoryCache.GetOrCreateExclusiveAsync(cacheKey, async (cacheEntry) => { cacheEntry.AddExpirationToken(ContentBlobCacheRegion.CreateChangeToken()); // If requested path is a directory we should always return true because Azure blob storage does not support checking if directories exist var result = string.IsNullOrEmpty(Path.GetExtension(path)); if (!result) { var url = GetAbsoluteUrl(path); try { result = await(await _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(url))).ExistsAsync(); } catch (Exception) { //Azure blob storage client does not provide method to check blob url exist without throwing exception } } return result; })); }
public async Task ProcessMessages(CancellationToken token) { foreach (CloudQueueMessage message in await _queue.GetMessagesAsync(1, TimeSpan.FromMinutes(1), null, null)) { var jsonMessage = JObject.Parse(message.AsString); Uri uploadedUri = new Uri(jsonMessage["data"]["url"].ToString()); string uploadedFile = Path.GetFileName(jsonMessage["data"]["url"].ToString()); Console.WriteLine("Blob available at: {0}", jsonMessage["data"]["url"]); var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uploadedUri); string destinationFile = Path.Combine(_downloadDestination, Path.GetFileName(uploadedFile)); Console.WriteLine("Downloading to {0}...", destinationFile); await cloudBlob.DownloadToFileAsync(destinationFile, FileMode.Create); Console.WriteLine("Done."); await _queue.DeleteMessageAsync(message); Console.WriteLine(); } if (!token.IsCancellationRequested) { await Task.Delay(1000); } }
private static async Task <List <string[]> > GetData(CloudBlobClient client, Uri uri, TraceWriter log) { log.Info($"Downloading: {uri}"); var blobRef = await client.GetBlobReferenceFromServerAsync(uri); var result = new List <string[]>(); using (var stream = new MemoryStream()) { await blobRef.DownloadToStreamAsync(stream); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var count = 0; var line = string.Empty; while ((line = await reader.ReadLineAsync()) != null) { // ignore header if (count != 0) { result.Add(line.Split(',')); } count++; } } } return(result); }
/// <summary> /// 스토리지 파일을 파일로 다운로드 /// </summary> /// <param name="sourceUri"></param> /// <param name="destfilename"></param> /// <param name="overwrite"></param> /// <param name="useSequencedName"></param> /// <returns></returns> public async Task <string> DownloadAsync(Uri sourceUri, string destfilename, bool overwrite = false, bool useSequencedName = true) { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri); if (overwrite == true) { await blob.DownloadToFileAsync(destfilename, FileMode.Create); FileInfo fi = new FileInfo(destfilename); if (fi.Exists) { return(fi.FullName); } else { return(null); } } else { FileInfo fileInfo = new FileInfo(destfilename); string destfilenameRe = destfilename; string filename = destfilename.Substring(0, destfilename.Length - fileInfo.Extension.Length); uint i = 0; while (true) { if (File.Exists(destfilenameRe) == true) { if (useSequencedName == true) { i++; destfilenameRe = filename + "[" + i.ToString() + "]" + fileInfo.Extension; continue; } else { throw new DuplicateFileException(); } } else { await blob.DownloadToFileAsync(destfilenameRe, FileMode.CreateNew); FileInfo fi = new FileInfo(destfilenameRe); if (fi.Exists) { return(fi.FullName); } else { return(null); } } } } }
protected Task <ICloudBlob> GetBlobRefAsync(string virtualPath) { string subPath = StripPrefix(virtualPath).Trim('/', '\\'); string relativeBlobURL = string.Format("{0}/{1}", CloudBlobClient.BaseUri.OriginalString.TrimEnd('/', '\\'), subPath); return(CloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(relativeBlobURL))); }
public async Task <byte[]> GetItem(Uri url) { var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(url); var stream = await blob.OpenReadAsync(); return(stream.CopyToBytes()); }
/// <summary> /// 스토리지 파일을 스트림에 다운로드 /// </summary> /// <param name="sourceUri"></param> /// <param name="deststream"></param> /// <returns></returns> public async Task <string> DownloadAsync(Uri sourceUri, Stream deststream) { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri); await blob.DownloadToStreamAsync(deststream); return(""); }
private Task<ICloudBlob> GetBlobRefAsync(string virtualPath) { var subPath = StripPrefixWithTenant(virtualPath); var fileName = EncodeFileName(subPath); var tenantPart = GetTenantPart(virtualPath); var relativeBlobUrl = $"{_cloudBlobClient.BaseUri.OriginalString.TrimEnd('/', '\\')}/{BlobContainerName}/{tenantPart}/{fileName}"; return _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(relativeBlobUrl)); }
public async Task <string> CheckPreprocessorStatus([ActivityTrigger] DocumentProcessingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec) { var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -"; string jobStatus = "Pending"; var orchestrationBlob = await orchestrationBlobClient.GetBlobReferenceFromServerAsync(new Uri(job.OrchestrationBlobUrl)); await orchestrationBlob.FetchAttributesAsync(); if (orchestrationBlob.CopyState.Status == CopyStatus.Pending) { jobStatus = "Pending"; } if (orchestrationBlob.CopyState.Status == CopyStatus.Success) { jobStatus = "Completed"; } log.LogInformation($"{snip} Job Status: {jobStatus}"); return(jobStatus); }
// Remember to dispose the stream when done with it. public async Task <MemoryStream> GetBlobStreamAsync(Uri blobUri) { _logger.Write("Getting blob: {0}", blobUri.ToString()); var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(blobUri); var stream = new MemoryStream(); await blob.DownloadToStreamAsync(stream); stream.Position = 0; return(stream); }
public static async Task DeleteBlobAsync(string account, string key, string blobUrl) { if (string.IsNullOrEmpty(blobUrl)) { return; } CloudBlobClient blobClient = Client.GetBlobClient(account, key); ICloudBlob blobRef = await blobClient.GetBlobReferenceFromServerAsync(new Uri(blobUrl)); await blobRef.DeleteAsync(); }
/// <summary> /// Generates a record in the jobstable and initializes the currentBlob /// </summary> private static bool InitializeJob() { //Initialize Job jobInfo = new DynamicTableEntity() { PartitionKey = conf.BatchId, RowKey = jobId }; jobInfo.Properties.Add("WorkerId", EntityProperty.GeneratePropertyForString(workerId)); jobInfo.Properties.Add("BlobUrl", EntityProperty.GeneratePropertyForString(currentBlobUrl)); jobInfo.Properties.Add("Status", EntityProperty.GeneratePropertyForString("Started")); jobInfo.Properties.Add("DownStarted", EntityProperty.GeneratePropertyForString(Utilities.Now2Log)); //preparing entity for inserting the job info in the jobs table var blobName = Utilities.BlobNameByUrl(currentBlobUrl); srcContainer = srcBlobClient.GetContainerReference(conf.SrcContainerName); bool sourceExists; if (sourceExists = srcContainer.ExistsAsync().Result) //Checking existence of source blob { var blobReference = srcContainer.GetBlobReference(blobName); if (sourceExists = blobReference.ExistsAsync().Result) { //checking existence of destiny blog. If it exists, the line parameter checking overwrites is evaluated var destBlobReference = destContainer.GetBlobReference(blobName); var destinyExists = destBlobReference.ExistsAsync().Result; var okToUpload = !destinyExists || conf.OverwriteIfExists; if (okToUpload) { currentBlob = (CloudBlob)srcBlobClient.GetBlobReferenceFromServerAsync(new Uri(currentBlobUrl)).Result; _ = currentBlob.FetchAttributesAsync(); jobInfo.Properties.Add("Size", EntityProperty.GeneratePropertyForLong(currentBlob.Properties.Length)); UpdateJobInfo(); return(true); } else { UpdateJobInfo("DestBlobExists"); LogUpdate($"Destiny {blobName} blob already exists and overwriting has not been declared in the batch parameter. Job will be removed from queue."); } } } if (!sourceExists) { UpdateJobInfo("SrcBlobMissing"); LogUpdate($"Source {blobName} blob missing. Job will be removed from queue."); } jobsQueue.DeleteMessageAsync(jobMessage); return(false); }
public async Task <string> GetBlobSignature(int clientId, string blobURI) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=vjdemodata;AccountKey=c0x/avvhx8/5Dw0W/K1PnqI4xaD85vxRyupDNG+sDJV5w2S0hOVjBNKFpVWCHi2swZQVqOdbIKGgt+HLo2gy+w==;EndpointSuffix=core.windows.net"); //Create the blob client object. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); //Get a reference to a container to use for the sample code, and create it if it does not exist. CloudBlob blob = (CloudBlob)blobClient.GetBlobReferenceFromServerAsync(new Uri(blobURI)).Result; return(GetBlobSasUri(blob)); }
public async Task GivenDataStream_WhenAppendToBlob_ThenDataShouldBeAppended() { IIntegrationDataStoreClientInitilizer <CloudBlobClient> initializer = GetClientInitializer(); CloudBlobClient client = await initializer.GetAuthorizedClientAsync(CancellationToken.None); string containerName = Guid.NewGuid().ToString("N"); string blobName = Guid.NewGuid().ToString("N"); Uri blobUri = new Uri(Path.Combine(client.StorageUri.PrimaryUri.ToString(), $"{containerName}/{blobName}")); try { AzureBlobIntegrationDataStoreClient blobClient = new AzureBlobIntegrationDataStoreClient(initializer, GetIntegrationDataStoreConfigurationOption(), new NullLogger <AzureBlobIntegrationDataStoreClient>()); await blobClient.PrepareResourceAsync(containerName, blobName, CancellationToken.None); long count = 30; List <string> blockIds = new List <string>(); for (long i = 0; i < count; ++i) { using Stream input = new MemoryStream(Encoding.UTF8.GetBytes(i.ToString() + "\r\n")); string blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); await blobClient.UploadBlockAsync(blobUri, input, blockId, CancellationToken.None); blockIds.Add(blockId); } await blobClient.CommitAsync(blobUri, blockIds.ToArray(), CancellationToken.None); ICloudBlob output = await client.GetBlobReferenceFromServerAsync(blobUri); using Stream outputStream = new MemoryStream(); await output.DownloadToStreamAsync(outputStream); outputStream.Position = 0; using StreamReader reader = new StreamReader(outputStream); long currentLine = 0; string content = null; while ((content = await reader.ReadLineAsync()) != null) { Assert.Equal(currentLine.ToString(), content); currentLine++; } Assert.Equal(count, currentLine); } finally { var container = client.GetContainerReference(containerName); await container.DeleteIfExistsAsync(); } }
public async Task <Uri> UploadAsync(string text, string replaceTo = null) { await _container.CreateIfNotExistsAsync(); CloudBlockBlob block; if (string.IsNullOrEmpty(replaceTo)) { var prefix = DateTimeOffset.Now.ToString("yyyy-MM-dd/hh/mm"); var guid = Guid.NewGuid().ToString(); block = _container.GetBlockBlobReference($"{prefix}/{guid}.log"); } else { block = (CloudBlockBlob)await _client.GetBlobReferenceFromServerAsync(new Uri(replaceTo)); } await block.UploadTextAsync(text); return(block.Uri); }
public async Task <string> GetSas(string BlobUri) { var blob = await _BlobClient.GetBlobReferenceFromServerAsync(new Uri(BlobUri)); SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); policy.SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(24); policy.Permissions = SharedAccessBlobPermissions.Read; string sasToken = blob.GetSharedAccessSignature(policy); return(BlobUri + sasToken); }
private async Task <CloudBlob> GetBlobAsync(Uri blobUri) { try { var blob = await _blobClient.GetBlobReferenceFromServerAsync(blobUri, accessCondition : null, options : _blobRequestOptions, operationContext : null); return(blob as CloudBlob); } catch (Exception) { return(null); } }
/// <summary> /// Get blob info by url /// </summary> /// <param name="blobUrl"></param> /// <returns></returns> public virtual async Task <BlobInfo> GetBlobInfoAsync(string blobUrl) { if (string.IsNullOrEmpty(blobUrl)) { throw new ArgumentNullException(nameof(blobUrl)); } var uri = blobUrl.IsAbsoluteUrl() ? new Uri(blobUrl) : new Uri(_cloudBlobClient.BaseUri, blobUrl.TrimStart('/')); BlobInfo retVal = null; try { var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri); retVal = ConvertBlobToBlobInfo(cloudBlob); } catch (Exception) { //Azure blob storage client does not provide method to check blob url exist without throwing exception } return(retVal); }