/// <summary>
/// Get blog info by url
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public virtual async Task <BlobInfo> GetBlobInfoAsync(string url)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException(nameof(url));
}
var uri = url.IsAbsoluteUrl() ? new Uri(url) : new Uri(_cloudBlobClient.BaseUri, url.TrimStart('/'));
BlobInfo retVal = null;
try
{
var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri);
retVal = AbstractTypeFactory <BlobInfo> .TryCreateInstance();
retVal.Url = Uri.EscapeUriString(cloudBlob.Uri.ToString());
retVal.Name = Path.GetFileName(Uri.UnescapeDataString(cloudBlob.Uri.ToString()));
retVal.ContentType = cloudBlob.Properties.ContentType;
retVal.Size = cloudBlob.Properties.Length;
retVal.ModifiedDate = cloudBlob.Properties.LastModified?.DateTime;
retVal.RelativeUrl = cloudBlob.Uri.LocalPath;
}
catch (Exception)
{
//Azure blob storage client does not provide method to check blob url exist without throwing exception
}
return(retVal);
}
public async Task <DocumentProcessingJob> StartPreprocessor([ActivityTrigger] DocumentProcessingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -";
var orchestrationContainer = orchestrationBlobClient.GetContainerReference($"{job.OrchestrationId}");
job.OrchestrationContainerName = orchestrationContainer.Name;
await orchestrationContainer.CreateIfNotExistsAsync();
log.LogTrace($"orchestrationContainerName={job.OrchestrationContainerName}");
var stagingBlobUri = new Uri(job.StagingBlobUrl);
var stagingContainerName = stagingBlobUri.Segments[1].Split('/')[0];
job.DocumentFormat = stagingContainerName;
var stagingContainer = stagingBlobClient.GetContainerReference(stagingContainerName);
string orchestrationBlobName = $"{stagingContainerName}-{Uri.UnescapeDataString(stagingBlobUri.Segments.Last())}";
job.OrchestrationBlobName = orchestrationBlobName;
log.LogTrace($"orchestrationBlobName={job.OrchestrationBlobName}");
var orchestrationBlob = orchestrationContainer.GetBlockBlobReference(orchestrationBlobName);
job.OrchestrationBlobUrl = orchestrationBlob.Uri.ToString();
var stagingBlob = await stagingBlobClient.GetBlobReferenceFromServerAsync(stagingBlobUri);
await orchestrationBlob.StartCopyAsync(new Uri(GetSharedAccessUri(stagingBlob.Name, stagingContainer)));
log.LogInformation($"{snip} - Completed successfully");
return(job);
}
/// <summary>
/// Delete the file from storage container if exists
/// </summary>
/// <param name="uri">The uri</param>
/// <returns></returns>
public async Task DeleteFileAsync(Uri uri)
{
_logger.LogInformation($"Delete File at {uri}");
var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri);
await blob.DeleteIfExistsAsync();
}
public async Task UploadBlob(StorageFile targetFile, string containerName, string prefix)
{
var container = blobClient.GetContainerReference(containerName);
var fullUrl = container.Uri.AbsolutePath + prefix + targetFile.Name;
var blobUri = new Uri(fullUrl);
var blobRef = await blobClient.GetBlobReferenceFromServerAsync(blobUri);
blobRef.UploadFromFileAsync(targetFile);
}
public async Task <byte[]> GetBlob(Uri key)
{
var blob = await _client.GetBlobReferenceFromServerAsync(key);
using (var stream = new MemoryStream())
{
await blob.DownloadToStreamAsync(stream);
return(stream.ToArray());
}
}
public Task DeleteAsync(CancellationToken cancellationToken = default)
{
return(Task.Run(async() =>
{
foreach (IListBlobItem listItem in _directory.ListBlobs())
{
ICloudBlob blob = await _client.GetBlobReferenceFromServerAsync(
listItem.Uri, cancellationToken)
.ConfigureAwait(false);
await blob.DeleteIfExistsAsync(cancellationToken)
.ConfigureAwait(false);
}
}, cancellationToken));
}
public ICloudBlob GetBlobReferenceFromServer(Uri blobUri)
{
var task = _client.GetBlobReferenceFromServerAsync(blobUri);
task.Wait();
return(task.Result);
}
public async Task <Dictionary <string, object> > GetPropertiesAsync(Uri resourceUri, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(resourceUri, nameof(resourceUri));
try
{
return(await _integrationStoreRetryExceptionPolicyFactory
.RetryPolicy
.ExecuteAsync(async() =>
{
CloudBlobClient cloudBlobClient = await _integrationDataStoreClientInitializer.GetAuthorizedClientAsync(cancellationToken);
ICloudBlob blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(resourceUri);
Dictionary <string, object> result = new Dictionary <string, object>();
result[IntegrationDataStoreClientConstants.BlobPropertyETag] = blob.Properties.ETag;
result[IntegrationDataStoreClientConstants.BlobPropertyLength] = blob.Properties.Length;
return result;
}));
}
catch (StorageException storageEx)
{
_logger.LogInformation(storageEx, "Failed to get properties of blob {0}", resourceUri);
HttpStatusCode statusCode = StorageExceptionParser.ParseStorageException(storageEx);
throw new IntegrationDataStoreException(storageEx.Message, statusCode);
}
}
public static async Task <string> Run([ActivityTrigger] DurableActivityContextBase myBlob, TraceWriter log)
{
//log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes");
log.Info("in final step......................");
string storagePath = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storagePath);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer destinationContainer = blobClient.GetContainerReference(Environment.GetEnvironmentVariable("AcceptDestinationPath"));
//await cloudBlobContainer.CreateAsync();
await destinationContainer.CreateIfNotExistsAsync();
// Set the permissions so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await destinationContainer.SetPermissionsAsync(permissions);
RequestApplication requestMetadata = myBlob.GetInput <RequestApplication>();
var sourceBlob = blobClient.GetBlobReferenceFromServerAsync(new Uri(requestMetadata.LocationUrl)).Result;
// var destinationContainer = blobClient.GetContainerReference(destinationContainer);
var destinationBlob = destinationContainer.GetBlobReference(sourceBlob.Name);
await destinationBlob.StartCopyAsync(sourceBlob.Uri);
Task.Delay(TimeSpan.FromSeconds(15)).Wait();
await sourceBlob.DeleteAsync();
return("Copied");
}
public async override Task <DocumentProcessingJob> Save(DocumentProcessingJob job, ILogger log, string snip)
{
var documentBlob = await orchestrationBlobClient.GetBlobReferenceFromServerAsync(new Uri(job.DocumentBlobUrl));
string documentBlobContents;
using (var memoryStream = new MemoryStream())
{
await documentBlob.DownloadToStreamAsync(memoryStream);
documentBlobContents = Encoding.UTF8.GetString(memoryStream.ToArray());
}
var document = JsonConvert.DeserializeObject <Document>(documentBlobContents);
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(cosmosDatabaseId);
ContainerProperties containerProperties = new ContainerProperties(cosmosContainerId, partitionKeyPath: "/Account");
Container container = await database.CreateContainerIfNotExistsAsync(
containerProperties,
throughput : 400);
_ = await container.CreateItemAsync(document, new PartitionKey(document.Account),
new ItemRequestOptions()
{
EnableContentResponseOnWrite = false
});
log.LogDebug($"{snip} document {document.DocumentNumber} was saved to Cosmos - database={cosmosDatabaseId}, container={cosmosContainerId})");
return(job);
}
public async Task DeleteBlob([ActivityTrigger] DataBusBlobData blobData, ILogger log)
{
var blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(blobData.Path));
log.LogInformation($"Deleting blob at {blobData.Path}");
await blob.DeleteIfExistsAsync();
}
/// <summary>
/// 스토리지 파일 제거
/// </summary>
/// <param name="sourceUri"></param>
public async Task DeleteAsync(Uri sourceUri)
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri);
await blob.DeleteAsync();
}
public async virtual Task <bool> PathExistsAsync(string path)
{
path = NormalizePath(path);
var cacheKey = CacheKey.With(GetType(), "PathExistsAsync", path);
return(await _memoryCache.GetOrCreateExclusiveAsync(cacheKey, async (cacheEntry) =>
{
cacheEntry.AddExpirationToken(ContentBlobCacheRegion.CreateChangeToken());
// If requested path is a directory we should always return true because Azure blob storage does not support checking if directories exist
var result = string.IsNullOrEmpty(Path.GetExtension(path));
if (!result)
{
var url = GetAbsoluteUrl(path);
try
{
result = await(await _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(url))).ExistsAsync();
}
catch (Exception)
{
//Azure blob storage client does not provide method to check blob url exist without throwing exception
}
}
return result;
}));
}
public async Task ProcessMessages(CancellationToken token)
{
foreach (CloudQueueMessage message in await _queue.GetMessagesAsync(1, TimeSpan.FromMinutes(1), null, null))
{
var jsonMessage = JObject.Parse(message.AsString);
Uri uploadedUri = new Uri(jsonMessage["data"]["url"].ToString());
string uploadedFile = Path.GetFileName(jsonMessage["data"]["url"].ToString());
Console.WriteLine("Blob available at: {0}", jsonMessage["data"]["url"]);
var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uploadedUri);
string destinationFile = Path.Combine(_downloadDestination, Path.GetFileName(uploadedFile));
Console.WriteLine("Downloading to {0}...", destinationFile);
await cloudBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
Console.WriteLine("Done.");
await _queue.DeleteMessageAsync(message);
Console.WriteLine();
}
if (!token.IsCancellationRequested)
{
await Task.Delay(1000);
}
}
private static async Task <List <string[]> > GetData(CloudBlobClient client, Uri uri, TraceWriter log)
{
log.Info($"Downloading: {uri}");
var blobRef = await client.GetBlobReferenceFromServerAsync(uri);
var result = new List <string[]>();
using (var stream = new MemoryStream())
{
await blobRef.DownloadToStreamAsync(stream);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var count = 0;
var line = string.Empty;
while ((line = await reader.ReadLineAsync()) != null)
{
// ignore header
if (count != 0)
{
result.Add(line.Split(','));
}
count++;
}
}
}
return(result);
}
/// <summary>
/// 스토리지 파일을 파일로 다운로드
/// </summary>
/// <param name="sourceUri"></param>
/// <param name="destfilename"></param>
/// <param name="overwrite"></param>
/// <param name="useSequencedName"></param>
/// <returns></returns>
public async Task <string> DownloadAsync(Uri sourceUri, string destfilename, bool overwrite = false, bool useSequencedName = true)
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri);
if (overwrite == true)
{
await blob.DownloadToFileAsync(destfilename, FileMode.Create);
FileInfo fi = new FileInfo(destfilename);
if (fi.Exists)
{
return(fi.FullName);
}
else
{
return(null);
}
}
else
{
FileInfo fileInfo = new FileInfo(destfilename);
string destfilenameRe = destfilename;
string filename = destfilename.Substring(0, destfilename.Length - fileInfo.Extension.Length);
uint i = 0;
while (true)
{
if (File.Exists(destfilenameRe) == true)
{
if (useSequencedName == true)
{
i++;
destfilenameRe = filename + "[" + i.ToString() + "]" + fileInfo.Extension;
continue;
}
else
{
throw new DuplicateFileException();
}
}
else
{
await blob.DownloadToFileAsync(destfilenameRe, FileMode.CreateNew);
FileInfo fi = new FileInfo(destfilenameRe);
if (fi.Exists)
{
return(fi.FullName);
}
else
{
return(null);
}
}
}
}
}
protected Task <ICloudBlob> GetBlobRefAsync(string virtualPath)
{
string subPath = StripPrefix(virtualPath).Trim('/', '\\');
string relativeBlobURL = string.Format("{0}/{1}", CloudBlobClient.BaseUri.OriginalString.TrimEnd('/', '\\'), subPath);
return(CloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(relativeBlobURL)));
}
public async Task <byte[]> GetItem(Uri url)
{
var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(url);
var stream = await blob.OpenReadAsync();
return(stream.CopyToBytes());
}
/// <summary>
/// 스토리지 파일을 스트림에 다운로드
/// </summary>
/// <param name="sourceUri"></param>
/// <param name="deststream"></param>
/// <returns></returns>
public async Task <string> DownloadAsync(Uri sourceUri, Stream deststream)
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ICloudBlob blob = await blobClient.GetBlobReferenceFromServerAsync(sourceUri);
await blob.DownloadToStreamAsync(deststream);
return("");
}
private Task<ICloudBlob> GetBlobRefAsync(string virtualPath)
{
var subPath = StripPrefixWithTenant(virtualPath);
var fileName = EncodeFileName(subPath);
var tenantPart = GetTenantPart(virtualPath);
var relativeBlobUrl = $"{_cloudBlobClient.BaseUri.OriginalString.TrimEnd('/', '\\')}/{BlobContainerName}/{tenantPart}/{fileName}";
return _cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(relativeBlobUrl));
}
public async Task <string> CheckPreprocessorStatus([ActivityTrigger] DocumentProcessingJob job, ILogger log, Microsoft.Azure.WebJobs.ExecutionContext ec)
{
var snip = $"Orchestration { job.OrchestrationId}: { ec.FunctionName} -";
string jobStatus = "Pending";
var orchestrationBlob = await orchestrationBlobClient.GetBlobReferenceFromServerAsync(new Uri(job.OrchestrationBlobUrl));
await orchestrationBlob.FetchAttributesAsync();
if (orchestrationBlob.CopyState.Status == CopyStatus.Pending)
{
jobStatus = "Pending";
}
if (orchestrationBlob.CopyState.Status == CopyStatus.Success)
{
jobStatus = "Completed";
}
log.LogInformation($"{snip} Job Status: {jobStatus}");
return(jobStatus);
}
// Remember to dispose the stream when done with it.
public async Task <MemoryStream> GetBlobStreamAsync(Uri blobUri)
{
_logger.Write("Getting blob: {0}", blobUri.ToString());
var blob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(blobUri);
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
stream.Position = 0;
return(stream);
}
public static async Task DeleteBlobAsync(string account, string key, string blobUrl)
{
if (string.IsNullOrEmpty(blobUrl))
{
return;
}
CloudBlobClient blobClient = Client.GetBlobClient(account, key);
ICloudBlob blobRef = await blobClient.GetBlobReferenceFromServerAsync(new Uri(blobUrl));
await blobRef.DeleteAsync();
}
/// <summary>
/// Generates a record in the jobstable and initializes the currentBlob
/// </summary>
private static bool InitializeJob()
{
//Initialize Job
jobInfo = new DynamicTableEntity()
{
PartitionKey = conf.BatchId,
RowKey = jobId
};
jobInfo.Properties.Add("WorkerId", EntityProperty.GeneratePropertyForString(workerId));
jobInfo.Properties.Add("BlobUrl", EntityProperty.GeneratePropertyForString(currentBlobUrl));
jobInfo.Properties.Add("Status", EntityProperty.GeneratePropertyForString("Started"));
jobInfo.Properties.Add("DownStarted", EntityProperty.GeneratePropertyForString(Utilities.Now2Log));
//preparing entity for inserting the job info in the jobs table
var blobName = Utilities.BlobNameByUrl(currentBlobUrl);
srcContainer = srcBlobClient.GetContainerReference(conf.SrcContainerName);
bool sourceExists;
if (sourceExists = srcContainer.ExistsAsync().Result) //Checking existence of source blob
{
var blobReference = srcContainer.GetBlobReference(blobName);
if (sourceExists = blobReference.ExistsAsync().Result)
{
//checking existence of destiny blog. If it exists, the line parameter checking overwrites is evaluated
var destBlobReference = destContainer.GetBlobReference(blobName);
var destinyExists = destBlobReference.ExistsAsync().Result;
var okToUpload = !destinyExists || conf.OverwriteIfExists;
if (okToUpload)
{
currentBlob = (CloudBlob)srcBlobClient.GetBlobReferenceFromServerAsync(new Uri(currentBlobUrl)).Result;
_ = currentBlob.FetchAttributesAsync();
jobInfo.Properties.Add("Size", EntityProperty.GeneratePropertyForLong(currentBlob.Properties.Length));
UpdateJobInfo();
return(true);
}
else
{
UpdateJobInfo("DestBlobExists");
LogUpdate($"Destiny {blobName} blob already exists and overwriting has not been declared in the batch parameter. Job will be removed from queue.");
}
}
}
if (!sourceExists)
{
UpdateJobInfo("SrcBlobMissing");
LogUpdate($"Source {blobName} blob missing. Job will be removed from queue.");
}
jobsQueue.DeleteMessageAsync(jobMessage);
return(false);
}
public async Task <string> GetBlobSignature(int clientId, string blobURI)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=vjdemodata;AccountKey=c0x/avvhx8/5Dw0W/K1PnqI4xaD85vxRyupDNG+sDJV5w2S0hOVjBNKFpVWCHi2swZQVqOdbIKGgt+HLo2gy+w==;EndpointSuffix=core.windows.net");
//Create the blob client object.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
//Get a reference to a container to use for the sample code, and create it if it does not exist.
CloudBlob blob = (CloudBlob)blobClient.GetBlobReferenceFromServerAsync(new Uri(blobURI)).Result;
return(GetBlobSasUri(blob));
}
public async Task GivenDataStream_WhenAppendToBlob_ThenDataShouldBeAppended()
{
IIntegrationDataStoreClientInitilizer <CloudBlobClient> initializer = GetClientInitializer();
CloudBlobClient client = await initializer.GetAuthorizedClientAsync(CancellationToken.None);
string containerName = Guid.NewGuid().ToString("N");
string blobName = Guid.NewGuid().ToString("N");
Uri blobUri = new Uri(Path.Combine(client.StorageUri.PrimaryUri.ToString(), $"{containerName}/{blobName}"));
try
{
AzureBlobIntegrationDataStoreClient blobClient = new AzureBlobIntegrationDataStoreClient(initializer, GetIntegrationDataStoreConfigurationOption(), new NullLogger <AzureBlobIntegrationDataStoreClient>());
await blobClient.PrepareResourceAsync(containerName, blobName, CancellationToken.None);
long count = 30;
List <string> blockIds = new List <string>();
for (long i = 0; i < count; ++i)
{
using Stream input = new MemoryStream(Encoding.UTF8.GetBytes(i.ToString() + "\r\n"));
string blockId = Convert.ToBase64String(Guid.NewGuid().ToByteArray());
await blobClient.UploadBlockAsync(blobUri, input, blockId, CancellationToken.None);
blockIds.Add(blockId);
}
await blobClient.CommitAsync(blobUri, blockIds.ToArray(), CancellationToken.None);
ICloudBlob output = await client.GetBlobReferenceFromServerAsync(blobUri);
using Stream outputStream = new MemoryStream();
await output.DownloadToStreamAsync(outputStream);
outputStream.Position = 0;
using StreamReader reader = new StreamReader(outputStream);
long currentLine = 0;
string content = null;
while ((content = await reader.ReadLineAsync()) != null)
{
Assert.Equal(currentLine.ToString(), content);
currentLine++;
}
Assert.Equal(count, currentLine);
}
finally
{
var container = client.GetContainerReference(containerName);
await container.DeleteIfExistsAsync();
}
}
public async Task <Uri> UploadAsync(string text, string replaceTo = null)
{
await _container.CreateIfNotExistsAsync();
CloudBlockBlob block;
if (string.IsNullOrEmpty(replaceTo))
{
var prefix = DateTimeOffset.Now.ToString("yyyy-MM-dd/hh/mm");
var guid = Guid.NewGuid().ToString();
block = _container.GetBlockBlobReference($"{prefix}/{guid}.log");
}
else
{
block = (CloudBlockBlob)await _client.GetBlobReferenceFromServerAsync(new Uri(replaceTo));
}
await block.UploadTextAsync(text);
return(block.Uri);
}
public async Task <string> GetSas(string BlobUri)
{
var blob = await _BlobClient.GetBlobReferenceFromServerAsync(new Uri(BlobUri));
SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
policy.SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(24);
policy.Permissions = SharedAccessBlobPermissions.Read;
string sasToken = blob.GetSharedAccessSignature(policy);
return(BlobUri + sasToken);
}
private async Task <CloudBlob> GetBlobAsync(Uri blobUri)
{
try
{
var blob = await _blobClient.GetBlobReferenceFromServerAsync(blobUri, accessCondition : null, options : _blobRequestOptions, operationContext : null);
return(blob as CloudBlob);
}
catch (Exception)
{
return(null);
}
}
/// <summary>
/// Get blob info by url
/// </summary>
/// <param name="blobUrl"></param>
/// <returns></returns>
public virtual async Task <BlobInfo> GetBlobInfoAsync(string blobUrl)
{
if (string.IsNullOrEmpty(blobUrl))
{
throw new ArgumentNullException(nameof(blobUrl));
}
var uri = blobUrl.IsAbsoluteUrl() ? new Uri(blobUrl) : new Uri(_cloudBlobClient.BaseUri, blobUrl.TrimStart('/'));
BlobInfo retVal = null;
try
{
var cloudBlob = await _cloudBlobClient.GetBlobReferenceFromServerAsync(uri);
retVal = ConvertBlobToBlobInfo(cloudBlob);
}
catch (Exception)
{
//Azure blob storage client does not provide method to check blob url exist without throwing exception
}
return(retVal);
}
