internal HttpMessage CreateSasUriRequest(BlobDetails blobDetails)
{
var message = _pipeline.CreateMessage();
var request = message.Request;
request.Method = RequestMethod.Post;
var uri = new RawRequestUriBuilder();
uri.Reset(_endpoint);
uri.AppendPath("/v1.0/subscriptions/", false);
uri.AppendPath(_subscriptionId, true);
uri.AppendPath("/resourceGroups/", false);
uri.AppendPath(_resourceGroupName, true);
uri.AppendPath("/providers/Microsoft.Quantum/workspaces/", false);
uri.AppendPath(_workspaceName, true);
uri.AppendPath("/storage/sasUri", false);
request.Uri = uri;
request.Headers.Add("Accept", "application/json");
request.Headers.Add("Content-Type", "application/json");
var content = new Utf8JsonRequestContent();
content.JsonWriter.WriteObjectValue(blobDetails);
request.Content = content;
return(message);
}
/// <summary>
/// Upload the blob content to azure.
/// </summary>
/// <param name="details">
/// Details of the blob.
/// </param>
/// <param name="containerName">
/// Name of the container.
/// </param>
/// <returns>
/// True, if the blob is successfully uploaded to azure;otherwise false.
/// </returns>
private bool UploadBlobContent(BlobDetails details, string containerName)
{
this.CheckNotNull(() => details);
try
{
var container = GetContainer(containerName);
// Seek to start.
details.Data.Position = 0;
// TODO: Check if the input file type and then use either block blob or page blob.
// For plate file we need to upload the file as page blob.
var blob = container.GetBlockBlobReference(details.BlobID.ToUpperInvariant());
blob.Properties.ContentType = details.MimeType;
blob.UploadFromStream(details.Data);
return(true);
}
catch (InvalidOperationException)
{
// "Error uploading blob {0}: {1}", ContainerUrl + _PathSeparator + blobName, se.Message
}
return(false);
}
/// <summary>
/// Checks the file from azure which is identified by file name.
/// </summary>
/// <param name="fileName">
/// file name.
/// </param>
/// <returns>
/// Operation Status
/// </returns>
public OperationStatus CheckIfAssetExists(string fileName)
{
OperationStatus operationStatus = null;
var fileBlob = new BlobDetails()
{
BlobID = fileName,
};
try
{
if (_blobDataRepository.CheckIfAssetExists(fileBlob))
{
operationStatus = OperationStatus.CreateSuccessStatus();
}
else
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage);
}
}
catch (Exception ex)
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage, ex);
}
return(operationStatus);
}
/// <summary>
/// Uploads file to azure.
/// </summary>
/// <param name="fileDetails">Details of the file.</param>
public OperationStatus UploadAsset(FileDetail fileDetails)
{
OperationStatus operationStatus = null;
this.CheckNotNull(() => new { fileDetails });
var fileBlob = new BlobDetails()
{
BlobID = fileDetails.Name,
Data = fileDetails.DataStream,
MimeType = fileDetails.MimeType
};
try
{
if (_blobDataRepository.UploadAsset(fileBlob))
{
operationStatus = OperationStatus.CreateSuccessStatus();
}
else
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage);
}
}
catch (Exception)
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage);
}
return(operationStatus);
}
/// <summary>
/// Uploads the associated file to temporary container.
/// </summary>
/// <param name="fileDetail">Details of the associated file.</param>
/// <returns>True if content is uploaded; otherwise false.</returns>
public OperationStatus UploadTemporaryFile(FileDetail fileDetail)
{
OperationStatus operationStatus = null;
// Make sure file detail is not null
this.CheckNotNull(() => new { fileDetail });
var fileBlob = new BlobDetails()
{
BlobID = fileDetail.AzureID.ToString(),
Data = fileDetail.DataStream,
MimeType = fileDetail.MimeType
};
try
{
_blobDataRepository.UploadTemporaryFile(fileBlob);
operationStatus = OperationStatus.CreateSuccessStatus();
}
catch (Exception)
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage);
}
return(operationStatus);
}
private async Task <List <BlobDetails> > CreateOrUpdateAnAttachment(List <IFormFile> files)
{
var lstBlobDetails = new List <BlobDetails>();
if (files != null)
{
foreach (var formFile in files)
{
var stream = formFile.OpenReadStream();
var length = (int)formFile.Length;
var data = new byte[length];
await stream.ReadAsync(data, 0, length);
var blobUri = await _blobStorageRepository.InsertResortDocument(data, formFile.FileName);
var fileExtIndex = formFile.FileName.IndexOf('.');
var blobName = formFile.FileName.Substring(0, fileExtIndex);
var contentType = formFile.ContentType;
var attachmentDetails = new BlobDetails
{
BlobUri = blobUri,
ContentType = contentType,
Name = blobName
};
lstBlobDetails.Add(attachmentDetails);
}
}
return(lstBlobDetails);
}
/// <summary>
/// Gets the file stream based on the blobDetails
/// </summary>
/// <param name="blobDetails">Details of the blob</param>
/// <returns>File Stream Result </returns>
private Stream GetFileStream(BlobDetails blobDetails)
{
// Update the response header.
Response.ContentType = blobDetails.MimeType;
// Set the position to Begin.
blobDetails.Data.Seek(0, SeekOrigin.Begin);
return(blobDetails.Data);
}
/// <summary>
/// Gets the file stream based on the blobDetails
/// </summary>
/// <param name="blobDetails">Details of the blob</param>
/// <returns>File Stream Result </returns>
private FileStreamResult GetFileStream(BlobDetails blobDetails)
{
// Update the response header.
Response.AddHeader("Content-Encoding", blobDetails.MimeType);
// Set the position to Begin.
blobDetails.Data.Seek(0, SeekOrigin.Begin);
return(new FileStreamResult(blobDetails.Data, blobDetails.MimeType));
}
private void DeleteTemporaryThumbnail(ContentDetails contentDetails)
{
var thumbnailBlob = new BlobDetails()
{
BlobID = contentDetails.Thumbnail.AzureID.ToString()
};
// Delete thumbnail from azure.
_blobDataRepository.DeleteThumbnail(thumbnailBlob);
}
public static CloudBlockBlob DestinationBlob(BlobDetails details)
{
var destConnectionString = CloudConfigurationManager.GetSetting(details.Destination);
var destStorageAccount =
CloudStorageAccount.Parse(destConnectionString);
var blobClient = destStorageAccount.CreateCloudBlobClient();
var containerReference = blobClient.GetContainerReference(details.ContainerName);
return(containerReference.GetBlockBlobReference(details.BlobName));
}
/// <summary>
/// Move temporary file to actual container in azure.
/// </summary>
/// <param name="fileDetails">Details of the file.</param>
private bool MoveAssetFile(FileDetail fileDetails)
{
var fileBlob = new BlobDetails()
{
BlobID = fileDetails.AzureID.ToString(),
MimeType = fileDetails.MimeType
};
return(_blobDataRepository.MoveAssetFile(fileBlob));
}
private static void MoveBlobs(BlobDetails details)
{
var sourceBlockBlob = AzureBlobReferences.SourceBlob(details);
var destBlockBlob = AzureBlobReferences.DestinationBlob(details);
using (var stream = sourceBlockBlob.OpenRead())
{
destBlockBlob.UploadFromStream(stream);
destBlockBlob.CreateSnapshot();
}
}
/// <summary>
/// Deletes file from azure.
/// </summary>
/// <param name="fileDetail">Details of the file.</param>
private void DeleteFile(FileDetail fileDetail)
{
var fileBlob = new BlobDetails()
{
BlobID = fileDetail.AzureID.ToString(),
MimeType = fileDetail.MimeType
};
// Delete file from azure.
_blobDataRepository.DeleteFile(fileBlob);
}
/// <summary>
/// Gets as SAS Uri for the linked storage account.
/// </summary>
/// <param name="containerName">Name of the container.</param>
/// <param name="blobName">Name of the BLOB.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>
/// Sas Uri.
/// </returns>
public async Task <string> GetSasUriAsync(string containerName, string blobName = null, CancellationToken cancellationToken = default)
{
BlobDetails details = new BlobDetails
{
ContainerName = containerName,
BlobName = blobName,
};
var response = await this.QuantumClient.Storage.SasUriAsync(details, cancellationToken);
return(response.SasUri);
}
/// <summary>
/// Deletes Thumbnail from azure.
/// </summary>
/// <param name="azureId">Id of the thumbnail to be deleted.</param>
private void DeleteThumbnail(Guid?azureId)
{
if (azureId.HasValue && !azureId.Equals(Guid.Empty))
{
var fileBlob = new BlobDetails()
{
BlobID = azureId.ToString(),
};
// Delete file from azure.
_blobDataRepository.DeleteThumbnail(fileBlob);
}
}
/// <summary>
/// Uploads the associated file to temporary container.
/// </summary>
/// <param name="fileDetail">Details of the associated file.</param>
/// <returns>True if content is uploaded; otherwise false.</returns>
public bool UploadTemporaryFile(FileDetail fileDetail)
{
// Make sure file detail is not null
this.CheckNotNull(() => new { fileDetail });
var fileBlob = new BlobDetails()
{
BlobID = fileDetail.AzureID.ToString(),
Data = fileDetail.DataStream,
MimeType = fileDetail.MimeType
};
return(_blobDataRepository.UploadTemporaryFile(fileBlob));
}
/// <summary>
/// Deletes the specified file pointed by the blob.
/// </summary>
/// <param name="details">
/// Details of the blob.
/// </param>
/// <param name="containerName">
/// Name of the container.
/// </param>
private static bool DeleteBlob(BlobDetails details, string containerName)
{
try
{
var container = GetContainer(containerName);
container.GetBlobReferenceFromServer(details.BlobID.ToUpperInvariant()).Delete();
return(true);
}
catch (InvalidOperationException)
{
// "Error deleting blob {0}: {1}", ContainerUrl + _PathSeparator + blobName, se.Message
return(false);
}
}
/// <summary>
/// Moves thumbnail from temporary storage to thumbnail storage in azure.
/// </summary>
/// <param name="fileDetails">Details of the thumbnail.</param>
private Guid MoveThumbnail(FileDetail fileDetails)
{
var thumbnailId = Guid.Empty;
if (fileDetails != null && fileDetails.AzureID != Guid.Empty)
{
var thumbnailBlob = new BlobDetails()
{
BlobID = fileDetails.AzureID.ToString()
};
thumbnailId = _blobDataRepository.MoveThumbnail(thumbnailBlob) ? fileDetails.AzureID : Guid.Empty;
}
return(thumbnailId);
}
/// <summary>
/// Moves thumbnail from temporary storage to thumbnail storage in azure.
/// </summary>
/// <param name="profileImageId">Guid of the temporary profile image.</param>
private Guid?MoveThumbnail(Guid?profileImageId)
{
Guid?thumbnailId = null;
if (profileImageId.HasValue && !profileImageId.Equals(Guid.Empty))
{
var thumbnailBlob = new BlobDetails()
{
BlobID = profileImageId.ToString()
};
thumbnailId = _blobDataRepository.MoveThumbnail(thumbnailBlob) ? profileImageId.Value : Guid.Empty;
}
return(thumbnailId);
}
/// <summary>
/// Deletes the file from azure which is identified by file name.
/// </summary>
/// <param name="fileName">
/// file name.
/// </param>
/// <returns>
/// Operation Status
/// </returns>
public OperationStatus DeleteAsset(string fileName)
{
OperationStatus operationStatus = null;
var fileBlob = new BlobDetails()
{
BlobID = fileName,
};
try
{
_blobDataRepository.DeleteAsset(fileBlob);
operationStatus = OperationStatus.CreateSuccessStatus();
}
catch (Exception)
{
operationStatus = OperationStatus.CreateFailureStatus(Resources.UnknownErrorMessage);
}
return(operationStatus);
}
/// <summary>
/// Checks if the blob content is present in azure or not.
/// </summary>
/// <param name="details">
/// Details of the blob.
/// </param>
/// <param name="containerName">
/// Name of the container.
/// </param>
/// <returns>
/// True, if the blob is successfully found to azure;otherwise false.
/// </returns>
private static bool ExistsBlobContent(BlobDetails details, string containerName)
{
try
{
var container = GetContainer(containerName);
// TODO: Check if the input file type and then use either block blob or page blob.
// For plate file we need to upload the file as page blob.
var blob = container.GetBlobReferenceFromServer(details.BlobID.ToUpperInvariant());
blob.FetchAttributes();
return(true);
}
catch (StorageException)
{
// "Error uploading blob {0}: {1}", ContainerUrl + _PathSeparator + blobName, se.Message
}
return(false);
}
public async Task <Response <SasUriResponse> > SasUriAsync(BlobDetails blobDetails, CancellationToken cancellationToken = default)
{
if (blobDetails == null)
{
throw new ArgumentNullException(nameof(blobDetails));
}
using var message = CreateSasUriRequest(blobDetails);
await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);
switch (message.Response.Status)
{
case 200:
{
SasUriResponse value = default;
using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);
value = SasUriResponse.DeserializeSasUriResponse(document.RootElement);
return(Response.FromValue(value, message.Response));
}
public async Task ExecutesFullIntegrationPass()
{
var jobStatusStore = new JobStatusStore(Configuration.StorageConnectionString, Configuration.JobStatusContainerName);
var blobReader = new AzureBlobReader(Configuration.StorageConnectionString, Configuration.FileUploadContainerName);
var extractor = new TextFileContentExtractor();
var searchIndex = new AzureSearchIndex(Configuration.SearchServiceName, Configuration.SearchAdminKey);
var docScorer = new TextDocumentScorer(searchIndex);
var workflow = new ParsingWorkflow(jobStatusStore, blobReader, extractor, searchIndex, docScorer);
var blobId = Guid.NewGuid().ToString().Replace("-", String.Empty);
var jobId = Guid.NewGuid().ToString().Replace("-", String.Empty);
var blobUri = String.Format("{0}/{1}", jobId, blobId);
var blobDetails = new BlobDetails();
blobDetails.ContainerName = Configuration.FileUploadContainerName;
blobDetails.FullBlobPath = blobUri;
blobDetails.DocumentId = blobId;
blobDetails.JobId = jobId;
var job = new JobStatus();
job.OriginalFileName = "not-real-file.txt";
job.IsComplete = false;
job.JobStartTime = DateTime.UtcNow;
job.JobId = jobId;
await jobStatusStore.UpdateStatusAsync(job);
await createSampleBlob(blobUri);
await workflow.ExecuteAsync(blobDetails);
job = await jobStatusStore.ReadStatusAsync(jobId);
var categoryCount = job.Categories.Length;
Assert.Equal(1, categoryCount);
Assert.Equal("Heavy Hitter", job.Categories[0]);
}
private async Task <bool> SaveToAzureBlob(BlobDetails blobDetails)
{
MemoryStream ms = null;
IFormFile file = null;
try
{
if (CloudStorageAccount.TryParse(_azureBlobConfig.Value.ConnectionString, out CloudStorageAccount storageAccount))
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = BlobUtilities.GetBlobContainer(blobClient, blobDetails.Type, _azureBlobConfig);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(BlobUtilities.GenerateUniqueAadhaarImageName(blobDetails.CustomerID, blobDetails.Type));
using (ms = new MemoryStream())
{
file = blobDetails.BlobFile;
file.CopyTo(ms);
var fileBytes = ms.ToArray();
await blockBlob.UploadFromByteArrayAsync(fileBytes, 0, fileBytes.Length);
}
return(true);
}
else
{
return(false);
}
}
catch
{
return(false);
}
//finally
//{
// await ms.DisposeAsync();
//}
}
private static bool MoveBlob(BlobDetails details, string sourceContainerName, string destinationContainerName)
{
try
{
var sourceContainer = GetContainer(sourceContainerName);
var destinationContainer = GetContainer(destinationContainerName);
// TODO: Check if the input file type and then use either block blob or page blob.
// For plate file we need to upload the file as page blob.
var sourceBlob = sourceContainer.GetBlockBlobReference(details.BlobID.ToUpperInvariant());
var destinationBlob = destinationContainer.GetBlockBlobReference(details.BlobID.ToUpperInvariant());
destinationBlob.StartCopyFromBlob(sourceBlob);
destinationBlob.Properties.ContentType = sourceBlob.Properties.ContentType;
sourceBlob.Delete();
return(true);
}
catch (Exception)
{
// "Error moving blob {0}: {1}", ContainerUrl + _PathSeparator + blobName, se.Message
}
return(false);
}
/// <summary>
/// Gets a URL with SAS token for a container/blob in the storage account
/// associated with the workspace. The SAS URL can be used to upload job input
/// and/or download job output.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='blobDetails'>
/// The details (name and container) of the blob to store or download data.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <SasUriResponse> SasUriAsync(this IStorageOperations operations, BlobDetails blobDetails, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.SasUriWithHttpMessagesAsync(blobDetails, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Gets a URL with SAS token for a container/blob in the storage account
/// associated with the workspace. The SAS URL can be used to upload job input
/// and/or download job output.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='blobDetails'>
/// The details (name and container) of the blob to store or download data.
/// </param>
public static SasUriResponse SasUri(this IStorageOperations operations, BlobDetails blobDetails)
{
return(operations.SasUriAsync(blobDetails).GetAwaiter().GetResult());
}
/// <summary>
/// Move a file from temporary container to asset container in azure.
/// </summary>
/// <param name="details">
/// Details of the file which has to be uploaded to azure.
/// </param>
/// <returns>
/// True if the file is moved successfully; otherwise false.
/// </returns>
public bool MoveAssetFile(BlobDetails details)
{
return(MoveBlob(details, Constants.TemporaryContainerName, Constants.AssetContainerName));
}
/// <summary>
/// Deletes a file from azure.
/// </summary>
/// <param name="details">
/// Details of the file which has to be deleted.
/// </param>
/// <returns>
/// True if the file is deleted successfully; otherwise false.
/// </returns>
public bool DeleteAsset(BlobDetails details)
{
return(DeleteBlob(details, Constants.AssetContainerName));
}
/// <summary>
/// Checks a file in azure.
/// </summary>
/// <param name="details">
/// Details of the file which has to be checked.
/// </param>
/// <returns>
/// True if the file is found successfully; otherwise false.
/// </returns>
public bool CheckIfAssetExists(BlobDetails details)
{
return(ExistsBlobContent(details, Constants.AssetContainerName));
}