/// <summary>
/// Gets from blob storage
/// NOTE: If you plan on getting the same blob over and over and quickly saving you will need to throttle and retry
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
/// <returns></returns>
public BlobResultModel GetBlob(string blobContainer, string fileName)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
if (blob.Exists())
{
BlobResultModel blobResultModel = new BlobResultModel();
blobResultModel.Stream = new MemoryStream();
blob.DownloadToStream(blobResultModel.Stream);
blobResultModel.eTag = blob.Properties.ETag;
return(blobResultModel);
}
else
{
return(null);
}
} // GetBlob
private async Task UploadOutputsAsync(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobTrigger, Guid id, string sampleName)
{
const string outputsContainer = "outputs";
try
{
var outputsResponse = await cromwellApiClient.GetOutputsAsync(id);
await storage.UploadFileTextAsync(
outputsResponse.Json,
outputsContainer,
$"{sampleName}.{id}.outputs.json");
var metadataResponse = await cromwellApiClient.GetMetadataAsync(id);
await storage.UploadFileTextAsync(
metadataResponse.Json,
outputsContainer,
$"{sampleName}.{id}.metadata.json");
}
catch (Exception exc)
{
logger.LogWarning(exc, $"Getting outputs and/or timing threw an exception for Id: {id}");
}
}
} // SaveBlob
/// <summary>
/// Verifies the eTag when saving
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
/// <param name="text"></param>
/// <param name="eTag"></param>
public void SaveBlob(string blobContainer, string fileName, string text, string eTag)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
// save
if (fileName.Contains("."))
{
blob.Properties.ContentType = GetMimeTypeFromExtension(fileName.Substring(fileName.LastIndexOf(".")));
}
else
{
blob.Properties.ContentType = "application/octet-stream";
}
Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = new Microsoft.WindowsAzure.Storage.AccessCondition();
accessCondition.IfMatchETag = eTag;
PutText(blob, text, accessCondition);
} // SaveBlob
/// <summary>
/// Saves to blob storage
/// NOTE: If you plan on getting the same blob over and over and quickly saving you will need to throttle and retry
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
/// <param name="memoryStream"></param>
public void SaveBlob(string blobContainer, string fileName, MemoryStream memoryStream)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
blobStorage.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 30, 0);
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
// save
if (fileName.Contains("."))
{
blob.Properties.ContentType = GetMimeTypeFromExtension(fileName.Substring(fileName.LastIndexOf(".")));
}
else
{
blob.Properties.ContentType = "application/octet-stream";
}
// save
memoryStream.Position = 0; // rewind
blob.UploadFromStream(memoryStream);
} // SaveBlob
private static string ExtractSampleName(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobTrigger)
{
var blobName = blobTrigger.Name.Substring(AzureStorage.WorkflowState.InProgress.ToString().Length + 1);
var withoutExtension = Path.GetFileNameWithoutExtension(blobName);
return(withoutExtension.Substring(0, withoutExtension.LastIndexOf('.')));
}
private static Guid ExtractWorkflowId(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobTrigger, AzureStorage.WorkflowState currentState)
{
var blobName = blobTrigger.Name.Substring(currentState.ToString().Length + 1);
var withoutExtension = Path.GetFileNameWithoutExtension(blobName);
var textId = withoutExtension.Substring(withoutExtension.LastIndexOf('.') + 1);
return(Guid.Parse(textId));
}
public async void StoreVideo(Entities.Video video)
{
var storageUrl = System.IO.Path.Combine("https://psmadd4asimagestorage.blob.core.windows.net/videos/", $"{video.Id.ToString()}-{video.Name}");
var credentials = new StorageCredentials(storageAccount, storageKey);
var blob = new Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob(new Uri(storageUrl), credentials);
await blob.UploadFromByteArrayAsync(video.Body, 0, video.Body.Length);
}
} // SaveBlob
/// <summary>
/// Writes text to a blob reference
/// </summary>
/// <param name="blob"></param>
/// <param name="text"></param>
public void PutText(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob, string text)
{
if (text == null)
{
text = string.Empty;
}
using (System.IO.MemoryStream memoryStream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(text)))
{
blob.UploadFromStream(memoryStream);
}
}
} // GetBlob
/// <summary>
/// Saves to blob storage
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
public void DeleteBlob(string blobContainer, string fileName)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
blob.DeleteIfExists();
} // DeleteBlob
/// <summary>
/// Gets from blob storage
/// NOTE: If you plan on getting the same blob over and over and quickly saving you will need to throttle and retry
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
/// <returns></returns>
public BlobResultModel GetBlobAsText(string blobContainer, string fileName)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
return(GetText(blob));
} // GetBlob
public long GetBlobSize(string blobContainer, string fileName)
{
long output = 0;
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = this.GetBlobHandle(blobContainer, fileName);
blob.FetchAttributes();
if (blob.Properties != null)
{
output = blob.Properties.Length;
}
return(output);
}
static async Task <OldCopyStatus> CheckOldSDKBlobStatus(OldCloudBlockBlob blob)
{
await blob.FetchAttributesAsync();
while (blob.CopyState.Status == OldCopyStatus.Pending)
{
await blob.FetchAttributesAsync();
Thread.Sleep(500);
}
return(blob.CopyState.Status);
}
public AzureContent(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob delegateContent,
IMultiPartUploadInfoFactory multiPartUploadInfoFactory,
IUploadIdentifierProvider uploadIdentifierProvider,
IContentNameProvider contentNameProvider,
IDownloadInfoFactory downloadInfoFactory,
IHttpPartUploadInfoFactory httpPartUploadInfoFactory)
{
_delegateContent = delegateContent;
_multiPartUploadInfoFactory = multiPartUploadInfoFactory;
_uploadIdentifierProvider = uploadIdentifierProvider;
_contentNameProvider = contentNameProvider;
_downloadInfoFactory = downloadInfoFactory;
_httpPartUploadInfoFactory = httpPartUploadInfoFactory;
}
protected void UpdateMetadata(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobReference, IDictionary <string, string> metadata)
{
foreach (var row in metadata)
{
if (blobReference.Metadata.ContainsKey(row.Key))
{
blobReference.Metadata[row.Key] = row.Value;
}
else
{
blobReference.Metadata.Add(row);
}
}
}
private async Task UploadTimingAsync(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobTrigger, Guid id, string sampleName)
{
const string outputsContainer = "outputs";
try
{
var timingResponse = await cromwellApiClient.GetTimingAsync(id);
await storage.UploadFileTextAsync(
timingResponse.Html,
outputsContainer,
$"{sampleName}.{id}.timing.html");
}
catch (Exception exc)
{
logger.LogWarning(exc, $"Getting timing threw an exception for Id: {id}");
}
}
} // GetBlob
/// <summary>
/// Returns the raw blob handle so you can obtain leases, read data, delete, etc...
/// </summary>
/// <param name="blobContainer"></param>
/// <param name="fileName"></param>
/// <returns></returns>
public Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob GetBlobHandle(string blobContainer, string fileName)
{
InitializeStorage(blobContainer);
string containerName = blobContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobStorage = blobStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
if (blob.Exists())
{
return(blob);
}
else
{
return(null);
}
} // GetBlobHandle
/// <summary>
/// Gets the text from a blob reference
/// </summary>
/// <param name="blob"></param>
/// <returns></returns>
public BlobResultModel GetText(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob)
{
if (blob.Exists())
{
string text = string.Empty;
BlobResultModel blobResultModel = new BlobResultModel();
using (System.IO.MemoryStream memoryStream = new System.IO.MemoryStream())
{
blob.DownloadToStream(memoryStream);
blobResultModel.Text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray());
blobResultModel.eTag = blob.Properties.ETag;
}
return(blobResultModel);
}
else
{
return(null);
}
}
void GetFiles()
{
var storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(Microsoft.Azure.CloudConfigurationManager.GetSetting("StorageConnection"));
var myClient = storageAccount.CreateCloudBlobClient();
var container = myClient.GetContainerReference("images-backup");
container.CreateIfNotExists(Microsoft.WindowsAzure.Storage.Blob.BlobContainerPublicAccessType.Blob);
var list = container.ListBlobs();
var blobs = list.OfType <Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob>().Where(b => System.IO.Path.GetExtension(b.Name).Equals(".png"));
foreach (var item in blobs)
{
string name = ((Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob)item).Name;
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blockBlob = container.GetBlockBlobReference(name);
string path = (@"C:\Users\mbcrump\Downloads\test\" + name);
blockBlob.DownloadToFile(path, System.IO.FileMode.OpenOrCreate);
}
}
/// <summary>
/// Writes text to a blob reference with an access condition
/// </summary>
/// <param name="blob"></param>
/// <param name="text"></param>
/// <param name="accessCondition"></param>
public void PutText(Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob, string text, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition)
{
using (System.IO.MemoryStream memoryStream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(text)))
{
try
{
blob.UploadFromStream(memoryStream, accessCondition);
}
catch (Microsoft.WindowsAzure.Storage.StorageException storageException)
{
if (storageException.Message.Contains("conditional header(s) is not met"))
{
throw new Sample.Azure.Common.CustomException.CustomConcurrencyException(System.Net.HttpStatusCode.PreconditionFailed, "The file has been updated by another user.");
}
else
{
throw;
}
}
}
}
public static async Task <IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, ILogger log, ExecutionContext context)
{
string Path = req.Query["Path"].ToString();
var sas = "https://adsgofasttransientstg.blob.core.windows.net/" + Path + "?";
foreach (var i in req.Query)
{
if (i.Key != "Path" && i.Key != "TargetSystemUidInPHI")
{
sas += "&" + i.Key + "=" + i.Value;
}
}
var cloudBlockBlob = new Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob(new Uri(sas));
await cloudBlockBlob.UploadTextAsync("Hello World");
//Write to Filelist table so that downstream tasks can be triggered efficiently
var _storageCredentials = new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials("?sv=2018-03-28&tn=Filelist&sig=MFbvVgbNLs3UjqAPfU%2BYwQqxcTYwCPnNKCwCUp4XRmo%3D&se=2021-09-06T23%3A15%3A57Z&sp=au");
var SourceStorageAccount = new Microsoft.WindowsAzure.Storage.CloudStorageAccount(storageCredentials: _storageCredentials, accountName: "adsgofasttransientstg", endpointSuffix: "core.windows.net", useHttps: true);
var client = SourceStorageAccount.CreateCloudTableClient();
Microsoft.WindowsAzure.Storage.Table.CloudTable table = client.GetTableReference("Filelist");
var _dict = new Dictionary <string, EntityProperty>();
_dict.Add("FilePath", new EntityProperty(Path));
var _tableEntity = new DynamicTableEntity(DateTime.UtcNow.ToString("yyyy-MM-dd hh:mm"), Guid.NewGuid().ToString(), null, _dict);
var _tableOperation = TableOperation.Insert(_tableEntity);
try
{
await table.ExecuteAsync(_tableOperation);
}
catch (Microsoft.WindowsAzure.Storage.StorageException ex)
{
}
return(new OkObjectResult(new { }));
}
public static Microsoft.AspNetCore.DataProtection.IDataProtectionBuilder PersistKeysToAzureBlobStorage(this Microsoft.AspNetCore.DataProtection.IDataProtectionBuilder builder, Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobReference)
{
throw null;
}
public static async Task <OldCopyStatus> MoveAsync(this OldCloudBlockBlob targetBlob, OldCloudBlockBlob originBlob)
{
await targetBlob.StartCopyAsync(originBlob);
return(await CheckOldSDKBlobStatus(targetBlob));
}
public static async Task <OldCopyStatus> MoveAsync(this OldCloudBlockBlob blob, Uri uri)
{
await blob.StartCopyAsync(uri);
return(await CheckOldSDKBlobStatus(blob));
}
public string AzureBlopforChunkUpload(System.IO.Stream inputStream, string fileName, string containerName, string accountKey)
{
if (fileName != null)
{
Repository.AzureBlobUpload objAzureUpload = new Repository.AzureBlobUpload();
// string accKey = lstBlob[0].Blob_Access_Key;//iConfigPro.GetConfigValue(lstBlob[0].Blob_Access_Key);
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = objAzureUpload.GetBlobContainerObject(containerName, accountKey);
container.CreateIfNotExists();
//to replace the file name with new GUID
//string newfileName = Guid.NewGuid().ToString() + Path.GetExtension(fileName);
// string newfileName = objBlInfra.GenerateFileName(fileName);
container.SetPermissions(new Microsoft.WindowsAzure.Storage.Blob.BlobContainerPermissions {
PublicAccess = Microsoft.WindowsAzure.Storage.Blob.BlobContainerPublicAccessType.Blob
});
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
int blockSize = 1024 * 1024; //256 kb
using (Stream fileStream = inputStream)
{
long fileSize = fileStream.Length;
//block count is the number of blocks + 1 for the last one
int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
//List of block ids; the blocks will be committed in the order of this list
List <string> blockIDs = new List <string>();
//starting block number - 1
int blockNumber = 0;
try
{
int bytesRead = 0; //number of bytes read so far
long bytesLeft = fileSize; //number of bytes left to read and upload
//do until all of the bytes are uploaded
while (bytesLeft > 0)
{
blockNumber++;
int bytesToRead;
if (bytesLeft >= blockSize)
{
//more than one block left, so put up another whole block
bytesToRead = blockSize;
}
else
{
//less than one block left, read the rest of it
bytesToRead = (int)bytesLeft;
}
//create a blockID from the block number, add it to the block ID list
//the block ID is a base64 string
string blockId =
Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}",
blockNumber.ToString("0000000"))));
blockIDs.Add(blockId);
//set up new buffer with the right size, and read that many bytes into it
byte[] bytes = new byte[bytesToRead];
fileStream.Read(bytes, 0, bytesToRead);
//calculate the MD5 hash of the byte array
//string blockHash = GetMD5HashFromStream(bytes);
var chunkStream = new MemoryStream(bytes);
//upload the block, provide the hash so Azure can verify it
blob.PutBlock(blockId, chunkStream, null, null,
new Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions()
{
RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(10), 3)
},
null);
//increment/decrement counters
bytesRead += bytesToRead;
bytesLeft -= bytesToRead;
}
//commit the blocks
blob.PutBlockList(blockIDs);
return(blob.Uri.ToString());
}
catch (Exception ex)
{
System.Diagnostics.Debug.Print("Exception thrown = {0}", ex);
return(null);
}
}
}
return(null);
}
public static void ProcessQueueMessage([QueueTrigger("merchantid-179")] string blobName, [Blob("merchantid-179/{queueTrigger}")] Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blobToDownload)
{
Console.WriteLine("Msg is {0}", blobName);
string[] words = blobName.Split('_');
MemoryStream ms;
try
{
ms = new MemoryStream();
blobToDownload.DownloadToStream(ms);
}
catch (Exception e)
{
Console.WriteLine("Blob Download to memory has Failed for Blob Name " + blobName + e.Message);
return;
}
int key4Index = GettWordAfteKey(words, "KEY4");
int key5Index = GettWordAfteKey(words, "KEY5");
int key1Index = GettWordAfteKey(words, "KEY1");
string storeid;
string posid;
string tablename;
if (key4Index != -1)
{
Console.WriteLine(" store id is = " + words[key4Index]);
storeid = (words[key4Index]);
}
else
{
Console.WriteLine(" store id is -1");
return;
}
if (key5Index != -1)
{
Console.WriteLine(" pos id is = " + words[key5Index]);
posid = (words[key5Index]);
}
else
{
Console.WriteLine(" pos id is -1");
return;
}
if (key1Index != -1)
{
Console.WriteLine(" table name is = " + words[key1Index]);
tablename = (words[key1Index]);
}
else
{
Console.WriteLine("table name is -1");
return;
}
try
{
if (blobName.Contains(".csv"))
{
ProcesssingAllTable(tablename, ms, storeid, posid, blobName);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
