public async Task <string> EvaluateData(string blobName)
{
try
{
double ModelVerificationPercent = 0;
string ModelValidationStorageContainerName = "";
string StorageConnection = _Engine.GetEnvironmentVariable("AzureWebJobsStorage", _Log);
string PendingEvaluationStorageContainerName = _Engine.GetEnvironmentVariable("pendingEvaluationStorageContainerName", _Log);
string EvaluatedDataStorageContainerName = _Engine.GetEnvironmentVariable("evaluatedDataStorageContainerName", _Log);
string JsonStorageContainerName = _Engine.GetEnvironmentVariable("jsonStorageContainerName", _Log);
string PendingSupervisionStorageContainerName = _Engine.GetEnvironmentVariable("pendingSupervisionStorageContainerName", _Log);
string ConfidenceJsonPath = _Engine.GetEnvironmentVariable("confidenceJSONPath", _Log);
double ConfidenceThreshold = Convert.ToDouble(_Engine.GetEnvironmentVariable("confidenceThreshold", _Log));
string modelType = _Engine.GetEnvironmentVariable("modelType", _Log);
if (modelType == "Trained")
{
string LabeledDataStorageContainerName = _Engine.GetEnvironmentVariable("labeledDataStorageContainerName", _Log);
ModelValidationStorageContainerName = _Engine.GetEnvironmentVariable("modelValidationStorageContainerName", _Log);
string PendingNewModelStorageContainerName = _Engine.GetEnvironmentVariable("pendingNewModelStorageContainerName", _Log);
ModelVerificationPercent = Convert.ToDouble(_Engine.GetEnvironmentVariable("modelVerificationPercentage", _Log));
}
//------------------------This section retrieves the blob needing evaluation and calls the evaluation service for processing.-----------------------
// Create Reference to Azure Storage Account and the container for data that is pending evaluation by the model.
CloudStorageAccount StorageAccount = CloudStorageAccount.Parse(StorageConnection);
CloudBlobClient BlobClient = StorageAccount.CreateCloudBlobClient();
CloudBlobContainer Container = BlobClient.GetContainerReference(PendingEvaluationStorageContainerName);
CloudBlockBlob rawDataBlob = Container.GetBlockBlobReference(blobName);
DataBlob dataEvaluating = new DataBlob(rawDataBlob, _Engine, _Search, _Log);
if (dataEvaluating == null)
{
throw (new MissingRequiredObject("\nMissing dataEvaluating blob object."));
}
//compute the file hash as this will be added to the meta data to allow for file version validation
//the blob has to be "touched" or the properties will all be null
if (dataEvaluating.AzureBlob.Exists() != true)
{
throw new MissingRequiredObject($"\ndataEvaluating does not exist {dataEvaluating.AzureBlob.Name}");
}
;
string blobMd5 = _Engine.EnsureMd5(dataEvaluating);
//****Currently only working with public access set on blob folders
//Generate a URL with SAS token to submit to analyze image API
//string dataEvaluatingSas = GetBlobSharedAccessSignature(dataEvaluating);
string DataEvaluatingUrl = dataEvaluating.AzureBlob.Uri.ToString(); //+ dataEvaluatingSas;
//string dataEvaluatingUrl = "test";
//package the file contents to send as http request content
//MemoryStream DataEvaluatingContent = new MemoryStream();
//DataEvaluating.AzureBlob.DownloadToStreamAsync(DataEvaluatingContent);
//HttpContent DataEvaluatingStream = new StreamContent(DataEvaluatingContent);
var content = new MultipartFormDataContent();
//content.Add(DataEvaluatingStream, "name");
//get environment variables used to construct the model request URL
string dataEvaluationServiceEndpoint = _Engine.GetEnvironmentVariable("DataEvaluationServiceEndpoint", _Log);
string evaluationDataParameterName = _Engine.GetEnvironmentVariable("evaluationDataParameterName", _Log);
string parameters = $"?{evaluationDataParameterName}={DataEvaluatingUrl}";
string evaluateDataUrl = _Engine.ConstructModelRequestUrl(dataEvaluationServiceEndpoint, parameters);
int retryLoops = 0;
string responseString = "";
do
{
//Make a request to the model service passing the file URL
responseString = _Engine.GetHttpResponseString(evaluateDataUrl, content);
if (responseString.Contains("iteration"))
{
_Log.LogInformation($"\nEvaluation response: {responseString}.");
break;
}
retryLoops++;
await Task.Delay(1000);
if (retryLoops == 5)
{
_Log.LogInformation($"\nEvaluation of {evaluateDataUrl} failed with 5 attempts.");
}
} while (retryLoops < 5);
string StrConfidence = null;
double Confidence = 0;
JProperty responseProperty = new JProperty("Response", responseString);
if (responseString == "Model not trained.")
{
Confidence = 0;
}
else
{
//deserialize response JSON, get confidence score and compare with confidence threshold
JObject analysisJson = JObject.Parse(responseString);
try
{
StrConfidence = (string)analysisJson.SelectToken(ConfidenceJsonPath);
}
catch
{
throw (new MissingRequiredObject($"\nInvalid response string {responseString} generated from URL: {evaluateDataUrl}."));
}
if (StrConfidence == null)
{
throw (new MissingRequiredObject("\nNo confidence value at " + ConfidenceJsonPath + " from environment variable ConfidenceJSONPath."));
}
Confidence = (double)analysisJson.SelectToken(ConfidenceJsonPath);
}
//----------------------------This section collects information about the blob being analyzed and packages it in JSON that is then written to blob storage for later processing-----------------------------------
_Log.LogInformation("\nStarting construction of json blob.");
//create environment JSON object
JProperty environmentProperty = _Engine.GetEnvironmentJson(_Log);
JProperty evaluationPass = new JProperty("pass",
new JObject(
new JProperty("date", DateTime.Now),
environmentProperty,
new JProperty("request", evaluateDataUrl),
responseProperty
)
);
//Note: all json files get writted to the same container as they are all accessed either by discrete name or by azure search index either GUID or Hash.
bool jsonBlobExists;
try
{
//does the json blob exist? if not error is thrown and we catch it and create a new json blob file.
JsonBlob jsonBlobTest = new JsonBlob(blobMd5, _Engine, _Search, _Log);
jsonBlobExists = true;
}
catch
{
_Log.LogInformation($"\nNo JSON blob found in seach index, creating new JSON blob for blob {dataEvaluating.AzureBlob.Name}.");
jsonBlobExists = false;
}
// If the Json blob already exists then update the blob with this pass iteration information
if (jsonBlobExists)
{
JsonBlob jsonBlob = new JsonBlob(blobMd5, _Engine, _Search, _Log);
JObject jsonBlobJObject = JObject.Parse(jsonBlob.AzureBlob.DownloadText());
JArray evaluationHistory = (JArray)jsonBlobJObject.SelectToken("passes");
JObject evaluationsObject = new JObject
{
evaluationPass
};
evaluationHistory.Add(evaluationsObject);
string serializedJsonBlob = JsonConvert.SerializeObject(jsonBlobJObject, Formatting.Indented, new JsonSerializerSettings {
});
Stream jsonBlobMemStream = new MemoryStream(Encoding.UTF8.GetBytes(serializedJsonBlob));
if (jsonBlobMemStream.Length != 0)
{
await jsonBlob.AzureBlob.UploadFromStreamAsync(jsonBlobMemStream);
}
else
{
throw (new ZeroLengthFileException("\nencoded JSON memory stream is zero length and cannot be writted to blob storage"));
}
}
// If the Json blob does not exist create one
else
{
// Add blob info to Json
JObject BlobAnalysis =
new JObject(
new JProperty("id", Guid.NewGuid().ToString()),
new JProperty("IsDeleted", false),
new JProperty("blobInfo",
new JObject(
new JProperty("name", blobName),
new JProperty("url", dataEvaluating.AzureBlob.Uri.ToString()),
new JProperty("modified", dataEvaluating.AzureBlob.Properties.LastModified.ToString()),
new JProperty("hash", blobMd5)
)
)
);
// Add pass infromation to Json blob
JArray evaluations = new JArray();
JObject evaluationsObject = new JObject
{
evaluationPass
};
evaluations.Add(evaluationsObject);
JProperty evaluationPasses = new JProperty("passes", evaluations);
BlobAnalysis.Add(evaluationPasses);
CloudBlockBlob JsonCloudBlob = _Search.GetBlob(StorageAccount, JsonStorageContainerName, (string)BlobAnalysis.SelectToken("id") + ".json");
JsonCloudBlob.Properties.ContentType = "application/json";
string serializedJson = JsonConvert.SerializeObject(BlobAnalysis, Newtonsoft.Json.Formatting.Indented, new JsonSerializerSettings {
});
Stream MemStream = new MemoryStream(Encoding.UTF8.GetBytes(serializedJson));
if (MemStream.Length != 0)
{
JsonCloudBlob.UploadFromStream(MemStream);
}
else
{
throw (new ZeroLengthFileException("\nencoded JSON memory stream is zero length and cannot be writted to blob storage"));
}
}
//--------------------------------This section processes the results of the analysis and transferes the blob to the container responsible for the next appropriate stage of processing.-------------------------------
//model successfully analyzed content
if (Confidence >= ConfidenceThreshold)
{
EvaluationPassed(blobName, ModelVerificationPercent, ModelValidationStorageContainerName, EvaluatedDataStorageContainerName, StorageAccount, dataEvaluating);
}
//model was not sufficiently confident in its analysis
else
{
EvaluationFailed(blobName, PendingSupervisionStorageContainerName, StorageAccount, dataEvaluating);
}
_Log.LogInformation($"C# Blob trigger function Processed blob\n Name:{blobName}");
}
catch (MissingRequiredObject e)
{
_Log.LogInformation("\n" + blobName + " could not be analyzed because of a MissingRequiredObject with message: " + e.Message);
}
catch (Exception e)
{
_Log.LogInformation("\n" + blobName + " could not be analyzed with message: " + e.Message);
}
return($"Evaluate data completed evaluating data blob: {blobName}");
}
public ImagePreviewResponse Preview(string imageId, string containerName, string imageFormat, int brightness, int contrast, int saturation, int sharpness, bool sepia, bool polaroid, bool greyscale)
{
var response = new ImagePreviewResponse();
//Create the id for the enhanced "preview" version of the source image
string enhancedImageId = Guid.NewGuid().ToString();
var outputFormatString = string.Empty;
ISupportedImageFormat outputFormatProcessorProperty = null;
System.Drawing.Imaging.ImageFormat outputFormatSystemrawingFormat = null;
#region Select image format, or send error
switch (imageFormat)
{
case "jpg":
outputFormatString = ".jpg";
outputFormatProcessorProperty = new JpegFormat {
Quality = 90
};
outputFormatSystemrawingFormat = System.Drawing.Imaging.ImageFormat.Jpeg;
break;
case "gif":
outputFormatString = ".gif";
outputFormatProcessorProperty = new GifFormat {
};
outputFormatSystemrawingFormat = System.Drawing.Imaging.ImageFormat.Gif;
break;
case "png":
outputFormatString = ".png";
outputFormatProcessorProperty = new PngFormat {
};
outputFormatSystemrawingFormat = System.Drawing.Imaging.ImageFormat.Png;
break;
case "bmp":
outputFormatString = ".bmp";
outputFormatProcessorProperty = new BitmapFormat {
};
outputFormatSystemrawingFormat = System.Drawing.Imaging.ImageFormat.Bmp;
break;
case "tiff":
outputFormatString = ".tiff";
outputFormatProcessorProperty = new TiffFormat {
};
outputFormatSystemrawingFormat = System.Drawing.Imaging.ImageFormat.Tiff;
break;
default:
return(new ImagePreviewResponse {
isSuccess = false, ErrorMessage = "Please please use a supported file format: jpg, png, gif, bmp or tiff."
});
}
#endregion
#region verify all input parameters, or send error
if (
brightness < -100 || brightness > 100 ||
sharpness < -100 || sharpness > 100 ||
contrast < -100 || contrast > 100 ||
saturation < -100 || saturation > 100
)
{
return(new ImagePreviewResponse {
isSuccess = false, ErrorMessage = "Please pass in a value between -100 and 100 for all enhancement parameters"
});
}
#endregion
//Build out response object
response.ImageID = enhancedImageId;
response.FileName = enhancedImageId + outputFormatString;
response.SourceFile = imageId + outputFormatString;
#region Connect to Intermediary Storage
// Credentials are from centralized CoreServiceSettings
StorageCredentials storageCredentials = new StorageCredentials(CoreServices.PlatformSettings.Storage.IntermediaryName, CoreServices.PlatformSettings.Storage.IntermediaryKey);
var storageAccount = new CloudStorageAccount(storageCredentials, false);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
#endregion
try
{
//Pull down the source image as MemoryStream from Blob storage and apply image enhancement properties
using (MemoryStream sourceStream = Common.GetAssetFromIntermediaryStorage(response.SourceFile, containerName, blobClient))
{
#region Image Processor & Storage
using (MemoryStream outStream = new MemoryStream())
{
using (ImageFactory imageFactory = new ImageFactory())
{
// Load, resize, set the format and quality and save an image.
// Applies all in order...
sourceStream.Position = 0;
imageFactory.Load(sourceStream);
if (brightness != 0)
{
imageFactory.Brightness(brightness);
}
if (contrast != 0)
{
imageFactory.Contrast(contrast);
}
if (saturation != 0)
{
imageFactory.Saturation(saturation);
}
//Sharpness
if (sharpness != 0)
{
imageFactory.GaussianSharpen(sharpness);
}
//Filters
if (sepia == true)
{
imageFactory.Filter(MatrixFilters.Sepia);
}
if (polaroid == true)
{
imageFactory.Filter(MatrixFilters.Polaroid);
}
if (greyscale == true)
{
imageFactory.Filter(MatrixFilters.GreyScale);
}
imageFactory.Save(outStream);
}
//Store each image size to BLOB STORAGE ---------------------------------
#region BLOB STORAGE
//Creat/Connect to the Blob Container
//blobClient.GetContainerReference(containerName).CreateIfNotExists(BlobContainerPublicAccessType.Blob); //<-- Create and make public
CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
//Get reference to the picture blob or create if not exists.
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(response.FileName);
//Save to storage
//Convert final BMP to byteArray
Byte[] finalByteArray;
finalByteArray = outStream.ToArray();
//Store the enhanced "preview" into the same container as the soure
blockBlob.UploadFromByteArray(finalByteArray, 0, finalByteArray.Length);
#endregion
}
#endregion
}
}
catch (Exception e)
{
return(new ImagePreviewResponse {
isSuccess = false, ErrorMessage = e.Message
});
}
//Marke as successful and return the new ImageID and FileName back to the client for previewing:
response.isSuccess = true;
return(response);
}
private void ProcessPackageEdits(int packageKey)
{
// Create a fresh entities context so that we work in isolation
var entitiesContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
// Get the list of edits for this package
// Do edit with a 'most recent edit to this package wins - other edits are deleted' strategy.
var editsForThisPackage = entitiesContext.Set <PackageEdit>()
.Where(pe => pe.PackageKey == packageKey && pe.TriedCount < 3)
.Include(pe => pe.Package)
.Include(pe => pe.Package.PackageRegistration)
.Include(pe => pe.User)
.OrderByDescending(pe => pe.Timestamp)
.ToList();
// List of Work to do:
// 1) Backup old blob, if the original has not been backed up yet
// 2) Downloads blob, create new NUPKG locally
// 3) Upload blob
// 4) Update the database
PackageEdit edit = editsForThisPackage.First();
var blobClient = StorageAccount.CreateCloudBlobClient();
var packagesContainer = Util.GetPackagesBlobContainer(blobClient);
var latestPackageFileName = Util.GetPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageFileName = Util.GetBackupOfOriginalPackageFileName(edit.Package.PackageRegistration.Id, edit.Package.Version);
var originalPackageBackupBlob = packagesContainer.GetBlockBlobReference(originalPackageFileName);
var latestPackageBlob = packagesContainer.GetBlockBlobReference(latestPackageFileName);
var edits = new List <Action <ManifestMetadata> >
{
(m) => { m.Authors = edit.Authors; },
(m) => { m.Copyright = edit.Copyright; },
(m) => { m.Description = edit.Description; },
(m) => { m.IconUrl = edit.IconUrl; },
(m) => { m.LicenseUrl = edit.LicenseUrl; },
(m) => { m.ProjectUrl = edit.ProjectUrl; },
(m) => { m.ReleaseNotes = edit.ReleaseNotes; },
(m) => { m.RequireLicenseAcceptance = edit.RequiresLicenseAcceptance; },
(m) => { m.Summary = edit.Summary; },
(m) => { m.Title = edit.Title; },
(m) => { m.Tags = edit.Tags; },
};
Log.Info(
"Processing Edit Key={0}, PackageId={1}, Version={2}, User={3}",
edit.Key,
edit.Package.PackageRegistration.Id,
edit.Package.Version,
edit.User.Username);
if (!WhatIf)
{
edit.TriedCount += 1;
int nr = entitiesContext.SaveChanges();
if (nr != 1)
{
throw new Exception(
String.Format("Something went terribly wrong, only one entity should be updated but actually {0} entities were updated", nr));
}
}
try
{
ArchiveOriginalPackageBlob(originalPackageBackupBlob, latestPackageBlob);
using (var readWriteStream = new MemoryStream())
{
// Download to memory
CloudBlockBlob downloadSourceBlob = WhatIf ? latestPackageBlob : originalPackageBackupBlob;
Log.Info("Downloading original package blob to memory {0}", downloadSourceBlob.Name);
downloadSourceBlob.DownloadToStream(readWriteStream);
// Rewrite in memory
Log.Info("Rewriting nupkg package in memory", downloadSourceBlob.Name);
NupkgRewriter.RewriteNupkgManifest(readWriteStream, edits);
// Get updated hash code, and file size
Log.Info("Computing updated hash code of memory stream");
var newPackageFileSize = readWriteStream.Length;
var hashAlgorithm = HashAlgorithm.Create("SHA512");
byte[] hashBytes = hashAlgorithm.ComputeHash(readWriteStream.GetBuffer());
var newHash = Convert.ToBase64String(hashBytes);
if (!WhatIf)
{
// Snapshot the blob
var blobSnapshot = latestPackageBlob.CreateSnapshot();
// Build up the changes in the entities context
edit.Apply(hashAlgorithm: "SHA512", hash: newHash, packageFileSize: newPackageFileSize);
foreach (var eachEdit in editsForThisPackage)
{
entitiesContext.DeleteOnCommit(eachEdit);
}
// Upload the blob before doing SaveChanges(). If blob update fails, we won't do SaveChanges() and the edit can be retried.
// If SaveChanges() fails we can undo the blob upload.
try
{
Log.Info("Uploading blob from memory {0}", latestPackageBlob.Name);
readWriteStream.Position = 0;
latestPackageBlob.UploadFromStream(readWriteStream);
}
catch (Exception e)
{
Log.Error("(error) - package edit blob update failed.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
throw; // To handler block that will record error in DB
}
try
{
// SaveChanges tries to commit changes to DB
entitiesContext.SaveChanges();
}
catch (Exception e)
{
// Commit changes to DB probably failed.
// Since our blob update wasn't part of the transaction (and doesn't AFAIK have a 'commit()' operator we can utilize for the type of blobs we are using)
// try, (single attempt) to roll back the blob update by restoring the previous snapshot.
Log.Error("(error) - package edit DB update failed. Trying to roll back the blob to its previous snapshot.");
Log.ErrorException("(exception)", e);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
try
{
latestPackageBlob.StartCopyFromBlob(blobSnapshot);
}
catch (Exception e2)
{
// If blob rollback fails it is not be the end of the world
// - the package metadata mismatches the edit now,
// but there should still an edit in the queue, waiting to be rerun and put everything back in synch.
Log.Error("(error) - rolling back the package blob to its previous snapshot failed.");
Log.ErrorException("(exception)", e2);
Log.Error("(note) - blob snapshot URL = " + blobSnapshot.Uri);
}
throw; // To handler block that will record error in DB
}
}
}
}
catch (Exception e)
{
if (!WhatIf)
{
try
{
Log.Info("Storing the error on package edit with key {0}", edit.Key);
// Try to record the error into the PackageEdit database record
// so that we can actually diagnose failures.
// This must be done on a fresh context to ensure no conflicts.
var errorContext = new EntitiesContext(ConnectionString.ConnectionString, readOnly: false);
var errorEdit = errorContext.Set <PackageEdit>().Where(pe => pe.Key == edit.Key).FirstOrDefault();
if (errorEdit != null)
{
errorEdit.LastError = string.Format("{0} : {1}", e.GetType(), e);
errorContext.SaveChanges();
}
else
{
Log.Info("The package edit with key {0} couldn't be found. It was likely canceled and deleted.", edit.Key);
}
}
catch (Exception errorException)
{
Log.ErrorException("(error) - couldn't save the last error on the edit that was being applied.", errorException);
}
}
}
}
public static async Task <object> GeneratesInitialSetup([ActivityTrigger] VideoAMS videoDto, TraceWriter log)
{
HttpResponseMessage httpResponse = new HttpResponseMessage();
Asset asset = new Asset();
Locator locator = new Locator();
string accessPolicyId = "";
// Getting the service authenticated
MediaServices mediaService = new MediaServices(_tenantDomain, restApiUrl: _restApiUrl, _clientId, _clientSecret, _storageConnection);
try
{
await mediaService.InitializeAccessTokenAsync();
log.Info("Authenication... Done.");
}
catch (Exception ex)
{
return(httpResponse.RequestMessage.CreateResponse(HttpStatusCode.Unauthorized, $"It wasn't possible get the service authenticated. \n {ex.StackTrace}"));
}
// Creating asset and locator
try
{
// Creating the asset
accessPolicyId = await mediaService.GenerateAccessPolicy(videoDto.AccessPolicyName, 100, 2);
asset = await mediaService.GenerateAsset(videoDto.AssetName, videoDto.StorageAccountName);
log.Info("Asset creation... Done.");
// Creating the locator
locator = await mediaService.GenerateLocator(accessPolicyId, asset.Id, DateTime.Now.AddMinutes(-5), 1);
log.Info("Locator creation... Done.");
}
catch (Exception ex)
{
return(httpResponse.RequestMessage.CreateResponse(HttpStatusCode.InternalServerError, $"It wasn't possible to create the asset. \n {ex.StackTrace}"));
}
// Moving the original video into the asset
try
{
// Converting string path into Uri
Uri uriSource = new Uri(videoDto.VideoPath, UriKind.Absolute);
// Moving file into the asset
CloudBlockBlob sourceBlob;
sourceBlob = new CloudBlockBlob(uriSource);
await MediaServices.MoveVideoToAssetLocator(sourceBlob, locator, _storageConnection);
await mediaService.GenerateFileInfo(asset.Id);
log.Info("Moving video... Done.");
}
catch (Exception ex)
{
return(httpResponse.RequestMessage.CreateResponse(HttpStatusCode.InternalServerError, $"It wasn't possible to upload the video. \n {ex.StackTrace}"));
}
return(new InitialSetupResult {
Asset = asset,
Locator = locator,
Video = videoDto
});
}
public async Task AddQueueMessage(string batchId, SalesCatalogueProductResponse salesCatalogueResponse, string callBackUri, string correlationId, CloudBlockBlob cloudBlockBlob, int instanceNumber, string storageAccountConnectionString, string expiryDate)
{
SalesCatalogueServiceResponseQueueMessage scsResponseQueueMessage = GetSalesCatalogueServiceResponseQueueMessage(batchId, salesCatalogueResponse, callBackUri, correlationId, cloudBlockBlob, expiryDate);
var scsResponseQueueMessageJSON = JsonConvert.SerializeObject(scsResponseQueueMessage);
await azureMessageQueueHelper.AddMessage(batchId, instanceNumber, storageAccountConnectionString, scsResponseQueueMessageJSON, correlationId);
}
public void ValidatePipelineICloudBlobWithInvalidBlobNameTest()
{
CloudBlockBlob blob = new CloudBlockBlob(new Uri("http://127.0.0.1/account/container/"));
AssertThrows <ArgumentException>(() => command.ValidatePipelineICloudBlob(blob), String.Format(Resources.InvalidBlobName, blob.Name));
}
public static async Task <object> Run([HttpTrigger(WebHookType = "genericJson")] HttpRequestMessage req, TraceWriter log)
{
log.Info($"AMS v2 Function - GetCaptionBlobSasUri was triggered!");
string jsonContent = await req.Content.ReadAsStringAsync();
dynamic data = JsonConvert.DeserializeObject(jsonContent);
// parameter handling
if (data.assetId == null)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, new { error = "Please pass assetId in the input object" }));
}
string assetId = data.assetId;
string timecodeOffset = null;
if (data.timecodeOffset != null)
{
timecodeOffset = data.timecodeOffset;
}
MediaServicesCredentials amsCredentials = new MediaServicesCredentials();
IAsset asset = null;
string vttContent = "";
string vttContentTimeOffset = "";
string vttBlobSasUri = null;
string ttmlContent = "";
string ttmlContentTimeOffset = "";
string ttmlBlobSasUri = null;
try
{
// Load AMS account context
log.Info($"Using AMS v2 REST API Endpoint : {amsCredentials.AmsRestApiEndpoint.ToString()}");
AzureAdTokenCredentials tokenCredentials = new AzureAdTokenCredentials(amsCredentials.AmsAadTenantDomain,
new AzureAdClientSymmetricKey(amsCredentials.AmsClientId, amsCredentials.AmsClientSecret),
AzureEnvironments.AzureCloudEnvironment);
AzureAdTokenProvider tokenProvider = new AzureAdTokenProvider(tokenCredentials);
_context = new CloudMediaContext(amsCredentials.AmsRestApiEndpoint, tokenProvider);
// Get the Asset
asset = _context.Assets.Where(a => a.Id == assetId).FirstOrDefault();
if (asset == null)
{
return(req.CreateResponse(HttpStatusCode.BadRequest, new { error = "Asset not found" }));
}
string destinationContainerName = asset.Uri.Segments[1];
var vttAssetFile = asset.AssetFiles.Where(a => a.Name.ToUpper().EndsWith(".VTT")).FirstOrDefault();
var ttmlAssetFile = asset.AssetFiles.Where(a => a.Name.ToUpper().EndsWith(".TTML")).FirstOrDefault();
if (vttAssetFile != null)
{
vttContent = MediaServicesHelper.GetContentFromAssetFile(vttAssetFile);
CloudBlobContainer destinationBlobContainer = BlobStorageHelper.GetCloudBlobContainer(BlobStorageHelper.AmsStorageAccountName, BlobStorageHelper.AmsStorageAccountKey, destinationContainerName);
CloudBlockBlob blobVTT = destinationBlobContainer.GetBlockBlobReference(vttAssetFile.Name);
if (timecodeOffset != null) // let's update the ttml with new timecode
{
var tcOffset = TimeSpan.Parse(timecodeOffset);
string arrow = " --> ";
System.Text.StringBuilder sb = new System.Text.StringBuilder();
string[] delim = { Environment.NewLine, "\n" }; // "\n" added in case you manually appended a newline
string[] vttlines = vttContent.Split(delim, StringSplitOptions.None);
foreach (string vttline in vttlines)
{
string line = vttline;
if (vttline.Contains(arrow))
{
TimeSpan begin = TimeSpan.Parse(vttline.Substring(0, vttline.IndexOf(arrow)));
TimeSpan end = TimeSpan.Parse(vttline.Substring(vttline.IndexOf(arrow) + 5));
line = (begin + tcOffset).ToString(@"d\.hh\:mm\:ss\.fff") + arrow + (end + tcOffset).ToString(@"d\.hh\:mm\:ss\.fff");
}
sb.AppendLine(line);
}
vttContentTimeOffset = sb.ToString();
if (vttContentTimeOffset != null)
{
string blobName = "Converted-" + vttAssetFile.Name;
blobVTT = MediaServicesHelper.WriteContentToBlob(asset, destinationBlobContainer, blobName, vttContentTimeOffset);
}
}
// Get Blob URI with SAS Token
var sasBlobPolicy = new SharedAccessBlobPolicy();
sasBlobPolicy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5);
sasBlobPolicy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(10);
sasBlobPolicy.Permissions = SharedAccessBlobPermissions.Read;
string sasBlobToken = blobVTT.GetSharedAccessSignature(sasBlobPolicy);
vttBlobSasUri = blobVTT.Uri + sasBlobToken;
}
if (ttmlAssetFile != null)
{
ttmlContent = MediaServicesHelper.GetContentFromAssetFile(ttmlAssetFile);
CloudBlobContainer destinationBlobContainer = BlobStorageHelper.GetCloudBlobContainer(BlobStorageHelper.AmsStorageAccountName, BlobStorageHelper.AmsStorageAccountKey, destinationContainerName);
CloudBlockBlob blobTTML = destinationBlobContainer.GetBlockBlobReference(ttmlAssetFile.Name);
if (timecodeOffset != null) // let's update the vtt with new timecode
{
var tcOffset = TimeSpan.Parse(timecodeOffset);
//log.Info("tsoffset : " + tcOffset.ToString(@"d\.hh\:mm\:ss\.fff"));
XNamespace xmlns = "http://www.w3.org/ns/ttml";
XDocument docXML = XDocument.Parse(ttmlContent);
var tt = docXML.Element(xmlns + "tt");
var ttmlElements = docXML.Element(xmlns + "tt").Element(xmlns + "body").Element(xmlns + "div").Elements(xmlns + "p");
foreach (var ttmlElement in ttmlElements)
{
var begin = TimeSpan.Parse((string)ttmlElement.Attribute("begin"));
var end = TimeSpan.Parse((string)ttmlElement.Attribute("end"));
ttmlElement.SetAttributeValue("end", (end + tcOffset).ToString(@"d\.hh\:mm\:ss\.fff"));
ttmlElement.SetAttributeValue("begin", (begin + tcOffset).ToString(@"d\.hh\:mm\:ss\.fff"));
}
ttmlContentTimeOffset = docXML.Declaration.ToString() + Environment.NewLine + docXML.ToString();
if (ttmlContentTimeOffset != null)
{
string blobName = "Converted-" + ttmlAssetFile.Name;
blobTTML = MediaServicesHelper.WriteContentToBlob(asset, destinationBlobContainer, blobName, ttmlContentTimeOffset);
}
}
// Get Blob URI with SAS Token
var sasBlobPolicy = new SharedAccessBlobPolicy();
sasBlobPolicy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5);
sasBlobPolicy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(10);
sasBlobPolicy.Permissions = SharedAccessBlobPermissions.Read;
string sasBlobToken = blobTTML.GetSharedAccessSignature(sasBlobPolicy);
ttmlBlobSasUri = blobTTML.Uri + sasBlobToken;
}
}
catch (Exception e)
{
log.Info($"Exception {e}");
return(req.CreateResponse(HttpStatusCode.BadRequest));
}
return(req.CreateResponse(HttpStatusCode.OK, new
{
vttBlobSasUri = vttBlobSasUri,
ttmlBlobSasUri = ttmlBlobSasUri
}));
}
/// <summary>
/// Basic operations to work with block blobs
/// </summary>
/// <returns>A Task object.</returns>
private static async Task BasicStorageBlockBlobOperationsWithAccountSASAsync()
{
const string ImageToUpload = "Tiger.png";
string containerName = ContainerPrefix + Guid.NewGuid();
// Get an account SAS token.
string sasToken = GetAccountSASToken();
// Use the account SAS token to create authentication credentials.
StorageCredentials accountSAS = new StorageCredentials(sasToken);
// Informational: Print the Account SAS Signature and Token.
Console.WriteLine();
Console.WriteLine("Account SAS Signature: " + accountSAS.SASSignature);
Console.WriteLine("Account SAS Token: " + accountSAS.SASToken);
Console.WriteLine();
// Get the URI for the container.
Uri containerUri = GetContainerUri(containerName);
// Get a reference to a container using the URI and the SAS token.
CloudBlobContainer container = new CloudBlobContainer(containerUri, accountSAS);
try
{
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container using Account SAS");
await container.CreateIfNotExistsAsync();
}
catch (StorageException e)
{
Console.WriteLine(e.ToString());
Console.WriteLine("If you are running with the default configuration, please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
Console.ReadLine();
throw;
}
try
{
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the image
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/democontainer/HelloWorld.png
// await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
// Upload a BlockBlob to the newly created container
Console.WriteLine("2. Uploading BlockBlob");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(ImageToUpload);
await blockBlob.UploadFromFileAsync(ImageToUpload);
// List all the blobs in the container
Console.WriteLine("3. List Blobs in Container");
BlobContinuationToken token = null;
do
{
BlobResultSegment resultSegment = await container.ListBlobsSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (IListBlobItem blob in resultSegment.Results)
{
// Blob type will be CloudBlockBlob, CloudPageBlob or CloudBlobDirectory
Console.WriteLine("{0} (type: {1}", blob.Uri, blob.GetType());
}
}while (token != null);
// Download a blob to your file system
Console.WriteLine("4. Download Blob from {0}", blockBlob.Uri.AbsoluteUri);
await blockBlob.DownloadToFileAsync(string.Format("./CopyOf{0}", ImageToUpload), FileMode.Create);
// Create a read-only snapshot of the blob
Console.WriteLine("5. Create a read-only snapshot of the blob");
CloudBlockBlob blockBlobSnapshot = await blockBlob.CreateSnapshotAsync(null, null, null, null);
// Delete the blob and its snapshots.
Console.WriteLine("6. Delete block Blob and all of its snapshots");
await blockBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
finally
{
// Clean up after the demo.
// Note that it is not necessary to delete all of the blobs in the container first; they will be deleted
// with the container.
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
}
public TestFixture()
{
RandomNameResolver nameResolver = new RandomNameResolver();
JobHostConfiguration hostConfiguration = new JobHostConfiguration()
{
NameResolver = nameResolver,
TypeLocator = new FakeTypeLocator(typeof(BlobBindingEndToEndTests)),
};
Config = hostConfiguration;
StorageAccount = CloudStorageAccount.Parse(hostConfiguration.StorageConnectionString);
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
BlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName));
Assert.False(BlobContainer.Exists());
BlobContainer.Create();
OutputBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(OutputContainerName));
CloudBlobContainer pageBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(pageBlobContainer.Exists());
pageBlobContainer.Create();
CloudBlobContainer hierarchicalBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(hierarchicalBlobContainer.Exists());
hierarchicalBlobContainer.Create();
Host = new JobHost(hostConfiguration);
Host.Start();
// upload some test blobs
CloudBlockBlob blob = BlobContainer.GetBlockBlobReference("blob1");
blob.UploadText(TestData);
blob = BlobContainer.GetBlockBlobReference("blob2");
blob.UploadText(TestData);
blob = BlobContainer.GetBlockBlobReference("blob3");
blob.UploadText(TestData);
blob = BlobContainer.GetBlockBlobReference("file1");
blob.UploadText(TestData);
blob = BlobContainer.GetBlockBlobReference("file2");
blob.UploadText(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob1");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob2");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/sub/blob3");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("blob4");
blob.UploadText(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
CloudPageBlob pageBlob = pageBlobContainer.GetPageBlobReference("blob1");
pageBlob.UploadFromByteArray(bytes, 0, bytes.Length);
pageBlob = pageBlobContainer.GetPageBlobReference("blob2");
pageBlob.UploadFromByteArray(bytes, 0, bytes.Length);
}
public MessageResponse convert(ConversionRequest reqValue)
{
LOGGER.Info("Conversion processing...");
if (reqValue == null || string.IsNullOrEmpty(reqValue.blobHeaderName) || string.IsNullOrEmpty(reqValue.blobOutputName) || string.IsNullOrEmpty(reqValue.containerName))
{
return(MessageResponse.info("[blobHeaderName, blobOutputName, containerName] are required."));
}
string BlobContainer = reqValue.containerName.Trim();
string BlobInputName = reqValue.blobHeaderName.Trim();
string BlobOutputName = reqValue.blobOutputName.Trim();
string AESKey = Vault.Current.AESKeyBLOB;
string storageConnectionString = Vault.Current.StorageConnectionString;
//validate parameter
if (string.IsNullOrEmpty(BlobContainer))
{
return(MessageResponse.info("Blob container is required."));
}
if (string.IsNullOrEmpty(BlobInputName))
{
return(MessageResponse.info("Blob file input is required."));
}
if (string.IsNullOrEmpty(BlobOutputName))
{
return(MessageResponse.info("Blob file output is required."));
}
if (string.IsNullOrEmpty(AESKey))
{
return(MessageResponse.info("AESKEY to encrypt/decrypt file is required."));
}
try
{
// Retrieve reference to a previously created container.
CloudBlobContainer blobContainer = BlobHelper.GetCloudBlobContainer(storageConnectionString, BlobContainer);
Task <bool> IsExitBlobContainer = blobContainer.ExistsAsync();
if (blobContainer == null || !IsExitBlobContainer.Result)
{
return(MessageResponse.info(string.Format("Can't find the BLOB container [{0}].", BlobContainer)));
}
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlobInput = blobContainer.GetBlockBlobReference(BlobInputName);
Task <bool> IsExitblockBlobInput = blockBlobInput.ExistsAsync();
if (blockBlobInput == null || !IsExitblockBlobInput.Result)
{
return(MessageResponse.info(string.Format("Can't find the BLOB file [{0}].", BlobInputName)));
}
//download file
byte[] fileInput = BlobHelper.DownloadFileToArrayByte(blockBlobInput, AESKey);
if (fileInput == null)
{
return(MessageResponse.info(string.Format("Can't find the content of the BLOB file [{0}].", BlobInputName)));
}
//do conversion
StringBuilder sbStandardFile = this.doConvert(fileInput);
//no error: write the file standard
if (this.ListErrorVendor == null || this.ListErrorVendor.Count == 0)
{
//upload the file standard file to azure
BlobHelper.UploadFile(blobContainer, BlobOutputName, sbStandardFile, AESKey);
//return the blob output name of file saved
//LOGGER.Info(string.Format("File [{0}] converted to the file [{1}] and saved successfully in the BLOB container [{2}]", BlobInputName, BlobOutputName, BlobContainer));
return(MessageResponse.ok(reqValue.blobOutputName));
}
//report errors if have
if (this.ListErrors == null)
{
this.ListErrors = new List <string>();
}
if (this.ListErrorVendor.Count > 0)
{
this.ListErrors.AddRange(this.ListErrorVendor);
}
return(MessageResponse.error(this.ListErrors));
}
catch (Exception ex)
{
LOGGER.Error(ex);
throw;
}
}
// Overload: Override the default file overwrite setting at the class level for this specific file
public async Task <string> DownloadStorageBlockBlobSegmentedOperationAsync(string MediaFile, bool overwrite)
{
try
{
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
WriteLine("Opening Blob Container in Azure Storage.");
CloudBlobContainer container = blobClient.GetContainerReference(BlobContainerName);
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException)
{
WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
throw;
}
// Access a specific blob in the container
WriteLine("Get Specific Blob in Container and its size");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(MediaFile);
int segmentSize = SegmentSizeKB * 1024; // SegmentSizeKB is set in the inspector chunk
if (blockBlob != null)
{
// Obtain the size of the blob
await blockBlob.FetchAttributesAsync();
long blobSize = blockBlob.Properties.Length;
long blobLengthRemaining = blobSize;
float completion = 0f;
long startPosition = 0;
WriteLine("3. Blob size (bytes):" + blobLengthRemaining.ToString());
// Download a blob to your file system
string path = "";
WriteLine(string.Format("Downloading Blob from {0}, please wait...", blockBlob.Uri.AbsoluteUri));
string fileName = MediaFile; // string.Format("CopyOf{0}", MediaFile);
bool fileExists = false;
#if WINDOWS_UWP
StorageFolder storageFolder = ApplicationData.Current.TemporaryFolder;
StorageFile sf;
try
{
CreationCollisionOption collisionoption = (overwrite ? CreationCollisionOption.ReplaceExisting : CreationCollisionOption.FailIfExists);
sf = await storageFolder.CreateFileAsync(fileName, collisionoption);
fileExists = false; // if the file existed but we were allowed to overwrite it, let's treat it as if it didn't exist
}
catch (Exception)
{
// The file already exists and we're not supposed to overwrite it
fileExists = true;
sf = await storageFolder.GetFileAsync(fileName); // Necessary to avoid a compilation error below
}
path = sf.Path;
#else
path = Path.Combine(Application.temporaryCachePath, fileName);
fileExists = File.Exists(path);
#endif
if (fileExists)
{
if (overwrite)
{
WriteLine(string.Format("Already exists. Deleting file {0}", fileName));
#if WINDOWS_UWP
// Nothing to do here in UWP, we already Replaced it when we created the StorageFile
#else
File.Delete(path);
#endif
}
else
{
WriteLine(string.Format("File {0} already exists and overwriting is disabled. Download operation cancelled.", fileName));
return(path);
}
}
ProgressBar.AddDownload();
#if WINDOWS_UWP
var fs = await sf.OpenAsync(FileAccessMode.ReadWrite);
#else
FileStream fs = new FileStream(path, FileMode.OpenOrCreate);
#endif
// Start the timer to measure performance
var sw = Stopwatch.StartNew();
do
{
long blockSize = Math.Min(segmentSize, blobLengthRemaining);
byte[] blobContents = new byte[blockSize];
using (MemoryStream ms = new MemoryStream())
{
await blockBlob.DownloadRangeToStreamAsync(ms, (long)startPosition, blockSize);
ms.Position = 0;
ms.Read(blobContents, 0, blobContents.Length);
#if WINDOWS_UWP
fs.Seek((ulong)startPosition);
await fs.WriteAsync(blobContents.AsBuffer());
#else
fs.Position = startPosition;
fs.Write(blobContents, 0, blobContents.Length);
#endif
}
completion = (float)startPosition / (float)blobSize;
WriteLine("Completed: " + (completion).ToString("P"));
ProgressBar.Value = (completion * 100);
startPosition += blockSize;
blobLengthRemaining -= blockSize;
}while (blobLengthRemaining > 0);
WriteLine("Completed: 100.00%");
ProgressBar.Value = 100;
ProgressBar.RemoveDownload();
#if !WINDOWS_UWP
// Required for Mono & .NET or we'll get a file IO access violation the next time we try to access it
fs.Close();
#else
fs.Dispose();
#endif
fs = null;
// Stop the timer and report back on completion + performance
sw.Stop();
TimeSpan time = sw.Elapsed;
WriteLine(string.Format("5. Blob file downloaded to {0} in {1}s", path, time.TotalSeconds.ToString()));
return(path);
}
else
{
WriteLine(string.Format("3. File {0} not found in blob {1}", MediaFile, blockBlob.Uri.AbsoluteUri));
return(string.Empty);
}
}
catch (Exception ex)
{
// Woops!
WriteLine(string.Format("Error while downloading file {0}", MediaFile));
WriteLine("Error: " + ex.ToString());
WriteLine("Error: " + ex.InnerException.ToString());
return(string.Empty);
}
}
// Overload: Override the default file overwrite setting at the class level for this specific file
public async Task <string> DownloadStorageBlockBlobBasicOperationAsync(string MediaFile, bool overwrite)
{
try
{
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
WriteLine("Opening Blob Container in Azure Storage.");
CloudBlobContainer container = blobClient.GetContainerReference(BlobContainerName);
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException)
{
WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
throw;
}
// Access a specific blob in the container
WriteLine("Getting Specific Blob in Container.");
// We assume the client app knows which asset to download by name
CloudBlockBlob blockBlob = container.GetBlockBlobReference(MediaFile);
if (blockBlob != null)
{
// Download a blob to your file system
string path = "";
WriteLine(string.Format("Downloading Blob from {0}, please wait...", blockBlob.Uri.AbsoluteUri));
string fileName = MediaFile; // string.Format("CopyOf{0}", MediaFile);
bool fileExists = false;
#if WINDOWS_UWP
StorageFolder storageFolder = ApplicationData.Current.TemporaryFolder;
StorageFile sf;
try
{
CreationCollisionOption collisionoption = (overwrite ? CreationCollisionOption.ReplaceExisting : CreationCollisionOption.FailIfExists);
sf = await storageFolder.CreateFileAsync(fileName, collisionoption);
fileExists = false; // if the file existed but we were allowed to overwrite it, let's treat it as if it didn't exist
path = sf.Path;
}
catch (Exception)
{
// The file already exists and we're not supposed to overwrite it
fileExists = true;
sf = await storageFolder.GetFileAsync(fileName); // Necessary to avoid a compilation error below
}
#else
path = Path.Combine(Application.temporaryCachePath, fileName);
fileExists = File.Exists(path);
#endif
if (fileExists)
{
if (overwrite)
{
WriteLine(string.Format("Already exists. Deleting file {0}", fileName));
#if WINDOWS_UWP
// Nothing to do here in UWP, we already Replaced it when we created the StorageFile
#else
File.Delete(path);
#endif
}
else
{
WriteLine(string.Format("File {0} already exists and overwriting is disabled. Download operation cancelled.", fileName));
return(path);
}
}
// Start the timer to measure performance
var sw = Stopwatch.StartNew();
//#if WINDOWS_UWP
// await blockBlob.DownloadToFileAsync(sf);
//#else
await blockBlob.DownloadToFileAsync(path, FileMode.Create);
//#endif
// Stop the timer and report back on completion + performance
sw.Stop();
TimeSpan time = sw.Elapsed;
WriteLine(string.Format("Blob file downloaded to {0} in {1}s.", path, time.TotalSeconds.ToString()));
return(path);
}
else
{
WriteLine(string.Format("File {0} not found in blob {1}.", MediaFile, blockBlob.Uri.AbsoluteUri));
return(string.Empty);
}
}
catch (Exception ex)
{
// Woops!
WriteLine(string.Format("Error while downloading file {0}.", MediaFile));
WriteLine("Error: " + ex.ToString());
WriteLine("Error: " + ex.InnerException.ToString());
return(string.Empty);
}
}
public AzureBlobStorage(CloudBlockBlob blob)
{
this.blob = blob ?? throw new ArgumentNullException(nameof(blob));
}
internal async Task UploadToBlobAsync(string blobName, Stream source, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var fileUploadRequest = new FileUploadRequest()
{
BlobName = blobName
};
var fileUploadResponse = await this.httpClientHelper.PostAsync <FileUploadRequest, FileUploadResponse>(
GetRequestUri(this.deviceId, CommonConstants.BlobUploadPathTemplate, null),
fileUploadRequest,
ExceptionHandlingHelper.GetDefaultErrorMapping(),
null,
cancellationToken).ConfigureAwait(false);
string putString = String.Format(
CultureInfo.InvariantCulture,
"https://{0}/{1}/{2}{3}",
fileUploadResponse.HostName,
fileUploadResponse.ContainerName,
Uri.EscapeDataString(fileUploadResponse.BlobName), // Pass URL encoded device name and blob name to support special characters
fileUploadResponse.SasToken);
var notification = new FileUploadNotificationResponse();
try
{
// 2. Use SAS URI to send data to Azure Storage Blob (PUT)
CloudBlockBlob blob = new CloudBlockBlob(new Uri(putString));
var uploadTask = blob.UploadFromStreamAsync(source);
await uploadTask.ConfigureAwait(false);
notification.CorrelationId = fileUploadResponse.CorrelationId;
notification.IsSuccess = uploadTask.IsCompleted;
notification.StatusCode = uploadTask.IsCompleted ? 0 : -1;
notification.StatusDescription = uploadTask.IsCompleted ? null : "Failed to upload to storage.";
// 3. POST to IoTHub with upload status
await this.httpClientHelper.PostAsync <FileUploadNotificationResponse>(
GetRequestUri(this.deviceId, CommonConstants.BlobUploadStatusPathTemplate + "notifications", null),
notification,
ExceptionHandlingHelper.GetDefaultErrorMapping(),
null,
cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex) when(!ex.IsFatal())
{
// 3. POST to IoTHub with upload status
notification.IsSuccess = false;
notification.StatusCode = -1;
notification.StatusDescription = ex.Message;
await this.httpClientHelper.PostAsync <FileUploadNotificationResponse>(
GetRequestUri(this.deviceId, CommonConstants.BlobUploadStatusPathTemplate + "notifications/" + fileUploadResponse.CorrelationId, null),
notification,
ExceptionHandlingHelper.GetDefaultErrorMapping(),
null,
cancellationToken).ConfigureAwait(false);
throw;
}
}
public void RemoveImage(string fileName)
{
CloudBlockBlob blockBlob = _container.GetBlockBlobReference(fileName);
blockBlob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots);
}
public static async Task Run([QueueTrigger("release-queue", Connection = "ReleaseStorage")] string myQueueItem, [Blob("releases/*", Connection = "ReleaseStorage")] CloudBlockBlob blobContainer, TraceWriter log)
{
//Parse queue message
var releaseDetails = myQueueItem.Split('|');
var releaseName = releaseDetails[0];
var releaseBody = releaseDetails[1];
var repoName = releaseDetails[2];
//Get issues and pull requests from release repo
string txtIssues = await GetReleaseDetails(IssueTypeQualifier.Issue, repoName);
string txtPulls = await GetReleaseDetails(IssueTypeQualifier.PullRequest, repoName);
//Get reference to blob container
var myBlobContainer = blobContainer.Container;
//Create a blob with the release name as the file name
var blob = myBlobContainer.GetBlockBlobReference(releaseName + ".md");
//Format release text and append to blob
var releaseText = String.Format("# {0} \n {1} \n\n" + "# Issues Closed:" + txtIssues + "\n\n# Changes Merged:" + txtPulls, releaseName, releaseBody);
await blob.UploadTextAsync(releaseText);
}
internal Task <string> DownloadAndDecompressAsBytesAsync(Uri blobUri)
{
CloudBlockBlob cloudBlockBlob = new CloudBlockBlob(blobUri, this.cloudBlobContainer.ServiceClient.Credentials);
return(DownloadAndDecompressAsBytesAsync(cloudBlockBlob));
}
/// <summary>
/// Basic operations to work with block blobs
/// </summary>
/// <returns>A Task object.</returns>
private static async Task BasicStorageBlockBlobOperationsAsync()
{
const string ImageToUpload = "Tiger.png";
string containerName = ContainerPrefix + Guid.NewGuid();
// Retrieve storage account information from connection string
CloudStorageAccount storageAccount = Cfg.GetAccount();
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container");
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
try
{
// The call below will fail if the sample is configured to use the storage emulator in the connection string, but
// the emulator is not running.
// Change the retry policy for this call so that if it fails, it fails quickly.
BlobRequestOptions requestOptions = new BlobRequestOptions()
{
RetryPolicy = new NoRetry()
};
await container.CreateIfNotExistsAsync(requestOptions, null);
}
catch (StorageException)
{
Console.WriteLine("If you are running with the default connection string, please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
Console.ReadLine();
throw;
}
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the image
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/democontainer/HelloWorld.png
// await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
// Upload a BlockBlob to the newly created container
Console.WriteLine("2. Uploading BlockBlob");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(ImageToUpload);
// Set the blob's content type so that the browser knows to treat it as an image.
blockBlob.Properties.ContentType = "image/png";
await blockBlob.UploadFromFileAsync(ImageToUpload);
// List all the blobs in the container.
/// Note that the ListBlobs method is called synchronously, for the purposes of the sample. However, in a real-world
/// application using the async/await pattern, best practices recommend using asynchronous methods consistently.
Console.WriteLine("3. List Blobs in Container");
BlobResultSegment segment = await container.ListBlobsSegmentedAsync(null);
foreach (IListBlobItem blob in segment.Results)
{
// Blob type will be CloudBlockBlob, CloudPageBlob or CloudBlobDirectory
// Use blob.GetType() and cast to appropriate type to gain access to properties specific to each type
Console.WriteLine("- {0} (type: {1})", blob.Uri, blob.GetType());
}
// Download a blob to your file system
Console.WriteLine("4. Download Blob from {0}", blockBlob.Uri.AbsoluteUri);
await blockBlob.DownloadToFileAsync(string.Format("./CopyOf{0}", ImageToUpload), FileMode.Create);
// Create a read-only snapshot of the blob
Console.WriteLine("5. Create a read-only snapshot of the blob");
CloudBlockBlob blockBlobSnapshot = await blockBlob.CreateSnapshotAsync(null, null, null, null);
// Clean up after the demo. This line is not strictly necessary as the container is deleted in the next call.
// It is included for the purposes of the example.
Console.WriteLine("6. Delete block blob and all of its snapshots");
await blockBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null);
// Note that deleting the container also deletes any blobs in the container, and their snapshots.
// In the case of the sample, we delete the blob and its snapshots, and then the container,
// to show how to delete each kind of resource.
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
public async Task DeleteFileAsync(string name)
{
CloudBlockBlob blob = GetBlob(name);
await blob.DeleteIfExistsAsync();
}
private SalesCatalogueServiceResponseQueueMessage GetSalesCatalogueServiceResponseQueueMessage(string batchId, SalesCatalogueProductResponse salesCatalogueResponse, string callBackUri, string correlationId, CloudBlockBlob cloudBlockBlob, string expiryDate)
{
long fileSize = CommonHelper.GetFileSize(salesCatalogueResponse);
var scsResponseQueueMessage = new SalesCatalogueServiceResponseQueueMessage()
{
BatchId = batchId,
ScsResponseUri = cloudBlockBlob.Uri.AbsoluteUri,
FileSize = fileSize,
CallbackUri = callBackUri == null ? string.Empty : callBackUri,
CorrelationId = correlationId,
ExchangeSetUrlExpiryDate = expiryDate
};
return(scsResponseQueueMessage);
}
public CloudBlockBlob GetBlob(string name)
{
CloudBlockBlob blob = container.GetBlockBlobReference(name);
return(blob);
}
private async Task UpstreamFromBlob(string container, ModuleClient moduleClient)
{
Console.WriteLine("***");
Console.WriteLine($"Upstream from Blob: {container}");
int count = 1;
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
cloudBlobContainer = cloudBlobClient.GetContainerReference(container);
if (!(await cloudBlobContainer.ExistsAsync()))
{
await cloudBlobContainer.CreateIfNotExistsAsync();
}
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
if (!shouldUpstream)
{
break;
}
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
Console.WriteLine(await blob.DownloadTextAsync());
var body = await blob.DownloadTextAsync();
if (!string.IsNullOrEmpty(body))
{
var messageBytes = Encoding.UTF8.GetBytes(body);
if (IsLimitsReached(messageBytes))
{
shouldUpstream = false;
}
else
{
var message = new Message(messageBytes);
message.ContentEncoding = "utf-8";
message.ContentType = "application/json";
if (container == anomalyContainer)
{
message.Properties.Add("IsAnomaly", "True");
}
await moduleClient.SendEventAsync("output1", message);
Console.WriteLine($"Message sent {count}: {body}");
count++;
}
}
}
}
} while (blobContinuationToken != null && shouldUpstream);
}
catch (StorageException ex)
{
Console.WriteLine($"Error returned from the service: {ex}");
}
}
}
public AzureCloudBlockBlob(Uri uri, StorageCredentials credentials)
{
_blob = new CloudBlockBlob(uri, credentials);
}
private async Task StartCopyFromFile(long taskId, IStorageBlobManagement destChannel, CloudFile srcFile, CloudBlockBlob destBlob)
{
await this.StartCopyFromUri(taskId, destChannel, srcFile.GenerateUriWithCredentials(), destBlob).ConfigureAwait(false);
}
public static async Task Run(
[BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "nsgSourceDataConnection")] CloudBlockBlob myBlob,
[Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable,
Binder nsgDataBlobBinder,
Binder cefLogBinder,
string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac,
ExecutionContext executionContext,
ILogger log)
{
log.LogDebug($"BlobTriggerIngestAndTransmit triggered: {executionContext.InvocationId} ");
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.LogError("Value for nsgSourceDataAccount is required.");
throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting.");
}
string blobContainerName = Util.GetEnvironmentVariable("blobContainerName");
if (blobContainerName.Length == 0)
{
log.LogError("Value for blobContainerName is required.");
throw new System.ArgumentNullException("blobContainerName", "Please provide setting.");
}
string outputBinding = Util.GetEnvironmentVariable("outputBinding");
if (outputBinding.Length == 0)
{
log.LogError("Value for outputBinding is required. Permitted values are: 'arcsight', 'splunk', 'eventhub'.");
throw new System.ArgumentNullException("outputBinding", "Please provide setting.");
}
var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac);
// get checkpoint
Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable);
var blockList = myBlob.DownloadBlockListAsync().Result;
var startingByte = blockList.Where((item, index) => index < checkpoint.CheckpointIndex).Sum(item => item.Length);
var endingByte = blockList.Where((item, index) => index < blockList.Count() - 1).Sum(item => item.Length);
var dataLength = endingByte - startingByte;
log.LogDebug("Blob: {0}, starting byte: {1}, ending byte: {2}, number of bytes: {3}", blobDetails.ToString(), startingByte, endingByte, dataLength);
if (dataLength == 0)
{
log.LogWarning(string.Format("Blob: {0}, triggered on completed hour.", blobDetails.ToString()));
return;
}
//foreach (var item in blockList)
//{
// log.LogInformation("Name: {0}, Length: {1}", item.Name, item.Length);
//}
var attributes = new Attribute[]
{
new BlobAttribute(string.Format("{0}/{1}", blobContainerName, myBlob.Name)),
new StorageAccountAttribute(nsgSourceDataAccount)
};
string nsgMessagesString = "";
var bytePool = ArrayPool <byte> .Shared;
byte[] nsgMessages = bytePool.Rent((int)dataLength);
try
{
CloudBlockBlob blob = nsgDataBlobBinder.BindAsync <CloudBlockBlob>(attributes).Result;
await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, startingByte, dataLength);
if (nsgMessages[0] == ',')
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 1, (int)(dataLength - 1));
}
else
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 0, (int)dataLength);
}
}
catch (Exception ex)
{
log.LogError(string.Format("Error binding blob input: {0}", ex.Message));
throw ex;
}
finally
{
bytePool.Return(nsgMessages);
}
//log.LogDebug(nsgMessagesString);
try
{
int bytesSent = await Util.SendMessagesDownstreamAsync(nsgMessagesString, executionContext, cefLogBinder, log);
log.LogInformation($"Sending {nsgMessagesString.Length} bytes (denormalized to {bytesSent} bytes) downstream via output binding {outputBinding}.");
}
catch (Exception ex)
{
log.LogError(string.Format("SendMessagesDownstreamAsync: Error {0}", ex.Message));
throw ex;
}
checkpoint.PutCheckpoint(checkpointTable, blockList.Count() - 1);
}
/// <summary>
/// Deletes the given blob
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobName"></param>
/// <returns></returns>
public async Task DeleteBlobAsync(string containerName, string blobName)
{
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
await blockBlob.DeleteAsync();
}
public async Task CheckBlobPermition(CloudBlockBlob abb)
{
await AzureStorageProviderHelper.CheckBlobPermition(abb, _permition);
}
public string AddLabeledData()
{
string TrainingDataUrl;
CloudStorageAccount StorageAccount = _Engine.StorageAccount;
CloudBlobClient BlobClient = StorageAccount.CreateCloudBlobClient();
string labeledDataStorageContainerName = _Engine.GetEnvironmentVariable("labeledDataStorageContainerName", _Log);
CloudBlobContainer LabeledDataContainer = BlobClient.GetContainerReference(labeledDataStorageContainerName);
foreach (IListBlobItem item in LabeledDataContainer.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob dataCloudBlockBlob = (CloudBlockBlob)item;
TrainingDataUrl = dataCloudBlockBlob.Uri.ToString();
string BindingHash = dataCloudBlockBlob.Properties.ContentMD5.ToString();
if (BindingHash == null)
{
//compute the file hash as this will be added to the meta data to allow for file version validation
string BlobMd5 = new DataBlob(dataCloudBlockBlob, _Engine, _Search, _Log).CalculateMD5Hash().ToString();
if (BlobMd5 == null)
{
_Log.LogInformation("\nWarning: Blob Hash calculation failed and will not be included in file information blob, continuing operation.");
}
else
{
dataCloudBlockBlob.Properties.ContentMD5 = BlobMd5;
}
}
//trim the 2 "equals" off the trailing end of the hash or the http send will fail either using the client or raw http calls.
BindingHash = BindingHash.Substring(0, BindingHash.Length - 2);
//Get the content from the bound JSON file and instanciate a JsonBlob class then retrieve the labels collection from the Json to add to the image.
JsonBlob boundJson = (JsonBlob)_Search.GetBlob("json", BindingHash);
//Note you cannot pull the URL from the JSON blob because it will have the original URL from the first container when the blob was added to ML Professoar
string labeledDataUrl = dataCloudBlockBlob.StorageUri.PrimaryUri.ToString();
string addLabeledDataParameters = $"?dataBlobUrl={labeledDataUrl}";
string trainingDataLabels = Uri.EscapeDataString(JsonConvert.SerializeObject(boundJson.Labels));
addLabeledDataParameters = $"{addLabeledDataParameters}&imageLabels={trainingDataLabels}";
//construct and call model URL then fetch response
// the model always sends the label set in the message body with the name LabelsJson. If your model needs other values in the URL then use
//{ {environment variable name}}.
// So the example load labels function in the sameple model package would look like this:
// https://branddetectionapp.azurewebsites.net/api/loadimagetags/?projectID={{ProjectID}}
// The orchestration engine appends the labels json file to the message body.
// http://localhost:7071/api/LoadImageTags/?projectID=8d9d12d1-5d5c-4893-b915-4b5b3201f78e&labelsJson={%22Labels%22:[%22Hemlock%22,%22Japanese%20Cherry%22]}
string labeledDataServiceEndpoint = _Engine.GetEnvironmentVariable("LabeledDataServiceEndpoint", _Log);
string addLabeledDataUrl = _Engine.ConstructModelRequestUrl(labeledDataServiceEndpoint, addLabeledDataParameters);
_Log.LogInformation($"\n Getting response from {addLabeledDataUrl}");
_Response = _Client.GetAsync(addLabeledDataUrl).Result;
_ResponseString = _Response.Content.ReadAsStringAsync().Result;
if (string.IsNullOrEmpty(_ResponseString))
{
throw (new MissingRequiredObject($"\nresponseString not generated from URL: {addLabeledDataUrl}"));
}
//the code below is for passing labels and conent as http content and not on the URL string.
//Format the Data Labels content
//HttpRequestMessage Request = new HttpRequestMessage(HttpMethod.Post, new Uri(AddLabeledDataUrl));
//HttpContent DataLabelsStringContent = new StringContent(trainingDataLabels, Encoding.UTF8, "application/x-www-form-urlencoded");
//MultipartFormDataContent LabeledDataContent = new MultipartFormDataContent();
//LabeledDataContent.Add(DataLabelsStringContent, "LabeledData");
//Format the data cotent
//*****TODO***** move to an async architecture
//*****TODO***** need to decide if there is value in sending the data as a binary stream in the post or if requireing the model data scienctist to accept URLs is sufficient. If accessing the data blob with a SAS url requires Azure classes then create a configuration to pass the data as a stream in the post. If there is then this should be a configurable option.
//MemoryStream dataBlobMemStream = new MemoryStream();
//dataBlob.DownloadToStream(dataBlobMemStream);
//HttpContent LabeledDataHttpContent = new StreamContent(dataBlobMemStream);
//LabeledDataContent.Add(LabeledDataContent, "LabeledData");
//Make the http call and get a response
//string AddLabelingTagsEndpoint = Engine.GetEnvironmentVariable("LabeledDataServiceEndpoint", log);
//if (string.IsNullOrEmpty(AddLabelingTagsEndpoint)) throw (new EnvironmentVariableNotSetException("LabeledDataServiceEndpoint environment variable not set"));
//string ResponseString = Helper.GetEvaluationResponseString(AddLabelingTagsEndpoint, LabeledDataContent, log);
//if (string.IsNullOrEmpty(ResponseString)) throw (new MissingRequiredObject("\nresponseString not generated from URL: " + AddLabelingTagsEndpoint));
_Log.LogInformation($"Completed call to add blob: {dataCloudBlockBlob.Name} with labels: {JsonConvert.SerializeObject(boundJson.Labels)} to model. The response string was: {_ResponseString}.");
}
}
return("Completed execution of AddLabeledData. See logs for success/fail details.");
}
private static async Task ProcessAsync()
{
IConfigurationRoot configRoot = new ConfigurationBuilder()
.AddJsonFile("settings.json")
.AddJsonFile("settings.local.json")
.Build();
string storageConnectionString = configRoot.GetSection("StorageConnectionString").Value;
// Retrieve storage account information from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the CloudBlobClient that represents the
// Blob storage endpoint for the storage account.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Create a container called 'quickstartblobs' and
// append a GUID value to it to make the name unique.
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference("quickstartblobs");
await cloudBlobContainer.CreateIfNotExistsAsync();
// Set the permissions so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await cloudBlobContainer.SetPermissionsAsync(permissions);
// Create a file in your local MyDocuments folder to upload to a blob.
string localPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string localFileName = "QuickStart_" + Guid.NewGuid().ToString() + ".txt";
string sourceFile = Path.Combine(localPath, localFileName);
// Write text to the file.
File.WriteAllText(sourceFile, "Hello, World!");
Console.WriteLine("Temp file = {0}", sourceFile);
Console.WriteLine("Uploading to Blob storage as blob '{0}'", localFileName);
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName for the blob name.
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(localFileName);
await cloudBlockBlob.UploadFromFileAsync(sourceFile);
// List the blobs in the container.
Console.WriteLine("List blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
// Download the blob to a local file, using the reference created earlier.
// Append the string "_DOWNLOADED" before the .txt extension so that you
// can see both files in MyDocuments.
string destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
Console.WriteLine("Downloading blob to {0}", destinationFile);
await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
// system properties and metadata
await ReadContainerPropertiesAsync(cloudBlobContainer);
await AddContainerMetadataAsync(cloudBlobContainer);
await ReadContainerMetadataAsync(cloudBlobContainer);
// list all containers
await ListContainersWithPrefixAsync(cloudBlobClient, null);
Console.WriteLine("Press the 'Enter' key to delete the example files, " +
"example container, and exit the application.");
Console.ReadLine();
// Clean up resources. This includes the container and the two temp files.
Console.WriteLine("Deleting the container");
if (cloudBlobContainer != null)
{
// await cloudBlobContainer.DeleteIfExistsAsync();
}
Console.WriteLine("Deleting the source, and downloaded files");
File.Delete(sourceFile);
File.Delete(destinationFile);
}
static async Task Main(string[] args)
{
var storageConnectionString = GetConnString();
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
string subContainer = "Images";
string subSubContainer = "Party";
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
// Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
cloudBlobContainer = cloudBlobClient.GetContainerReference("quick-blobcontainer");
if (await cloudBlobContainer.CreateIfNotExistsAsync())
{
Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name);
Console.WriteLine();
}
// Upload #1
var blobName = $"{subContainer}/{Guid.NewGuid()}";
Console.WriteLine("Uploading to Blob storage as blob '{0}'", blobName);
Console.WriteLine();
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(blobName);
await cloudBlockBlob.UploadTextAsync("Hello, world!");
// Upload #2
blobName = $"{subContainer}/{subSubContainer}/{Guid.NewGuid()}";
Console.WriteLine("Uploading to Blob storage as blob '{0}'", blobName);
Console.WriteLine();
cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(blobName);
await cloudBlockBlob.UploadTextAsync("Hello, world!");
}
catch (StorageException ex)
{
Console.WriteLine("Error returned from the service: {0}", ex.Message);
}
catch (Exception ex)
{
Console.WriteLine("Unexpected exception: {0}", ex.Message);
}
finally
{
Console.WriteLine("Press any key to delete the sample blobs and example container.");
Console.ReadLine();
// Clean up resources. This includes the container and the two temp files.
Console.WriteLine("Deleting the container and any blobs it contains");
if (cloudBlobContainer != null)
{
await cloudBlobContainer.DeleteIfExistsAsync();
}
}
}
}
/// <summary>
/// Updates the BLOB metadata.
/// </summary>
/// <param name="package">The package.</param>
/// <param name="blob">The BLOB.</param>
private void UpdateBlobMetadata(IPackage package, CloudBlockBlob blob)
{
blob.Metadata[Constants.LatestModificationDate] = DateTimeOffset.Now.ToString();
var azurePackage = Mapper.Map<AzurePackage>(package);
//blob.Metadata[AzurePropertiesConstants.Package] = JsonConvert.SerializeObject(azurePackage);
_packageSerializer.SaveToMetadata(azurePackage, blob);
blob.SetMetadata();
}
