public AudibleFileLoader(TableHelper tc,BlobHelper bh)
{
m_TableHelper = tc;
m_LogBlob = bh;
m_TableHelper.CreateTable("ISBN");
m_LogBlob.CreateContainer("log");
//m_LogBlob.SetContainerACL("log", "private");
CloudBlobContainer container = m_LogBlob.BlobClient.GetContainerReference("log");
/*
CloudBlobContainer blobContainer = blobClient.GetContainerReference("azurecontainer");
CloudBlob blob = Container.GetBlobReference(blobname + ".res");
BlobStream bs = blob.OpenWrite();
TextWriter tw = new StreamWriter(bs);
string append = resultsLine + ", ";
tw.WriteLine(append);
*/
CloudBlob blob = container.GetBlobReference("AudibleLoader.Log");
BlobStream bs = blob.OpenWrite();
TextWriter tw = new StreamWriter(bs);
tw.WriteLine("test");
tw.Flush();
//content = new UTF8Encoding().GetString(data);
bs.Close();
//BlobStream OpenWrite ()
}
public void ListsContainers()
{
var blobHelper = new BlobHelper();
var listTask = blobHelper.ListContainers2();
listTask.Wait();
Assert.False(listTask.IsFaulted);
Assert.True(listTask.Result.Any(c => c.Name.Equals("public", StringComparison.OrdinalIgnoreCase)));
}
public GoogleLoader(TableHelper tc, BlobHelper bh)
{
m_TableHelper = tc;
m_LogBlob = bh;
m_TableHelper.CreateTable("ISBN");
m_LogBlob.CreateContainer("log");
CloudBlobContainer container = m_LogBlob.BlobClient.GetContainerReference("log");
CloudBlob blob = container.GetBlobReference("AudibleLoader.Log");
BlobStream bs = blob.OpenWrite();
TextWriter tw = new StreamWriter(bs);
tw.WriteLine("test");
tw.Flush();
//content = new UTF8Encoding().GetString(data);
bs.Close();
//BlobStream OpenWrite ()
}
/// <summary>
/// Shows how to get item-to-item recommendations in batch.
/// You can learn more about batch scoring at https://azure.microsoft.com/en-us/documentation/articles/cognitive-services-recommendations-batch-scoring/
/// Before you can use this method, you need to provide your blob account name, blob account key, and the input container name.
/// </summary>
/// <param name="recommender">Wrapper that maintains API key</param>
/// <param name="modelId">Model ID</param>
/// <param name="buildId">Build ID</param>
public static void GetRecommendationsBatch(RecommendationsApiWrapper recommender, string modelId, long buildId)
{
#region setup
// Set storage credentials and copy input file that defines items we want to get recommendations to the Blob Container.
string blobStorageAccountName = ""; // enter your account name here.
string blobStorageAccountKey = ""; // enter your account key here
const string containerName = ""; // enter your container name here
string outputContainerName = containerName;
string baseLocation = "https://" + blobStorageAccountName + ".blob.core.windows.net/";
string inputFileName = "batchInput.json"; // the batch input
string outputFileName = "batchOutput.json"; // the batch output
string errorFileName = "batchError.json"; // the batch error
// Validate user entered credentials.
if (String.IsNullOrEmpty(blobStorageAccountKey) || String.IsNullOrEmpty(blobStorageAccountKey) || String.IsNullOrEmpty(containerName))
{
Console.WriteLine("GetRecommendationsBatch: Provide your blob account name, blob account key, and the input container name.");
Console.WriteLine("Press any key to continue.");
Console.ReadKey();
}
string connectionString = "DefaultEndpointsProtocol=https;AccountName=" + blobStorageAccountName + ";AccountKey=" + blobStorageAccountKey;
// Copy input file from resources directory to blob storate
var sourceStorageAccount = CloudStorageAccount.Parse(connectionString);
BlobHelper bh = new BlobHelper(sourceStorageAccount, containerName);
var resourcesDir = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Resources");
bh.PutBlockBlob(containerName, inputFileName, File.ReadAllText(Path.Combine(resourcesDir, inputFileName)));
var inputSas = BlobHelper.GenerateBlobSasToken(connectionString, containerName, inputFileName);
var outputSas = BlobHelper.GenerateBlobSasToken(connectionString, outputContainerName, outputFileName);
var errorSas = BlobHelper.GenerateBlobSasToken(connectionString, outputContainerName, errorFileName);
// Now we need to define the batch job to perform.
BatchJobsRequestInfo batchJobsRequestInfo = new BatchJobsRequestInfo
{
Input = new StorageBlobInfo
{
AuthenticationType = "PublicOrSas",
BaseLocation = baseLocation,
RelativeLocation = containerName + "/" + inputFileName,
SasBlobToken = inputSas
},
Output = new StorageBlobInfo
{
AuthenticationType = "PublicOrSas",
BaseLocation = baseLocation,
RelativeLocation = containerName + "/" + outputFileName,
SasBlobToken = outputSas
},
Error = new StorageBlobInfo
{
AuthenticationType = "PublicOrSas",
BaseLocation = baseLocation,
RelativeLocation = containerName + "/" + errorFileName,
SasBlobToken = errorSas
},
// You may modify the information below to meet your request needs.
// Note that currently only "ItemRecommend" is supported.
Job = new JobInfo
{
ApiName = "ItemRecommend",
ModelId = modelId, //staging model id for books
BuildId = buildId,
NumberOfResults = 10,
IncludeMetadata = false,
MinimalScore = 0
}
};
#endregion
#region start the job, wait for completion
// kick start the batch job.
string operationLocationHeader = "";
var jobId = recommender.StartBatchJob(batchJobsRequestInfo, out operationLocationHeader);
// Monitor the batch job and wait for completion.
Console.WriteLine("Monitoring batch job {0}", jobId);
var batchInfo = recommender.WaitForOperationCompletion <BatchJobInfo>(RecommendationsApiWrapper.GetOperationId(operationLocationHeader));
Console.WriteLine("Batch {0} ended with status {1}.\n", jobId, batchInfo.Status);
if (String.Compare(batchInfo.Status, "Succeeded", StringComparison.OrdinalIgnoreCase) != 0)
{
Console.WriteLine("Batch job {0} did not end successfully, the sample app will stop here.", jobId);
Console.WriteLine("Press any key to end");
Console.ReadKey();
return;
}
else
{
// Copy the output file from blob starage into the local machine.
Stream reader = bh.GetBlobReader(outputContainerName, outputFileName);
string outputFullPath = Path.Combine(resourcesDir, outputFileName);
using (var fileStream = File.Create(outputFullPath))
{
reader.Seek(0, SeekOrigin.Begin);
reader.CopyTo(fileStream);
}
Console.WriteLine("The output of the blob operation has been saved to: {0}", outputFullPath);
}
#endregion
}
public async Task ExtractRegistrationNumber(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger logger
)
{
StorageBlobCreatedEventData blobCreatedEventData =
((JObject)eventGridEvent.Data).ToObject <StorageBlobCreatedEventData>();
string blobName = BlobHelper.GetBlobName(blobCreatedEventData.Url);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberStarted),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberStarted.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Started.ToString(),
"Execution Started"
);
#endregion
CustomEventData customEventData = new CustomEventData
{
ImageUrl = blobCreatedEventData.Url,
TicketNumber = blobName,
};
try
{
string registrationNumber = await _computerVisionHandler
.ExtractRegistrationNumberWithUrlAsync(blobCreatedEventData.Url)
.ConfigureAwait(false);
var metadata = await _blobHandler
.GetBlobMetadataAsync(blobCreatedEventData.Url);
customEventData.DistrictOfInfraction = metadata["districtofinfraction"];
customEventData.DateOfInfraction = eventGridEvent.EventTime.ToString("dd-MM-yyyy");
if (!string.IsNullOrWhiteSpace(registrationNumber))
{
customEventData.VehicleRegistrationNumber = registrationNumber;
customEventData.CustomEvent = CustomEvent.NumberExtractionCompleted.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Finished.ToString(),
"Execution Finished"
);
#endregion
}
else
{
customEventData.CustomEvent = CustomEvent.Exceptioned.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Failed.ToString(),
"Execution Failed. Reason: Failed to extract number plate from the image"
);
#endregion
}
}
catch (Exception ex)
{
customEventData.CustomEvent = CustomEvent.Exceptioned.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.ExtractRegistrationNumberFinished),
LoggingConstants.Template,
LoggingConstants.EventId.ExtractRegistrationNumberFinished.ToString(),
blobName,
LoggingConstants.ProcessingFunction.ExtractRegistrationNumber.ToString(),
LoggingConstants.ProcessStatus.Failed.ToString(),
$"Execution Failed. Reason: {ex.Message}"
);
#endregion
}
await _eventHandler
.PublishEventToTopicAsync(customEventData)
.ConfigureAwait(false);
}
public PhotoRepository()
{
//Blobber = new BlobHelper(@"DefaultEndpointsProtocol=http;AccountName=%1;AccountKey=%2".Replace("%1", accountname).Replace("%2", accountkey));
Blobber = new BlobHelper(Settings.Default.ConnectionString);
}
public void TestInitiaize()
{
// Creates a sample-blob-file.xml file in blob storage for integration testing purposes.
// Set your file name property to BlobHelper.BlobName
BlobHelper.CreateSampleBlobFile();
}
public AzureBlob(string accountName, string accountKey)
{
this.accountName = accountName;
this.accountKey = accountKey;
this.blobHelper = new BlobHelper(accountName, accountKey);
}
/// This is the function that can be called for deleting all assets created on save of a flow: consumergroup, secrets in Key Vault, cosmosDB products document such that this flow stops showing up in the UI under Flows and blobs
/// Please note if a a new asset is created on Azure as part of saving the flow or any other action taken by user in the UI, this function will need to be updated so that the asset can be deleted on delete of the flow.
/// </summary>
/// <param name="jObject">jObject requires: Subscription; Name; EventhubConnectionString; IsIotHub;EventhubName;userID</param>
/// <returns>Returns result - success or failure as the case maybe</returns>
public async Task <ApiResult> DeleteFlow(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ConfigName = diag.Name;
bool errorExists = false;
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
///Delete consumer group
_logger.LogInformation($"For FlowId: {ConfigName} Deleting flow specific consumer group.. ");
var inputEventhubConnection = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
_inputEventhubConnectionStringRef = Helper.IsKeyVault(diag.EventhubConnectionString) ? diag.EventhubConnectionString : Helper.GenerateNewSecret(_keySecretList, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName + "-input-eventhubconnectionstring", _engineEnvironment.EngineFlowConfig.SparkType, diag.EventhubConnectionString, false);
diag.EventhubConnectionString = _inputEventhubConnectionStringRef;
if (diag.InputType == Constants.InputType_EventHub && !string.IsNullOrEmpty(inputEventhubConnection))
{
var ehName = Helper.ParseEventHub(inputEventhubConnection);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (string.IsNullOrWhiteSpace(ehName) || string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The connection string for Event Hub input type must contain Endpoint, SharedAccessKeyName, SharedAccessKey, and EntityPath";
_logger.LogError(error);
errorExists = true;
}
_eventHubNames = new List <string>()
{
ehName
};
}
else if (diag.InputType == Constants.InputType_IoTHub && !string.IsNullOrEmpty(diag.EventhubNames) && !string.IsNullOrEmpty(inputEventhubConnection))
{
_eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (_eventHubNames.Count < 1)
{
string error = "The event hub-compatible name for IoT Hub input type must be defined";
_logger.LogError(error);
errorExists = true;
}
if (string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The event hub-compatible endpoint for IoT Hub input type must contain Endpoint, SharedAccessKeyName, and SharedAccessKey";
_logger.LogError(error);
errorExists = true;
}
}
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && _eventHubNames != null && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
foreach (string ehName in _eventHubNames)
{
var result = EventHub.DeleteConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, _eventHubNamespace, ehName, ConsumerGroupName, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully deleted flow specific consumer group");
}
}
}
///Delete cosmosDB document related to a flow
response = await CosmosDB.DeleteConfigFromDocumentDB(_engineEnvironment.CosmosDBDatabaseName, _engineEnvironment.CosmosDBEndPoint, _engineEnvironment.CosmosDBUserName, _engineEnvironment.CosmosDBPassword, "flows", ConfigName);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific cosmosDB entry");
}
///Delete configs stored in blobs
// ruleDefinitions
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(RuleDefinitionPath, RuleDefinitionFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific rules definition blob");
}
// outputTemplates
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(OutputTemplatePath, OutputTemplateFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific output template blob");
}
string resourceGroupLocation = _engineEnvironment.EngineFlowConfig.ResourceGroupLocation;
string resourceGroupName = _engineEnvironment.EngineFlowConfig.ResourceGroupName;
string storageAccountName = _engineEnvironment.EngineFlowConfig.StorageAccountName;
string containerPath = Path.Combine(_flowContainerName, _engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName);
string subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific blobs under the folder {ConfigName} under container {_Centralprocessing}");
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _flowContainerName, Path.Combine(_engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific productconfig: {ProductConfigName} and {JobConfigName} for {ConfigName}.");
/// Delete sample data and the checkpoints folder if it exists for that flow
var hashValue = Helper.GetHashCode(diag.UserName);
await BlobHelper.DeleteBlob(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsSamplePath, $"{ConfigName}-{hashValue}.json"));
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific sampledata file: {ConfigName}-{ hashValue}.json");
await BlobHelper.DeleteAllBlobsInAContainer(_engineEnvironment.OpsBlobConnectionString, $"{_engineEnvironment.CheckPointContainerNameHelper(ConfigName)}-checkpoints", _engineEnvironment.EngineFlowConfig.OpsBlobDirectory);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific checkpoints for {ConfigName}.");
_logger.LogInformation("Deleting flow specific secrets..");
///Delete secrets specific to a flow from KeyVault
KeyVault.GetSecretsAndDeleteFromKeyvault(_engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific secrets");
if (!errorExists)
{
return(ApiResult.CreateSuccess("Deleted!"));
}
else
{
return(ApiResult.CreateError("Deleted but with some error. Please check logs for details"));
}
}
public virtual void BeforeBatch(string local, string remote)
{
BlobHelper.CreateContainer(remote);
BlobHelper.CleanupContainer(remote);
}
public virtual void Before(string containerName, string fileName)
{
BlobHelper.CreateContainer(containerName);
BlobHelper.DeleteBlob(containerName, fileName);
}
// GET: MultiFaceDetection
public async Task <ActionResult> Index()
{
try
{
// Step 1. Get images from blob storage.
BlobHelper BlobHelper = new BlobHelper(StorageAccount, StorageKey);
List <string> blobs = BlobHelper.ListBlobs(Container);
List <string> images = new List <string>();
foreach (var blobName in blobs)
{
images.Add(blobName);
}
// Step 2. For each image, run the face api detection algorithm.
var faceServiceClient = new FaceServiceClient(ServiceKey, "https://westcentralus.api.cognitive.microsoft.com/face/v1.0");
for (int i = 0; i < blobs.Count; i++)
{
var detectedFaces = new ObservableCollection <vmFace>();
var resultCollection = new ObservableCollection <vmFace>();
using (WebClient client = new WebClient())
{
byte[] fileBytes = client.DownloadData(string.Concat("https://faceapiweustorage.blob.core.windows.net/cloudprojectsampleimages/", images[i]));
bool exists = System.IO.Directory.Exists(Server.MapPath(directory));
if (!exists)
{
try
{
Directory.CreateDirectory(Server.MapPath(directory));
}
catch (Exception ex)
{
ex.ToString();
}
}
string imageRelativePath = "../MultiDetectedFiles" + '/' + images[i];
string imageFullPath = Server.MapPath(directory) + '/' + images[i] as string;
System.IO.File.WriteAllBytes(imageFullPath, fileBytes);
using (var stream = client.OpenRead(string.Concat("https://faceapiweustorage.blob.core.windows.net/cloudprojectsampleimages/", images[i])))
{
Face[] faces = await faceServiceClient.DetectAsync(stream, true, true, new FaceAttributeType[] { FaceAttributeType.Gender, FaceAttributeType.Age, FaceAttributeType.Smile, FaceAttributeType.Glasses });
Bitmap CroppedFace = null;
foreach (var face in faces)
{
//Create & Save Cropped Images
var croppedImg = Convert.ToString(Guid.NewGuid()) + ".jpeg" as string;
var croppedImgPath = "../MultiDetectedFiles" + '/' + croppedImg as string;
var croppedImgFullPath = Server.MapPath(directory) + '/' + croppedImg as string;
CroppedFace = CropBitmap(
(Bitmap)Bitmap.FromFile(imageFullPath),
face.FaceRectangle.Left,
face.FaceRectangle.Top,
face.FaceRectangle.Width,
face.FaceRectangle.Height);
CroppedFace.Save(croppedImgFullPath, ImageFormat.Jpeg);
if (CroppedFace != null)
{
((IDisposable)CroppedFace).Dispose();
}
detectedFaces.Add(new vmFace()
{
ImagePath = imageRelativePath,
FileName = images[i],
FilePath = croppedImgPath,
Left = face.FaceRectangle.Left,
Top = face.FaceRectangle.Top,
Width = face.FaceRectangle.Width,
Height = face.FaceRectangle.Height,
FaceId = face.FaceId.ToString(),
Gender = face.FaceAttributes.Gender,
Age = string.Format("{0:#} years old", face.FaceAttributes.Age),
IsSmiling = face.FaceAttributes.Smile > 0.0 ? "Smile" : "Not Smile",
Glasses = face.FaceAttributes.Glasses.ToString(),
});
}
// Convert detection result into UI binding object for rendering.
var imageInfo = UIHelper.GetImageInfoForRenderingFromStream(stream);
var rectFaces = UIHelper.CalculateFaceRectangleForRendering(faces, MaxImageSize, imageInfo);
foreach (var face in rectFaces)
{
resultCollection.Add(face);
}
var faceModal = new FaceDetectionModal {
DetectedFaces = detectedFaces, ResultCollection = resultCollection
};
this.finalModal.Items.Add(faceModal);
}
}
}
}
catch (Exception ex)
{
Console.Write(ex);
throw;
}
return(View(this.finalModal));
}
public static async Task <ChunkHolder <MyColor> > GetRawColors(Angle lat, Angle lon, TraceListener log)
{
ImageFileMetadata fileInfo = (await GetChunkMetadata(log))
.Where(p => Utils.Contains(p.Points, lat.DecimalDegree, lon.DecimalDegree))
.FirstOrDefault();
if (fileInfo == null)
{
throw new MountainViewException("Need more data for " + lat.ToLatString() + ", " + lon.ToLonString() + "!");
}
if (!File.Exists(Path.Combine(Path.GetTempPath(), fileInfo.LocalName)))
{
using (var ms = await BlobHelper.TryGetStreamAsync(cachedFileContainer, fileInfo.FileName, log))
{
if (ms == null)
{
log?.WriteLine("File should exist: '" + fileInfo.FileName + "'");
log?.WriteLine("Assuming the tile is empty");
return(null);
}
using (var fileStream = File.Create(Path.Combine(Path.GetTempPath(), fileInfo.LocalName)))
{
ms.Stream.Position = 0;
ms.Stream.CopyTo(fileStream);
}
}
}
string fileName = fileInfo.LocalName;
if (cache.TryGetValue(fileName, out ChunkHolder <MyColor> ret))
{
cacheLastTouched[fileName] = DateTime.Now;
return(ret);
}
object specificLock = null;
lock (generalLock)
{
if (!specificLocks.TryGetValue(fileName, out specificLock))
{
specificLock = new object();
specificLocks.Add(fileName, specificLock);
}
}
lock (specificLock)
{
if (cache.TryGetValue(fileName, out ret))
{
cacheLastTouched[fileName] = DateTime.Now;
return(ret);
}
// Now the real compute starts
log?.WriteLine("Reading into cache: " + fileName);
FIBITMAP sdib = FreeImage.LoadEx(Path.Combine(Path.GetTempPath(), fileName));
try
{
var width = (int)FreeImage.GetWidth(sdib);
var height = (int)FreeImage.GetHeight(sdib);
ret = new ChunkHolder <MyColor>(height, width,
Angle.FromDecimalDegrees(fileInfo.Points.Min(p => p.Item1)),
Angle.FromDecimalDegrees(fileInfo.Points.Min(p => p.Item2)),
Angle.FromDecimalDegrees(fileInfo.Points.Max(p => p.Item1)),
Angle.FromDecimalDegrees(fileInfo.Points.Max(p => p.Item2)),
(i, j) =>
{
FreeImage.GetPixelColor(sdib, (uint)(width - 1 - j), (uint)(i), out RGBQUAD value);
return(new MyColor(value.rgbRed, value.rgbGreen, value.rgbBlue));
},
Utils.ColorToDoubleArray,
Utils.ColorFromDoubleArray);
}
finally
{
FreeImage.UnloadEx(ref sdib);
}
cache.Add(fileName, ret);
cacheLastTouched.Add(fileName, DateTime.Now);
if (cache.Count > 8)
{
var toRemove = cacheLastTouched.OrderBy(p => p.Value).First().Key;
cache.Remove(toRemove);
cacheLastTouched.Remove(toRemove);
}
}
return(ret);
}
public static async Task Run([QueueTrigger(Constants.ChunkQueueName, Connection = "ConnectionString2")] string myQueueItem, TraceWriter log)
{
log.Info($"ProcessQueueChunk processed: {myQueueItem}");
string computerName = Environment.GetEnvironmentVariable("ComputerName", EnvironmentVariableTarget.Process);
log.Info($"Computer name: {computerName}");
var chunkMetadata = JsonConvert.DeserializeObject <ChunkMetadata>(myQueueItem);
var config = chunkMetadata.Config.GetConfig();
string cs = Environment.GetEnvironmentVariable("ConnectionString", EnvironmentVariableTarget.Process);
BlobHelper.SetConnectionString(cs);
var log2 = new MyTraceLister(log);
var imageFile = chunkMetadata.SessionId + "." + chunkMetadata.ChunkKey + ".png";
int numParts = (int)((config.ElevationViewMax.Radians - config.ElevationViewMin.Radians) / config.AngularResolution.Radians);
MyColor[][] resultImage = null;
View.ColorHeight[][] view = null;
if (chunkMetadata.ChunkKey == 0)
{
resultImage = View.ProcessImageBackdrop(config.NumTheta, numParts);
}
else
{
var chunkView = await View.GetPolarData(config, chunkMetadata.ChunkKey, chunkMetadata.HeightOffset, log2);
if (chunkView.Count() == 0)
{
log.Error($"ERROR: No pixels from GetPolarData");
}
view = new View.ColorHeight[config.NumTheta][];
for (int i = 0; i < view.Length; i++)
{
view[i] = new View.ColorHeight[numParts];
}
foreach (var pixel in chunkView)
{
view[pixel.iTheta][pixel.iViewElev] = pixel.ToColorHeight();
}
resultImage = await View.ProcessImage(view, log2);
if (resultImage.SelectMany(p => p).Count(p => p.R != 0 || p.G != 0 || p.B != 0) == 0)
{
log.Error($"ERROR: Only black pixels in result image");
}
}
Utils.WriteImageFile(resultImage, Path.Combine(Path.GetTempPath(), imageFile), a => a, OutputType.PNG);
string location = await BlobHelper.WriteStream("share", imageFile, Path.Combine(Path.GetTempPath(), imageFile), log2);
string maptxt = "";
//// id = 0;
// if (id == 0)
// {
// maptxt = View.ProcessImageMapBackdrop(location);
// }
// else
// {
// maptxt = View.ProcessImageMap(view3, location);
// }
string cs2 = Environment.GetEnvironmentVariable("ConnectionString2", EnvironmentVariableTarget.Process);
TableHelper.SetConnectionString(cs2);
try
{
TableHelper.Insert(chunkMetadata.SessionId, chunkMetadata.Order, location, maptxt);
}
catch (Exception ex)
{
log.Error(ex.ToString());
}
// return req.CreateResponse(HttpStatusCode.OK, maptxt);
}
public void DeleteBlob(string blobAddress)
{
var bh = new BlobHelper();
bh.DeleteBlob(new Uri(blobAddress));
}
private void CreateEventProcessorHostClient(ref EventHubSettings eventHubSettings)
{
Trace.TraceInformation("Creating EventProcessorHost: {0}, {1}, {2}", this.Server.MachineName, eventHubSettings.name, eventHubSettings.consumerGroup);
try
{
eventHubSettings.client = EventHubClient.CreateFromConnectionString(eventHubSettings.connectionString,
eventHubSettings.name);
// Delete and recreate the consumer group
// this allows to ensure we will start receiving only fresh messages when the site starts
foreach (ConsumerGroupDescription consumerGroupDesc in eventHubSettings.namespaceManager.GetConsumerGroups(eventHubSettings.client.Path))
{
// We remove any previously created consumergroups containing the word WebSite in the name
if (consumerGroupDesc.Name.ToLowerInvariant().Contains("website") &&
!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME"))
||
consumerGroupDesc.Name.ToLowerInvariant().Contains("local") &&
String.IsNullOrEmpty(Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME")))
{
eventHubSettings.namespaceManager.DeleteConsumerGroup(eventHubSettings.name, consumerGroupDesc.Name);
}
}
//Workaround to delete old blobs related to old consumer groups
CloudBlobContainer eventHubBlobContainer = BlobHelper.SetUpContainer(eventHubSettings.storageConnectionString, eventHubSettings.name);
string blobPerfix = String.IsNullOrEmpty(Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME")) ? "local" : "website";
IEnumerable <CloudBlockBlob> oldBlobs = eventHubBlobContainer.ListBlobs(blobPerfix, true, BlobListingDetails.All).OfType <CloudBlockBlob>();
foreach (var blob in oldBlobs)
{
try
{
blob.DeleteIfExists();
}
catch (Exception)
{
Debug.Print("Error happened while trying to delete old ConsumerGroup related blob.");
}
}
}
catch
{
// Error happened while trying to delete old ConsumerGroups.
Debug.Print("Error happened while trying to delete old ConsumerGroups");
}
finally
{
try
{
// We create a new consumer group with a new mame each time to
eventHubSettings.consumerGroup += DateTime.UtcNow.Ticks.ToString();
eventHubSettings.namespaceManager.CreateConsumerGroupIfNotExists(eventHubSettings.name,
eventHubSettings.consumerGroup);
}
catch (Exception ex)
{
Debug.Print("Error happened: " + ex.Message);
}
}
try
{
eventHubSettings.processorHost = new EventProcessorHost(this.Server.MachineName,
eventHubSettings.client.Path,
eventHubSettings.consumerGroup.ToLowerInvariant(),
eventHubSettings.connectionString,
eventHubSettings.storageConnectionString);
eventHubSettings.processorHostOptions = new EventProcessorOptions();
eventHubSettings.processorHostOptions.ExceptionReceived += WebSocketEventProcessor.ExceptionReceived;
eventHubSettings.processorHostOptions.InitialOffsetProvider = (partitionId) => DateTime.UtcNow;
//eventHubSettings.processorHostOptions.InitialOffsetProvider = partitionId =>
//{
// return eventHubSettings.namespaceManager.GetEventHubPartition(eventHubSettings.client.Path, partitionId).LastEnqueuedOffset;
//};
Trace.TraceInformation("Registering EventProcessor for " + eventHubSettings.name);
eventHubSettings.processorHost.RegisterEventProcessorAsync <WebSocketEventProcessor>(
eventHubSettings.processorHostOptions).Wait();
}
catch
{
Debug.Print("Error happened while trying to connect Event Hub");
}
}
public async Task <bool> ExistsComputedChunk(StandardChunkMetadata template, TraceListener log)
{
string filename = GetShortFilename(template);
return(await BlobHelper.BlobExists(cachedFileContainer, GetFullFileName(template, filename), log));
}
private async Task <List <StationImageUploadModel> > UploadBlobsToAzure(IList <HttpContent> files, StationImageUploadModel uploadedModel)
{
// NOTE: FileData is a property of MultipartFileStreamProvider and is a list of multipart
// files that have been uploaded and saved to disk in the Path.GetTempPath() location.
foreach (var fileData in files)
{
if (!string.IsNullOrEmpty(fileData.Headers.ContentDisposition.FileName))
{
string mediaType = fileData.Headers.ContentType.MediaType.ToString();
//If the file an image media type
if (System.Text.RegularExpressions.Regex.IsMatch(mediaType, "image/\\S+"))
{
// Sometimes the filename has a leading and trailing double-quote character
// when uploaded, so we trim it; otherwise, we get an illegal character exception
var fileName = Path.GetFileName(fileData.Headers.ContentDisposition.FileName.Trim('"'));
var path = HttpRuntime.AppDomainAppPath;
string directoryName = Path.Combine(path, "StationImage");
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(@directoryName);
}
string filePath = Path.Combine(directoryName, fileName);
//Deletion exists file
if (File.Exists(filePath))
{
File.Delete(filePath);
}
// Retrieve reference to a blob
var blobContainer = BlobHelper.GetBlobContainer();
var blob = blobContainer.GetBlockBlobReference(fileName);
bool blobExists = blob.Exists();
//if is doesn't exist, then add it.
if (!blobExists)
{
// Set the blob content type
blob.Properties.ContentType = fileData.Headers.ContentType.MediaType;
Stream input = await fileData.ReadAsStreamAsync();
//Directory.CreateDirectory(@directoryName);
using (Stream file = File.OpenWrite(filePath))
{
input.CopyTo(file);
//close file
file.Close();
}
// Upload file into blob storage, basically copying it from local disk into Azure
using (var fs = File.OpenRead(filePath))
{
long fileSizeInKB = (long)(fs.Length / 1024);
//If the image is greater than 1 MB don't save it
if (fileSizeInKB > 1001)
{
continue;
}
blob.UploadFromStream(fs);
}
// Delete local file from disk
File.Delete(filePath);
}
// Create blob upload model with properties from blob info
var stationImageUploadModel = new StationImageUploadModel
{
FileName = blob.Name,
FileUrl = blob.Uri.AbsoluteUri,
FileSizeInBytes = blob.Properties.Length,
StationID = uploadedModel.StationID,
User = uploadedModel.User,
Primary = uploadedModel.Primary,
ImageTypeID = uploadedModel.ImageTypeID,
Description = uploadedModel.Description,
PhysHabYear = uploadedModel.PhysHabYear
};
// Add uploaded blob to the list
Uploads.Add(stationImageUploadModel);
}
}
}
return(Uploads);
}
public async Task DetectAndBlurFaces(
[EventGridTrigger] EventGridEvent eventGridEvent,
ILogger logger
)
{
CustomEventData inputEventData =
((JObject)eventGridEvent.Data).ToObject <CustomEventData>();
var correlationId = LoggingHelper.GetCorrelationId(inputEventData);
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.DetectAndBlurFacesStarted),
LoggingConstants.Template,
LoggingConstants.EventId.DetectAndBlurFacesStarted.ToString(),
correlationId,
LoggingConstants.ProcessingFunction.DetectAndBlurFaces.ToString(),
LoggingConstants.ProcessStatus.Started,
"Execution Started"
);
#endregion
CustomEventData outputEventData = new CustomEventData
{
ImageUrl = inputEventData.ImageUrl,
TicketNumber = inputEventData.TicketNumber,
DistrictOfInfraction = inputEventData.DistrictOfInfraction,
DateOfInfraction = inputEventData.DateOfInfraction
};
try
{
var detectedFaces = await _faceHandler
.DetectFacesWithUrlAsync(inputEventData.ImageUrl)
.ConfigureAwait(false);
var imageBytes = await _blobHandler
.DownloadBlobAsync(inputEventData.ImageUrl)
.ConfigureAwait(false);
var blurredImageBytes = await _faceHandler
.BlurFacesAsync(imageBytes, detectedFaces.ToList());
using (MemoryStream ms = new MemoryStream(blurredImageBytes))
{
await _blobHandler.UploadStreamAsBlobAsync(
containerName : _options.BlurredImageContainerName,
stream : ms,
contentType : _options.UploadContentType,
blobName : BlobHelper.GetBlobNameWithExtension(inputEventData.TicketNumber)
)
.ConfigureAwait(false);
}
outputEventData.CustomEvent = CustomEvent.FaceDetectionAndBlurringCompleted.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.DetectAndBlurFacesFinished),
LoggingConstants.Template,
LoggingConstants.EventId.DetectAndBlurFacesFinished.ToString(),
correlationId,
LoggingConstants.ProcessingFunction.DetectAndBlurFaces.ToString(),
LoggingConstants.ProcessStatus.Finished,
"Execution Finished"
);
#endregion
}
catch (Exception ex)
{
outputEventData.CustomEvent = CustomEvent.Exceptioned.ToString();
#region Logging
logger.LogInformation(
new EventId((int)LoggingConstants.EventId.DetectAndBlurFacesFinished),
LoggingConstants.Template,
LoggingConstants.EventId.DetectAndBlurFacesFinished.ToString(),
correlationId,
LoggingConstants.ProcessingFunction.DetectAndBlurFaces.ToString(),
LoggingConstants.ProcessStatus.Failed,
$"Execution Failed. Reason: {ex.Message}"
);
#endregion
}
await _eventhandler.PublishEventToTopicAsync(outputEventData)
.ConfigureAwait(false);
}
public ImageConverter(string queueName)
{
dataRepository = new TableHelper("table");
blobRepository = new BlobHelper("blob");
queue = new QueueHelper(queueName);
}
public void DeleteBlob(string blobUrl)
{
BlobHelper.Delete(Request.Cookies[SiteHelper.ACCOUNT].Value, Request.Cookies[SiteHelper.KEY].Value, blobUrl);
grdBlobs.DataBind();
}
public List <BlobHelper> GetContainerBlobs(string account, string key, string container)
{
return(BlobHelper.GetAll(container, account, key));
}
public AzureBlob(string accountName, string accountKey)
{
this.accountName = accountName;
this.accountKey = accountKey;
this.blobHelper = new BlobHelper(accountName, accountKey);
}
public void TestInitiaize()
{
BlobHelper.CreateSampleBlobFile();
}
private static List <Airport> LoadAirportData(out string errmsg)
{
var errors = new StringBuilder( );
try
{
if (_airports != null && _airports.Count > 0)
{
errmsg = "";
return(_airports);
}
var key = "get-airport-data";
var obj = CacheHelper.GetCachedItem(key);
if (obj == null)
{
var arr = BlobHelper.GetBlob(Invariables.SwitchStorageConnection, "airportdata.csv", "system-defaults");
var lines = Encoding.ASCII.GetString(arr).Split('\n');
var i = 0;
foreach (var line in lines)
{
try
{
if (i != 0)
{
var tokens = StringSplit(line.Replace("\r", ""));
if (tokens.Length > 0)
{
if (_airports == null)
{
_airports = new List <Airport>( );
}
_airports.Add(new Airport
{
Code = tokens[0],
AirportName = tokens[1],
Latitude = double.Parse(tokens[2]),
Longitude = double.Parse(tokens[3]),
Area = tokens[4]
});
}
}
}
catch (Exception)
{
errors.AppendLine("Error line # " + i + Environment.NewLine);
Logger.ErrorLog("WeatherUtility", "LoadAirportData", "Error line # " + i);
}
finally
{
i++;
}
}
if (_airports != null && _airports.Count > 0)
{
CacheHelper.AddToCache(key, _airports);
}
if (_airports != null)
{
Logger.LocalLog("WeatherUtility", "LoadAirportData", "LogMessage", "Loaded " + _airports.Count + " airports");
errmsg = errors.Length > 5 ? errors.ToString() : "";
return(_airports);
}
errmsg = "";
return(_airports);
}
errmsg = "";
_airports = (List <Airport>)obj;
return(_airports);
}
catch (Exception exception)
{
errmsg = "Error: " + exception;
Logger.ErrorLog("DataHelper", "LoadAirportData", exception.ToString( ));
return(null);
}
}
protected override async Task <InvokeResponse> OnInvokeActivityAsync(ITurnContext <IInvokeActivity> turnContext, CancellationToken cancellationToken)
{
if (Constants.ComposeExtensionFetch.Equals(turnContext.Activity.Name))
{
MessagingExtensionAction action = new MessagingExtensionAction();
var actionJson = JsonConvert.DeserializeObject <MessagingExtensionActionDeserializer>(turnContext.Activity.Value.ToString());
action.CommandId = actionJson.commandId;
var task = await this.OnTeamsMessagingExtensionFetchTaskAsync(turnContext, action, cancellationToken);
return(CreateInvokeResponse(task));
}
else if (Constants.ComposeExtensionSubmit.Equals(turnContext.Activity.Name))
{
MessagingExtensionAction action = new MessagingExtensionAction();
var task = await this.OnTeamsMessagingExtensionSubmitActionAsync(turnContext, action, cancellationToken);
return(CreateInvokeResponse(task));
}
else if (Constants.TaskModuleFetch.Equals(turnContext.Activity.Name))
{
var actionJson = JsonConvert.DeserializeObject <ActionBase>(turnContext.Activity.Value.ToString());
var task = await ReturnViewDetails(actionJson.data.reqId);
return(CreateInvokeResponse(task));
}
else if (Constants.AdaptiveCardAction.Equals(turnContext.Activity.Name))
{
var data = JsonConvert.DeserializeObject <ActionType>(turnContext.Activity.Value.ToString());
string channel = turnContext.Activity.ChannelId;
string verb = data.action.verb;
var blobInfo = BlobHelper.GetBlob(data.action.data.reqId, verb).Result;
string cardJsonString;
if (Status.Refresh.Equals(verb) && Status.Pending.Equals(blobInfo.status))
{
CardHelper.CreateAdaptiveCardAttachment(blobInfo.status, blobInfo, channel, out cardJsonString);
var cardResponse = JObject.Parse(cardJsonString);
var res = new AdaptiveCardInvokeResponse()
{
StatusCode = 200,
Type = AdaptiveCard.ContentType,
Value = cardResponse
};
return(CreateInvokeResponse(res));
}
else
{
var adaptiveCardAttachment = CardHelper.CreateAdaptiveCardAttachment(blobInfo.status, blobInfo, channel, out cardJsonString);
UpdateCardInTeams(blobInfo, adaptiveCardAttachment);
var cardResponse = JObject.Parse(cardJsonString);
var res = new AdaptiveCardInvokeResponse()
{
StatusCode = 200,
Type = AdaptiveCard.ContentType,
Value = cardResponse
};
return(CreateInvokeResponse(res));
}
}
return(null);
}
