public async Task <IActionResult> PredictEmail(string messageId)
{
var(mail, uniqueId) = await _mailKitRepository.GetUnreadEmailByIdAsync(messageId).ConfigureAwait(false);
var headerEmailSentence = new StringBuilder();
var authorizationResultsHeaders = mail.Headers.Where(header => header.Field == "Authentication-Results");
var authorizationResultsHeadersValues = authorizationResultsHeaders.Select(authorizationResultsHeader => authorizationResultsHeader.Value).ToList();
var senderPolicyFrameworkProtocolsResults = _informationExtractorService.GetSenderPolicyFrameworkResults(authorizationResultsHeadersValues);
var domainKeysIdentifiedMailProtocolsResults = _informationExtractorService.GetDomainKeysIdentifiedMailResults(authorizationResultsHeadersValues);
var domainBasedMessageAuthenticationReportingConformanceProtocolsResults = _informationExtractorService.GetDomainBasedMessageAuthenticationReportingConformanceResults(authorizationResultsHeadersValues);
//Add attachments info
foreach (var attachment in mail.Attachments)
{
var file = attachment.ContentDisposition.FileName;
var fileSize = attachment.ContentDisposition.Size;
var fileName = Path.GetFileName(file).ToLower();
var fileExtension = Path.GetExtension(file).ToLower();
headerEmailSentence.Append($"attachment filename {fileName} extension {fileExtension} size {fileSize} ");
}
//Add protocol result info
foreach (var protocolResult in senderPolicyFrameworkProtocolsResults)
{
headerEmailSentence.Append($"spfProtocol result {protocolResult} ");
}
foreach (var protocolResult in domainKeysIdentifiedMailProtocolsResults)
{
headerEmailSentence.Append($"dkimProtocol result {protocolResult} ");
}
foreach (var protocolResult in domainBasedMessageAuthenticationReportingConformanceProtocolsResults)
{
headerEmailSentence.Append($"dmarcProtocol result {protocolResult} ");
}
//Add body info
var bodyHtmlText = mail.HtmlBody;
var bodySentence = _textPreparerService.Prepare(GetBodyText(bodyHtmlText)).ToList();
var bodyFullSentence = string.Join(" ", bodySentence);
var bodyPrediction = await _grannyModelAccessorService.PredictRawEmail(bodyFullSentence).ConfigureAwait(false);
var predictionResultViewModel = new PredictionResultViewModel()
{
BodyPredictionResult = bodyPrediction.First(),
HeaderPredictionResult = 0,
SubjectPredictionResult = 0
};
await _mailKitRepository.MarkEmailAsSeen(uniqueId).ConfigureAwait(false);
return(View(predictionResultViewModel));
}
async Task <List <CroppedImage> > CropImageAsync(Bitmap original, PredictionResultViewModel predictionResultModel, double probability)
{
var croppedImages = new List <CroppedImage>();
// An empty bitmap which will hold the cropped image
foreach (var image in predictionResultModel.Predictions)
{
if (image.Probability > probability)
{
using (var bmp = new Bitmap((int)(predictionResultModel.OriginalWidth * image.BoundingBox.Width), (int)(predictionResultModel.OriginalHeight * image.BoundingBox.Height)))
{
var section = new Rectangle((int)(predictionResultModel.OriginalWidth * image.BoundingBox.Left),
(int)(predictionResultModel.OriginalHeight * image.BoundingBox.Top),
(int)(predictionResultModel.OriginalWidth * image.BoundingBox.Width),
(int)(predictionResultModel.OriginalHeight * image.BoundingBox.Height));
var g = Graphics.FromImage(bmp);
// Draw the given area (section) of the source image
// at location 0,0 on the empty bitmap (bmp)
g.DrawImage(original, 0, 0, section, GraphicsUnit.Pixel);
using (var memoryStream = new MemoryStream())
{
bmp.Save(memoryStream, ImageFormat.Jpeg);
var handwrittenResult = await _handwrittenRecognitionService.AnalyzeImageAsync(memoryStream.ToArray());
if (handwrittenResult == null ||
handwrittenResult.RecognitionResult == null ||
handwrittenResult.RecognitionResult.Lines == null ||
handwrittenResult.RecognitionResult.Lines.Count == 0 ||
handwrittenResult.RecognitionResult.Lines.Count(l => l.Words.Count == 0) > 0)
{
continue;
}
var id = Guid.NewGuid();
string fileName = $"{ original.Tag }_{ id }.jpg";
string imageUrl = await _uploadToAzureStorageService.UploadFileAsync(memoryStream.ToArray(), fileName,
await _keyVaultService.GetSecretAsync("AzureStorageAccountAccessKey"),
"croppedimages");
croppedImages.Add(new CroppedImage
{
ImageUrl = imageUrl,
HandwrittenResult = handwrittenResult
});
}
}
}
}
return(croppedImages);
}
public static async Task <List <CroppedImage> > CropImage(Bitmap original, PredictionResultViewModel predictionResultModel, double probability)
{
var croppedImages = new List <CroppedImage>();
// An empty bitmap which will hold the cropped image
foreach (var image in predictionResultModel.Predictions)
{
if (image.Probability > probability)
{
Bitmap bmp = new Bitmap((int)(predictionResultModel.OriginalWidth * image.BoundingBox.Width), (int)(predictionResultModel.OriginalHeight * image.BoundingBox.Height));
Rectangle section = new Rectangle((int)(predictionResultModel.OriginalWidth * image.BoundingBox.Left),
(int)(predictionResultModel.OriginalHeight * image.BoundingBox.Top),
(int)(predictionResultModel.OriginalWidth * image.BoundingBox.Width),
(int)(predictionResultModel.OriginalHeight * image.BoundingBox.Height));
Graphics g = Graphics.FromImage(bmp);
// Draw the given area (section) of the source image
// at location 0,0 on the empty bitmap (bmp)
g.DrawImage(original, 0, 0, section, GraphicsUnit.Pixel);
using (MemoryStream memoryStream = new MemoryStream())
{
bmp.Save(memoryStream, ImageFormat.Jpeg);
var handwrittenResult = await MakeRequest(memoryStream);
if (handwrittenResult.RecognitionResult == null || handwrittenResult.RecognitionResult.Lines.Count == 0 ||
handwrittenResult.RecognitionResult.Lines.Count(l => l.Words.Count == 0) > 0)
{
continue;
}
var id = Guid.NewGuid();
croppedImages.Add(new CroppedImage
{
Bitmap = bmp,
Guid = id,
HandwrittenResult = handwrittenResult,
});
string fileName = "D:\\future-now-images\\" + original.Tag + "\\" + id + ".jpg";
bmp.Save(fileName, ImageFormat.Jpeg);
}
//bmp.Dispose();
}
}
return(croppedImages);
}
public async Task <List <CroppedImage> > AnalyzeImageAsync(IFormFile imageFile, double probability)
{
var stream = imageFile.OpenReadStream();
int height = 0;
int width = 0;
string url = $"https://southcentralus.api.cognitive.microsoft.com/customvision/v2.0/Prediction/{ await _keyVaultService.GetSecretAsync("CognitiveServiceKey1") }/image?iterationId={ await _keyVaultService.GetSecretAsync("IterationId") }";
byte[] data;
var result = new PredictionResultViewModel();
var croppedImages = new List <CroppedImage>();
using (var client = new HttpClient())
using (var br = new BinaryReader(stream))
{
data = br.ReadBytes((int)stream.Length);
var image = Image.FromStream(stream);
var bitmap = new Bitmap(image);
height = bitmap.Height;
width = bitmap.Width;
if (data.Length > 0)
{
using (var content = new ByteArrayContent(data))
{
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
client.DefaultRequestHeaders.Add("Prediction-Key", await _keyVaultService.GetSecretAsync("PredictionKey"));
using (var response = await client.PostAsync(url, content))
{
result = await response.Content.ReadAsAsync <PredictionResultViewModel>();
result.OriginalHeight = height;
result.OriginalWidth = width;
return(await CropImageAsync(bitmap, result, probability / 100d));
}
}
}
}
return(new List <CroppedImage>());
}
public PredictionResult(MovieShort movie) : this()
{
BindingContext = new PredictionResultViewModel(movie, new PageService());
}