private async void OnCaptureButton_Click(Object sender, RoutedEventArgs e)
{
Visibility = Visibility.Hidden;
try
{
CaptureWindow captureWindow = new CaptureWindow();
Boolean result = captureWindow.ShowDialog().Value;
if (result)
{ // The region is captured. Run OCR and Translate
MemoryStream ms;
var bitmapSource = CopyScreen(captureWindow.Rect, out ms);
captureImage.Source = bitmapSource;
Visibility = Visibility.Visible;
initialized = true;
InMemoryRandomAccessStream inMemoryStream = new InMemoryRandomAccessStream();
ms.Position = 0;
Byte[] byteArray = ms.GetBuffer();
inMemoryStream.AsStream().Write(byteArray, 0, byteArray.Length);
inMemoryStream.AsStream().Flush();
Windows.Graphics.Imaging.BitmapDecoder bitmapDecoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(inMemoryStream);
SoftwareBitmap softwareBitmap = await bitmapDecoder.GetSoftwareBitmapAsync().AsTask().ConfigureAwait(true);
captureImage.Tag = softwareBitmap; // for later use
await RunOcrAndTranslate();
}
}
catch { }
Visibility = Visibility.Visible;
}
public static async Task <OcrResult?> GetOcrResultFromRegion(Rectangle region)
{
Language?selectedLanguage = GetOCRLanguage();
if (selectedLanguage == null)
{
return(null);
}
Bitmap bmp = new(region.Width, region.Height, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
Graphics g = Graphics.FromImage(bmp);
g.CopyFromScreen(region.Left, region.Top, 0, 0, bmp.Size, CopyPixelOperation.SourceCopy);
OcrResult?ocrResult;
await using (MemoryStream memory = new())
{
bmp.Save(memory, ImageFormat.Bmp);
memory.Position = 0;
BitmapDecoder bmpDecoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream());
SoftwareBitmap softwareBmp = await bmpDecoder.GetSoftwareBitmapAsync();
OcrEngine ocrEngine = OcrEngine.TryCreateFromLanguage(selectedLanguage);
ocrResult = await ocrEngine.RecognizeAsync(softwareBmp);
}
return(ocrResult);
}
private async void OnRecognize(object sender, RoutedEventArgs e)
{
OpenFileDialog dialog = new OpenFileDialog();
if (dialog.ShowDialog() == true)
{
string fileName = dialog.FileName;
var selectedStorageFile = await StorageFile.GetFileFromPathAsync(dialog.FileName);
SoftwareBitmap softwareBitmap;
using (IRandomAccessStream stream = await selectedStorageFile.OpenAsync(FileAccessMode.Read))
{
// Create the decoder from the stream
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(stream);
// Get the SoftwareBitmap representation of the file in BGRA8 format
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
// Display the image
//SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
//await imageSource.SetBitmapAsync(softwareBitmap);
//PreviewImage.Source = imageSource;
PreviewImage.Source = new System.Windows.Media.Imaging.BitmapImage(new Uri(fileName));
// Encapsulate the image in the WinML image type (VideoFrame) to be bound and evaluated
VideoFrame inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
await EvaluateVideoFrameAsync(inputImage);
}
}
public static async Task <string> GetFullText2(string pathToImg)
{
Bitmap bmp = new Bitmap(pathToImg);
int width = bmp.Width;
int height = bmp.Height;
using (Graphics g = Graphics.FromImage(bmp))
{
// copy rectangle from screen (doesn't include cursor)
using (var stream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
//These steps to get to a SoftwareBitmap are aweful!
bmp.Save(stream.AsStream(), System.Drawing.Imaging.ImageFormat.Bmp);//choose the specific image format by your own bitmap source
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(stream);
SoftwareBitmap bitmap = await decoder.GetSoftwareBitmapAsync();
OcrResult ocrResult = await _ocrEngine.RecognizeAsync(bitmap);
for (int i = 0; i < ocrResult.Lines.Count; i++)
{
OcrLine line = ocrResult.Lines[i];
}
}
}
return(string.Empty);
}
public static async Task <string> GetFullTextFromImage(Image img)
{
try
{
StringBuilder b = new StringBuilder();
using (var stream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
img.Save(stream.AsStream(), System.Drawing.Imaging.ImageFormat.Bmp);
//These steps to get to a SoftwareBitmap are aweful!
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(stream);
SoftwareBitmap bitmap = await decoder.GetSoftwareBitmapAsync();
OcrResult ocrResult = await _ocrEngine.RecognizeAsync(bitmap);
for (int i = 0; i < ocrResult.Lines.Count; i++)
{
b.Append(ocrResult.Lines[i].Text);
b.Append(" ");
}
}
return(b.ToString());
}catch (Exception ex)
{
Logger.Error(ex.ToString());
}
return(string.Empty);
}
public async Task <SoftwareBitmap> CreateSoftwareBitmap(Windows.Storage.StorageFile file, BitmapImage bitmap)
{
Windows.Storage.Streams.IRandomAccessStream random = await Windows.Storage.Streams.RandomAccessStreamReference.CreateFromFile(file).OpenReadAsync();
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(random);
var swBitmap = new SoftwareBitmap(BitmapPixelFormat.Rgba8, bitmap.PixelWidth, bitmap.PixelHeight);
return(swBitmap = await decoder.GetSoftwareBitmapAsync());
}
public async Task <SoftwareBitmap> GetSoftwareSnapShot(int pattern)
{
var target = this.GetTargetWindow();
var bitmapSource = target?.GetClientBitmap();
var bitmap = new System.Drawing.Bitmap(
bitmapSource.PixelWidth,
bitmapSource.PixelHeight,
System.Drawing.Imaging.PixelFormat.Format32bppPArgb
);
var bitmapData = bitmap.LockBits(
new System.Drawing.Rectangle(System.Drawing.Point.Empty, bitmap.Size),
System.Drawing.Imaging.ImageLockMode.WriteOnly,
System.Drawing.Imaging.PixelFormat.Format32bppPArgb
);
bitmapSource.CopyPixels(
System.Windows.Int32Rect.Empty,
bitmapData.Scan0,
bitmapData.Height * bitmapData.Stride,
bitmapData.Stride
);
bitmap.UnlockBits(bitmapData);
bitmap = trimmingBitmap(bitmap, pattern);
var folder = Directory.GetCurrentDirectory();
var imageName = "ScreenCapture.bmp";
StorageFolder appFolder = await StorageFolder.GetFolderFromPathAsync(@folder);
bitmap.Save(folder + "\\" + imageName, ImageFormat.Bmp);
SoftwareBitmap softwareBitmap;
var bmpFile = await appFolder.GetFileAsync(imageName);
using (IRandomAccessStream stream = await bmpFile.OpenAsync(FileAccessMode.Read))
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
}
File.Delete(folder + "\\" + imageName);
return(softwareBitmap);
}
public static async Task <string> ExtractText(Bitmap bmp, System.Windows.Point?singlePoint = null)
{
Language?selectedLanguage = GetOCRLanguage();
if (selectedLanguage == null)
{
return("");
}
XmlLanguage lang = XmlLanguage.GetLanguage(selectedLanguage.LanguageTag);
CultureInfo culture = lang.GetEquivalentCulture();
bool scaleBMP = true;
if (singlePoint != null ||
bmp.Width * 1.5 > OcrEngine.MaxImageDimension)
{
scaleBMP = false;
}
Bitmap scaledBitmap;
if (scaleBMP)
{
scaledBitmap = ScaleBitmapUniform(bmp, 1.5);
}
else
{
scaledBitmap = ScaleBitmapUniform(bmp, 1.0);
}
StringBuilder text = new();
await using (MemoryStream memory = new())
{
scaledBitmap.Save(memory, ImageFormat.Bmp);
memory.Position = 0;
BitmapDecoder bmpDecoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream());
SoftwareBitmap softwareBmp = await bmpDecoder.GetSoftwareBitmapAsync();
OcrEngine ocrEngine = OcrEngine.TryCreateFromLanguage(selectedLanguage);
OcrResult ocrResult = await ocrEngine.RecognizeAsync(softwareBmp);
if (singlePoint == null)
{
foreach (OcrLine line in ocrResult.Lines)
{
text.AppendLine(line.Text);
}
}
else
{
Windows.Foundation.Point fPoint = new Windows.Foundation.Point(singlePoint.Value.X, singlePoint.Value.Y);
foreach (OcrLine ocrLine in ocrResult.Lines)
{
foreach (OcrWord ocrWord in ocrLine.Words)
{
if (ocrWord.BoundingRect.Contains(fPoint))
{
_ = text.Append(ocrWord.Text);
}
}
}
}
}
if (culture.TextInfo.IsRightToLeft)
{
string[] textListLines = text.ToString().Split(new char[] { '\n', '\r' });
_ = text.Clear();
foreach (string textLine in textListLines)
{
List <string> wordArray = textLine.Split().ToList();
wordArray.Reverse();
_ = text.Append(string.Join(' ', wordArray));
if (textLine.Length > 0)
{
_ = text.Append('\n');
}
}
return(text.ToString());
}
else
{
return(text.ToString());
}
}