private async void TakePhoto_Click(object sender, RoutedEventArgs e)
{
try
{
photo = await captureUI.CaptureFileAsync(CameraCaptureUIMode.Photo);
if (photo == null)
{
return;
}
else
{
imageStream = await photo.OpenAsync(FileAccessMode.Read);
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(imageStream);
Windows.Graphics.Imaging.SoftwareBitmap softwareBitmap = await decoder.GetSoftwareBitmapAsync();
Windows.Graphics.Imaging.SoftwareBitmap softwareBitmapBRG = Windows.Graphics.Imaging.SoftwareBitmap.Convert(softwareBitmap, Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8,
Windows.Graphics.Imaging.BitmapAlphaMode.Premultiplied);
SoftwareBitmapSource bitmapSource = new SoftwareBitmapSource();
await bitmapSource.SetBitmapAsync(softwareBitmapBRG);
image.Source = bitmapSource;
}
}
catch
{
output.Text = "Error carnal";
}
}
private async Task RecognizeImageAsync()
{
while (_isRunning)
{
await Task.Delay(100);
}
if (this.Image1.Source == null)
{
return;
}
// 表示している画像を SoftwareBitmap として取得
UwpSoftwareBitmap bitmap
= await UwpSoftwareBitmapHelper.ConvertFrom(this.Image1.Source as BitmapFrame);
if (bitmap == null)
{
return;
}
// 認識言語を変えて再認識させたいときのために保持
if (_lastRecognizedBitmap != bitmap)
{
_lastRecognizedBitmap?.Dispose();
_lastRecognizedBitmap = bitmap;
}
// SoftwareBitmap を OCR に掛ける
await RecognizeBitmapAsync(bitmap);
}
private async Task ReRecognizeAsync()
{
UwpSoftwareBitmap bitmap = _lastRecognizedBitmap;
if (bitmap == null)
{
return;
}
await RecognizeBitmapAsync(bitmap);
}
// SoftwareBitmap を OCR に掛ける
private async Task RecognizeBitmapAsync(UwpSoftwareBitmap bitmap)
{
this.RecognizedTextTextBox.Text = string.Empty;
var ocrEngine = UwpOcrEngine.TryCreateFromLanguage(this.LangComboBox.SelectedItem as UwpLanguage);
UwpOcrResult ocrResult = await ocrEngine.RecognizeAsync(bitmap);
foreach (var ocrLine in ocrResult.Lines)
{
this.RecognizedTextTextBox.Text += (ocrLine.Text + "\n");
}
}
async Task <OcrResult> RecognizeBitmapAsync(Bitmap b)
{
// Need to marshall from Drawing.Bitmap to UWP SoftwareBitmap
using (var stream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
b.Save(stream.AsStream(), System.Drawing.Imaging.ImageFormat.Bmp); //choose the specific image format by your own bitmap source
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(stream);
Windows.Graphics.Imaging.SoftwareBitmap softwareBitmap = await decoder.GetSoftwareBitmapAsync();
return(await engine.RecognizeAsync(softwareBitmap));
}
}
public async Task <string> ExtractText(Bitmap bmp, string languageCode, System.Windows.Point?singlePoint = null)
{
if (!GlobalizationPreferences.Languages.Contains(languageCode))
{
throw new ArgumentOutOfRangeException($"{languageCode} is not installed.");
}
StringBuilder text = new StringBuilder();
await using (MemoryStream memory = new MemoryStream())
{
bmp.Save(memory, System.Drawing.Imaging.ImageFormat.Bmp);
memory.Position = 0;
BitmapDecoder bmpDecoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream());
Windows.Graphics.Imaging.SoftwareBitmap softwareBmp = await bmpDecoder.GetSoftwareBitmapAsync();
OcrEngine ocrEngine = OcrEngine.TryCreateFromLanguage(new Language(languageCode));
OcrResult ocrResult = await ocrEngine.RecognizeAsync(softwareBmp);
if (singlePoint == null)
{
foreach (OcrLine line in ocrResult.Lines)
{
text.AppendLine(line.Text);
}
}
else
{
Windows.Foundation.Point fPoint = new Windows.Foundation.Point(singlePoint.Value.X, singlePoint.Value.Y);
foreach (OcrLine ocrLine in ocrResult.Lines)
{
foreach (OcrWord ocrWord in ocrLine.Words)
{
if (ocrWord.BoundingRect.Contains(fPoint))
{
text.Append(ocrWord.Text);
}
}
}
}
}
return(text.ToString());
}
/// <summary>
/// Save As <see cref="Windows.Graphics.Imaging.SoftwareBitmap"/>
/// </summary>
public unsafe Windows.Graphics.Imaging.SoftwareBitmap SaveAsSoftwareBitmap()
{
var size = bmp.Size;
var sbitmap = new Windows.Graphics.Imaging.SoftwareBitmap(Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8, size.Width, size.Height, Windows.Graphics.Imaging.BitmapAlphaMode.Premultiplied);
using (var sbitbuffer = sbitmap.LockBuffer(Windows.Graphics.Imaging.BitmapBufferAccessMode.Write))
{
using (var reference = sbitbuffer.CreateReference())
{
byte *dataInBytes;
uint capacity;
((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacity);
// Fill-in the BGRA plane
var bufferLayout = sbitbuffer.GetPlaneDescription(0);
var intptr = new IntPtr(dataInBytes);
Bitmap.CopyPixels(bufferLayout.Stride, intptr, (int)capacity);
}
}
return(sbitmap);
}
private async void GetPhotoButton_Click(object sender, RoutedEventArgs e)
{
// This is to select a file to open.
//Windows.Storage.Pickers.FileOpenPicker openPicker = new Windows.Storage.Pickers.FileOpenPicker();
openPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
openPicker.ViewMode = PickerViewMode.Thumbnail;
// Filter to include a sample subset of file types.
openPicker.FileTypeFilter.Clear();
openPicker.FileTypeFilter.Add(".bmp");
openPicker.FileTypeFilter.Add(".png");
openPicker.FileTypeFilter.Add(".jpeg");
openPicker.FileTypeFilter.Add(".jpg");
// Open the file picker .
photo = await openPicker.PickSingleFileAsync();
// The file is null if user cancels the file picker.
if (photo != null)
{
// Here we open a stream for the selected file.
// The 'using' block ensures the stream is disposed
// after the image is loaded.
imageStream = await photo.OpenAsync(FileAccessMode.Read);
Windows.Graphics.Imaging.BitmapDecoder decoder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(imageStream);
Windows.Graphics.Imaging.SoftwareBitmap softwareBitmap = await decoder.GetSoftwareBitmapAsync();
Windows.Graphics.Imaging.SoftwareBitmap softwareBitmapBRG = Windows.Graphics.Imaging.SoftwareBitmap.Convert(softwareBitmap, Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8,
Windows.Graphics.Imaging.BitmapAlphaMode.Premultiplied);
SoftwareBitmapSource bitmapSource = new SoftwareBitmapSource();
await bitmapSource.SetBitmapAsync(softwareBitmapBRG);
image.Source = bitmapSource;
}
}
/// <summary></summary>
public void CreateFromSoftwareBitmap(ICanvasResourceCreator resourceCreator, Windows.Graphics.Imaging.SoftwareBitmap sourceBitmap)
{
throw new System.NotImplementedException();
}
public async Task RecognizeAsync(BitmapFrame bitmapFrame)
{
if (!CanExecute)
{
return;
}
ClearResults();
// SoftwareBitmapを取得
UwpSoftwareBitmap bitmap
= await UwpSoftwareBitmapHelper.ConvertFrom(bitmapFrame).ConfigureAwait(true);
if (bitmap == null)
{
return;
}
// OCR実行
UwpOcrResult ocrResult = await engine.RecognizeAsync(bitmap);
bitmap.Dispose();
// Angle
ocrAngle = ocrResult.TextAngle ?? 0.0;
// Line
foreach (var ocrLine in ocrResult.Lines)
{
WinOcrResult result = new WinOcrResult();
string words = "";
double left = ocrLine.Words[0].BoundingRect.Left;
double right = ocrLine.Words[0].BoundingRect.Right;
double top = ocrLine.Words[0].BoundingRect.Top;
double bottom = ocrLine.Words[0].BoundingRect.Bottom;
foreach (var word in ocrLine.Words)
{
words += word.Text;
if (word.BoundingRect.Left < left)
{
left = word.BoundingRect.Left;
}
if (right < word.BoundingRect.Right)
{
right = word.BoundingRect.Right;
}
if (word.BoundingRect.Top < top)
{
top = word.BoundingRect.Top;
}
if (bottom < word.BoundingRect.Bottom)
{
bottom = word.BoundingRect.Bottom;
}
}
result.Words = words;
result.RectLeft = left;
result.RectTop = top;
result.RectWidth = right - left;
result.RectHeight = bottom - top;
this.ocrResults.Add(result);
}
}
private async void HandleRequestOCR(byte[] message)
{
// first 2 bytes is number of images
int numImages = BitConverter.ToInt16(message, 0);
// get images
List <Windows.Graphics.Imaging.SoftwareBitmap> bitmaps = new List <Windows.Graphics.Imaging.SoftwareBitmap> {
};
int k = 2;
for (int img = 0; img < numImages; ++img)
{
int width = BitConverter.ToInt16(message, k);
int height = BitConverter.ToInt16(message, k + 2);
k += 4;
// bgr -> bgra
byte[] image = new byte[width * height * 4];
for (int j = 0; j < image.Length;)
{
Buffer.BlockCopy(message, k, image, j, 3); j += 3;
image[j++] = 0;
k += 3;
}
// bytes -> software bitmap
var bitmap = new Windows.Graphics.Imaging.SoftwareBitmap(Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8, width, height);
bitmap.CopyFromBuffer(image.AsBuffer());
bitmaps.Add(bitmap);
}
// conduct ocr
string results = "";
string languageDelimiter = " &&& ";
string imageDelimiter = " ;;; ";
foreach (var bitmap in bitmaps)
{
if (bitmap.PixelWidth > OcrEngine.MaxImageDimension || bitmap.PixelHeight > OcrEngine.MaxImageDimension)
{
results += imageDelimiter;
continue;
}
foreach (var engine in this.ocrEngine)
{
var ocrResult = await engine.RecognizeAsync(bitmap);
results += ocrResult.Text + languageDelimiter;
}
results = results.Remove(results.Length - languageDelimiter.Length);
results += imageDelimiter;
}
if (results.Length != 0)
{
results = results.Remove(results.Length - imageDelimiter.Length);
}
// e.g results = "detected message &&& message in language 2 ;;; nothing detected in next image ;;; ;;; detected message"
// send results
this.client.Publish("/kinect/response/ocr", Encoding.UTF8.GetBytes(results));
}
public async Task <string> ExtractText(Bitmap bmp, string languageCode, System.Windows.Point?singlePoint = null)
{
Language selectedLanguage = new Language(languageCode);
List <Language> possibleOCRLangs = OcrEngine.AvailableRecognizerLanguages.ToList();
if (possibleOCRLangs.Count < 1)
{
throw new ArgumentOutOfRangeException($"No possible OCR languages are installed.");
}
if (possibleOCRLangs.Where(l => l.LanguageTag == selectedLanguage.LanguageTag).Count() < 1)
{
List <Language> similarLanguages = possibleOCRLangs.Where(la => la.AbbreviatedName == selectedLanguage.AbbreviatedName).ToList();
if (similarLanguages.Count() > 0)
{
selectedLanguage = similarLanguages.FirstOrDefault();
}
else
{
selectedLanguage = possibleOCRLangs.FirstOrDefault();
}
}
bool scaleBMP = true;
if (singlePoint != null ||
bmp.Width * 1.5 > OcrEngine.MaxImageDimension)
{
scaleBMP = false;
}
Bitmap scaledBitmap;
if (scaleBMP)
{
scaledBitmap = ScaleBitmapUniform(bmp, 1.5);
}
else
{
scaledBitmap = ScaleBitmapUniform(bmp, 1.0);
}
StringBuilder text = new StringBuilder();
XmlLanguage lang = XmlLanguage.GetLanguage(languageCode);
CultureInfo culture = lang.GetEquivalentCulture();
await using (MemoryStream memory = new MemoryStream())
{
scaledBitmap.Save(memory, ImageFormat.Bmp);
memory.Position = 0;
BitmapDecoder bmpDecoder = await BitmapDecoder.CreateAsync(memory.AsRandomAccessStream());
Windows.Graphics.Imaging.SoftwareBitmap softwareBmp = await bmpDecoder.GetSoftwareBitmapAsync();
OcrEngine ocrEngine = OcrEngine.TryCreateFromLanguage(selectedLanguage);
OcrResult ocrResult = await ocrEngine.RecognizeAsync(softwareBmp);
if (singlePoint == null)
{
foreach (OcrLine line in ocrResult.Lines)
{
text.AppendLine(line.Text);
}
}
else
{
Windows.Foundation.Point fPoint = new Windows.Foundation.Point(singlePoint.Value.X, singlePoint.Value.Y);
foreach (OcrLine ocrLine in ocrResult.Lines)
{
foreach (OcrWord ocrWord in ocrLine.Words)
{
if (ocrWord.BoundingRect.Contains(fPoint))
{
text.Append(ocrWord.Text);
}
}
}
}
}
if (culture.TextInfo.IsRightToLeft)
{
List <string> textListLines = text.ToString().Split(new char[] { '\n', '\r' }).ToList();
text.Clear();
foreach (string textLine in textListLines)
{
List <string> wordArray = textLine.Split().ToList();
wordArray.Reverse();
text.Append(string.Join(' ', wordArray));
if (textLine.Length > 0)
{
text.Append('\n');
}
}
return(text.ToString());
}
else
{
return(text.ToString());
}
}