static async Task <string> SubmitBitmapToCloudGetResultLocationAsync(
string cloudEndpoint,
string cloudApiKey,
DeviceOcrResult deviceOcrResult)
{
string resultLocation = null;
// First, encode the software bitmap as a Jpeg...
using (var memoryStream = new InMemoryRandomAccessStream())
{
var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, memoryStream);
encoder.SetSoftwareBitmap(deviceOcrResult.BestOcrSoftwareBitmap);
await encoder.FlushAsync();
memoryStream.Seek(0);
// Now, send it off to the computer vision API.
using (var httpClient = new HttpClient())
using (var httpContent = new HttpStreamContent(memoryStream))
{
httpContent.Headers["Content-Type"] = "application/octet-stream";
httpClient.DefaultRequestHeaders[API_SUBSCRIPTION_KEY_HEADER] = cloudApiKey;
using (var response = await httpClient.PostAsync(new Uri(cloudEndpoint), httpContent))
{
if (response.IsSuccessStatusCode)
{
resultLocation = response.Headers["Operation-Location"];
}
}
}
}
return(resultLocation);
}
public async Task <GeneralOcrResult> MatchOnCloudAsync(
string cloudEndpoint,
string cloudApiKey,
DeviceOcrResult deviceOcrResult,
TimeSpan timeBetweenCalls,
TimeSpan timeout)
{
GeneralOcrResult ocrResult = new GeneralOcrResult()
{
ResultType = OcrMatchResult.CloudCallFailed
};
if (deviceOcrResult.ResultType != OcrMatchResult.TimedOutCloudCallAvailable)
{
throw new InvalidOperationException();
}
var resultLocation = await SubmitBitmapToCloudGetResultLocationAsync(
cloudEndpoint, cloudApiKey, deviceOcrResult);
if (!string.IsNullOrEmpty(resultLocation))
{
ocrResult = await PollCloudResultLocation(resultLocation, cloudApiKey, timeBetweenCalls, timeout);
}
return(ocrResult);
}
async Task OcrProcessFrameAsync(
MediaFrameReader reader,
OcrEngine ocrEngine,
DeviceOcrResult ocrDeviceResult)
{
using (var frame = reader.TryAcquireLatestFrame())
{
if (frame?.VideoMediaFrame != null)
{
using (var bitmap = frame.VideoMediaFrame.SoftwareBitmap)
{
var result = await ocrEngine.RecognizeAsync(bitmap);
if (result?.Text != null)
{
var matchingResults = this.matchExpression.Matches(result.Text);
var matched = matchingResults?.Count > 0;
if (matched)
{
// We take the first one, we don't do multiple (yet).
ocrDeviceResult.MatchedText = matchingResults[0].Value;
ocrDeviceResult.ResultType = OcrMatchResult.Succeeded;
ocrDeviceResult.BestOcrSoftwareBitmap?.Dispose();
ocrDeviceResult.BestOcrSoftwareBitmap = null;
}
else if (result.Text.Length > ocrDeviceResult.BestOcrTextLengthFound)
{
ocrDeviceResult.BestOcrTextLengthFound = result.Text.Length;
ocrDeviceResult.BestOcrSoftwareBitmap?.Dispose();
ocrDeviceResult.BestOcrSoftwareBitmap = SoftwareBitmap.Copy(bitmap);
ocrDeviceResult.ResultType = OcrMatchResult.TimedOutCloudCallAvailable;
}
}
}
}
}
}
/// <summary>
/// We scan frames from the device, running OCR and trying to find text that matches the pattern
/// passed until the point that timeout expires.
/// We also try and store the 'best' frame of text that we have seen in case we
/// need to later try to submit to the cloud.
/// </summary>
/// <param name="searchPattern"></param>
/// What to look for.
/// <param name="timeout"></param>
/// How long to spend on it.
/// <returns></returns>
public async Task <DeviceOcrResult> MatchOnDeviceAsync(TimeSpan timeout)
{
var deviceOcrResult = new DeviceOcrResult()
{
ResultType = OcrMatchResult.TimeOutNoCloudCallAvailable
};
using (var mediaCapture = new MediaCapture())
{
await mediaCapture.InitializeAsync(
new MediaCaptureInitializationSettings()
{
StreamingCaptureMode = StreamingCaptureMode.Video,
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
SourceGroup = this.mediaFrameSourceFinder.FrameSourceGroup
}
);
var frameSource = mediaCapture.FrameSources.Single(
fs => fs.Value.Info.Id == this.mediaFrameSourceFinder.FrameSourceInfo.Id);
// BGRA8 here is intended to line up with what I think the OCR engine supports.
using (var reader = await mediaCapture.CreateFrameReaderAsync(frameSource.Value,
MediaEncodingSubtypes.Bgra8))
{
TaskCompletionSource <bool> completedTask = new TaskCompletionSource <bool>();
var timeoutTask = Task.Delay(timeout);
var ocrEngine = OcrEngine.TryCreateFromUserProfileLanguages();
reader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime;
int occupiedFlag = 0;
TypedEventHandler <MediaFrameReader, MediaFrameArrivedEventArgs> handler =
async(s, e) =>
{
if (Interlocked.CompareExchange(ref occupiedFlag, 1, 0) == 0)
{
await OcrProcessFrameAsync(
reader, ocrEngine, deviceOcrResult);
if (deviceOcrResult.ResultType == OcrMatchResult.Succeeded)
{
completedTask.SetResult(true);
}
Interlocked.Exchange(ref occupiedFlag, 0);
}
};
reader.FrameArrived += handler;
await reader.StartAsync();
var timedOut = (await Task.WhenAny(completedTask.Task, timeoutTask)) == timeoutTask;
reader.FrameArrived -= handler;
await reader.StopAsync();
}
}
return(deviceOcrResult);
}