public void VNSupportedRevisionsTwoTest()
{
// Tests '*RequestRevision.Two' given to APIs.
var rect = new CGRect(0, 0, 10, 10);
Assert.DoesNotThrow(() => {
var detectedObjectObservation = VNDetectedObjectObservation.FromBoundingBox(VNDetectedObjectObservationRequestRevision.Two, rect);
Assert.NotNull(detectedObjectObservation, "detectedObjectObservation is null");
Assert.That(detectedObjectObservation.BoundingBox, Is.EqualTo(rect));
var faceObservation = VNFaceObservation.FromBoundingBox(VNFaceObservationRequestRevision.Two, rect);
Assert.NotNull(faceObservation, "faceObservation is null");
Assert.That(faceObservation.BoundingBox, Is.EqualTo(rect));
var recognizedObjectObservation = VNRecognizedObjectObservation.FromBoundingBox(VNRecognizedObjectObservationRequestRevision.Two, rect);
Assert.NotNull(recognizedObjectObservation, "recognizedObjectObservation is null");
Assert.That(recognizedObjectObservation.BoundingBox, Is.EqualTo(rect));
var rectangleObservation = VNRectangleObservation.FromBoundingBox(VNRectangleObservationRequestRevision.Two, rect);
Assert.NotNull(rectangleObservation, "rectangleObservation is null");
Assert.That(rectangleObservation.BoundingBox, Is.EqualTo(rect));
var textObservation = VNTextObservation.FromBoundingBox(VNTextObservationRequestRevision.Two, rect);
Assert.NotNull(textObservation, "textObservation is null");
Assert.That(textObservation.BoundingBox, Is.EqualTo(rect));
var barcodeObservation = VNBarcodeObservation.FromBoundingBox(VNBarcodeObservationRequestRevision.Two, rect);
Assert.NotNull(barcodeObservation, "barcodeObservation is null");
Assert.That(barcodeObservation.BoundingBox, Is.EqualTo(rect));
}, "*RequestRevision.Two throw");
}
/// <summary>
/// Asynchronously called by the Vision subsystem subsequent to `Perform` in `OnFrameCaptured`
/// </summary>
/// <param name="request">The request sent to the Vision subsystem.</param>
/// <param name="err">If not null, describes an error in Vision.</param>
private void ObjectTracked(VNRequest request, NSError err)
{
if (err != null)
{
Console.Error.WriteLine(err);
InvokeOnMainThread(() => overlay.Message = err.ToString());
return;
}
InvokeOnMainThread(() =>
{
overlay.Clear();
overlay.StrokeColor = UIColor.Green.CGColor;
var observations = request.GetResults <VNDetectedObjectObservation>();
var o = observations.FirstOrDefault();
if (o != null)
{
// o is a succesfully tracked object, so draw it on the `overlay`
overlay.Message = "Locked";
if (o.Confidence < 0.5)
{
overlay.StrokeColor = UIColor.Red.CGColor;
}
if (o.Confidence < 0.8)
{
overlay.StrokeColor = UIColor.Yellow.CGColor;
}
var quad = new[] {
new CGPoint(o.BoundingBox.Left, o.BoundingBox.Top),
new CGPoint(o.BoundingBox.Right, o.BoundingBox.Top),
new CGPoint(o.BoundingBox.Right, o.BoundingBox.Bottom),
new CGPoint(o.BoundingBox.Left, o.BoundingBox.Bottom)
};
ObjectTracked(quad);
trackedRectangle = o;
}
});
}
public void VNSupportedRevisionsTwoTest()
{
// Tests '*RequestRevision.Two' given to APIs.
var rect = new CGRect(0, 0, 10, 10);
{
var detectedObjectObservation = VNDetectedObjectObservation.FromBoundingBox(VNDetectedObjectObservationRequestRevision.Two, rect);
Assert.NotNull(detectedObjectObservation, "detectedObjectObservation is null");
Assert.That(detectedObjectObservation.BoundingBox, Is.EqualTo(rect));
var faceObservation = VNFaceObservation.FromBoundingBox(VNFaceObservationRequestRevision.Two, rect);
Assert.NotNull(faceObservation, "faceObservation is null");
Assert.That(faceObservation.BoundingBox, Is.EqualTo(rect));
var recognizedObjectObservation = VNRecognizedObjectObservation.FromBoundingBox(VNRecognizedObjectObservationRequestRevision.Two, rect);
if (TestRuntime.CheckXcodeVersion(11, 0) && !TestRuntime.CheckXcodeVersion(12, 0))
{
Assert.Null(recognizedObjectObservation, "recognizedObjectObservation is null");
}
else
{
Assert.NotNull(recognizedObjectObservation, "recognizedObjectObservation is null");
Assert.That(recognizedObjectObservation.BoundingBox, Is.EqualTo(rect));
}
var rectangleObservation = VNRectangleObservation.FromBoundingBox(VNRectangleObservationRequestRevision.Two, rect);
Assert.NotNull(rectangleObservation, "rectangleObservation is null");
Assert.That(rectangleObservation.BoundingBox, Is.EqualTo(rect));
var textObservation = VNTextObservation.FromBoundingBox(VNTextObservationRequestRevision.Two, rect);
Assert.NotNull(textObservation, "textObservation is null");
Assert.That(textObservation.BoundingBox, Is.EqualTo(rect));
var barcodeObservation = VNBarcodeObservation.FromBoundingBox(VNBarcodeObservationRequestRevision.Two, rect);
Assert.NotNull(barcodeObservation, "barcodeObservation is null");
Assert.That(barcodeObservation.BoundingBox, Is.EqualTo(rect));
}
}
public void VNSupportedRevisionsUnsupportedTest()
{
var allSupported = new List <(string type, IEnumerable revs)> {
(nameof(VNCoreMLRequest), VNCoreMLRequest.SupportedRevisions),
(nameof(VNDetectBarcodesRequest), VNDetectBarcodesRequest.SupportedRevisions),
(nameof(VNDetectFaceLandmarksRequest), VNDetectFaceLandmarksRequest.SupportedRevisions),
(nameof(VNDetectFaceRectanglesRequest), VNDetectFaceRectanglesRequest.SupportedRevisions),
(nameof(VNDetectHorizonRequest), VNDetectHorizonRequest.SupportedRevisions),
(nameof(VNDetectRectanglesRequest), VNDetectRectanglesRequest.SupportedRevisions),
(nameof(VNDetectTextRectanglesRequest), VNDetectTextRectanglesRequest.SupportedRevisions),
(nameof(VNTranslationalImageRegistrationRequest), VNTranslationalImageRegistrationRequest.SupportedRevisions),
(nameof(VNHomographicImageRegistrationRequest), VNHomographicImageRegistrationRequest.SupportedRevisions),
(nameof(VNTrackObjectRequest), VNTrackObjectRequest.SupportedRevisions),
(nameof(VNTrackRectangleRequest), VNTrackRectangleRequest.SupportedRevisions),
};
foreach (var revisions in allSupported)
{
var type = revisions.type;
foreach (object rev in revisions.revs)
{
Assert.That(Convert.ChangeType(rev, Enum.GetUnderlyingType(rev.GetType())), Is.Not.EqualTo(0), $"SupportedRevisions Unspecified found: {type}");
}
}
var allDefault = new List <(string type, object rev)> {
(nameof(VNCoreMLRequest), VNCoreMLRequest.DefaultRevision),
(nameof(VNDetectBarcodesRequest), VNDetectBarcodesRequest.DefaultRevision),
(nameof(VNDetectFaceLandmarksRequest), VNDetectFaceLandmarksRequest.DefaultRevision),
(nameof(VNDetectFaceRectanglesRequest), VNDetectFaceRectanglesRequest.DefaultRevision),
(nameof(VNDetectHorizonRequest), VNDetectHorizonRequest.DefaultRevision),
(nameof(VNDetectRectanglesRequest), VNDetectRectanglesRequest.DefaultRevision),
(nameof(VNDetectTextRectanglesRequest), VNDetectTextRectanglesRequest.DefaultRevision),
(nameof(VNTranslationalImageRegistrationRequest), VNTranslationalImageRegistrationRequest.DefaultRevision),
(nameof(VNHomographicImageRegistrationRequest), VNHomographicImageRegistrationRequest.DefaultRevision),
(nameof(VNTrackObjectRequest), VNTrackObjectRequest.DefaultRevision),
(nameof(VNTrackRectangleRequest), VNTrackRectangleRequest.DefaultRevision),
};
foreach (var defrev in allDefault)
{
Assert.That(Convert.ChangeType(defrev.rev, Enum.GetUnderlyingType(defrev.rev.GetType())), Is.Not.EqualTo(0), $"DefaultRevision Unspecified found: {defrev.type}");
}
var allCurrent = new List <(string type, object rev)> {
(nameof(VNCoreMLRequest), VNCoreMLRequest.CurrentRevision),
(nameof(VNDetectBarcodesRequest), VNDetectBarcodesRequest.CurrentRevision),
(nameof(VNDetectFaceLandmarksRequest), VNDetectFaceLandmarksRequest.CurrentRevision),
(nameof(VNDetectFaceRectanglesRequest), VNDetectFaceRectanglesRequest.CurrentRevision),
(nameof(VNDetectHorizonRequest), VNDetectHorizonRequest.CurrentRevision),
(nameof(VNDetectRectanglesRequest), VNDetectRectanglesRequest.CurrentRevision),
(nameof(VNDetectTextRectanglesRequest), VNDetectTextRectanglesRequest.CurrentRevision),
(nameof(VNTranslationalImageRegistrationRequest), VNTranslationalImageRegistrationRequest.CurrentRevision),
(nameof(VNHomographicImageRegistrationRequest), VNHomographicImageRegistrationRequest.CurrentRevision),
(nameof(VNTrackObjectRequest), VNTrackObjectRequest.CurrentRevision),
(nameof(VNTrackRectangleRequest), VNTrackRectangleRequest.CurrentRevision),
};
foreach (var currev in allCurrent)
{
Assert.That(Convert.ChangeType(currev.rev, Enum.GetUnderlyingType(currev.rev.GetType())), Is.Not.EqualTo(0), $"CurrentRevision Unspecified found: {currev.type}");
}
// Tests 'VNRequestRevision.Unspecified' given to APIs.
var rect = new CGRect(0, 0, 10, 10);
{
var detectedObjectObservation = VNDetectedObjectObservation.FromBoundingBox(VNDetectedObjectObservationRequestRevision.Unspecified, rect);
Assert.NotNull(detectedObjectObservation, "detectedObjectObservation is null");
Assert.That(detectedObjectObservation.BoundingBox, Is.EqualTo(rect));
var faceObservation = VNFaceObservation.FromBoundingBox(VNFaceObservationRequestRevision.Unspecified, rect);
Assert.NotNull(faceObservation, "faceObservation is null");
Assert.That(faceObservation.BoundingBox, Is.EqualTo(rect));
var recognizedObjectObservation = VNRecognizedObjectObservation.FromBoundingBox(VNRecognizedObjectObservationRequestRevision.Unspecified, rect);
if (TestRuntime.CheckXcodeVersion(11, 0) && !TestRuntime.CheckXcodeVersion(12, 0))
{
Assert.IsNull(recognizedObjectObservation, "recognizedObjectObservation is null");
}
else
{
Assert.NotNull(recognizedObjectObservation, "recognizedObjectObservation is null");
Assert.That(recognizedObjectObservation.BoundingBox, Is.EqualTo(rect));
}
var rectangleObservation = VNRectangleObservation.FromBoundingBox(VNRectangleObservationRequestRevision.Unspecified, rect);
Assert.NotNull(rectangleObservation, "rectangleObservation is null");
Assert.That(rectangleObservation.BoundingBox, Is.EqualTo(rect));
var textObservation = VNTextObservation.FromBoundingBox(VNTextObservationRequestRevision.Unspecified, rect);
Assert.NotNull(textObservation, "textObservation is null");
Assert.That(textObservation.BoundingBox, Is.EqualTo(rect));
var barcodeObservation = VNBarcodeObservation.FromBoundingBox(VNBarcodeObservationRequestRevision.Unspecified, rect);
Assert.NotNull(barcodeObservation, "barcodeObservation is null");
Assert.That(barcodeObservation.BoundingBox, Is.EqualTo(rect));
}
// Tests random request revision
{
var detectedObjectObservation = VNDetectedObjectObservation.FromBoundingBox((VNDetectedObjectObservationRequestRevision)5000, rect);
Assert.NotNull(detectedObjectObservation, "randomRevision detectedObjectObservation is null");
Assert.That(detectedObjectObservation.BoundingBox, Is.EqualTo(rect));
var faceObservation = VNFaceObservation.FromBoundingBox((VNFaceObservationRequestRevision)5000, rect);
Assert.NotNull(faceObservation, "randomRevision faceObservation is null");
Assert.That(faceObservation.BoundingBox, Is.EqualTo(rect));
var recognizedObjectObservation = VNRecognizedObjectObservation.FromBoundingBox((VNRecognizedObjectObservationRequestRevision)5000, rect);
if (TestRuntime.CheckXcodeVersion(11, 0) && !TestRuntime.CheckXcodeVersion(12, 0))
{
Assert.IsNull(recognizedObjectObservation, "randomRevision recognizedObjectObservation is null");
}
else
{
Assert.NotNull(recognizedObjectObservation, "randomRevision recognizedObjectObservation is null");
Assert.That(recognizedObjectObservation.BoundingBox, Is.EqualTo(rect));
}
var rectangleObservation = VNRectangleObservation.FromBoundingBox((VNRectangleObservationRequestRevision)5000, rect);
Assert.NotNull(rectangleObservation, "randomRevision rectangleObservation is null");
Assert.That(rectangleObservation.BoundingBox, Is.EqualTo(rect));
var textObservation = VNTextObservation.FromBoundingBox((VNTextObservationRequestRevision)5000, rect);
Assert.NotNull(textObservation, "randomRevision textObservation is null");
Assert.That(textObservation.BoundingBox, Is.EqualTo(rect));
var barcodeObservation = VNBarcodeObservation.FromBoundingBox((VNBarcodeObservationRequestRevision)5000, rect);
Assert.NotNull(barcodeObservation, "randomRevision barcodeObservation is null");
Assert.That(barcodeObservation.BoundingBox, Is.EqualTo(rect));
}
}
public TrackedPolyRect(VNDetectedObjectObservation observation, UIColor color, TrackedPolyRectStyle style = TrackedPolyRectStyle.Solid)
: this(observation.BoundingBox, color, style)
{
}
/// <summary>
/// Perform Requests
/// </summary>
public void PerformTracking(TrackedObjectType type, out NSError error)
{
var videoReader = VideoReader.Create(videoAsset);
if (videoReader != null)
{
if (videoReader.NextFrame() != null)
{
this.cancelRequested = false;
// Create initial observations
var inputObservations = new Dictionary <NSUuid, VNDetectedObjectObservation>();
var trackedObjects = new Dictionary <NSUuid, TrackedPolyRect>();
switch (type)
{
case TrackedObjectType.Object:
foreach (var rect in this.ObjectsToTrack)
{
var inputObservation = VNDetectedObjectObservation.FromBoundingBox(rect.BoundingBox);
inputObservations[inputObservation.Uuid] = inputObservation;
trackedObjects[inputObservation.Uuid] = rect;
}
break;
case TrackedObjectType.Rectangle:
foreach (var rectangleObservation in this.initialRectObservations)
{
inputObservations[rectangleObservation.Uuid] = rectangleObservation;
var rectColor = TrackedObjectsPalette.Color(trackedObjects.Count);
trackedObjects[rectangleObservation.Uuid] = new TrackedPolyRect(rectangleObservation, rectColor);
}
break;
}
var requestHandler = new VNSequenceRequestHandler();
var frames = 1;
var trackingFailedForAtLeastOneObject = false;
CVPixelBuffer frame = null;
while (true)
{
if (this.cancelRequested || (frame = videoReader.NextFrame()) == null)
{
break;
}
this.Delegate?.DisplayFrameCounter(frames);
frames += 1;
var rects = new List <TrackedPolyRect>();
var trackingRequests = new List <VNRequest>();
foreach (var inputObservation in inputObservations)
{
VNTrackingRequest request = null;
switch (type)
{
case TrackedObjectType.Object:
request = new VNTrackObjectRequest(inputObservation.Value);
break;
case TrackedObjectType.Rectangle:
if (inputObservation.Value is VNRectangleObservation rectObservation)
{
request = new VNTrackRectangleRequest(rectObservation);
}
else
{
continue;
}
break;
}
request.TrackingLevel = this.TrackingLevel;
trackingRequests.Add(request);
}
// Perform array of requests
requestHandler.Perform(trackingRequests.ToArray(), frame, videoReader.Orientation, out NSError performError);
trackingFailedForAtLeastOneObject = performError != null;
foreach (var processedRequest in trackingRequests)
{
var results = processedRequest.GetResults <VNObservation>();
if (results == null || !results.Any())
{
continue;
}
if (results.FirstOrDefault() is VNDetectedObjectObservation observation)
{
// Assume threshold = 0.5f
var rectStyle = observation.Confidence > 0.5f ? TrackedPolyRectStyle.Solid : TrackedPolyRectStyle.Dashed;
var knownRect = trackedObjects[observation.Uuid];
switch (type)
{
case TrackedObjectType.Object:
rects.Add(new TrackedPolyRect(observation, knownRect.Color, rectStyle));
break;
case TrackedObjectType.Rectangle:
if (observation is VNRectangleObservation rectObservation)
{
rects.Add(new TrackedPolyRect(rectObservation, knownRect.Color, rectStyle));
}
break;
}
// Initialize inputObservation for the next iteration
inputObservations[observation.Uuid] = observation;
}
}
// Draw results
this.Delegate?.DisplayFrame(frame, videoReader.AffineTransform, rects);
var miliseconds = videoReader.FrameRateInSeconds / 1000;
System.Threading.Thread.Sleep((int)miliseconds);
}
this.Delegate?.DidFinifshTracking();
error = trackingFailedForAtLeastOneObject ? new VisionTrackerProcessorError(VisionTrackerProcessorErrorType.FirstFrameReadFailed)
: null;
}
else
{
error = new VisionTrackerProcessorError(VisionTrackerProcessorErrorType.FirstFrameReadFailed);
}
}
else
{
error = new VisionTrackerProcessorError(VisionTrackerProcessorErrorType.ReaderInitializationFailed);
}
}
/// <summary> /// Track the specified observation. /// </summary> internal void Track(VNDetectedObjectObservation observation) => trackedRectangle = observation;