public HandsAnalyzerTests()
{
this.handsAnalyzer = new HandsAnalyzer();
this.left = BgrImage.GetBlank(640, 480, Color.Red);
this.leftPoi = new List <Point>();
this.right = BgrImage.GetBlank(640, 480, Color.Green);
this.rightPoi = new List <Point>();
}
/// <summary>
/// Sets the image coming from the computer's primary camera.
/// </summary>
/// <param name="image">The image from the computer's primary camera.</param>
public void SetComputerCameraImage(BgrImage image)
{
Application.Current?.Dispatcher.Invoke(() =>
{
this.webCameraFrame = this.BitmapToBitmapSource(image.Bitmap, PixelFormats.Bgr24);
this.OnPropertyChanged(nameof(this.CurrentWebCameraFrame));
});
}
public void Merge_TheMergedImageHasTheCorrectSize()
{
BgrImage bgrImageUpsideDown = new BgrImage(this.emguImage.Flip(FlipType.Horizontal));
BgrImage imageMerged = bgrImageUpsideDown.Merge(this.bgrImage);
Assert.Equal(this.bgrImage.Width, imageMerged.Width);
Assert.Equal(this.bgrImage.Height, imageMerged.Height);
}
public void KeyInputIsSentAfterHandsInput_KeyInputIsNotIgnored()
{
var skinColorHandsDetectorMock = new Mock <ISkinColorHandsDetector>();
skinColorHandsDetectorMock.Setup(schd => schd.Tuned).Returns(true);
var left = new NormalizedHand(0.15, 0.2, 0.18);
var right = new NormalizedHand(0.85, 0.2, 0.18);
var hands = new NormalizedHands(left, right);
var handsDetectorResult = new HandsDetectorResult(hands, null);
skinColorHandsDetectorMock.Setup(schd => schd.DetectHands(It.IsAny <BgrImage>()))
.Returns(handsDetectorResult);
var keyInputEvaluator = new GeneralDroneKeyInputEvaluator();
var handsInputEvaluator = new DroneControllerHandsInputEvaluator();
var neutralLeft = new NormalizedHand(0.15, 0.5, 0.18);
var neutralRight = new NormalizedHand(0.85, 0.5, 0.18);
var neutralHands = new NormalizedHands(neutralLeft, neutralRight);
handsInputEvaluator.Tune(neutralHands);
this.caduhdApp = new CaduhdApp(null, skinColorHandsDetectorMock.Object, this.tello, keyInputEvaluator, handsInputEvaluator);
this.caduhdApp.Input(BgrImage.GetBlank(1, 1, Color.Black));
this.udpClient.Client.ReceiveTimeout = 1000;
string response = string.Empty;
bool handsInputIgnored = false;
try
{
response = this.udpClient.Receive(ref this.anyEndPoint).AsString();
Assert.Equal($"rc 0 0 {this.tello.Speed} 0", response);
}
catch (SocketException)
{
}
this.caduhdApp.Input(new KeyInfo(Key.D, KeyState.Down));
try
{
response = this.udpClient.Receive(ref this.anyEndPoint).AsString();
}
catch (SocketException e)
{
if (e.SocketErrorCode == SocketError.TimedOut)
{
handsInputIgnored = true;
}
}
this.tello.Dispose();
Assert.False(handsInputIgnored);
Assert.Equal($"rc 0 0 0 {this.tello.Speed}", response);
}
/// <summary>
/// This is the entry point of the hands into the Caduhd App.
/// </summary>
/// <param name="image">The image, possibly containing hands.</param>
public void Input(BgrImage image)
{
if (Interlocked.CompareExchange(ref this.isWebCameraFrameProcessorBusy, YES, NO) == NO)
{
if (this.handsAnalyzer != null)
{
this.uiConnector.SetHandsAnalyzerState(this.handsAnalyzer.State);
}
BgrImage frame = image;
MoveCommand moveCommand = null;
if (this.skinColorHandsDetector.Tuned)
{
HandsDetectorResult result = this.skinColorHandsDetector.DetectHands(frame);
frame = result.Image;
var hands = result.Hands;
moveCommand =
(this.handsDroneController.ProcessHandsInput(hands) as DroneControllerHandsInputProcessResult)?.Result;
}
else
{
if (this.handsAnalyzer.State == HandsAnalyzerState.ReadyToAnalyzeLeft || this.handsAnalyzer.State == HandsAnalyzerState.AnalyzingLeft)
{
if (this.handsAnalyzer.State == HandsAnalyzerState.AnalyzingLeft)
{
this.handsAnalyzer.AnalyzeLeft(frame, this.handsDroneController.HandsInputEvaluator.TunerHands["left"]["poi"]);
this.handsAnalyzer.AdvanceState();
}
frame.MarkPoints(this.handsDroneController.HandsInputEvaluator.TunerHands["left"]["outline"], Color.Yellow);
}
else if (this.handsAnalyzer.State == HandsAnalyzerState.ReadyToAnalyzeRight || this.handsAnalyzer.State == HandsAnalyzerState.AnalyzingRight)
{
if (this.handsAnalyzer.State == HandsAnalyzerState.AnalyzingRight)
{
this.handsAnalyzer.AnalyzeRight(frame, this.handsDroneController.HandsInputEvaluator.TunerHands["right"]["poi"]);
this.handsAnalyzer.AdvanceState();
}
frame.MarkPoints(this.handsDroneController.HandsInputEvaluator.TunerHands["right"]["outline"], Color.Yellow);
}
else if (this.handsAnalyzer.State == HandsAnalyzerState.Tuning)
{
HandsAnalyzerResult result = this.handsAnalyzer.Result;
NormalizedHands neutralHands = this.skinColorHandsDetector.Tune(result);
this.handsDroneController.HandsInputEvaluator.Tune(neutralHands);
}
}
this.uiConnector?.SetComputerCameraImage(frame);
this.uiConnector?.SetEvaluatedHandsInput(moveCommand);
Interlocked.Exchange(ref this.isWebCameraFrameProcessorBusy, NO);
}
}
public HandsAnalyzerResultTests()
{
var histogramMock = new Mock <IHistogram>();
var colorMap = new ColorMap(histogramMock.Object, histogramMock.Object, histogramMock.Object);
this.handsColorMaps = new HandsColorMaps(colorMap, colorMap);
this.handsBackground = BgrImage.GetBlank(100, 100, Color.Red);
this.handsForeground = BgrImage.GetBlank(100, 100, Color.Green);
this.handsAnalyzerResult = new HandsAnalyzerResult(this.handsColorMaps, this.handsBackground, this.handsForeground);
}
private void SplitChannels()
{
var channels = BgrImage.Split();
_channelB = channels[0];
_channelG = channels[1];
_channelR = channels[2];
_isChannelSplit = true;
}
/// <summary>
/// Resets the <see cref="HandsAnalyzer"/> object.
/// </summary>
public void Reset()
{
this.leftHandColorMap = null;
this.rightHandColorMap = null;
this.leftHandAnalysisImage = null;
this.rightHandAnalysisImage = null;
this.State = HandsAnalyzerState.ReadyToAnalyzeLeft;
}
public BgrImageTests()
{
Assembly executingAssembly = Assembly.GetExecutingAssembly();
string resource = executingAssembly.GetManifestResourceNames()
.FirstOrDefault(r => r.Contains("colorful_houses.jpg"));
Stream resourceStream = executingAssembly.GetManifestResourceStream(resource);
this.bitmap = new Bitmap(resourceStream);
this.emguImage = new Image <Bgr, byte>(this.bitmap);
this.bgrImage = new BgrImage(this.emguImage);
}
/// <summary>
/// Detect hand based on color on the left and right third of the image.
/// </summary>
/// <param name="image">The image in which the hands should detected.</param>
/// <returns>The result of the hand detection.</returns>
public HandsDetectorResult DetectHands(BgrImage image)
{
if (image == null)
{
throw new ArgumentNullException("The image sent for hand detection was null.");
}
BgrImage imagePreProcessed = this.PreProcess(image);
return(this.DetectHandsInternally(imagePreProcessed));
}
public void DifferentHandsInputsAfterEachOther_BothSent()
{
var skinColorHandsDetectorMock = new Mock <ISkinColorHandsDetector>();
skinColorHandsDetectorMock.Setup(schd => schd.Tuned).Returns(true);
var firstLeft = new NormalizedHand(0.15, 0.2, 0.18);
var firstRight = new NormalizedHand(0.85, 0.2, 0.18);
var firstHands = new NormalizedHands(firstLeft, firstRight);
var firstHandsDetectorResult = new HandsDetectorResult(firstHands, null);
var secondLeft = new NormalizedHand(0.15, 0.8, 0.18);
var secondRight = new NormalizedHand(0.85, 0.7, 0.18);
var secondHands = new NormalizedHands(secondLeft, secondRight);
var secondandsDetectorResult = new HandsDetectorResult(secondHands, null);
skinColorHandsDetectorMock.SetupSequence(schd => schd.DetectHands(It.IsAny <BgrImage>()))
.Returns(firstHandsDetectorResult)
.Returns(secondandsDetectorResult);
var keyInputEvaluator = new GeneralDroneKeyInputEvaluator();
var handsInputEvaluator = new DroneControllerHandsInputEvaluator();
var neutralLeft = new NormalizedHand(0.15, 0.5, 0.18);
var neutralRight = new NormalizedHand(0.85, 0.5, 0.18);
var neutralHands = new NormalizedHands(neutralLeft, neutralRight);
handsInputEvaluator.Tune(neutralHands);
this.caduhdApp = new CaduhdApp(null, skinColorHandsDetectorMock.Object, this.tello, keyInputEvaluator, handsInputEvaluator);
this.caduhdApp.Input(BgrImage.GetBlank(1, 1, Color.Black));
this.caduhdApp.Input(BgrImage.GetBlank(1, 1, Color.Black));
this.udpClient.Client.ReceiveTimeout = 1000;
int counter = 0;
try
{
string response = this.udpClient.Receive(ref this.anyEndPoint).AsString();
Assert.Equal($"rc 0 0 {this.tello.Speed} 0", response);
counter++;
response = this.udpClient.Receive(ref this.anyEndPoint).AsString();
Assert.Equal($"rc 0 0 -{this.tello.Speed} 0", response);
counter++;
}
catch (SocketException)
{
}
this.tello.Dispose();
Assert.Equal(2, counter);
}
/// <summary>
/// Analyses the left hand.
/// </summary>
/// <param name="image">The image in which the left hand and the right hand background can be found.</param>
/// <param name="poi">The points of interest.</param>
public void AnalyzeLeft(BgrImage image, List <Point> poi)
{
if (this.State != HandsAnalyzerState.AnalyzingLeft)
{
throw new InvalidOperationException($"Hands analyzer has to be in {HandsAnalyzerState.AnalyzingLeft} state in order to analyze left hand.");
}
if (image == null)
{
throw new ArgumentNullException("The image used for left hand analysis was null.");
}
this.leftHandAnalysisImage = image.Copy();
this.leftHandColorMap = this.ExtractColorMap(image, poi);
}
private ColorMap ExtractColorMap(BgrImage image, List <Point> poi)
{
IHistogram blues = new Histogram(0, 255, 64);
IHistogram greens = new Histogram(0, 255, 64);
IHistogram reds = new Histogram(0, 255, 64);
foreach (Point point in poi)
{
BgrPixel pixel = image.GetPixel(point.X, point.Y);
blues.Insert(pixel.Blue);
greens.Insert(pixel.Green);
reds.Insert(pixel.Red);
}
return(new ColorMap(blues, greens, reds));
}
public SkinColorHandsDetectorTests()
{
int blue = 44;
int green = 87;
int red = 123;
this.skinColor = Color.FromArgb(red, green, blue);
this.handsBackground = BgrImage.GetBlank(WIDTH, HEIGHT, Color.White);
this.handsForeground = this.handsBackground.Copy();
var blueHistogramMock = new Mock <IHistogram>();
blueHistogramMock.Setup(bh => bh.Smallest).Returns(blue - PIXEL_TOLERANCE);
blueHistogramMock.Setup(bh => bh.Greatest).Returns(blue + PIXEL_TOLERANCE);
var greenHistogramMock = new Mock <IHistogram>();
greenHistogramMock.Setup(bh => bh.Smallest).Returns(green - PIXEL_TOLERANCE);
greenHistogramMock.Setup(bh => bh.Greatest).Returns(green + PIXEL_TOLERANCE);
var redHistogramMock = new Mock <IHistogram>();
redHistogramMock.Setup(bh => bh.Smallest).Returns(red - PIXEL_TOLERANCE);
redHistogramMock.Setup(bh => bh.Greatest).Returns(red + PIXEL_TOLERANCE);
var leftColorMap = new ColorMap(blueHistogramMock.Object,
greenHistogramMock.Object,
redHistogramMock.Object);
var rightColorMap = new ColorMap(blueHistogramMock.Object,
greenHistogramMock.Object,
redHistogramMock.Object);
var handsColorMaps = new HandsColorMaps(leftColorMap, rightColorMap);
var tuningMock = new Mock <IHandsDetectorTuning>();
tuningMock.Setup(t => t.HandsBackground).Returns(this.handsBackground);
tuningMock.Setup(t => t.HandsForeground).Returns(this.handsForeground);
tuningMock.Setup(t => t.HandsColorMaps).Returns(handsColorMaps);
this.handsDetector = new SkinColorHandsDetector();
this.handsDetector.Tune(tuningMock.Object);
}
public void HandsInputAnalyzerTuned_HandsInput_MoveDownwards_HandsInputEvaluated_CorrectMoveCommandSent()
{
var skinColorHandsDetectorMock = new Mock <ISkinColorHandsDetector>();
skinColorHandsDetectorMock.Setup(schd => schd.Tuned).Returns(true);
var neutralLeft = new NormalizedHand(0.15, 0.5, 0.18);
var neutralRight = new NormalizedHand(0.85, 0.5, 0.18);
var neutralHands = new NormalizedHands(neutralLeft, neutralRight);
var left = new NormalizedHand(0.15, 0.8, 0.18);
var right = new NormalizedHand(0.85, 0.8, 0.18);
var hands = new NormalizedHands(left, right);
var handsDetectorResult = new HandsDetectorResult(hands, null);
skinColorHandsDetectorMock.Setup(schd => schd.DetectHands(It.IsAny <BgrImage>()))
.Returns(handsDetectorResult);
var keyInputEvaluator = new GeneralDroneKeyInputEvaluator();
var handsInputEvaluator = new DroneControllerHandsInputEvaluator();
handsInputEvaluator.Tune(neutralHands);
this.caduhdApp = new CaduhdApp(null, skinColorHandsDetectorMock.Object, this.tello, keyInputEvaluator, handsInputEvaluator);
string response = string.Empty;
this.caduhdApp.Input(BgrImage.GetBlank(1, 1, Color.Black));
try
{
response = this.udpClient.Receive(ref this.anyEndPoint).AsString();
}
catch (SocketException)
{
}
// it needs to be disposed of here
this.tello.Dispose();
Assert.Equal($"rc 0 0 -{this.tello.Speed} 0", response);
}
private bool CompareImagesPixelByPixel(Image <Bgr, byte> emguImage, BgrImage bgrImage)
{
if (emguImage.Width != bgrImage.Width || emguImage.Height != bgrImage.Height)
{
return(false);
}
for (int y = 0; y < emguImage.Height; y++)
{
for (int x = 0; x < bgrImage.Width; x++)
{
if (!this.BgrEqualsBgrPixel(emguImage[y, x], bgrImage.GetPixel(x, y)))
{
return(false);
}
}
}
return(true);
}
private bool CompareImagesPixelByPixel(BgrImage a, BgrImage b)
{
if (a.Width != b.Width || a.Height != b.Height)
{
return(false);
}
for (int y = 0; y < a.Height; y++)
{
for (int x = 0; x < b.Width; x++)
{
if (!this.AreBgrPixelsTheSame(a.GetPixel(x, y), b.GetPixel(x, y)))
{
return(false);
}
}
}
return(true);
}
public void Merge_BgrImagesMergedCorrectly()
{
BgrImage bgrImageUpsideDown = new BgrImage(this.emguImage.Flip(FlipType.Horizontal));
BgrImage imageMerged = bgrImageUpsideDown.Merge(this.bgrImage);
Assert.False(this.CompareImagesPixelByPixel(this.bgrImage, imageMerged));
int half = imageMerged.Width / 2;
bool success = true;
int row = 0;
int column = 0;
while (row < imageMerged.Height && success)
{
while (column < imageMerged.Width && success)
{
if (column < half)
{
if (!this.AreBgrPixelsTheSame(bgrImageUpsideDown.GetPixel(column, row), imageMerged.GetPixel(column, row)))
{
success = false;
}
}
else
{
if (!this.AreBgrPixelsTheSame(this.bgrImage.GetPixel(column, row), imageMerged.GetPixel(column, row)))
{
success = false;
}
}
column++;
}
column = 0;
row++;
}
Assert.True(success);
}
private NormalizedHand EvaluatePixels(BgrImage image, int startX, int width, ColorMap colorMap, NormalizedHand neutralHand)
{
HandBuilder handBuilder = new HandBuilder();
for (int y = 0; y < image.Height; y++)
{
for (int x = startX; x < startX + width; x++)
{
BgrPixel pixel = image.GetPixel(x, y);
if (this.IsBackground(pixel, x, y))
{
image.SetPixel(this.backgroundPixel, x, y);
}
else if (!colorMap.Satisfies(pixel))
{
image.SetPixel(this.unknownPixel, x, y);
}
else
{
image.SetPixel(this.skinPixel, x, y);
handBuilder.Append(x, y);
}
}
}
NormalizedHand detectedHand = new HandNormalizer(image.Width, image.Height).Normalize(handBuilder.Build());
if (this.Tuned && neutralHand != null && detectedHand.Weight < neutralHand.Weight / DIVISOR_TO_GET_VALID_HAND_WEIGHT_LOWER_BOUND_FROM_NEUTRAL_HAND_WEIGHT)
{
// after tuning the neutral hands are cached
// if the weight of the hand is under a threshold
// (which probably means that there is no hand on the image but noise)
// then we're going to use the cached neutral hand instead of the freshly detected (wrong) one
detectedHand = new NormalizedHand(neutralHand.X, neutralHand.Y, neutralHand.Weight);
}
return(detectedHand);
}
private HandsDetectorResult DetectHandsInternally(BgrImage image)
{
int handAreaWidth = image.Width / DIVISOR_TO_GET_HAND_AREA_WIDTH_FROM_IMAGE_WIDTH;
NormalizedHand left = this.EvaluatePixels(image, 0, handAreaWidth, this.tuning.HandsColorMaps.Left, this.neutralHands?.Left);
NormalizedHand right = this.EvaluatePixels(image, image.Width - handAreaWidth, handAreaWidth, this.tuning.HandsColorMaps.Right, this.neutralHands?.Right);
int radius = image.Width * image.Height / DIVISOR_TO_GET_HAND_MARKER_CIRCLE_RADIUS_FROM_IMAGE_SIZE;
if (this.Tuned && this.neutralHands != null)
{
image.DrawLineSegment(left.X, left.Y, this.neutralHands.Center.X, this.neutralHands.Center.Y, Color.Yellow, radius);
image.DrawLineSegment(this.neutralHands.Center.X, this.neutralHands.Center.Y, right.X, right.Y, Color.Yellow, radius);
}
image.DrawCircle(left.X, left.Y, Color.Blue, radius, BgrImage.FILL_DRAWING);
image.DrawCircle(right.X, right.Y, Color.Blue, radius, BgrImage.FILL_DRAWING);
NormalizedHands normalizedHands = new NormalizedHands(left, right);
return(new HandsDetectorResult(normalizedHands, image));
}
/// <summary>
/// Initializes a new instance of the <see cref="DroneControllerHandsInputEvaluator"/> class.
/// </summary>
public DroneControllerHandsInputEvaluator()
{
this.TunerHands = new Dictionary <string, Dictionary <string, List <Point> > >();
this.TunerHands.Add("left", new Dictionary <string, List <Point> >());
this.TunerHands["left"].Add("poi", new List <Point>());
this.TunerHands["left"].Add("outline", new List <Point>());
this.TunerHands.Add("right", new Dictionary <string, List <Point> >());
this.TunerHands["right"].Add("poi", new List <Point>());
this.TunerHands["right"].Add("outline", new List <Point>());
var assembly = Assembly.GetExecutingAssembly();
var resources = assembly
.GetManifestResourceNames()
.Where(r => r.EndsWith("_neutral_hand.png"));
string resource = string.Empty;
BgrImage left;
BgrImage right;
resource = resources.Single(r => r.Contains("left_neutral_hand.png"));
using (var stream = assembly.GetManifestResourceStream(resource))
{
left = new BgrImage(new Bitmap(stream));
}
resource = resources.Single(r => r.Contains("right_neutral_hand.png"));
using (var stream = assembly.GetManifestResourceStream(resource))
{
right = new BgrImage(new Bitmap(stream));
}
for (int y = 0; y < left.Height; y++)
{
for (int x = 0; x < right.Width; x++)
{
var leftPixel = left.GetPixel(x, y);
var rightPixel = right.GetPixel(x, y);
if (!leftPixel.Equals(Color.White))
{
if (leftPixel.Equals(Color.Black))
{
this.TunerHands["left"]["outline"].Add(new Point(x, y));
}
else
{
this.TunerHands["left"]["poi"].Add(new Point(x, y));
}
}
if (!rightPixel.Equals(Color.White))
{
if (rightPixel.Equals(Color.Black))
{
this.TunerHands["right"]["outline"].Add(new Point(x, y));
}
else if (rightPixel.Equals(Color.Blue))
{
this.TunerHands["right"]["poi"].Add(new Point(x, y));
}
}
}
}
}
private BgrImage PreProcess(BgrImage image)
{
// no pre process at the moment
return(image);
}
/// <summary>
/// Initializes a new instance of the <see cref="HandsDetectorResult"/> class.
/// </summary>
/// <param name="hands">The detected hands.</param>
/// <param name="image">The image of the hand detection.</param>
public HandsDetectorResult(NormalizedHands hands, BgrImage image)
{
this.Hands = hands;
this.Image = image;
}
public void Resize(double scale)
{
BgrImage = BgrImage.Resize(scale, Inter.Linear);
GrayImage = GrayImage.Resize(scale, Inter.Linear);
BitMapImage = BitMapConverter.Convert(BgrImage.ToBitmap());
}
/// <summary>
/// Initializes a new instance of the <see cref="NewDroneCameraFrameEventArgs"/> class.
/// </summary>
/// <param name="frame">Gets the new frame as <see cref="BgrImage"/>.</param>
public NewDroneCameraFrameEventArgs(BgrImage frame)
{
this.Frame = frame;
}
/// <summary>
/// Initializes a new instance of the <see cref="HandsAnalyzerResult"/> class.
/// </summary>
/// <param name="handsColorMaps">The hands color maps.</param>
/// <param name="handsBackgrounds">The background of the hands.</param>
/// <param name="handsForeground">The foreground of the hands.</param>
public HandsAnalyzerResult(HandsColorMaps handsColorMaps, BgrImage handsBackgrounds, BgrImage handsForeground)
{
this.HandsColorMaps = handsColorMaps;
this.HandsBackground = handsBackgrounds;
this.HandsForeground = handsForeground;
}
public HandsDetectorResultTests()
{
this.hands = new NormalizedHands(new NormalizedHand(), new NormalizedHand());
this.image = BgrImage.GetBlank(100, 100, Color.White);
this.handsDetectorResult = new HandsDetectorResult(this.hands, this.image);
}
public Image(IInputArray inputImage)
{
BgrImage = inputImage.GetInputArray().GetMat().ToImage <Bgr, byte>();
GrayImage = inputImage.GetInputArray().GetMat().ToImage <Gray, byte>();
BitMapImage = BitMapConverter.Convert(BgrImage.ToBitmap());
}